carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chenliang...@apache.org
Subject [43/52] [partial] incubator-carbondata git commit: Renamed packages to org.apache.carbondata and fixed errors
Date Mon, 15 Aug 2016 07:09:28 GMT
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java
new file mode 100644
index 0000000..80a39f1
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java
@@ -0,0 +1,425 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.path;
+
+import java.io.File;
+
+import org.apache.carbondata.core.carbon.CarbonTableIdentifier;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFileFilter;
+import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+
+import static org.apache.carbondata.core.constants.CarbonCommonConstants.INVALID_SEGMENT_ID;
+
+import org.apache.hadoop.fs.Path;
+
+
+/**
+ * Helps to get Table content paths.
+ */
+public class CarbonTablePath extends Path {
+
+  protected static final String METADATA_DIR = "Metadata";
+  protected static final String DICTIONARY_EXT = ".dict";
+  protected static final String DICTIONARY_META_EXT = ".dictmeta";
+  protected static final String SORT_INDEX_EXT = ".sortindex";
+  protected static final String SCHEMA_FILE = "schema";
+  protected static final String TABLE_STATUS_FILE = "tablestatus";
+  protected static final String FACT_DIR = "Fact";
+  protected static final String AGGREGATE_TABLE_PREFIX = "Agg";
+  protected static final String SEGMENT_PREFIX = "Segment_";
+  protected static final String PARTITION_PREFIX = "Part";
+  protected static final String CARBON_DATA_EXT = ".carbondata";
+  protected static final String DATA_PART_PREFIX = "part";
+  protected static final String INDEX_FILE_EXT = ".carbonindex";
+
+  protected String tablePath;
+  protected CarbonTableIdentifier carbonTableIdentifier;
+
+  /**
+   *
+   * @param carbonTableIdentifier
+   * @param tablePathString
+   */
+  public CarbonTablePath(CarbonTableIdentifier carbonTableIdentifier, String tablePathString) {
+    super(tablePathString);
+    this.carbonTableIdentifier = carbonTableIdentifier;
+    this.tablePath = tablePathString;
+  }
+
+  /**
+   * The method returns the folder path containing the carbon file.
+   *
+   * @param carbonFilePath
+   */
+  public static String getFolderContainingFile(String carbonFilePath) {
+    return carbonFilePath.substring(0, carbonFilePath.lastIndexOf(File.separator));
+  }
+
+  /**
+   * @param columnId unique column identifier
+   * @return name of dictionary file
+   */
+  public static String getDictionaryFileName(String columnId) {
+    return columnId + DICTIONARY_EXT;
+  }
+
+  /**
+   * whether carbonFile is dictionary file or not
+   *
+   * @param carbonFile
+   * @return
+   */
+  public static Boolean isDictionaryFile(CarbonFile carbonFile) {
+    return (!carbonFile.isDirectory()) && (carbonFile.getName().endsWith(DICTIONARY_EXT));
+  }
+
+  /**
+   * check if it is carbon data file matching extension
+   *
+   * @param fileNameWithPath
+   * @return boolean
+   */
+  public static boolean isCarbonDataFile(String fileNameWithPath) {
+    int pos = fileNameWithPath.lastIndexOf('.');
+    if (pos != -1) {
+      return fileNameWithPath.substring(pos).startsWith(CARBON_DATA_EXT);
+    }
+    return false;
+  }
+
+  /**
+   * check if it is carbon index file matching extension
+   *
+   * @param fileNameWithPath
+   * @return boolean
+   */
+  public static boolean isCarbonIndexFile(String fileNameWithPath) {
+    int pos = fileNameWithPath.lastIndexOf('.');
+    if (pos != -1) {
+      return fileNameWithPath.substring(pos).startsWith(INDEX_FILE_EXT);
+    }
+    return false;
+  }
+
+  /**
+   * gets table path
+   */
+  public String getPath() {
+    return tablePath;
+  }
+
+  /**
+   * @param columnId unique column identifier
+   * @return absolute path of dictionary file
+   */
+  public String getDictionaryFilePath(String columnId) {
+    return getMetaDataDir() + File.separator + getDictionaryFileName(columnId);
+  }
+
+  /**
+   * @return it return relative directory
+   */
+  public String getRelativeDictionaryDirectory() {
+    return carbonTableIdentifier.getDatabaseName() + File.separator + carbonTableIdentifier
+        .getTableName();
+  }
+
+  /**
+   * This method will return the metadata directory location for a table
+   *
+   * @return
+   */
+  public String getMetadataDirectoryPath() {
+    return getMetaDataDir();
+  }
+
+  /**
+   * @param columnId unique column identifier
+   * @return absolute path of dictionary meta file
+   */
+  public String getDictionaryMetaFilePath(String columnId) {
+    return getMetaDataDir() + File.separator + columnId + DICTIONARY_META_EXT;
+  }
+
+  /**
+   * @param columnId unique column identifier
+   * @return absolute path of sort index file
+   */
+  public String getSortIndexFilePath(String columnId) {
+    return getMetaDataDir() + File.separator + columnId + SORT_INDEX_EXT;
+  }
+
+  /**
+   *
+   * @param columnId
+   * @param dictOffset
+   * @return absolute path of sortindex with appeneded dictionary offset
+   */
+  public String getSortIndexFilePath(String columnId, long dictOffset) {
+    return getMetaDataDir() + File.separator + columnId + "_" + dictOffset + SORT_INDEX_EXT;
+  }
+
+  /**
+   * @return absolute path of schema file
+   */
+  public String getSchemaFilePath() {
+    return getMetaDataDir() + File.separator + SCHEMA_FILE;
+  }
+
+  /**
+   * @return absolute path of table status file
+   */
+  public String getTableStatusFilePath() {
+    return getMetaDataDir() + File.separator + TABLE_STATUS_FILE;
+  }
+
+  /**
+   * Gets absolute path of data file
+   *
+   * @param partitionId         unique partition identifier
+   * @param segmentId           unique partition identifier
+   * @param filePartNo          data file part number
+   * @param factUpdateTimeStamp unique identifier to identify an update
+   * @return absolute path of data file stored in carbon data format
+   */
+  public String getCarbonDataFilePath(String partitionId, String segmentId, Integer filePartNo,
+      Integer taskNo, String factUpdateTimeStamp) {
+    return getSegmentDir(partitionId, segmentId) + File.separator + getCarbonDataFileName(
+        filePartNo, taskNo, factUpdateTimeStamp);
+  }
+
+  /**
+   * Below method will be used to get the index file present in the segment folder
+   * based on task id
+   *
+   * @param taskId      task id of the file
+   * @param partitionId partition number
+   * @param segmentId   segment number
+   * @return full qualified carbon index path
+   */
+  public String getCarbonIndexFilePath(final String taskId, final String partitionId,
+      final String segmentId) {
+    String segmentDir = getSegmentDir(partitionId, segmentId);
+    CarbonFile carbonFile =
+        FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir));
+
+    CarbonFile[] files = carbonFile.listFiles(new CarbonFileFilter() {
+      @Override public boolean accept(CarbonFile file) {
+        return file.getName().startsWith(taskId) && file.getName().endsWith(INDEX_FILE_EXT);
+      }
+    });
+    return files[0].getAbsolutePath();
+  }
+
+  /**
+   * Gets absolute path of data file
+   *
+   * @param partitionId unique partition identifier
+   * @param segmentId   unique partition identifier
+   * @return absolute path of data file stored in carbon data format
+   */
+  public String getCarbonDataDirectoryPath(String partitionId, String segmentId) {
+    return getSegmentDir(partitionId, segmentId);
+  }
+
+  /**
+   * Gets absolute path of data file of given aggregate table
+   *
+   * @param aggTableID          unique aggregate table identifier
+   * @param partitionId         unique partition identifier
+   * @param segmentId           unique partition identifier
+   * @param filePartNo          data file part number
+   * @param factUpdateTimeStamp unique identifier to identify an update
+   * @return absolute path of data file stored in carbon data format
+   */
+  public String getCarbonAggDataFilePath(String aggTableID, String partitionId, String segmentId,
+      Integer filePartNo, Integer taskNo, String factUpdateTimeStamp) {
+    return getAggSegmentDir(aggTableID, partitionId, segmentId) + File.separator
+        + getCarbonDataFileName(filePartNo, taskNo, factUpdateTimeStamp);
+  }
+
+  /**
+   * Gets data file name only with out path
+   *
+   * @param filePartNo          data file part number
+   * @param taskNo              task identifier
+   * @param factUpdateTimeStamp unique identifier to identify an update
+   * @return gets data file name only with out path
+   */
+  public String getCarbonDataFileName(Integer filePartNo, Integer taskNo,
+      String factUpdateTimeStamp) {
+    return DATA_PART_PREFIX + "-" + filePartNo + "-" + taskNo + "-" + factUpdateTimeStamp
+        + CARBON_DATA_EXT;
+  }
+
+  /**
+   * Below method will be used to get the carbon index filename
+   *
+   * @param taskNo               task number
+   * @param factUpdatedTimeStamp time stamp
+   * @return filename
+   */
+  public String getCarbonIndexFileName(int taskNo, String factUpdatedTimeStamp) {
+    return taskNo + "-" + factUpdatedTimeStamp + INDEX_FILE_EXT;
+  }
+
+  private String getSegmentDir(String partitionId, String segmentId) {
+    return getPartitionDir(partitionId) + File.separator + SEGMENT_PREFIX + segmentId;
+  }
+
+  public String getPartitionDir(String partitionId) {
+    return getFactDir() + File.separator + PARTITION_PREFIX + partitionId;
+  }
+
+  private String getAggSegmentDir(String aggTableID, String partitionId, String segmentId) {
+    return getAggPartitionDir(aggTableID, partitionId) + File.separator + SEGMENT_PREFIX
+        + segmentId;
+  }
+
+  private String getAggPartitionDir(String aggTableID, String partitionId) {
+    return getAggregateTableDir(aggTableID) + File.separator + PARTITION_PREFIX + partitionId;
+  }
+
+  private String getMetaDataDir() {
+    return tablePath + File.separator + METADATA_DIR;
+  }
+
+  public String getFactDir() {
+    return tablePath + File.separator + FACT_DIR;
+  }
+
+  private String getAggregateTableDir(String aggTableId) {
+    return tablePath + File.separator + AGGREGATE_TABLE_PREFIX + aggTableId;
+  }
+
+  @Override public boolean equals(Object o) {
+    if (!(o instanceof CarbonTablePath)) {
+      return false;
+    }
+    CarbonTablePath path = (CarbonTablePath) o;
+    return tablePath.equals(path.tablePath) && super.equals(o);
+  }
+
+  @Override public int hashCode() {
+    return super.hashCode() + tablePath.hashCode();
+  }
+
+  /**
+   * To manage data file name and composition
+   */
+  public static class DataFileUtil {
+
+    /**
+     * gets updated timestamp information from given carbon data file name
+     */
+    public static String getUpdateTimeStamp(String carbonDataFileName) {
+      // Get the file name from path
+      String fileName = getFileName(carbonDataFileName);
+      // + 1 for size of "-"
+      int firstDashPos = fileName.indexOf("-");
+      int secondDashPos = fileName.indexOf("-", firstDashPos + 1);
+      int startIndex = fileName.indexOf("-", secondDashPos + 1) + 1;
+      int endIndex = fileName.indexOf(".");
+      return fileName.substring(startIndex, endIndex);
+    }
+
+    /**
+     * gets file part number information from given carbon data file name
+     */
+    public static String getPartNo(String carbonDataFileName) {
+      // Get the file name from path
+      String fileName = getFileName(carbonDataFileName);
+      // + 1 for size of "-"
+      int startIndex = fileName.indexOf("-") + 1;
+      int endIndex = fileName.indexOf("-", startIndex);
+      return fileName.substring(startIndex, endIndex);
+    }
+
+    /**
+     * gets updated timestamp information from given carbon data file name
+     */
+    public static String getTaskNo(String carbonDataFileName) {
+      // Get the file name from path
+      String fileName = getFileName(carbonDataFileName);
+      // + 1 for size of "-"
+      int firstDashPos = fileName.indexOf("-");
+      int startIndex = fileName.indexOf("-", firstDashPos + 1) + 1;
+      int endIndex = fileName.indexOf("-", startIndex);
+      return fileName.substring(startIndex, endIndex);
+    }
+
+    /**
+     * Gets the file name from file path
+     */
+    private static String getFileName(String carbonDataFileName) {
+      int endIndex = carbonDataFileName.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR);
+      if (endIndex > -1) {
+        return carbonDataFileName.substring(endIndex + 1, carbonDataFileName.length());
+      } else {
+        return carbonDataFileName;
+      }
+    }
+  }
+
+  /**
+   * To manage data path and composition
+   */
+  public static class DataPathUtil {
+
+    /**
+     * gets segement id from given absolute data file path
+     */
+    public static String getSegmentId(String dataFileAbsolutePath) {
+      // find segment id from last of data file path
+      int endIndex = dataFileAbsolutePath.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR);
+      // + 1 for size of "/"
+      int startIndex =
+          dataFileAbsolutePath.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR, endIndex - 1) + 1;
+      String segmentDirStr = dataFileAbsolutePath.substring(startIndex, endIndex);
+      //identify id in segment_<id>
+      String[] segmentDirSplits = segmentDirStr.split("_");
+      try {
+        if (segmentDirSplits.length == 2) {
+          return segmentDirSplits[1];
+        }
+      } catch (Exception e) {
+        return INVALID_SEGMENT_ID;
+      }
+      return INVALID_SEGMENT_ID;
+    }
+  }
+
+  /**
+   * Below method will be used to get sort index file present in mentioned folder
+   *
+   * @param sortIndexDir directory where sort index file resides
+   * @param columnUniqueId   columnunique id
+   * @return sort index carbon files
+   */
+  public CarbonFile[] getSortIndexFiles(CarbonFile sortIndexDir, final String columnUniqueId) {
+    CarbonFile[] files = sortIndexDir.listFiles(new CarbonFileFilter() {
+      @Override public boolean accept(CarbonFile file) {
+        return file.getName().startsWith(columnUniqueId) && file.getName().endsWith(SORT_INDEX_EXT);
+      }
+    });
+    return files;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatistic.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatistic.java b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatistic.java
new file mode 100644
index 0000000..39198c8
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatistic.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.querystatistics;
+
+import java.io.Serializable;
+
+/**
+ * Wrapper class to maintain the query statistics for each phase of the query
+ */
+public class QueryStatistic implements Serializable {
+
+  /**
+   * serialization id
+   */
+  private static final long serialVersionUID = -5667106646135905848L;
+
+  /**
+   * statistic message
+   */
+  private String message;
+
+  /**
+   * total time take of the phase
+   */
+  private long timeTaken;
+
+  /**
+   * starttime of the phase
+   */
+  private long startTime;
+
+  public QueryStatistic() {
+    this.startTime = System.currentTimeMillis();
+  }
+
+  /**
+   * below method will be used to add the statistic
+   *
+   * @param message     Statistic message
+   * @param currentTime current time
+   */
+  public void addStatistics(String message, long currentTime) {
+    this.timeTaken = currentTime - startTime;
+    this.message = message;
+  }
+
+  /**
+   * Below method will be used to add fixed time statistic.
+   * For example total time taken for scan or result preparation
+   *
+   * @param message   statistic message
+   * @param timetaken
+   */
+  public void addFixedTimeStatistic(String message, long timetaken) {
+    this.timeTaken = timetaken;
+    this.message = message;
+  }
+
+  /**
+   * Below method will be used to get the statistic message, which will
+   * be used to log
+   *
+   * @param queryWithTaskId query with task id to append in the message
+   * @return statistic message
+   */
+  public String getStatistics(String queryWithTaskId) {
+    return message + " for the taskid : " + queryWithTaskId + " Is : " + timeTaken;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsRecorder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsRecorder.java b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsRecorder.java
new file mode 100644
index 0000000..ce12cae
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsRecorder.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.querystatistics;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+
+/**
+ * Class will be used to record and log the query statistics
+ */
+public class QueryStatisticsRecorder implements Serializable {
+
+  private static final LogService LOGGER =
+      LogServiceFactory.getLogService(QueryStatisticsRecorder.class.getName());
+  /**
+   * serialization version
+   */
+  private static final long serialVersionUID = -5719752001674467864L;
+
+  /**
+   * list for statistics to record time taken
+   * by each phase of the query for example aggregation
+   * scanning,block loading time etc.
+   */
+  private List<QueryStatistic> queryStatistics;
+
+  /**
+   * query with taskd
+   */
+  private String queryIWthTask;
+
+  public QueryStatisticsRecorder(String queryId) {
+    queryStatistics = new ArrayList<QueryStatistic>();
+    this.queryIWthTask = queryId;
+  }
+
+  /**
+   * Below method will be used to add the statistics
+   *
+   * @param statistic
+   */
+  public synchronized void recordStatistics(QueryStatistic statistic) {
+    queryStatistics.add(statistic);
+  }
+
+  /**
+   * Below method will be used to log the statistic
+   */
+  public void logStatistics() {
+    for (QueryStatistic statistic : queryStatistics) {
+      LOGGER.statistic(statistic.getStatistics(queryIWthTask));
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
new file mode 100644
index 0000000..4fa77ba
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -0,0 +1,892 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.constants;
+
+public final class CarbonCommonConstants {
+  /**
+   * integer size in bytes
+   */
+  public static final int INT_SIZE_IN_BYTE = 4;
+  /**
+   * short size in bytes
+   */
+  public static final int SHORT_SIZE_IN_BYTE = 2;
+  /**
+   * DOUBLE size in bytes
+   */
+  public static final int DOUBLE_SIZE_IN_BYTE = 8;
+  /**
+   * LONG size in bytes
+   */
+  public static final int LONG_SIZE_IN_BYTE = 8;
+  /**
+   * byte to KB conversion factor
+   */
+  public static final int BYTE_TO_KB_CONVERSION_FACTOR = 1024;
+  /**
+   * BYTE_ENCODING
+   */
+  public static final String BYTE_ENCODING = "ISO-8859-1";
+  /**
+   * measure meta data file name
+   */
+  public static final String MEASURE_METADATA_FILE_NAME = "/msrMetaData_";
+  /**
+   * location of the carbon member, hierarchy and fact files
+   */
+  public static final String STORE_LOCATION = "carbon.storelocation";
+  /**
+   * blocklet size in carbon file
+   */
+  public static final String BLOCKLET_SIZE = "carbon.blocklet.size";
+  /**
+   * TODO: max number of blocklets written in a single file?
+   */
+  public static final String MAX_FILE_SIZE = "carbon.max.file.size";
+  /**
+   * Number of cores to be used
+   */
+  public static final String NUM_CORES = "carbon.number.of.cores";
+  /**
+   * carbon sort size
+   */
+  public static final String SORT_SIZE = "carbon.sort.size";
+  /**
+   * default location of the carbon member, hierarchy and fact files
+   */
+  public static final String STORE_LOCATION_DEFAULT_VAL = "../carbon.store";
+  /**
+   * the folder name of kettle home path
+   */
+  public static final String KETTLE_HOME_NAME = "carbonplugins";
+  /**
+   * CARDINALITY_INCREMENT_DEFAULT_VALUE
+   */
+  public static final int CARDINALITY_INCREMENT_VALUE_DEFAULT_VAL = 10;
+  /**
+   * default blocklet size
+   */
+  public static final String BLOCKLET_SIZE_DEFAULT_VAL = "120000";
+  /**
+   * min blocklet size
+   */
+  public static final int BLOCKLET_SIZE_MIN_VAL = 50;
+  /**
+   * max blocklet size
+   */
+  public static final int BLOCKLET_SIZE_MAX_VAL = 12000000;
+  /**
+   * TODO: default value of max number of blocklet written in a single file?
+   */
+  public static final String MAX_FILE_SIZE_DEFAULT_VAL = "1024";
+  /**
+   * TODO: min value of max number of blocklets written in a single file?
+   */
+  public static final int MAX_FILE_SIZE_DEFAULT_VAL_MIN_VAL = 1;
+  /**
+   * max allowed block size for a file. If block size is greater than this value
+   * then the value is reset to default block size for a file
+   */
+  public static final int MAX_FILE_SIZE_DEFAULT_VAL_MAX_VAL = 2048;
+  /**
+   * default value of number of cores to be used
+   */
+  public static final String NUM_CORES_DEFAULT_VAL = "2";
+  /**
+   * min value of number of cores to be used
+   */
+  public static final int NUM_CORES_MIN_VAL = 1;
+  /**
+   * max value of number of cores to be used
+   */
+  public static final int NUM_CORES_MAX_VAL = 32;
+  /**
+   * default carbon sort size
+   */
+  public static final String SORT_SIZE_DEFAULT_VAL = "100000";
+  /**
+   * min carbon sort size
+   */
+  public static final int SORT_SIZE_MIN_VAL = 1000;
+  /**
+   * carbon properties file path
+   */
+  public static final String CARBON_PROPERTIES_FILE_PATH = "../../../conf/carbon.properties";
+  /**
+   * CARBON_DDL_BASE_HDFS_URL
+   */
+  public static final String CARBON_DDL_BASE_HDFS_URL = "carbon.ddl.base.hdfs.url";
+  /**
+   * Slice Meta data file.
+   */
+  public static final String SLICE_METADATA_FILENAME = "sliceMetaData";
+  /**
+   * Load Folder Name
+   */
+  public static final String LOAD_FOLDER = "Segment_";
+  /**
+   * RESTructure Folder
+   */
+  public static final String RESTRUCTRE_FOLDER = "RS_";
+  /**
+   * BYTEBUFFER_SIZE
+   */
+
+  public static final int BYTEBUFFER_SIZE = 24 * 1024;
+  /**
+   * Average constant
+   */
+  public static final String AVERAGE = "avg";
+  /**
+   * Count constant
+   */
+  public static final String COUNT = "count";
+  /**
+   * Count constant
+   */
+  public static final String COUNT_STAR = "countstar";
+  /**
+   * Max constant
+   */
+  public static final String MAX = "max";
+  /**
+   * Min constant
+   */
+  public static final String MIN = "min";
+  /**
+   * distinct count
+   */
+  public static final String DISTINCT_COUNT = "distinct-count";
+  /**
+   * CUSTOM
+   */
+  public static final String CUSTOM = "custom";
+  /**
+   * SUM
+   */
+  public static final String SUM = "sum";
+  /**
+   * DUMMY aggregation function
+   */
+  public static final String DUMMY = "dummy";
+  /**
+   * MEMBER_DEFAULT_VAL
+   */
+  public static final String MEMBER_DEFAULT_VAL = "@NU#LL$!";
+  /**
+   * BLANK_LINE_FLAG
+   */
+  public static final String BLANK_LINE_FLAG = "@NU#LL$!BLANKLINE";
+  /**
+   * FILE STATUS IN-PROGRESS
+   */
+  public static final String FILE_INPROGRESS_STATUS = ".inprogress";
+  /**
+   * CARBON_BADRECORDS_LOCATION
+   */
+  public static final String CARBON_BADRECORDS_LOC = "carbon.badRecords.location";
+  /**
+   * CARBON_BADRECORDS_LOCATION_DEFAULT
+   */
+  public static final String CARBON_BADRECORDS_LOC_DEFAULT_VAL =
+      "../unibi-solutions/system/carbon/badRecords";
+  /**
+   * HIERARCHY_FILE_EXTENSION
+   */
+  public static final String HIERARCHY_FILE_EXTENSION = ".hierarchy";
+  /**
+   * SORT_TEMP_FILE_LOCATION
+   */
+  public static final String SORT_TEMP_FILE_LOCATION = "sortrowtmp";
+  /**
+   * CARBON_RESULT_SIZE_DEFAULT
+   */
+  public static final String LEVEL_FILE_EXTENSION = ".level";
+  /**
+   * FACT_FILE_EXT
+   */
+  public static final String FACT_FILE_EXT = ".carbondata";
+  /**
+   * MEASUREMETADATA_FILE_EXT
+   */
+  public static final String MEASUREMETADATA_FILE_EXT = ".msrmetadata";
+  /**
+   * GRAPH_ROWSET_SIZE
+   */
+  public static final String GRAPH_ROWSET_SIZE = "carbon.graph.rowset.size";
+  /**
+   * GRAPH_ROWSET_SIZE_DEFAULT
+   */
+  public static final String GRAPH_ROWSET_SIZE_DEFAULT = "500";
+  /**
+   * Comment for <code>TYPE_MYSQL</code>
+   */
+  public static final String TYPE_MYSQL = "MYSQL";
+  /**
+   * Comment for <code>TYPE_MSSQL</code>
+   */
+  public static final String TYPE_MSSQL = "MSSQL";
+  /**
+   * Comment for <code>TYPE_ORACLE</code>
+   */
+  public static final String TYPE_ORACLE = "ORACLE";
+  /**
+   * Comment for <code>TYPE_SYBASE</code>
+   */
+  public static final String TYPE_SYBASE = "SYBASE";
+  /**
+   * SORT_INTERMEDIATE_FILES_LIMIT
+   */
+  public static final String SORT_INTERMEDIATE_FILES_LIMIT = "carbon.sort.intermediate.files.limit";
+  /**
+   * SORT_INTERMEDIATE_FILES_LIMIT_DEFAULT_VALUE
+   */
+  public static final String SORT_INTERMEDIATE_FILES_LIMIT_DEFAULT_VALUE = "20";
+  /**
+   * MERGERD_EXTENSION
+   */
+  public static final String MERGERD_EXTENSION = ".merge";
+  /**
+   * SORT_FILE_BUFFER_SIZE
+   */
+  public static final String SORT_FILE_BUFFER_SIZE = "carbon.sort.file.buffer.size";
+  /**
+   * no.of records after which counter to be printed
+   */
+  public static final String DATA_LOAD_LOG_COUNTER = "carbon.load.log.counter";
+  /**
+   * DATA_LOAD_LOG_COUNTER_DEFAULT_COUNTER
+   */
+  public static final String DATA_LOAD_LOG_COUNTER_DEFAULT_COUNTER = "500000";
+  /**
+   * SORT_FILE_WRITE_BUFFER_SIZE
+   */
+  public static final String CARBON_SORT_FILE_WRITE_BUFFER_SIZE =
+      "carbon.sort.file.write.buffer.size";
+  /**
+   * SORT_FILE_WRITE_BUFFER_SIZE_DEFAULT_VALUE
+   */
+  public static final String CARBON_SORT_FILE_WRITE_BUFFER_SIZE_DEFAULT_VALUE = "50000";
+  /**
+   * Number of cores to be used while loading
+   */
+  public static final String NUM_CORES_LOADING = "carbon.number.of.cores.while.loading";
+  /**
+   * Number of cores to be used while compacting
+   */
+  public static final String NUM_CORES_COMPACTING = "carbon.number.of.cores.while.compacting";
+  /**
+   * Number of cores to be used for block sort
+   */
+  public static final String NUM_CORES_BLOCK_SORT = "carbon.number.of.cores.block.sort";
+  /**
+   * Default value of number of cores to be used for block sort
+   */
+  public static final String NUM_CORES_BLOCK_SORT_DEFAULT_VAL = "7";
+  /**
+   * Max value of number of cores to be used for block sort
+   */
+  public static final int NUM_CORES_BLOCK_SORT_MAX_VAL = 12;
+  /**
+   * Min value of number of cores to be used for block sort
+   */
+  public static final int NUM_CORES_BLOCK_SORT_MIN_VAL = 1;
+  /**
+   * CSV_READ_BUFFER_SIZE
+   */
+  public static final String CSV_READ_BUFFER_SIZE = "carbon.csv.read.buffersize.byte";
+  /**
+   * CSV_READ_BUFFER_SIZE
+   */
+  public static final String CSV_READ_BUFFER_SIZE_DEFAULT = "50000";
+  /**
+   * CSV_READ_COPIES
+   */
+  public static final String DEFAULT_NUMBER_CORES = "2";
+  /**
+   * CSV_FILE_EXTENSION
+   */
+  public static final String CSV_FILE_EXTENSION = ".csv";
+  /**
+   * COLON_SPC_CHARACTER
+   */
+  public static final String COLON_SPC_CHARACTER = ":!@#COLON#@!:";
+  /**
+   * HASH_SPC_CHARATER
+   */
+  public static final String HASH_SPC_CHARACTER = "#!@:HASH:@!#";
+  /**
+   * SEMICOLON_SPC_CHARATER
+   */
+  public static final String SEMICOLON_SPC_CHARACTER = ";#!@:SEMIC:@!#;";
+  /**
+   * AMPERSAND_SPC_CHARATER
+   */
+  public static final String AMPERSAND_SPC_CHARACTER = "&#!@:AMPER:@!#&";
+  /**
+   * ATTHERATE_SPC_CHARATER
+   */
+  public static final String COMA_SPC_CHARACTER = ",#!:COMA:!#,";
+  /**
+   * HYPHEN_SPC_CHARACTER
+   */
+  public static final String HYPHEN_SPC_CHARACTER = "-#!:HYPHEN:!#-";
+  /**
+   * CARBON_DECIMAL_POINTERS_DEFAULT
+   */
+  public static final byte CARBON_DECIMAL_POINTERS_DEFAULT = 5;
+  /**
+   * SORT_TEMP_FILE_EXT
+   */
+  public static final String SORT_TEMP_FILE_EXT = ".sorttemp";
+  /**
+   * CARBON_MERGE_SORT_READER_THREAD
+   */
+  public static final String CARBON_MERGE_SORT_READER_THREAD = "carbon.merge.sort.reader.thread";
+  /**
+   * CARBON_MERGE_SORT_READER_THREAD_DEFAULTVALUE
+   */
+  public static final String CARBON_MERGE_SORT_READER_THREAD_DEFAULTVALUE = "3";
+  /**
+   * IS_SORT_TEMP_FILE_COMPRESSION_ENABLED
+   */
+  public static final String IS_SORT_TEMP_FILE_COMPRESSION_ENABLED =
+      "carbon.is.sort.temp.file.compression.enabled";
+  /**
+   * IS_SORT_TEMP_FILE_COMPRESSION_ENABLED_DEFAULTVALUE
+   */
+  public static final String IS_SORT_TEMP_FILE_COMPRESSION_ENABLED_DEFAULTVALUE = "false";
+  /**
+   * SORT_TEMP_FILE_NO_OF_RECORDS_FOR_COMPRESSION
+   */
+  public static final String SORT_TEMP_FILE_NO_OF_RECORDS_FOR_COMPRESSION =
+      "carbon.sort.temp.file.no.of.records.for.compression";
+  /**
+   * SORT_TEMP_FILE_NO_OF_RECORD_FOR_COMPRESSION_DEFAULTVALUE
+   */
+  public static final String SORT_TEMP_FILE_NO_OF_RECORD_FOR_COMPRESSION_DEFAULTVALUE = "50";
+  /**
+   * DEFAULT_COLLECTION_SIZE
+   */
+  public static final int DEFAULT_COLLECTION_SIZE = 16;
+  /**
+   * CARBON_TIMESTAMP_DEFAULT_FORMAT
+   */
+  public static final String CARBON_TIMESTAMP_DEFAULT_FORMAT = "yyyy-MM-dd HH:mm:ss";
+  /**
+   * CARBON_TIMESTAMP_DEFAULT_FORMAT
+   */
+  public static final String CARBON_TIMESTAMP_FORMAT = "carbon.timestamp.format";
+  /**
+   * STORE_LOCATION_HDFS
+   */
+  public static final String STORE_LOCATION_HDFS = "carbon.storelocation.hdfs";
+  /**
+   * STORE_LOCATION_TEMP_PATH
+   */
+  public static final String STORE_LOCATION_TEMP_PATH = "carbon.tempstore.location";
+  /**
+   * IS_COLUMNAR_STORAGE_DEFAULTVALUE
+   */
+  public static final String IS_COLUMNAR_STORAGE_DEFAULTVALUE = "true";
+  /**
+   * DIMENSION_SPLIT_VALUE_IN_COLUMNAR_DEFAULTVALUE
+   */
+  public static final String DIMENSION_SPLIT_VALUE_IN_COLUMNAR_DEFAULTVALUE = "1";
+  /**
+   * IS_FULLY_FILLED_BITS_DEFAULT_VALUE
+   */
+  public static final String IS_FULLY_FILLED_BITS_DEFAULT_VALUE = "true";
+  /**
+   * IS_INT_BASED_INDEXER
+   */
+  public static final String AGGREAGATE_COLUMNAR_KEY_BLOCK = "aggregate.columnar.keyblock";
+  /**
+   * IS_INT_BASED_INDEXER_DEFAULTVALUE
+   */
+  public static final String AGGREAGATE_COLUMNAR_KEY_BLOCK_DEFAULTVALUE = "true";
+  /**
+   * TIME_STAT_UTIL_TYPE
+   */
+  public static final String ENABLE_DATA_LOADING_STATISTICS = "enable.data.loading.statistics";
+  /**
+   * TIME_STAT_UTIL_TYPE_DEFAULT
+   */
+  public static final String ENABLE_DATA_LOADING_STATISTICS_DEFAULT = "false";
+  /**
+   * IS_INT_BASED_INDEXER
+   */
+  public static final String HIGH_CARDINALITY_VALUE = "high.cardinality.value";
+  /**
+   * IS_INT_BASED_INDEXER_DEFAULTVALUE
+   */
+  public static final String HIGH_CARDINALITY_VALUE_DEFAULTVALUE = "100000";
+  /**
+   * CONSTANT_SIZE_TEN
+   */
+  public static final int CONSTANT_SIZE_TEN = 10;
+  /**
+   * LEVEL_METADATA_FILE
+   */
+  public static final String LEVEL_METADATA_FILE = "levelmetadata_";
+  public static final String ENABLE_BASE64_ENCODING = "enable.base64.encoding";
+  public static final String ENABLE_BASE64_ENCODING_DEFAULT = "false";
+  /**
+   * LOAD_STATUS SUCCESS
+   */
+  public static final String STORE_LOADSTATUS_SUCCESS = "Success";
+  /**
+   * LOAD_STATUS FAILURE
+   */
+  public static final String STORE_LOADSTATUS_FAILURE = "Failure";
+  /**
+   * LOAD_STATUS PARTIAL_SUCCESS
+   */
+  public static final String STORE_LOADSTATUS_PARTIAL_SUCCESS = "Partial Success";
+  /**
+   * LOAD_STATUS
+   */
+  public static final String CARBON_METADATA_EXTENSION = ".metadata";
+  /**
+   * LOAD_STATUS
+   */
+  public static final String CARBON_DEFAULT_STREAM_ENCODEFORMAT = "UTF-8";
+  /**
+   * AGGREGATE_TABLE_START_TAG
+   */
+  public static final String AGGREGATE_TABLE_START_TAG = "agg";
+  /**
+   * COMMA
+   */
+  public static final String COMMA = ",";
+  /**
+   * UNDERSCORE
+   */
+  public static final String UNDERSCORE = "_";
+  /**
+   * POINT
+   */
+  public static final String POINT = ".";
+  /**
+   * File separator
+   */
+  public static final String FILE_SEPARATOR = "/";
+  /**
+   * MAX_QUERY_EXECUTION_TIME
+   */
+  public static final String MAX_QUERY_EXECUTION_TIME = "max.query.execution.time";
+  /**
+   * CARBON_TIMESTAMP
+   */
+  public static final String CARBON_TIMESTAMP = "dd-MM-yyyy HH:mm:ss";
+  /**
+   * METADATA_LOCK
+   */
+  public static final String METADATA_LOCK = "meta.lock";
+  /**
+   * NUMBER_OF_TRIES_FOR_LOAD_METADATA_LOCK
+   */
+  public static final int NUMBER_OF_TRIES_FOR_LOAD_METADATA_LOCK_DEFAULT = 3;
+  /**
+   * MAX_TIMEOUT_FOR_LOAD_METADATA_LOCK
+   */
+  public static final int MAX_TIMEOUT_FOR_LOAD_METADATA_LOCK_DEFAULT = 5;
+  /**
+   * NUMBER_OF_TRIES_FOR_LOAD_METADATA_LOCK
+   */
+  public static final String NUMBER_OF_TRIES_FOR_LOAD_METADATA_LOCK =
+      "carbon.load.metadata.lock.retries";
+  /**
+   * MAX_TIMEOUT_FOR_LOAD_METADATA_LOCK
+   */
+  public static final String MAX_TIMEOUT_FOR_LOAD_METADATA_LOCK =
+      "carbon.load.metadata.lock.retry.timeout.sec";
+  /**
+   * MARKED_FOR_DELETION
+   */
+  public static final String MARKED_FOR_DELETE = "Marked for Delete";
+  public static final String MARKED_FOR_UPDATE = "Marked for Update";
+  public static final String STRING_TYPE = "StringType";
+  public static final String INTEGER_TYPE = "IntegerType";
+  public static final String LONG_TYPE = "LongType";
+  public static final String DOUBLE_TYPE = "DoubleType";
+  public static final String FLOAT_TYPE = "FloatType";
+  public static final String DATE_TYPE = "DateType";
+  public static final String BOOLEAN_TYPE = "BooleanType";
+  public static final String TIMESTAMP_TYPE = "TimestampType";
+  public static final String BYTE_TYPE = "ByteType";
+  public static final String SHORT_TYPE = "ShortType";
+  public static final String BINARY_TYPE = "BinaryType";
+  public static final String DECIMAL_TYPE = "DecimalType";
+  public static final String STRING = "String";
+  public static final String COLUMNAR = "columnar";
+
+  public static final String INTEGER = "Integer";
+  public static final String SHORT = "Short";
+  public static final String NUMERIC = "Numeric";
+  public static final String TIMESTAMP = "Timestamp";
+  public static final String ARRAY = "ARRAY";
+  public static final String STRUCT = "STRUCT";
+  public static final String INCLUDE = "include";
+  public static final String FROM = "from";
+  public static final String WITH = "with";
+  /**
+   * FACT_UPDATE_EXTENSION.
+   */
+  public static final String FACT_UPDATE_EXTENSION = ".carbondata_update";
+  public static final String FACT_DELETE_EXTENSION = "_delete";
+  /**
+   * MARKED_FOR_UPDATION
+   */
+  public static final String FACT_FILE_UPDATED = "update";
+  /**
+   * MAX_QUERY_EXECUTION_TIME
+   */
+  public static final int DEFAULT_MAX_QUERY_EXECUTION_TIME = 60;
+  /**
+   * LOADMETADATA_FILENAME
+   */
+  public static final String LOADMETADATA_FILENAME = "tablestatus";
+  public static final String SUM_DISTINCT = "sum-distinct";
+  /**
+   * INMEMORY_REOCRD_SIZE
+   */
+  public static final String INMEMORY_REOCRD_SIZE = "carbon.inmemory.record.size";
+  public static final int INMEMORY_REOCRD_SIZE_DEFAULT = 240000;
+
+  /**
+   * INMEMORY_REOCRD_SIZE
+   */
+  public static final String DETAIL_QUERY_BATCH_SIZE = "carbon.detail.batch.size";
+  public static final int DETAIL_QUERY_BATCH_SIZE_DEFAULT = 10000;
+  /**
+   * SPILL_OVER_DISK_PATH
+   */
+  public static final String SCHEMAS_MODIFIED_TIME_FILE = "modifiedTime.mdt";
+  public static final String DEFAULT_INVISIBLE_DUMMY_MEASURE = "default_dummy_measure";
+  /**
+   * max level cache size upto which level cache will be loaded in memory
+   */
+  public static final String CARBON_MAX_LEVEL_CACHE_SIZE = "carbon.max.level.cache.size";
+  /**
+   * max level cache size default value in GB
+   */
+  public static final String CARBON_MAX_LEVEL_CACHE_SIZE_DEFAULT = "-1";
+  /**
+   * DOUBLE_VALUE_MEASURE
+   */
+  public static final char SUM_COUNT_VALUE_MEASURE = 'n';
+  /**
+   * BYTE_VALUE_MEASURE
+   */
+  public static final char BYTE_VALUE_MEASURE = 'c';
+  /**
+   * BIG_DECIMAL_MEASURE
+   */
+  public static final char BIG_DECIMAL_MEASURE = 'b';
+
+  /**
+   * BIG_INT_MEASURE
+   */
+  public static final char BIG_INT_MEASURE = 'l';
+
+  /**
+   * This determines the size of array to be processed in data load steps. one
+   * for dimensions , one of ignore dictionary dimensions , one for measures.
+   */
+  public static final int ARRAYSIZE = 3;
+  /**
+   * CARBON_PREFETCH_BUFFERSIZE
+   */
+  public static final int CARBON_PREFETCH_BUFFERSIZE = 20000;
+  /**
+   * CARBON_PREFETCH_IN_MERGE
+   */
+  public static final boolean CARBON_PREFETCH_IN_MERGE_VALUE = false;
+  /**
+   * TEMPWRITEFILEEXTENSION
+   */
+  public static final String TEMPWRITEFILEEXTENSION = ".write";
+  /**
+   * ENABLE_AUTO_LOAD_MERGE
+   */
+  public static final String ENABLE_AUTO_LOAD_MERGE = "carbon.enable.auto.load.merge";
+  /**
+   * DEFAULT_ENABLE_AUTO_LOAD_MERGE
+   */
+  public static final String DEFAULT_ENABLE_AUTO_LOAD_MERGE = "false";
+
+  /**
+   * ZOOKEEPER_ENABLE_LOCK if this is set to true then zookeeper will be used to handle locking
+   * mechanism of carbon
+   */
+  public static final String LOCK_TYPE = "carbon.lock.type";
+
+  /**
+   * ZOOKEEPER_ENABLE_DEFAULT the default value for zookeeper will be true for carbon
+   */
+  public static final String LOCK_TYPE_DEFAULT = "LOCALLOCK";
+
+  /**
+   * ZOOKEEPER_LOCATION this is the location in zookeeper file system where locks are created.
+   * mechanism of carbon
+   */
+  public static final String ZOOKEEPER_LOCATION = "/CarbonLocks";
+
+  /**
+   * maximum dictionary chunk size that can be kept in memory while writing dictionary file
+   */
+  public static final String DICTIONARY_ONE_CHUNK_SIZE = "carbon.dictionary.chunk.size";
+
+  /**
+   * dictionary chunk default size
+   */
+  public static final String DICTIONARY_ONE_CHUNK_SIZE_DEFAULT = "10000";
+
+  /**
+   * xxhash algorithm property for hashmap
+   */
+  public static final String ENABLE_XXHASH = "carbon.enableXXHash";
+
+  /**
+   * xxhash algorithm property for hashmap. Default value false
+   */
+  public static final String ENABLE_XXHASH_DEFAULT = "true";
+
+  /**
+   * default charset to be used for reading and writing
+   */
+  public static final String DEFAULT_CHARSET = "UTF-8";
+
+  /**
+   * surrogate key that will be sent whenever in the dictionary chunks
+   * a valid surrogate key is not found for a given dictionary value
+   */
+  public static final int INVALID_SURROGATE_KEY = -1;
+
+  /**
+   * surrogate key for MEMBER_DEFAULT_VAL
+   */
+  public static final int MEMBER_DEFAULT_VAL_SURROGATE_KEY = 1;
+
+  public static final String INVALID_SEGMENT_ID = "-1";
+
+  /**
+   * Size of Major Compaction in MBs
+   */
+  public static final String MAJOR_COMPACTION_SIZE = "carbon.major.compaction.size";
+
+  /**
+   * By default size of major compaction in MBs.
+   */
+  public static final String DEFAULT_MAJOR_COMPACTION_SIZE = "1024";
+
+  /**
+   * This property is used to tell how many segments to be preserved from merging.
+   */
+  public static final java.lang.String PRESERVE_LATEST_SEGMENTS_NUMBER =
+      "carbon.numberof.preserve.segments";
+
+  /**
+   * If preserve property is enabled then 2 segments will be preserved.
+   */
+  public static final String DEFAULT_PRESERVE_LATEST_SEGMENTS_NUMBER = "0";
+
+  /**
+   * This property will determine the loads of how many days can be compacted.
+   */
+  public static final java.lang.String DAYS_ALLOWED_TO_COMPACT = "carbon.allowed.compaction.days";
+
+  /**
+   * Default value of 1 day loads can be compacted
+   */
+  public static final String DEFAULT_DAYS_ALLOWED_TO_COMPACT = "0";
+
+  /**
+   * space reserved for writing block meta data in carbon data file
+   */
+  public static final String CARBON_BLOCK_META_RESERVED_SPACE =
+      "carbon.block.meta.size.reserved.percentage";
+
+  /**
+   * default value for space reserved for writing block meta data in carbon data file
+   */
+  public static final String CARBON_BLOCK_META_RESERVED_SPACE_DEFAULT = "10";
+
+  /**
+   * property to enable min max during filter query
+   */
+  public static final String CARBON_QUERY_MIN_MAX_ENABLED = "carbon.enableMinMax";
+
+  /**
+   * default value to enable min or max during filter query execution
+   */
+  public static final String MIN_MAX_DEFAULT_VALUE = "true";
+
+  /**
+   * this variable is to enable/disable prefetch of data during merge sort while
+   * reading data from sort temp files
+   */
+  public static final String CARBON_MERGE_SORT_PREFETCH = "carbon.merge.sort.prefetch";
+  public static final String CARBON_MERGE_SORT_PREFETCH_DEFAULT = "true";
+
+  /**
+   *  default name of data base
+   */
+  public static final String DATABASE_DEFAULT_NAME = "default";
+
+  // tblproperties
+  public static final String COLUMN_GROUPS = "column_groups";
+  public static final String DICTIONARY_EXCLUDE = "dictionary_exclude";
+  public static final String DICTIONARY_INCLUDE = "dictionary_include";
+  public static final String PARTITIONCLASS = "partitionclass";
+  public static final String PARTITIONCOUNT = "partitioncount";
+  public static final String COLUMN_PROPERTIES = "columnproperties";
+
+  /**
+   * this variable is to enable/disable identify high cardinality during first data loading
+   */
+  public static final String HIGH_CARDINALITY_IDENTIFY_ENABLE =
+      "high.cardinality.identify.enable";
+  public static final String HIGH_CARDINALITY_IDENTIFY_ENABLE_DEFAULT = "true";
+
+  /**
+   * threshold of high cardinality
+   */
+  public static final String HIGH_CARDINALITY_THRESHOLD = "high.cardinality.threshold";
+  public static final String HIGH_CARDINALITY_THRESHOLD_DEFAULT = "1000000";
+  public static final int HIGH_CARDINALITY_THRESHOLD_MIN = 10000;
+
+  /**
+   * percentage of cardinality in row count
+   */
+  public static final String HIGH_CARDINALITY_IN_ROW_COUNT_PERCENTAGE =
+      "high.cardinality.row.count.percentage";
+  public static final String HIGH_CARDINALITY_IN_ROW_COUNT_PERCENTAGE_DEFAULT = "80";
+
+  /**
+   * 16 mb size
+   */
+  public static final long CARBON_16MB = 16*1024*1024;
+  /**
+   * 256 mb size
+   */
+  public static final long CARBON_256MB = 256*1024*1024;
+
+  /**
+   * Data type String.
+   */
+  public static final String DATATYPE_STRING = "STRING";
+
+  /**
+   * SEGMENT_COMPACTED is property to indicate whether seg is compacted or not.
+   */
+  public static final String SEGMENT_COMPACTED = "Compacted";
+
+  /**
+   * property for number of core to load the blocks in driver
+   */
+  public static final String NUMBER_OF_CORE_TO_LOAD_DRIVER_SEGMENT =
+      "no.of.cores.to.load.blocks.in.driver";
+  /**
+   * default number of cores
+   */
+  public static final int NUMBER_OF_CORE_TO_LOAD_DRIVER_SEGMENT_DEFAULT_VALUE = 10;
+
+  /**
+   * ZOOKEEPERLOCK TYPE
+   */
+  public static final String CARBON_LOCK_TYPE_ZOOKEEPER =
+      "ZOOKEEPERLOCK";
+
+  /**
+   * LOCALLOCK TYPE
+   */
+  public static final String CARBON_LOCK_TYPE_LOCAL =
+      "LOCALLOCK";
+
+  /**
+   * HDFSLOCK TYPE
+   */
+  public static final String CARBON_LOCK_TYPE_HDFS =
+      "HDFSLOCK";
+
+  /**
+   * Lock file in zoo keeper will be of this name.
+   */
+  public static final String ZOOKEEPER_LOCK = "zookeeperLock";
+
+  /**
+   * Invalid filter member log string
+   */
+  public static final String FILTER_INVALID_MEMBER = " Invalid Record(s) are present "
+                                                     + "while filter evaluation. ";
+
+  /**
+   * Number of unmerged segments to be merged.
+   */
+  public static final String COMPACTION_SEGMENT_LEVEL_THRESHOLD =
+      "carbon.compaction.level.threshold";
+
+  /**
+   * Default count for Number of segments to be merged in levels is 4,3
+   */
+  public static final String DEFAULT_SEGMENT_LEVEL_THRESHOLD = "4,3";
+
+  /**
+   * default location of the carbon metastore db
+   */
+  public static final String METASTORE_LOCATION_DEFAULT_VAL = "../carbon.metastore";
+
+  /**
+   * hive connection url
+   */
+  public static final String HIVE_CONNECTION_URL = "javax.jdo.option.ConnectionURL";
+
+  /**
+   * Rocord size in case of compaction.
+   */
+  public static final int COMPACTION_INMEMORY_RECORD_SIZE = 120000;
+
+  /**
+   * If the level 2 compaction is done in minor then new compacted segment will end with .2
+   */
+  public static String LEVEL2_COMPACTION_INDEX = ".2";
+
+  /**
+   * Indicates compaction
+   */
+  public static String COMPACTION_KEY_WORD = "COMPACTION";
+
+  /**
+   * hdfs temporary directory key
+   */
+  public static final String HDFS_TEMP_LOCATION = "hadoop.tmp.dir";
+
+  /**
+   * zookeeper url key
+   */
+  public static final String ZOOKEEPER_URL = "spark.deploy.zookeeper.url";
+
+  private CarbonCommonConstants() {
+  }
+}
+
+

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/constants/IgnoreDictionary.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/IgnoreDictionary.java b/core/src/main/java/org/apache/carbondata/core/constants/IgnoreDictionary.java
new file mode 100644
index 0000000..6a0105c
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/constants/IgnoreDictionary.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.constants;
+
+/**
+ * This enum is used for determining the indexes of the
+ * dimension,ignoreDictionary,measure columns.
+ */
+public enum IgnoreDictionary {
+  /**
+   * POSITION WHERE DIMENSIONS R STORED IN OBJECT ARRAY.
+   */
+  DIMENSION_INDEX_IN_ROW(0),
+
+  /**
+   * POSITION WHERE BYTE[] (high cardinality) IS STORED IN OBJECT ARRAY.
+   */
+  BYTE_ARRAY_INDEX_IN_ROW(1),
+
+  /**
+   * POSITION WHERE MEASURES R STORED IN OBJECT ARRAY.
+   */
+  MEASURES_INDEX_IN_ROW(2);
+
+  private final int index;
+
+  IgnoreDictionary(int index) {
+    this.index = index;
+  }
+
+  public int getIndex() {
+    return this.index;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/datastorage/store/FileHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/FileHolder.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/FileHolder.java
new file mode 100644
index 0000000..9f8d8ed
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/FileHolder.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.store;
+
+
+public interface FileHolder {
+  /**
+   * This method will be used to read the byte array from file based on offset
+   * and length(number of bytes) need to read
+   *
+   * @param filePath fully qualified file path
+   * @param offset   reading start position,
+   * @param length   number of bytes to be read
+   * @return read byte array
+   */
+  byte[] readByteArray(String filePath, long offset, int length);
+
+  /**
+   * This method will be used to read the byte array from file based on length(number of bytes)
+   *
+   * @param filePath fully qualified file path
+   * @param length   number of bytes to be read
+   * @return read byte array
+   */
+  byte[] readByteArray(String filePath, int length);
+
+  /**
+   * This method will be used to read int from file from postion(offset), here
+   * length will be always 4 bacause int byte size if 4
+   *
+   * @param filePath fully qualified file path
+   * @param offset   reading start position,
+   * @return read int
+   */
+  int readInt(String filePath, long offset);
+
+  /**
+   * This method will be used to read long from file from postion(offset), here
+   * length will be always 8 bacause int byte size is 8
+   *
+   * @param filePath fully qualified file path
+   * @param offset   reading start position,
+   * @return read long
+   */
+  long readLong(String filePath, long offset);
+
+  /**
+   * This method will be used to read int from file from postion(offset), here
+   * length will be always 4 bacause int byte size if 4
+   *
+   * @param filePath fully qualified file path
+   * @return read int
+   */
+  int readInt(String filePath);
+
+  /**
+   * This method will be used to read long value from file from postion(offset), here
+   * length will be always 8 because long byte size if 4
+   *
+   * @param filePath fully qualified file path
+   * @param offset   reading start position,
+   * @return read long
+   */
+  long readDouble(String filePath, long offset);
+
+  /**
+   * This method will be used to close all the streams currently present in the cache
+   */
+  void finish();
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/datastorage/store/MeasureDataWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/MeasureDataWrapper.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/MeasureDataWrapper.java
new file mode 100644
index 0000000..80a4374
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/MeasureDataWrapper.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.store;
+
+import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
+
+/**
+ * MeasureDataWrapper, interface.
+ */
+public interface MeasureDataWrapper {
+  CarbonReadDataHolder[] getValues();
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeKeyStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeKeyStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeKeyStore.java
new file mode 100644
index 0000000..42c5071
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeKeyStore.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.store;
+
+public interface NodeKeyStore {
+  /**
+   * This method will be used to get the actual mdkeys array present in the
+   * store store
+   *
+   * @param fileHolder
+   * @return mdkey
+   */
+  byte[] getBackArray(FileHolder fileHolder);
+
+  /**
+   * This method will be used to insert mdkey to store
+   *
+   * @param index index of mdkey
+   * @param value mdkey
+   */
+  void put(int index, byte[] value);
+
+  /**
+   * This method will be used to get the writable key array.
+   * writable key array will hold below information:
+   * <size of key array><key array>
+   * total length will be 4 bytes for size + key array length
+   *
+   * @return writable array (compressed or normal)
+   */
+  byte[] getWritableKeyArray();
+
+  /**
+   * This method will be used to get the mdkkey array based on index
+   *
+   * @param index      index in store
+   * @param fileHolder file holder will be used to read the file
+   * @return mdkey
+   */
+  byte[] get(int index, FileHolder fileHolder);
+
+  /**
+   * This method will clear the store and create the new empty store
+   */
+  void clear();
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java
new file mode 100644
index 0000000..f2fe9d2
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.store;
+
+import org.apache.carbondata.core.datastorage.store.dataholder.CarbonWriteDataHolder;
+
+public interface NodeMeasureDataStore {
+  /**
+   * This method will be used to get the writable key array.
+   * writable measure data array will hold below information:
+   * <size of measure data array><measure data array>
+   * total length will be 4 bytes for size + measure data array length
+   *
+   * @return writable array (compressed or normal)
+   */
+  byte[][] getWritableMeasureDataArray(CarbonWriteDataHolder[] dataHolderArray);
+
+  MeasureDataWrapper getBackData(int[] cols, FileHolder fileHolder);
+
+  MeasureDataWrapper getBackData(int cols, FileHolder fileHolder);
+
+  short getLength();
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForInt.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForInt.java
new file mode 100644
index 0000000..013d873
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForInt.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.store.columnar;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.ByteUtil;
+
+public class BlockIndexerStorageForInt implements IndexStorage<int[]> {
+  private boolean alreadySorted;
+
+  private int[] dataAfterComp;
+
+  private int[] indexMap;
+
+  private byte[][] keyBlock;
+
+  private int[] dataIndexMap;
+
+  private int totalSize;
+
+  public BlockIndexerStorageForInt(byte[][] keyBlock, boolean compressData, boolean isNoDictionary,
+      boolean isSortRequired) {
+    ColumnWithIntIndex[] columnWithIndexs = createColumnWithIndexArray(keyBlock, isNoDictionary);
+    if (isSortRequired) {
+      Arrays.sort(columnWithIndexs);
+    }
+    compressMyOwnWay(extractDataAndReturnIndexes(columnWithIndexs, keyBlock));
+    if (compressData) {
+      compressDataMyOwnWay(columnWithIndexs);
+    }
+  }
+
+  /**
+   * Create an object with each column array and respective index
+   *
+   * @return
+   */
+  private ColumnWithIntIndex[] createColumnWithIndexArray(byte[][] keyBlock,
+      boolean isNoDictionary) {
+    ColumnWithIntIndex[] columnWithIndexs;
+    if (isNoDictionary) {
+      columnWithIndexs = new ColumnWithIntIndexForHighCard[keyBlock.length];
+      for (int i = 0; i < columnWithIndexs.length; i++) {
+        columnWithIndexs[i] = new ColumnWithIntIndexForHighCard(keyBlock[i], i);
+      }
+
+    } else {
+      columnWithIndexs = new ColumnWithIntIndex[keyBlock.length];
+      for (int i = 0; i < columnWithIndexs.length; i++) {
+        columnWithIndexs[i] = new ColumnWithIntIndex(keyBlock[i], i);
+      }
+    }
+
+    return columnWithIndexs;
+  }
+
+  private int[] extractDataAndReturnIndexes(ColumnWithIntIndex[] columnWithIndexs,
+      byte[][] keyBlock) {
+    int[] indexes = new int[columnWithIndexs.length];
+    for (int i = 0; i < indexes.length; i++) {
+      indexes[i] = columnWithIndexs[i].getIndex();
+      keyBlock[i] = columnWithIndexs[i].getColumn();
+    }
+    this.keyBlock = keyBlock;
+    return indexes;
+  }
+
+  /**
+   * It compresses depends up on the sequence numbers.
+   * [1,2,3,4,6,8,10,11,12,13] is translated to [1,4,6,8,10,13] and [0,6]. In
+   * first array the start and end of sequential numbers and second array
+   * keeps the indexes of where sequential numbers starts. If there is no
+   * sequential numbers then the same array it returns with empty second
+   * array.
+   *
+   * @param indexes
+   */
+  public void compressMyOwnWay(int[] indexes) {
+    List<Integer> list = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+    List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+    int k = 0;
+    int i = 1;
+    for (; i < indexes.length; i++) {
+      if (indexes[i] - indexes[i - 1] == 1) {
+        k++;
+      } else {
+        if (k > 0) {
+          map.add((list.size()));
+          list.add(indexes[i - k - 1]);
+          list.add(indexes[i - 1]);
+        } else {
+          list.add(indexes[i - 1]);
+        }
+        k = 0;
+      }
+    }
+    if (k > 0) {
+      map.add((list.size()));
+      list.add(indexes[i - k - 1]);
+      list.add(indexes[i - 1]);
+    } else {
+      list.add(indexes[i - 1]);
+    }
+    dataAfterComp = convertToArray(list);
+    if (indexes.length == dataAfterComp.length) {
+      indexMap = new int[0];
+    } else {
+      indexMap = convertToArray(map);
+    }
+    if (dataAfterComp.length == 2 && indexMap.length == 1) {
+      alreadySorted = true;
+    }
+  }
+
+  private int[] convertToArray(List<Integer> list) {
+    int[] shortArray = new int[list.size()];
+    for (int i = 0; i < shortArray.length; i++) {
+      shortArray[i] = list.get(i);
+    }
+    return shortArray;
+  }
+
+  /**
+   * @return the alreadySorted
+   */
+  public boolean isAlreadySorted() {
+    return alreadySorted;
+  }
+
+  /**
+   * @return the dataAfterComp
+   */
+  public int[] getDataAfterComp() {
+    return dataAfterComp;
+  }
+
+  /**
+   * @return the indexMap
+   */
+  public int[] getIndexMap() {
+    return indexMap;
+  }
+
+  /**
+   * @return the keyBlock
+   */
+  public byte[][] getKeyBlock() {
+    return keyBlock;
+  }
+
+  private void compressDataMyOwnWay(ColumnWithIntIndex[] indexes) {
+    byte[] prvKey = indexes[0].getColumn();
+    List<ColumnWithIntIndex> list =
+        new ArrayList<ColumnWithIntIndex>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+    list.add(indexes[0]);
+    int counter = 1;
+    int start = 0;
+    List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+    for (int i = 1; i < indexes.length; i++) {
+      if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, indexes[i].getColumn()) != 0) {
+        prvKey = indexes[i].getColumn();
+        list.add(indexes[i]);
+        map.add(start);
+        map.add(counter);
+        start += counter;
+        counter = 1;
+        continue;
+      }
+      counter++;
+    }
+    map.add(start);
+    map.add(counter);
+    this.keyBlock = convertToKeyArray(list);
+    if (indexes.length == keyBlock.length) {
+      dataIndexMap = new int[0];
+    } else {
+      dataIndexMap = convertToArray(map);
+    }
+  }
+
+  private byte[][] convertToKeyArray(List<ColumnWithIntIndex> list) {
+    byte[][] shortArray = new byte[list.size()][];
+    for (int i = 0; i < shortArray.length; i++) {
+      shortArray[i] = list.get(i).getColumn();
+      totalSize += shortArray[i].length;
+    }
+    return shortArray;
+  }
+
+  @Override public int[] getDataIndexMap() {
+    return dataIndexMap;
+  }
+
+  @Override public int getTotalSize() {
+    return totalSize;
+  }
+
+  @Override public byte[] getMin() {
+    return keyBlock[0];
+  }
+
+  @Override public byte[] getMax() {
+    return keyBlock[keyBlock.length - 1];
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForNoInvertedIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForNoInvertedIndex.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForNoInvertedIndex.java
new file mode 100644
index 0000000..c7d43cf
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForNoInvertedIndex.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.store.columnar;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.ByteUtil;
+
+public class BlockIndexerStorageForNoInvertedIndex implements IndexStorage<int[]> {
+  private byte[][] keyBlock;
+  private byte[][] sortedBlock;
+  private int totalSize;
+  private int[] dataIndexMap;
+
+  public BlockIndexerStorageForNoInvertedIndex(byte[][] keyBlockInput, boolean compressData,
+      boolean isNoDictionary) {
+    // without invertedindex but can be RLE
+    if (compressData) {
+      // with RLE
+      byte[] prvKey = keyBlockInput[0];
+      List<byte[]> list = new ArrayList<byte[]>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+      list.add(keyBlockInput[0]);
+      int counter = 1;
+      int start = 0;
+      List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+      int length = keyBlockInput.length;
+      for(int i = 1; i < length; i++) {
+        if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, keyBlockInput[i]) != 0) {
+          prvKey = keyBlockInput[i];
+          list.add(keyBlockInput[i]);
+          map.add(start);
+          map.add(counter);
+          start += counter;
+          counter = 1;
+          continue;
+        }
+        counter++;
+      }
+      map.add(start);
+      map.add(counter);
+      this.keyBlock = convertToKeyArray(list);
+      if (keyBlockInput.length == this.keyBlock.length) {
+        dataIndexMap = new int[0];
+      } else {
+        dataIndexMap = convertToArray(map);
+      }
+    } else {
+      this.keyBlock = keyBlockInput;
+      dataIndexMap = new int[0];
+    }
+
+    this.sortedBlock = new byte[keyBlock.length][];
+    System.arraycopy(keyBlock, 0, sortedBlock, 0, keyBlock.length);
+    if (isNoDictionary) {
+      Arrays.sort(sortedBlock, new Comparator<byte[]>() {
+        @Override
+        public int compare(byte[] col1, byte[] col2) {
+          return ByteUtil.UnsafeComparer.INSTANCE
+              .compareTo(col1, 2, col1.length - 2, col2, 2, col2.length - 2);
+        }
+      });
+    } else {
+      Arrays.sort(sortedBlock, new Comparator<byte[]>() {
+        @Override
+        public int compare(byte[] col1, byte[] col2) {
+          return ByteUtil.UnsafeComparer.INSTANCE.compareTo(col1, col2);
+        }
+      });
+    }
+
+  }
+
+  private int[] convertToArray(List<Integer> list) {
+    int[] shortArray = new int[list.size()];
+    for(int i = 0; i < shortArray.length; i++) {
+      shortArray[i] = list.get(i);
+    }
+    return shortArray;
+  }
+
+  private byte[][] convertToKeyArray(List<byte[]> list) {
+    byte[][] shortArray = new byte[list.size()][];
+    for (int i = 0; i < shortArray.length; i++) {
+      shortArray[i] = list.get(i);
+      totalSize += shortArray[i].length;
+    }
+    return shortArray;
+  }
+
+  @Override
+  public int[] getDataIndexMap() {
+    return dataIndexMap;
+  }
+
+  @Override
+  public int getTotalSize() {
+    return totalSize;
+  }
+
+  @Override
+  public boolean isAlreadySorted() {
+    return true;
+  }
+
+  /**
+   * no use
+   * @return
+   */
+  @Override
+  public int[] getDataAfterComp() {
+    return new int[0];
+  }
+
+  /**
+   * no use
+   * @return
+   */
+  @Override
+  public int[] getIndexMap() {
+    return new int[0];
+  }
+
+  /**
+   * @return the keyBlock
+   */
+  public byte[][] getKeyBlock() {
+    return keyBlock;
+  }
+
+  @Override public byte[] getMin() {
+    return sortedBlock[0];
+  }
+
+  @Override public byte[] getMax() {
+    return sortedBlock[sortedBlock.length - 1];
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java
new file mode 100644
index 0000000..cf9ba40
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.datastorage.store.columnar;
+
+public class ColumnGroupModel {
+
+  /**
+   * number of columns in columnar block
+   */
+  private int[] columnSplit;
+
+  /**
+   * total number of columns
+   */
+  private int noOfColumnsStore;
+
+  /**
+   * whether given index is columnar or not
+   * true: columnar
+   * false: row block
+   */
+  private boolean[] columnarStore;
+
+  /**
+   * column groups
+   * e.g
+   * {{0,1,2},3,4,{5,6}}
+   */
+  private int[][] columnGroups;
+
+  /**
+   * return columnSplit
+   *
+   * @return
+   */
+  public int[] getColumnSplit() {
+    return columnSplit;
+  }
+
+  /**
+   * set columnSplit
+   *
+   * @param split
+   */
+  public void setColumnSplit(int[] split) {
+    this.columnSplit = split;
+  }
+
+  /**
+   * @return no of columnar block
+   */
+  public int getNoOfColumnStore() {
+    return this.noOfColumnsStore;
+  }
+
+  /**
+   * set no of columnar block
+   *
+   * @param noOfColumnsStore
+   */
+  public void setNoOfColumnStore(int noOfColumnsStore) {
+    this.noOfColumnsStore = noOfColumnsStore;
+  }
+
+  /**
+   * it's an identifier for row block or single column block
+   *
+   * @param columnarStore
+   */
+  public void setColumnarStore(boolean[] columnarStore) {
+    this.columnarStore = columnarStore;
+  }
+
+  /**
+   * set column groups
+   *
+   * @param columnGroups
+   */
+  public void setColumnGroup(int[][] columnGroups) {
+    this.columnGroups = columnGroups;
+  }
+
+  /**
+   * check if given column group is columnar
+   *
+   * @param colGroup
+   * @return true if given block is columnar
+   */
+  public boolean isColumnar(int colGroup) {
+    return columnarStore[colGroup];
+  }
+
+  /**
+   * @return columngroups
+   */
+  public int[][] getColumnGroup() {
+    return this.columnGroups;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnWithIntIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnWithIntIndex.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnWithIntIndex.java
new file mode 100644
index 0000000..36606a5
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnWithIntIndex.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.store.columnar;
+
+import java.util.Arrays;
+
+import org.apache.carbondata.core.util.ByteUtil;
+
+public class ColumnWithIntIndex implements Comparable<ColumnWithIntIndex> {
+  protected byte[] column;
+
+  private int index;
+
+  public ColumnWithIntIndex(byte[] column, int index) {
+    this.column = column;
+    this.index = index;
+  }
+
+  public ColumnWithIntIndex() {
+  }
+
+  /**
+   * @return the column
+   */
+  public byte[] getColumn() {
+    return column;
+  }
+
+  /**
+   * @param column the column to set
+   */
+  public void setColumn(byte[] column) {
+    this.column = column;
+  }
+
+  /**
+   * @return the index
+   */
+  public int getIndex() {
+    return index;
+  }
+
+  /**
+   * @param index the index to set
+   */
+  public void setIndex(int index) {
+    this.index = index;
+  }
+
+  @Override public int compareTo(ColumnWithIntIndex o) {
+    return ByteUtil.UnsafeComparer.INSTANCE.compareTo(column, o.column);
+  }
+
+  @Override public boolean equals(Object obj) {
+    if(obj == null || getClass() != obj.getClass()) {
+      return false;
+    }
+    ColumnWithIntIndex o = (ColumnWithIntIndex)obj;
+    return Arrays.equals(column, o.column) && index == o.index;
+  }
+
+  @Override public int hashCode() {
+    return Arrays.hashCode(column) + index;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnWithIntIndexForHighCard.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnWithIntIndexForHighCard.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnWithIntIndexForHighCard.java
new file mode 100644
index 0000000..61a1165
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnWithIntIndexForHighCard.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.store.columnar;
+
+import java.util.Arrays;
+
+import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
+
+public class ColumnWithIntIndexForHighCard extends ColumnWithIntIndex
+    implements Comparable<ColumnWithIntIndex> {
+
+  public ColumnWithIntIndexForHighCard(byte[] column, int index) {
+    super(column, index);
+  }
+
+  @Override public int compareTo(ColumnWithIntIndex o) {
+    return UnsafeComparer.INSTANCE
+        .compareTo(column, 2, column.length - 2, o.column, 2, o.column.length - 2);
+  }
+
+  @Override public boolean equals(Object obj) {
+    if(obj == null || getClass() != obj.getClass()) {
+      return false;
+    }
+    ColumnWithIntIndexForHighCard o = (ColumnWithIntIndexForHighCard)obj;
+    return Arrays.equals(column, o.column) && getIndex() == o.getIndex();
+  }
+
+  @Override public int hashCode() {
+    return Arrays.hashCode(column) + getIndex();
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStore.java
new file mode 100644
index 0000000..9e12847
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStore.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.store.columnar;
+
+import org.apache.carbondata.core.datastorage.store.FileHolder;
+
+public interface ColumnarKeyStore {
+  /**
+   * This method will be used to get the actual mdkeys array present in the
+   * carbon store, it will read and uncomnpress the key
+   *
+   * @param fileHolder
+   * @return mdkey
+   * @noDictionaryValKeyIndexes, directkey indexes for determining the NO_DICTIONARY
+   * Col inorder to process the direct surrogates data.
+   */
+  ColumnarKeyStoreDataHolder[] getUnCompressedKeyArray(FileHolder fileHolder, int[] blockIndex,
+      boolean[] needCompressedData, int[] noDictionaryValKeyIndexes);
+
+  /**
+   * This method will be used to get the actual mdkeys array present in the
+   * carbon store, it will read and uncomnpress the key
+   *
+   * @param fileHolder
+   * @return mdkey
+   */
+  ColumnarKeyStoreDataHolder getUnCompressedKeyArray(FileHolder fileHolder, int blockIndex,
+      boolean needCompressedData, int[] noDictionaryValKeyIndexes);
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolder.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolder.java
new file mode 100644
index 0000000..b66d957
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolder.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.store.columnar;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+
+public class ColumnarKeyStoreDataHolder {
+  private byte[] keyblockData;
+  private List<byte[]> noDictionaryValBasedKeyBlockData;
+  private ColumnarKeyStoreMetadata columnarKeyStoreMetadata;
+
+  public ColumnarKeyStoreDataHolder(final byte[] keyblockData,
+      final ColumnarKeyStoreMetadata columnarKeyStoreMetadata) {
+    this.keyblockData = keyblockData;
+    this.columnarKeyStoreMetadata = columnarKeyStoreMetadata;
+  }
+
+  //Added constructor for holding noDictionaryValBasedKeyBlockData
+  public ColumnarKeyStoreDataHolder(final List<byte[]> noDictionaryValBasedKeyBlockData,
+      final ColumnarKeyStoreMetadata columnarKeyStoreMetadata) {
+    this.noDictionaryValBasedKeyBlockData = noDictionaryValBasedKeyBlockData;
+    this.columnarKeyStoreMetadata = columnarKeyStoreMetadata;
+  }
+
+  public byte[] getKeyBlockData() {
+    return keyblockData;
+  }
+
+  /**
+   * @return the columnarKeyStoreMetadata
+   */
+  public ColumnarKeyStoreMetadata getColumnarKeyStoreMetadata() {
+    return columnarKeyStoreMetadata;
+  }
+
+  public void unCompress() {
+    if (columnarKeyStoreMetadata.isUnCompressed()) {
+      return;
+    }
+    this.keyblockData = UnBlockIndexer
+        .uncompressData(keyblockData, columnarKeyStoreMetadata.getDataIndex(),
+            columnarKeyStoreMetadata.getEachRowSize());
+    columnarKeyStoreMetadata.setUnCompressed(true);
+  }
+
+  public int getSurrogateKey(int columnIndex) {
+    byte[] actual = new byte[4];
+    int startIndex;
+    if (null != columnarKeyStoreMetadata.getColumnReverseIndex()) {
+      startIndex =
+          columnarKeyStoreMetadata.getColumnReverseIndex()[columnIndex] * columnarKeyStoreMetadata
+              .getEachRowSize();
+    } else {
+      startIndex = columnIndex * columnarKeyStoreMetadata.getEachRowSize();
+    }
+    int destPos = 4 - columnarKeyStoreMetadata.getEachRowSize();
+    System.arraycopy(keyblockData, startIndex, actual, destPos,
+        columnarKeyStoreMetadata.getEachRowSize());
+    return ByteBuffer.wrap(actual).getInt();
+  }
+
+  /**
+   * get the byte[] for high cardinality column block
+   *
+   * @return List<byte[]>.
+   */
+  public List<byte[]> getNoDictionaryValBasedKeyBlockData() {
+    return noDictionaryValBasedKeyBlockData;
+  }
+
+  /**
+   * set the byte[] for high cardinality column block
+   *
+   * @param noDictionaryValBasedKeyBlockData
+   */
+  public void setNoDictionaryValBasedKeyBlockData(List<byte[]> noDictionaryValBasedKeyBlockData) {
+    this.noDictionaryValBasedKeyBlockData = noDictionaryValBasedKeyBlockData;
+  }
+}



Mime
View raw message