carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chenliang...@apache.org
Subject [46/52] [partial] incubator-carbondata git commit: Renamed packages to org.apache.carbondata and fixed errors
Date Mon, 15 Aug 2016 07:09:31 GMT
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/TableTaskInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/TableTaskInfo.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/TableTaskInfo.java
new file mode 100644
index 0000000..1f8caf0
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/TableTaskInfo.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.block;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+/**
+ * This class is responsible for maintaining the mapping of tasks of a node.
+ */
+public class TableTaskInfo extends Distributable {
+
+  private final List<TableBlockInfo> tableBlockInfoList;
+  private final String taskId;
+  public String getTaskId() {
+    return taskId;
+  }
+
+  public List<TableBlockInfo> getTableBlockInfoList() {
+    return tableBlockInfoList;
+  }
+
+  public TableTaskInfo(String taskId, List<TableBlockInfo> tableBlockInfoList){
+    this.taskId = taskId;
+    this.tableBlockInfoList = tableBlockInfoList;
+  }
+
+  @Override public String[] getLocations() {
+    Set<String> locations = new HashSet<String>();
+    for(TableBlockInfo tableBlockInfo: tableBlockInfoList){
+      locations.addAll(Arrays.asList(tableBlockInfo.getLocations()));
+    }
+    locations.toArray(new String[locations.size()]);
+    List<String> nodes =  TableTaskInfo.maxNoNodes(tableBlockInfoList);
+    return nodes.toArray(new String[nodes.size()]);
+  }
+
+  @Override public int compareTo(Distributable o) {
+    return taskId.compareTo(((TableTaskInfo)o).getTaskId());
+  }
+
+  /**
+   * Finding which node has the maximum number of blocks for it.
+   * @param blockList
+   * @return
+   */
+  public static List<String> maxNoNodes(List<TableBlockInfo> blockList) {
+    boolean useIndex = true;
+    Integer maxOccurence = 0;
+    String maxNode = null;
+    Map<String, Integer> nodeAndOccurenceMapping = new TreeMap<>();
+
+    // populate the map of node and number of occurences of that node.
+    for (TableBlockInfo block : blockList) {
+      for (String node : block.getLocations()) {
+        Integer nodeOccurence = nodeAndOccurenceMapping.get(node);
+        if (null == nodeOccurence) {
+          nodeAndOccurenceMapping.put(node, 1);
+        } else {
+          nodeOccurence++;
+        }
+      }
+    }
+    Integer previousValueOccurence = null;
+
+    // check which node is occured maximum times.
+    for (Map.Entry<String, Integer> entry : nodeAndOccurenceMapping.entrySet()) {
+      // finding the maximum node.
+      if (entry.getValue() > maxOccurence) {
+        maxOccurence = entry.getValue();
+        maxNode = entry.getKey();
+      }
+      // first time scenario. initialzing the previous value.
+      if (null == previousValueOccurence) {
+        previousValueOccurence = entry.getValue();
+      } else {
+        // for the case where all the nodes have same number of blocks then
+        // we need to return complete list instead of max node.
+        if (previousValueOccurence != entry.getValue()) {
+          useIndex = false;
+        }
+      }
+    }
+
+    // if all the nodes have equal occurence then returning the complete key set.
+    if (useIndex) {
+      return new ArrayList<>(nodeAndOccurenceMapping.keySet());
+    }
+
+    // if any max node is found then returning the max node.
+    List<String> node =  new ArrayList<>(1);
+    node.add(maxNode);
+    return node;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/TaskBlockInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/TaskBlockInfo.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/TaskBlockInfo.java
new file mode 100644
index 0000000..1221cc1
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/TaskBlockInfo.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.carbon.datastore.block;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+
+/**
+ * This class contains blocks info of each task
+ */
+public class TaskBlockInfo {
+
+  // stores TableBlockInfo list of each task
+  private Map<String, List<TableBlockInfo>> taskBlockInfoMapping;
+
+  public TaskBlockInfo(){
+
+    taskBlockInfoMapping = new HashMap<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+  }
+
+  /**
+   * returns task set
+   * @return
+   */
+  public Set<String> getTaskSet() {
+    return taskBlockInfoMapping.keySet();
+  }
+
+
+  /**
+   * returns TableBlockInfoList of given task
+   * @return
+   */
+  public List<TableBlockInfo> getTableBlockInfoList(String task) {
+    return taskBlockInfoMapping.get(task);
+  }
+
+  /**
+   *  maps TableBlockInfoList to respective task
+   * @param task
+   * @param tableBlockInfoList
+   */
+  public void addTableBlockInfoList(String task, List<TableBlockInfo> tableBlockInfoList) {
+    taskBlockInfoMapping.put(task, tableBlockInfoList);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/DimensionChunkAttributes.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/DimensionChunkAttributes.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/DimensionChunkAttributes.java
new file mode 100644
index 0000000..4dcf083
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/DimensionChunkAttributes.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk;
+
+/**
+ * Dimension chunk attributes which holds all the
+ * property about the dimension chunk data
+ */
+public class DimensionChunkAttributes {
+
+  /**
+   * inverted index of the data
+   */
+  private int[] invertedIndexes;
+
+  /**
+   * reverse index of the data
+   */
+  private int[] invertedIndexesReverse;
+
+  /**
+   * each row size
+   */
+  private int columnValueSize;
+
+  /**
+   * is no dictionary
+   */
+  private boolean isNoDictionary;
+
+  /**
+   * @return the invertedIndexes
+   */
+  public int[] getInvertedIndexes() {
+    return invertedIndexes;
+  }
+
+  /**
+   * @param invertedIndexes the invertedIndexes to set
+   */
+  public void setInvertedIndexes(int[] invertedIndexes) {
+    this.invertedIndexes = invertedIndexes;
+  }
+
+  /**
+   * @return the invertedIndexesReverse
+   */
+  public int[] getInvertedIndexesReverse() {
+    return invertedIndexesReverse;
+  }
+
+  /**
+   * @param invertedIndexesReverse the invertedIndexesReverse to set
+   */
+  public void setInvertedIndexesReverse(int[] invertedIndexesReverse) {
+    this.invertedIndexesReverse = invertedIndexesReverse;
+  }
+
+  /**
+   * @return the eachRowSize
+   */
+  public int getColumnValueSize() {
+    return columnValueSize;
+  }
+
+  /**
+   * @param eachRowSize the eachRowSize to set
+   */
+  public void setEachRowSize(int eachRowSize) {
+    this.columnValueSize = eachRowSize;
+  }
+
+  /**
+   * @return the isNoDictionary
+   */
+  public boolean isNoDictionary() {
+    return isNoDictionary;
+  }
+
+  /**
+   * @param isNoDictionary the isNoDictionary to set
+   */
+  public void setNoDictionary(boolean isNoDictionary) {
+    this.isNoDictionary = isNoDictionary;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/DimensionColumnDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/DimensionColumnDataChunk.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/DimensionColumnDataChunk.java
new file mode 100644
index 0000000..ddc76c0
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/DimensionColumnDataChunk.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk;
+
+import org.apache.carbondata.scan.executor.infos.KeyStructureInfo;
+
+/**
+ * Interface for dimension column chunk.
+ */
+public interface DimensionColumnDataChunk<T> {
+
+  /**
+   * Below method will be used to fill the data based on offset and row id
+   *
+   * @param data   data to filed
+   * @param offset offset from which data need to be filed
+   * @param rowId  row id of the chunk
+   * @return how many bytes was copied
+   */
+  int fillChunkData(byte[] data, int offset, int columnIndex, KeyStructureInfo restructuringInfo);
+
+  /**
+   * It uses to convert column data to dictionary integer value
+   * @param rowId
+   * @param columnIndex
+   * @param row
+   * @param restructuringInfo  @return
+   */
+  int fillConvertedChunkData(int rowId, int columnIndex, int[] row,
+      KeyStructureInfo restructuringInfo);
+
+  /**
+   * Below method to get  the data based in row id
+   *
+   * @param row id
+   *            row id of the data
+   * @return chunk
+   */
+  byte[] getChunkData(int columnIndex);
+
+  /**
+   * Below method will be used get the chunk attributes
+   *
+   * @return chunk attributes
+   */
+  DimensionChunkAttributes getAttributes();
+
+  /**
+   * Below method will be used to return the complete data chunk
+   * This will be required during filter query
+   *
+   * @return complete chunk
+   */
+  T getCompleteDataChunk();
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/MeasureColumnDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/MeasureColumnDataChunk.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/MeasureColumnDataChunk.java
new file mode 100644
index 0000000..fbe6e95
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/MeasureColumnDataChunk.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk;
+
+import org.apache.carbondata.core.carbon.metadata.blocklet.datachunk.PresenceMeta;
+import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
+
+/**
+ * Holder for measure column chunk
+ * it will have data and its attributes which will
+ * be required for processing
+ */
+public class MeasureColumnDataChunk {
+
+  /**
+   * measure chunk
+   */
+  private CarbonReadDataHolder measureDataHolder;
+
+  /**
+   * below to hold null value holds this information
+   * about the null value index this will be helpful in case of
+   * to remove the null value while aggregation
+   */
+  private PresenceMeta nullValueIndexHolder;
+
+  /**
+   * @return the measureDataHolder
+   */
+  public CarbonReadDataHolder getMeasureDataHolder() {
+    return measureDataHolder;
+  }
+
+  /**
+   * @param measureDataHolder the measureDataHolder to set
+   */
+  public void setMeasureDataHolder(CarbonReadDataHolder measureDataHolder) {
+    this.measureDataHolder = measureDataHolder;
+  }
+
+  /**
+   * @return the nullValueIndexHolder
+   */
+  public PresenceMeta getNullValueIndexHolder() {
+    return nullValueIndexHolder;
+  }
+
+  /**
+   * @param nullValueIndexHolder the nullValueIndexHolder to set
+   */
+  public void setNullValueIndexHolder(PresenceMeta nullValueIndexHolder) {
+    this.nullValueIndexHolder = nullValueIndexHolder;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java
new file mode 100644
index 0000000..77fb163
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.impl;
+
+import org.apache.carbondata.core.carbon.datastore.chunk.DimensionChunkAttributes;
+import org.apache.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.scan.executor.infos.KeyStructureInfo;
+
+/**
+ * This class is holder of the dimension column chunk data of the fixed length
+ * key size
+ */
+public class ColumnGroupDimensionDataChunk implements DimensionColumnDataChunk<byte[]> {
+
+  /**
+   * dimension chunk attributes
+   */
+  private DimensionChunkAttributes chunkAttributes;
+
+  /**
+   * data chunks
+   */
+  private byte[] dataChunk;
+
+  /**
+   * Constructor for this class
+   *
+   * @param dataChunk       data chunk
+   * @param chunkAttributes chunk attributes
+   */
+  public ColumnGroupDimensionDataChunk(byte[] dataChunk, DimensionChunkAttributes chunkAttributes) {
+    this.chunkAttributes = chunkAttributes;
+    this.dataChunk = dataChunk;
+  }
+
+  /**
+   * Below method will be used to fill the data based on offset and row id
+   *
+   * @param data             data to filed
+   * @param offset           offset from which data need to be filed
+   * @param rowId            row id of the chunk
+   * @param restructuringInfo define the structure of the key
+   * @return how many bytes was copied
+   */
+  @Override public int fillChunkData(byte[] data, int offset, int rowId,
+      KeyStructureInfo restructuringInfo) {
+    byte[] maskedKey =
+        getMaskedKey(dataChunk, rowId * chunkAttributes.getColumnValueSize(), restructuringInfo);
+    System.arraycopy(maskedKey, 0, data, offset, maskedKey.length);
+    return maskedKey.length;
+  }
+
+  /**
+   * Converts to column dictionary integer value
+   */
+  @Override public int fillConvertedChunkData(int rowId, int columnIndex, int[] row,
+      KeyStructureInfo info) {
+    int start = rowId * chunkAttributes.getColumnValueSize();
+    long[] keyArray = info.getKeyGenerator().getKeyArray(dataChunk, start);
+    int[] ordinal = info.getMdkeyQueryDimensionOrdinal();
+    for (int i = 0; i < ordinal.length; i++) {
+      row[columnIndex++] = (int)keyArray[ordinal[i]];
+    }
+    return columnIndex;
+  }
+
+  /**
+   * Below method masks key
+   *
+   */
+  public byte[] getMaskedKey(byte[] data, int offset, KeyStructureInfo info) {
+    byte[] maskedKey = new byte[info.getMaskByteRanges().length];
+    int counter = 0;
+    int byteRange = 0;
+    for (int i = 0; i < info.getMaskByteRanges().length; i++) {
+      byteRange = info.getMaskByteRanges()[i];
+      maskedKey[counter++] = (byte) (data[byteRange + offset] & info.getMaxKey()[byteRange]);
+    }
+    return maskedKey;
+  }
+
+  /**
+   * Below method to get the data based in row id
+   *
+   * @param rowId row id of the data
+   * @return chunk
+   */
+  @Override public byte[] getChunkData(int rowId) {
+    byte[] data = new byte[chunkAttributes.getColumnValueSize()];
+    System.arraycopy(dataChunk, rowId * data.length, data, 0, data.length);
+    return data;
+  }
+
+  /**
+   * Below method will be used get the chunk attributes
+   *
+   * @return chunk attributes
+   */
+  @Override public DimensionChunkAttributes getAttributes() {
+    return chunkAttributes;
+  }
+
+  /**
+   * Below method will be used to return the complete data chunk
+   * This will be required during filter query
+   *
+   * @return complete chunk
+   */
+  @Override public byte[] getCompleteDataChunk() {
+    return dataChunk;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/impl/FixedLengthDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
new file mode 100644
index 0000000..2867d76
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.impl;
+
+import org.apache.carbondata.core.carbon.datastore.chunk.DimensionChunkAttributes;
+import org.apache.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.scan.executor.infos.KeyStructureInfo;
+
+/**
+ * This class is holder of the dimension column chunk data of the fixed length
+ * key size
+ */
+public class FixedLengthDimensionDataChunk implements DimensionColumnDataChunk<byte[]> {
+
+  /**
+   * dimension chunk attributes
+   */
+  private DimensionChunkAttributes chunkAttributes;
+
+  /**
+   * data chunks
+   */
+  private byte[] dataChunk;
+
+  /**
+   * Constructor for this class
+   *
+   * @param dataChunk       data chunk
+   * @param chunkAttributes chunk attributes
+   */
+  public FixedLengthDimensionDataChunk(byte[] dataChunk, DimensionChunkAttributes chunkAttributes) {
+    this.chunkAttributes = chunkAttributes;
+    this.dataChunk = dataChunk;
+  }
+
+  /**
+   * Below method will be used to fill the data based on offset and row id
+   *
+   * @param data             data to filed
+   * @param offset           offset from which data need to be filed
+   * @param index            row id of the chunk
+   * @param keyStructureInfo define the structure of the key
+   * @return how many bytes was copied
+   */
+  @Override public int fillChunkData(byte[] data, int offset, int index,
+      KeyStructureInfo keyStructureInfo) {
+    if (chunkAttributes.getInvertedIndexes() != null) {
+      index = chunkAttributes.getInvertedIndexesReverse()[index];
+    }
+    System.arraycopy(dataChunk, index * chunkAttributes.getColumnValueSize(), data, offset,
+        chunkAttributes.getColumnValueSize());
+    return chunkAttributes.getColumnValueSize();
+  }
+
+  /**
+   * Converts to column dictionary integer value
+   */
+  @Override public int fillConvertedChunkData(int rowId, int columnIndex, int[] row,
+      KeyStructureInfo restructuringInfo) {
+    if (chunkAttributes.getInvertedIndexes() != null) {
+      rowId = chunkAttributes.getInvertedIndexesReverse()[rowId];
+    }
+    int start = rowId * chunkAttributes.getColumnValueSize();
+    int dict = 0;
+    for (int i = start; i < start + chunkAttributes.getColumnValueSize(); i++) {
+      dict <<= 8;
+      dict ^= dataChunk[i] & 0xFF;
+    }
+    row[columnIndex] = dict;
+    return columnIndex + 1;
+  }
+
+  /**
+   * Below method to get the data based in row id
+   *
+   * @param index row id of the data
+   * @return chunk
+   */
+  @Override public byte[] getChunkData(int index) {
+    byte[] data = new byte[chunkAttributes.getColumnValueSize()];
+    if (chunkAttributes.getInvertedIndexes() != null) {
+      index = chunkAttributes.getInvertedIndexesReverse()[index];
+    }
+    System.arraycopy(dataChunk, index * chunkAttributes.getColumnValueSize(), data, 0,
+        chunkAttributes.getColumnValueSize());
+    return data;
+  }
+
+  /**
+   * Below method will be used get the chunk attributes
+   *
+   * @return chunk attributes
+   */
+  @Override public DimensionChunkAttributes getAttributes() {
+    return chunkAttributes;
+  }
+
+  /**
+   * Below method will be used to return the complete data chunk
+   * This will be required during filter query
+   *
+   * @return complete chunk
+   */
+  @Override public byte[] getCompleteDataChunk() {
+    return dataChunk;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/impl/VariableLengthDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
new file mode 100644
index 0000000..6d2d400
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.impl;
+
+import java.util.List;
+
+import org.apache.carbondata.core.carbon.datastore.chunk.DimensionChunkAttributes;
+import org.apache.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.scan.executor.infos.KeyStructureInfo;
+
+/**
+ * This class is holder of the dimension column chunk data of the variable
+ * length key size
+ */
+public class VariableLengthDimensionDataChunk implements DimensionColumnDataChunk<List<byte[]>> {
+
+  /**
+   * dimension chunk attributes
+   */
+  private DimensionChunkAttributes chunkAttributes;
+
+  /**
+   * data chunk
+   */
+  private List<byte[]> dataChunk;
+
+  /**
+   * Constructor for this class
+   *
+   * @param dataChunk       data chunk
+   * @param chunkAttributes chunk attributes
+   */
+  public VariableLengthDimensionDataChunk(List<byte[]> dataChunk,
+      DimensionChunkAttributes chunkAttributes) {
+    this.chunkAttributes = chunkAttributes;
+    this.dataChunk = dataChunk;
+  }
+
+  /**
+   * Below method will be used to fill the data based on offset and row id
+   *
+   * @param data             data to filed
+   * @param offset           offset from which data need to be filed
+   * @param index            row id of the chunk
+   * @param restructuringInfo define the structure of the key
+   * @return how many bytes was copied
+   */
+  @Override public int fillChunkData(byte[] data, int offset, int index,
+      KeyStructureInfo restructuringInfo) {
+    // no required in this case because this column chunk is not the part if
+    // mdkey
+    return 0;
+  }
+
+  /**
+   * Converts to column dictionary integer value
+   * @param rowId
+   * @param columnIndex
+   * @param row
+   * @param restructuringInfo  @return
+   */
+  @Override public int fillConvertedChunkData(int rowId, int columnIndex, int[] row,
+      KeyStructureInfo restructuringInfo) {
+    return columnIndex + 1;
+  }
+
+  /**
+   * Below method to get the data based in row id
+   *
+   * @param index row id of the data
+   * @return chunk
+   */
+  @Override public byte[] getChunkData(int index) {
+    if (null != chunkAttributes.getInvertedIndexes()) {
+      index = chunkAttributes.getInvertedIndexesReverse()[index];
+    }
+    return dataChunk.get(index);
+  }
+
+  /**
+   * Below method will be used get the chunk attributes
+   *
+   * @return chunk attributes
+   */
+  @Override public DimensionChunkAttributes getAttributes() {
+    return chunkAttributes;
+  }
+
+  /**
+   * Below method will be used to return the complete data chunk
+   * This will be required during filter query
+   *
+   * @return complete chunk
+   */
+  @Override public List<byte[]> getCompleteDataChunk() {
+    return dataChunk;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/DimensionColumnChunkReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/DimensionColumnChunkReader.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/DimensionColumnChunkReader.java
new file mode 100644
index 0000000..b958245
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/DimensionColumnChunkReader.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.reader;
+
+import org.apache.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastorage.store.FileHolder;
+
+/**
+ * Interface for reading the data chunk
+ * Its concrete implementation can be used to read the chunk.
+ * compressed or uncompressed chunk
+ */
+public interface DimensionColumnChunkReader {
+
+  /**
+   * Below method will be used to read the chunk based on block indexes
+   *
+   * @param fileReader   file reader to read the blocks from file
+   * @param blockIndexes blocks to be read
+   * @return dimension column chunks
+   */
+  DimensionColumnDataChunk[] readDimensionChunks(FileHolder fileReader, int... blockIndexes);
+
+  /**
+   * Below method will be used to read the chunk based on block index
+   *
+   * @param fileReader file reader to read the blocks from file
+   * @param blockIndex block to be read
+   * @return dimension column chunk
+   */
+  DimensionColumnDataChunk readDimensionChunk(FileHolder fileReader, int blockIndex);
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/MeasureColumnChunkReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/MeasureColumnChunkReader.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/MeasureColumnChunkReader.java
new file mode 100644
index 0000000..8a7c8ef
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/MeasureColumnChunkReader.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.reader;
+
+import org.apache.carbondata.core.carbon.datastore.chunk.MeasureColumnDataChunk;
+import org.apache.carbondata.core.datastorage.store.FileHolder;
+
+/**
+ * Reader interface for reading the measure blocks from file
+ */
+public interface MeasureColumnChunkReader {
+
+  /**
+   * Method to read the blocks data based on block indexes
+   *
+   * @param fileReader   file reader to read the blocks
+   * @param blockIndexes blocks to be read
+   * @return measure data chunks
+   */
+  MeasureColumnDataChunk[] readMeasureChunks(FileHolder fileReader, int... blockIndexes);
+
+  /**
+   * Method to read the blocks data based on block index
+   *
+   * @param fileReader file reader to read the blocks
+   * @param blockIndex block to be read
+   * @return measure data chunk
+   */
+  MeasureColumnDataChunk readMeasureChunk(FileHolder fileReader, int blockIndex);
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/AbstractChunkReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/AbstractChunkReader.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/AbstractChunkReader.java
new file mode 100644
index 0000000..59dcd38
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/AbstractChunkReader.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.reader.dimension;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.core.carbon.datastore.chunk.reader.DimensionColumnChunkReader;
+import org.apache.carbondata.core.carbon.metadata.blocklet.datachunk.DataChunk;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastorage.store.compression.Compressor;
+import org.apache.carbondata.core.datastorage.store.compression.SnappyCompression;
+import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor;
+import org.apache.carbondata.core.util.CarbonProperties;
+
+/**
+ * Class which will have all the common properties and behavior among all type
+ * of reader
+ */
+public abstract class AbstractChunkReader implements DimensionColumnChunkReader {
+
+  /**
+   * compressor will be used to uncompress the data
+   */
+  protected static final Compressor<byte[]> COMPRESSOR =
+      SnappyCompression.SnappyByteCompression.INSTANCE;
+
+  /**
+   * data chunk list which holds the information
+   * about the data block metadata
+   */
+  protected List<DataChunk> dimensionColumnChunk;
+
+  /**
+   * size of the each column value
+   * for no dictionary column it will be -1
+   */
+  protected int[] eachColumnValueSize;
+
+  /**
+   * full qualified path of the data file from
+   * which data will be read
+   */
+  protected String filePath;
+
+  /**
+   * this will be used to uncompress the
+   * row id and rle chunk
+   */
+  protected NumberCompressor numberComressor;
+
+  /**
+   * number of element in each chunk
+   */
+  private int numberOfElement;
+
+  /**
+   * Constructor to get minimum parameter to create
+   * instance of this class
+   *
+   * @param dimensionColumnChunk dimension chunk metadata
+   * @param eachColumnValueSize  size of the each column value
+   * @param filePath             file from which data will be read
+   */
+  public AbstractChunkReader(List<DataChunk> dimensionColumnChunk, int[] eachColumnValueSize,
+      String filePath) {
+    this.dimensionColumnChunk = dimensionColumnChunk;
+    this.eachColumnValueSize = eachColumnValueSize;
+    this.filePath = filePath;
+    int numberOfElement = 0;
+    try {
+      numberOfElement = Integer.parseInt(CarbonProperties.getInstance()
+          .getProperty(CarbonCommonConstants.BLOCKLET_SIZE,
+              CarbonCommonConstants.BLOCKLET_SIZE_DEFAULT_VAL));
+    } catch (NumberFormatException exception) {
+      numberOfElement = Integer.parseInt(CarbonCommonConstants.BLOCKLET_SIZE_DEFAULT_VAL);
+    }
+    this.numberComressor = new NumberCompressor(numberOfElement);
+  }
+
+  /**
+   * Below method will be used to create the inverted index reverse
+   * this will be used to point to actual data in the chunk
+   *
+   * @param invertedIndex inverted index
+   * @return reverse inverted index
+   */
+  protected int[] getInvertedReverseIndex(int[] invertedIndex) {
+    int[] columnIndexTemp = new int[invertedIndex.length];
+
+    for (int i = 0; i < invertedIndex.length; i++) {
+      columnIndexTemp[invertedIndex[i]] = i;
+    }
+    return columnIndexTemp;
+  }
+
+  /**
+   * In case of no dictionary column size of the each column value
+   * will not be same, so in case of filter query we can not take
+   * advantage of binary search as length with each value will be also
+   * store with the data, so converting this data to two dimension
+   * array format filter query processing will be faster
+   *
+   * @param dataChunkWithLength no dictionary column chunk
+   *                            <Lenght><Data><Lenght><data>
+   *                            Length will store in 2 bytes
+   * @return list of data chuck, one value in list will represent one column value
+   */
+  protected List<byte[]> getNoDictionaryDataChunk(byte[] dataChunkWithLength) {
+    List<byte[]> dataChunk = new ArrayList<byte[]>(numberOfElement);
+    // wrapping the chunk to byte buffer
+    ByteBuffer buffer = ByteBuffer.wrap(dataChunkWithLength);
+    buffer.rewind();
+    byte[] data = null;
+    // iterating till all the elements are read
+    while (buffer.hasRemaining()) {
+      // as all the data is stored with length(2 bytes)
+      // first reading the size and then based on size
+      // we need to read the actual value
+      data = new byte[buffer.getShort()];
+      buffer.get(data);
+      dataChunk.add(data);
+    }
+    return dataChunk;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/CompressedDimensionChunkFileBasedReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/CompressedDimensionChunkFileBasedReader.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/CompressedDimensionChunkFileBasedReader.java
new file mode 100644
index 0000000..209217b
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/CompressedDimensionChunkFileBasedReader.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.reader.dimension;
+
+import java.util.List;
+
+import org.apache.carbondata.core.carbon.datastore.chunk.DimensionChunkAttributes;
+import org.apache.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.carbon.datastore.chunk.impl.ColumnGroupDimensionDataChunk;
+import org.apache.carbondata.core.carbon.datastore.chunk.impl.FixedLengthDimensionDataChunk;
+import org.apache.carbondata.core.carbon.datastore.chunk.impl.VariableLengthDimensionDataChunk;
+import org.apache.carbondata.core.carbon.metadata.blocklet.datachunk.DataChunk;
+import org.apache.carbondata.core.carbon.metadata.encoder.Encoding;
+import org.apache.carbondata.core.datastorage.store.FileHolder;
+import org.apache.carbondata.core.datastorage.store.columnar.UnBlockIndexer;
+import org.apache.carbondata.core.util.CarbonUtil;
+
+/**
+ * Compressed dimension chunk reader class
+ */
+public class CompressedDimensionChunkFileBasedReader extends AbstractChunkReader {
+
+  /**
+   * Constructor to get minimum parameter to create instance of this class
+   *
+   * @param dimensionColumnChunk dimension chunk metadata
+   * @param eachColumnValueSize  size of the each column value
+   * @param filePath             file from which data will be read
+   */
+  public CompressedDimensionChunkFileBasedReader(List<DataChunk> dimensionColumnChunk,
+      int[] eachColumnValueSize, String filePath) {
+    super(dimensionColumnChunk, eachColumnValueSize, filePath);
+  }
+
+  /**
+   * Below method will be used to read the chunk based on block indexes
+   *
+   * @param fileReader   file reader to read the blocks from file
+   * @param blockIndexes blocks to be read
+   * @return dimension column chunks
+   */
+  @Override public DimensionColumnDataChunk[] readDimensionChunks(FileHolder fileReader,
+      int... blockIndexes) {
+    // read the column chunk based on block index and add
+    DimensionColumnDataChunk[] dataChunks =
+        new DimensionColumnDataChunk[dimensionColumnChunk.size()];
+    for (int i = 0; i < blockIndexes.length; i++) {
+      dataChunks[blockIndexes[i]] = readDimensionChunk(fileReader, blockIndexes[i]);
+    }
+    return dataChunks;
+  }
+
+  /**
+   * Below method will be used to read the chunk based on block index
+   *
+   * @param fileReader file reader to read the blocks from file
+   * @param blockIndex block to be read
+   * @return dimension column chunk
+   */
+  @Override public DimensionColumnDataChunk readDimensionChunk(FileHolder fileReader,
+      int blockIndex) {
+    byte[] dataPage = null;
+    int[] invertedIndexes = null;
+    int[] invertedIndexesReverse = null;
+    int[] rlePage = null;
+
+    // first read the data and uncompressed it
+    dataPage = COMPRESSOR.unCompress(fileReader
+        .readByteArray(filePath, dimensionColumnChunk.get(blockIndex).getDataPageOffset(),
+            dimensionColumnChunk.get(blockIndex).getDataPageLength()));
+    // if row id block is present then read the row id chunk and uncompress it
+    if (CarbonUtil.hasEncoding(dimensionColumnChunk.get(blockIndex).getEncodingList(),
+        Encoding.INVERTED_INDEX)) {
+      invertedIndexes = CarbonUtil
+          .getUnCompressColumnIndex(dimensionColumnChunk.get(blockIndex).getRowIdPageLength(),
+              fileReader.readByteArray(filePath,
+                  dimensionColumnChunk.get(blockIndex).getRowIdPageOffset(),
+                  dimensionColumnChunk.get(blockIndex).getRowIdPageLength()), numberComressor);
+      // get the reverse index
+      invertedIndexesReverse = getInvertedReverseIndex(invertedIndexes);
+    }
+    // if rle is applied then read the rle block chunk and then uncompress
+    //then actual data based on rle block
+    if (CarbonUtil
+        .hasEncoding(dimensionColumnChunk.get(blockIndex).getEncodingList(), Encoding.RLE)) {
+      // read and uncompress the rle block
+      rlePage = numberComressor.unCompress(fileReader
+          .readByteArray(filePath, dimensionColumnChunk.get(blockIndex).getRlePageOffset(),
+              dimensionColumnChunk.get(blockIndex).getRlePageLength()));
+      // uncompress the data with rle indexes
+      dataPage = UnBlockIndexer.uncompressData(dataPage, rlePage, eachColumnValueSize[blockIndex]);
+      rlePage = null;
+    }
+    // fill chunk attributes
+    DimensionChunkAttributes chunkAttributes = new DimensionChunkAttributes();
+    chunkAttributes.setEachRowSize(eachColumnValueSize[blockIndex]);
+    chunkAttributes.setInvertedIndexes(invertedIndexes);
+    chunkAttributes.setInvertedIndexesReverse(invertedIndexesReverse);
+    DimensionColumnDataChunk columnDataChunk = null;
+
+    if (dimensionColumnChunk.get(blockIndex).isRowMajor()) {
+      // to store fixed length column chunk values
+      columnDataChunk = new ColumnGroupDimensionDataChunk(dataPage, chunkAttributes);
+    }
+    // if no dictionary column then first create a no dictionary column chunk
+    // and set to data chunk instance
+    else if (!CarbonUtil
+        .hasEncoding(dimensionColumnChunk.get(blockIndex).getEncodingList(), Encoding.DICTIONARY)) {
+      columnDataChunk =
+          new VariableLengthDimensionDataChunk(getNoDictionaryDataChunk(dataPage), chunkAttributes);
+      chunkAttributes.setNoDictionary(true);
+    } else {
+      // to store fixed length column chunk values
+      columnDataChunk = new FixedLengthDimensionDataChunk(dataPage, chunkAttributes);
+    }
+    return columnDataChunk;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/AbstractMeasureChunkReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/AbstractMeasureChunkReader.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/AbstractMeasureChunkReader.java
new file mode 100644
index 0000000..dc8771f
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/AbstractMeasureChunkReader.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.reader.measure;
+
+import java.util.List;
+
+import org.apache.carbondata.core.carbon.datastore.chunk.reader.MeasureColumnChunkReader;
+import org.apache.carbondata.core.carbon.metadata.blocklet.datachunk.DataChunk;
+import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel;
+import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
+import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder.UnCompressValue;
+
+/**
+ * Measure block reader abstract class
+ */
+public abstract class AbstractMeasureChunkReader implements MeasureColumnChunkReader {
+
+  /**
+   * metadata which was to used to compress and uncompress the measure value
+   */
+  protected ValueCompressionModel compressionModel;
+
+  /**
+   * file path from which blocks will be read
+   */
+  protected String filePath;
+
+  /**
+   * measure chunk have the information about the metadata present in the file
+   */
+  protected List<DataChunk> measureColumnChunk;
+
+  /**
+   * type of valu comprssion model selected for each measure column
+   */
+  protected UnCompressValue[] values;
+
+  /**
+   * Constructor to get minimum parameter to create instance of this class
+   *
+   * @param measureColumnChunk measure chunk metadata
+   * @param compression        model metadata which was to used to compress and uncompress
+   *                           the measure value
+   * @param filePath           file from which data will be read
+   * @param isInMemory         in case of in memory it will read and holds the data and when
+   *                           query request will come it will uncompress and the data
+   */
+  public AbstractMeasureChunkReader(List<DataChunk> measureColumnChunk,
+      ValueCompressionModel compressionModel, String filePath, boolean isInMemory) {
+    this.measureColumnChunk = measureColumnChunk;
+    this.compressionModel = compressionModel;
+    this.filePath = filePath;
+    values =
+        new ValueCompressonHolder.UnCompressValue[compressionModel.getUnCompressValues().length];
+    for (int i = 0; i < values.length; i++) {
+      values[i] = compressionModel.getUnCompressValues()[i].getNew().getCompressorObject();
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/CompressedMeasureChunkFileBasedReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/CompressedMeasureChunkFileBasedReader.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/CompressedMeasureChunkFileBasedReader.java
new file mode 100644
index 0000000..31c470d
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/CompressedMeasureChunkFileBasedReader.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.reader.measure;
+
+import java.util.List;
+
+import org.apache.carbondata.core.carbon.datastore.chunk.MeasureColumnDataChunk;
+import org.apache.carbondata.core.carbon.metadata.blocklet.datachunk.DataChunk;
+import org.apache.carbondata.core.datastorage.store.FileHolder;
+import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel;
+import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
+import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
+
+/**
+ * Compressed measure chunk reader
+ */
+public class CompressedMeasureChunkFileBasedReader extends AbstractMeasureChunkReader {
+
+  /**
+   * Constructor to get minimum parameter to create instance of this class
+   *
+   * @param measureColumnChunk measure chunk metadata
+   * @param compression        model metadata which was to used to compress and uncompress
+   *                           the measure value
+   * @param filePath           file from which data will be read
+   */
+  public CompressedMeasureChunkFileBasedReader(List<DataChunk> measureColumnChunk,
+      ValueCompressionModel compressionModel, String filePath) {
+    super(measureColumnChunk, compressionModel, filePath, false);
+  }
+
+  /**
+   * Method to read the blocks data based on block indexes
+   *
+   * @param fileReader   file reader to read the blocks
+   * @param blockIndexes blocks to be read
+   * @return measure data chunks
+   */
+  @Override public MeasureColumnDataChunk[] readMeasureChunks(FileHolder fileReader,
+      int... blockIndexes) {
+    MeasureColumnDataChunk[] datChunk = new MeasureColumnDataChunk[values.length];
+    for (int i = 0; i < blockIndexes.length; i++) {
+      datChunk[blockIndexes[i]] = readMeasureChunk(fileReader, blockIndexes[i]);
+    }
+    return datChunk;
+  }
+
+  /**
+   * Method to read the blocks data based on block index
+   *
+   * @param fileReader file reader to read the blocks
+   * @param blockIndex block to be read
+   * @return measure data chunk
+   */
+  @Override public MeasureColumnDataChunk readMeasureChunk(FileHolder fileReader, int blockIndex) {
+    MeasureColumnDataChunk datChunk = new MeasureColumnDataChunk();
+    // create a new uncompressor
+    ValueCompressonHolder.UnCompressValue copy = values[blockIndex].getNew();
+    // read data from file and set to uncompressor
+    copy.setValue(fileReader
+        .readByteArray(filePath, measureColumnChunk.get(blockIndex).getDataPageOffset(),
+            measureColumnChunk.get(blockIndex).getDataPageLength()));
+    // get the data holder after uncompressing
+    CarbonReadDataHolder measureDataHolder =
+        copy.uncompress(compressionModel.getChangedDataType()[blockIndex])
+            .getValues(compressionModel.getDecimal()[blockIndex],
+                compressionModel.getMaxValue()[blockIndex]);
+    // set the data chunk
+    datChunk.setMeasureDataHolder(measureDataHolder);
+    // set the enun value indexes
+    datChunk
+        .setNullValueIndexHolder(measureColumnChunk.get(blockIndex).getNullValueIndexForColumn());
+    return datChunk;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/exception/IndexBuilderException.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/exception/IndexBuilderException.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/exception/IndexBuilderException.java
new file mode 100644
index 0000000..a9c7343
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/exception/IndexBuilderException.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.carbon.datastore.exception;
+
+import java.util.Locale;
+
+/**
+ * Exception class for block builder
+ *
+ * @author Administrator
+ */
+public class IndexBuilderException extends Exception {
+  /**
+   * default serial version ID.
+   */
+  private static final long serialVersionUID = 1L;
+
+  /**
+   * The Error message.
+   */
+  private String msg = "";
+
+  /**
+   * Constructor
+   *
+   * @param errorCode The error code for this exception.
+   * @param msg       The error message for this exception.
+   */
+  public IndexBuilderException(String msg) {
+    super(msg);
+    this.msg = msg;
+  }
+
+  /**
+   * Constructor
+   *
+   * @param msg       exception message
+   * @param throwable detail exception
+   */
+  public IndexBuilderException(String msg, Throwable throwable) {
+    super(msg, throwable);
+    this.msg = msg;
+  }
+
+  /**
+   * Constructor
+   *
+   * @param throwable exception
+   */
+  public IndexBuilderException(Throwable throwable) {
+    super(throwable);
+  }
+
+  /**
+   * This method is used to get the localized message.
+   *
+   * @param locale - A Locale object represents a specific geographical,
+   *               political, or cultural region.
+   * @return - Localized error message.
+   */
+  public String getLocalizedMessage(Locale locale) {
+    return "";
+  }
+
+  /**
+   * getLocalizedMessage
+   */
+  @Override public String getLocalizedMessage() {
+    return super.getLocalizedMessage();
+  }
+
+  /**
+   * getMessage
+   */
+  public String getMessage() {
+    return this.msg;
+  }
+}
+

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeBuilder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeBuilder.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeBuilder.java
new file mode 100644
index 0000000..d414d3e
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeBuilder.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.impl.btree;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.core.carbon.datastore.BtreeBuilder;
+import org.apache.carbondata.core.carbon.datastore.IndexKey;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.CarbonProperties;
+
+/**
+ * Abstract Btree based builder
+ */
+public abstract class AbstractBTreeBuilder implements BtreeBuilder {
+
+  /**
+   * default Number of keys per page
+   */
+  private static final int DEFAULT_NUMBER_OF_ENTRIES_NONLEAF = 32;
+
+  /**
+   * Maximum number of entries in intermediate nodes
+   */
+  protected int maxNumberOfEntriesInNonLeafNodes;
+
+  /**
+   * Number of leaf nodes
+   */
+  protected int nLeaf;
+
+  /**
+   * root node of a btree
+   */
+  protected BTreeNode root;
+
+  public AbstractBTreeBuilder() {
+    maxNumberOfEntriesInNonLeafNodes = Integer.parseInt(CarbonProperties.getInstance()
+        .getProperty("com.huawei.datastore.internalnodesize",
+            DEFAULT_NUMBER_OF_ENTRIES_NONLEAF + ""));
+  }
+
+  /**
+   * Below method is to build the intermediate node of the btree
+   *
+   * @param curNode              current node
+   * @param childNodeGroups      children group which will have all the children for
+   *                             particular intermediate node
+   * @param currentGroup         current group
+   * @param interNSKeyList       list if keys
+   * @param numberOfInternalNode number of internal node
+   */
+  protected void addIntermediateNode(BTreeNode curNode, List<BTreeNode[]> childNodeGroups,
+      BTreeNode[] currentGroup, List<List<IndexKey>> interNSKeyList, int numberOfInternalNode) {
+
+    int groupCounter;
+    // Build internal nodes level by level. Each upper node can have
+    // upperMaxEntry keys and upperMaxEntry+1 children
+    int remainder;
+    int nHigh = numberOfInternalNode;
+    boolean bRootBuilt = false;
+    remainder = nLeaf % (maxNumberOfEntriesInNonLeafNodes);
+    List<IndexKey> interNSKeys = null;
+    while (nHigh > 1 || !bRootBuilt) {
+      List<BTreeNode[]> internalNodeGroups =
+          new ArrayList<BTreeNode[]>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+      List<List<IndexKey>> interNSKeyTmpList =
+          new ArrayList<List<IndexKey>>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+      numberOfInternalNode = 0;
+      for (int i = 0; i < nHigh; i++) {
+        // Create a new internal node
+        curNode = new BTreeNonLeafNode();
+        // Allocate a new node group if current node group is full
+        groupCounter = i % (maxNumberOfEntriesInNonLeafNodes);
+        if (groupCounter == 0) {
+          // Create new node group
+          currentGroup = new BTreeNonLeafNode[maxNumberOfEntriesInNonLeafNodes];
+          internalNodeGroups.add(currentGroup);
+          numberOfInternalNode++;
+          interNSKeys = new ArrayList<IndexKey>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+          interNSKeyTmpList.add(interNSKeys);
+        }
+
+        // Add the new internal node to current group
+        if (null != currentGroup) {
+          currentGroup[groupCounter] = curNode;
+        }
+        int nNodes;
+
+        if (i == nHigh - 1 && remainder != 0) {
+          nNodes = remainder;
+        } else {
+          nNodes = maxNumberOfEntriesInNonLeafNodes;
+        }
+        // Point the internal node to its children node group
+        curNode.setChildren(childNodeGroups.get(i));
+        // Fill the internal node with keys based on its child nodes
+        for (int j = 0; j < nNodes; j++) {
+          curNode.setKey(interNSKeyList.get(i).get(j));
+          if (j == 0 && null != interNSKeys) {
+            interNSKeys.add(interNSKeyList.get(i).get(j));
+
+          }
+        }
+      }
+      // If nHigh is 1, we have the root node
+      if (nHigh == 1) {
+        bRootBuilt = true;
+      }
+
+      remainder = nHigh % (maxNumberOfEntriesInNonLeafNodes);
+      nHigh = numberOfInternalNode;
+      childNodeGroups = internalNodeGroups;
+      interNSKeyList = interNSKeyTmpList;
+    }
+    root = curNode;
+  }
+
+  /**
+   * Below method is to convert the start key
+   * into fixed and variable length key.
+   * data format<lenght><fixed length key><length><variable length key>
+   *
+   * @param startKey
+   * @return Index key
+   */
+  protected IndexKey convertStartKeyToNodeEntry(byte[] startKey) {
+    ByteBuffer buffer = ByteBuffer.wrap(startKey);
+    buffer.rewind();
+    int dictonaryKeySize = buffer.getInt();
+    int nonDictonaryKeySize = buffer.getInt();
+    byte[] dictionaryKey = new byte[dictonaryKeySize];
+    buffer.get(dictionaryKey);
+    byte[] nonDictionaryKey = new byte[nonDictonaryKeySize];
+    buffer.get(nonDictionaryKey);
+    IndexKey entry = new IndexKey(dictionaryKey, nonDictionaryKey);
+    return entry;
+  }
+
+  /**
+   * Below method will be used to get the first data block
+   * in Btree case it will be root node
+   */
+  @Override public BTreeNode get() {
+    return root;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeLeafNode.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeLeafNode.java
new file mode 100644
index 0000000..de476ad
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeLeafNode.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.impl.btree;
+
+import org.apache.carbondata.core.carbon.datastore.DataRefNode;
+import org.apache.carbondata.core.carbon.datastore.IndexKey;
+import org.apache.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.carbon.datastore.chunk.MeasureColumnDataChunk;
+import org.apache.carbondata.core.datastorage.store.FileHolder;
+
+/**
+ * Non leaf node abstract class
+ */
+public abstract class AbstractBTreeLeafNode implements BTreeNode {
+
+  /**
+   * number of keys in a btree
+   */
+  protected int numberOfKeys;
+
+  /**
+   * node number
+   */
+  protected long nodeNumber;
+
+  /**
+   * Next node of the leaf
+   */
+  protected BTreeNode nextNode;
+
+  /**
+   * max key of the column this will be used to check whether this leaf will
+   * be used for scanning or not
+   */
+  protected byte[][] maxKeyOfColumns;
+
+  /**
+   * min key of the column this will be used to check whether this leaf will
+   * be used for scanning or not
+   */
+  protected byte[][] minKeyOfColumns;
+
+  /**
+   * Method to get the next block this can be used while scanning when
+   * iterator of this class can be used iterate over blocks
+   *
+   * @return next block
+   */
+  @Override public int nodeSize() {
+    return this.numberOfKeys;
+  }
+
+  /**
+   * below method will used to set the next node
+   *
+   * @param nextNode
+   */
+  @Override public void setNextNode(BTreeNode nextNode) {
+    this.nextNode = nextNode;
+  }
+
+  /**
+   * Below method is to get the children based on index
+   *
+   * @param index children index
+   * @return btree node
+   */
+  @Override public BTreeNode getChild(int index) {
+    throw new UnsupportedOperationException("Operation not supported in case of leaf node");
+  }
+
+  /**
+   * below method to set the node entry
+   *
+   * @param key node entry
+   */
+  @Override public void setKey(IndexKey key) {
+    throw new UnsupportedOperationException("Operation not supported in case of leaf node");
+  }
+
+  /**
+   * Method can be used to get the block index .This can be used when multiple
+   * thread can be used scan group of blocks in that can we can assign the
+   * some of the blocks to one thread and some to other
+   *
+   * @return block number
+   */
+  @Override public long nodeNumber() {
+    return nodeNumber;
+  }
+
+  /**
+   * This method will be used to get the max value of all the columns this can
+   * be used in case of filter query
+   *
+   * @param max value of all the columns
+   */
+  @Override public byte[][] getColumnsMaxValue() {
+    return maxKeyOfColumns;
+  }
+
+  /**
+   * This method will be used to get the max value of all the columns this can
+   * be used in case of filter query
+   *
+   * @param max value of all the columns
+   */
+  @Override public byte[][] getColumnsMinValue() {
+    return minKeyOfColumns;
+  }
+
+  /**
+   * to check whether node in a btree is a leaf node or not
+   *
+   * @return leaf node or not
+   */
+  @Override public boolean isLeafNode() {
+    return true;
+  }
+
+  /**
+   * Method to get the next block this can be used while scanning when
+   * iterator of this class can be used iterate over blocks
+   *
+   * @return next block
+   */
+  @Override public DataRefNode getNextDataRefNode() {
+    return nextNode;
+  }
+
+  /**
+   * below method will return the one node indexes
+   *
+   * @return node entry array
+   */
+  @Override public IndexKey[] getNodeKeys() {
+    // as this is a leaf node so this method implementation is not required
+    throw new UnsupportedOperationException("Operation not supported in case of leaf node");
+  }
+
+  /**
+   * below method will be used to set the children of intermediate node
+   *
+   * @param children array
+   */
+  @Override public void setChildren(BTreeNode[] children) {
+    // no required in case of leaf node as leaf node will not have any children
+    throw new UnsupportedOperationException("Operation not supported in case of leaf node");
+  }
+
+  /**
+   * Below method will be used to get the dimension chunks
+   *
+   * @param fileReader   file reader to read the chunks from file
+   * @param blockIndexes indexes of the blocks need to be read
+   * @return dimension data chunks
+   */
+  @Override public DimensionColumnDataChunk[] getDimensionChunks(FileHolder fileReader,
+      int[] blockIndexes) {
+    // No required here as leaf which will will be use this class will implement its own get
+    // dimension chunks
+    return null;
+  }
+
+  /**
+   * Below method will be used to get the dimension chunk
+   *
+   * @param fileReader file reader to read the chunk from file
+   * @param blockIndex block index to be read
+   * @return dimension data chunk
+   */
+  @Override public DimensionColumnDataChunk getDimensionChunk(FileHolder fileReader,
+      int blockIndex) {
+    // No required here as leaf which will will be use this class will implement
+    // its own get dimension chunks
+    return null;
+  }
+
+  /**
+   * Below method will be used to get the measure chunk
+   *
+   * @param fileReader   file reader to read the chunk from file
+   * @param blockIndexes block indexes to be read from file
+   * @return measure column data chunk
+   */
+  @Override public MeasureColumnDataChunk[] getMeasureChunks(FileHolder fileReader,
+      int[] blockIndexes) {
+    // No required here as leaf which will will be use this class will implement its own get
+    // measure chunks
+    return null;
+  }
+
+  /**
+   * Below method will be used to read the measure chunk
+   *
+   * @param fileReader file read to read the file chunk
+   * @param blockIndex block index to be read from file
+   * @return measure data chunk
+   */
+  @Override public MeasureColumnDataChunk getMeasureChunk(FileHolder fileReader, int blockIndex) {
+    // No required here as leaf which will will be use this class will implement its own get
+    // measure chunks
+    return null;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeDataRefNodeFinder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeDataRefNodeFinder.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeDataRefNodeFinder.java
new file mode 100644
index 0000000..e443182
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeDataRefNodeFinder.java
@@ -0,0 +1,264 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.impl.btree;
+
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.core.carbon.datastore.DataRefNode;
+import org.apache.carbondata.core.carbon.datastore.DataRefNodeFinder;
+import org.apache.carbondata.core.carbon.datastore.IndexKey;
+import org.apache.carbondata.core.util.ByteUtil;
+
+/**
+ * Below class will be used to find a block in a btree
+ */
+public class BTreeDataRefNodeFinder implements DataRefNodeFinder {
+
+  /**
+   * no dictionary column value is of variable length so in each column value
+   * it will -1
+   */
+  private static final int NO_DCITIONARY_COLUMN_VALUE = -1;
+
+  /**
+   * sized of the short value in bytes
+   */
+  private static final short SHORT_SIZE_IN_BYTES = 2;
+  /**
+   * this will holds the information about the size of each value of a column,
+   * this will be used during Comparison of the btree node value and the
+   * search value if value is more than zero then its a fixed length column
+   * else its variable length column. So as data of both type of column store
+   * separately so this value size array will be used for both purpose
+   * comparison and jumping(which type value we need to compare)
+   */
+  private int[] eachColumnValueSize;
+
+  /**
+   * this will be used during search for no dictionary column
+   */
+  private int numberOfNoDictionaryColumns;
+
+  public BTreeDataRefNodeFinder(int[] eachColumnValueSize) {
+    this.eachColumnValueSize = eachColumnValueSize;
+
+    for (int i = 0; i < eachColumnValueSize.length; i++) {
+      if (eachColumnValueSize[i] == -1) {
+        numberOfNoDictionaryColumns++;
+      }
+    }
+  }
+
+  /**
+   * Below method will be used to get the first tentative data block based on
+   * search key
+   *
+   * @param dataBlocks complete data blocks present
+   * @param serachKey  key to be search
+   * @return data block
+   */
+  @Override public DataRefNode findFirstDataBlock(DataRefNode dataRefBlock, IndexKey searchKey) {
+    // as its for btree type cast it to btree interface
+    BTreeNode rootNode = (BTreeNode) dataRefBlock;
+    while (!rootNode.isLeafNode()) {
+      rootNode = findFirstLeafNode(searchKey, rootNode);
+    }
+    return rootNode;
+  }
+
+  /**
+   * Below method will be used to get the last data tentative block based on
+   * search key
+   *
+   * @param dataBlocks complete data blocks present
+   * @param serachKey  key to be search
+   * @return data block
+   */
+  @Override public DataRefNode findLastDataBlock(DataRefNode dataRefBlock, IndexKey searchKey) {
+    // as its for btree type cast it to btree interface
+    BTreeNode rootNode = (BTreeNode) dataRefBlock;
+    while (!rootNode.isLeafNode()) {
+      rootNode = findLastLeafNode(searchKey, rootNode);
+    }
+    return rootNode;
+  }
+
+  /**
+   * Binary search used to get the first tentative block of the btree based on
+   * search key
+   *
+   * @param key  search key
+   * @param node root node of btree
+   * @return first tentative block
+   */
+  private BTreeNode findFirstLeafNode(IndexKey key, BTreeNode node) {
+    int childNodeIndex;
+    int low = 0;
+    int high = node.nodeSize() - 1;
+    int mid = 0;
+    int compareRes = -1;
+    IndexKey[] nodeKeys = node.getNodeKeys();
+    //
+    while (low <= high) {
+      mid = (low + high) >>> 1;
+      // compare the entries
+      compareRes = compareIndexes(key, nodeKeys[mid]);
+      if (compareRes < 0) {
+        high = mid - 1;
+      } else if (compareRes > 0) {
+        low = mid + 1;
+      } else {
+        // if key is matched then get the first entry
+        int currentPos = mid;
+        while (currentPos - 1 >= 0 && compareIndexes(key, nodeKeys[currentPos - 1]) == 0) {
+          currentPos--;
+        }
+        mid = currentPos;
+        break;
+      }
+    }
+    // if compare result is less than zero then we
+    // and mid is more than 0 then we need to previous block as duplicates
+    // record can be present
+    if (compareRes < 0) {
+      if (mid > 0) {
+        mid--;
+      }
+      childNodeIndex = mid;
+    } else {
+      childNodeIndex = mid;
+    }
+    // get the leaf child
+    node = node.getChild(childNodeIndex);
+    return node;
+  }
+
+  /**
+   * Binary search used to get the last tentative block of the btree based on
+   * search key
+   *
+   * @param key  search key
+   * @param node root node of btree
+   * @return first tentative block
+   */
+  private BTreeNode findLastLeafNode(IndexKey key, BTreeNode node) {
+    int childNodeIndex;
+    int low = 0;
+    int high = node.nodeSize() - 1;
+    int mid = 0;
+    int compareRes = -1;
+    IndexKey[] nodeKeys = node.getNodeKeys();
+    //
+    while (low <= high) {
+      mid = (low + high) >>> 1;
+      // compare the entries
+      compareRes = compareIndexes(key, nodeKeys[mid]);
+      if (compareRes < 0) {
+        high = mid - 1;
+      } else if (compareRes > 0) {
+        low = mid + 1;
+      } else {
+        int currentPos = mid;
+        // if key is matched then get the first entry
+        while (currentPos + 1 < node.nodeSize()
+            && compareIndexes(key, nodeKeys[currentPos + 1]) == 0) {
+          currentPos++;
+        }
+        mid = currentPos;
+        break;
+      }
+    }
+    // if compare result is less than zero then we
+    // and mid is more than 0 then we need to previous block as duplicates
+    // record can be present
+    if (compareRes < 0) {
+      if (mid > 0) {
+        mid--;
+      }
+      childNodeIndex = mid;
+    } else {
+      childNodeIndex = mid;
+    }
+    node = node.getChild(childNodeIndex);
+    return node;
+  }
+
+  /**
+   * Comparison of index key will be following format of key <Dictionary> key
+   * will be in byte array No dictionary key Index of FirstKey (2
+   * bytes)><Index of SecondKey (2 bytes)><Index of NKey (2 bytes)> <First Key
+   * ByteArray><2nd Key ByteArray><N Key ByteArray> in each column value size
+   * of no dictionary column will be -1 if in each column value is not -1 then
+   * compare the byte array based on size and increment the offset to
+   * dictionary column size if size is -1 then its a no dictionary key so to
+   * get the length subtract the size of current with next key offset it will
+   * give the actual length if it is at last position or only one key is
+   * present then subtract with length
+   *
+   * @param first  key
+   * @param second key
+   * @return comparison value
+   */
+  private int compareIndexes(IndexKey first, IndexKey second) {
+    int dictionaryKeyOffset = 0;
+    int nonDictionaryKeyOffset = 0;
+    int compareResult = 0;
+    int processedNoDictionaryColumn = numberOfNoDictionaryColumns;
+    ByteBuffer firstNoDictionaryKeyBuffer = ByteBuffer.wrap(first.getNoDictionaryKeys());
+    ByteBuffer secondNoDictionaryKeyBuffer = ByteBuffer.wrap(second.getNoDictionaryKeys());
+    int actualOffset = 0;
+    int firstNoDcitionaryLength = 0;
+    int secondNodeDictionaryLength = 0;
+
+    for (int i = 0; i < eachColumnValueSize.length; i++) {
+
+      if (eachColumnValueSize[i] != NO_DCITIONARY_COLUMN_VALUE) {
+        compareResult = ByteUtil.UnsafeComparer.INSTANCE
+            .compareTo(first.getDictionaryKeys(), dictionaryKeyOffset, eachColumnValueSize[i],
+                second.getDictionaryKeys(), dictionaryKeyOffset, eachColumnValueSize[i]);
+        dictionaryKeyOffset += eachColumnValueSize[i];
+      } else {
+        if (processedNoDictionaryColumn > 1) {
+          actualOffset = firstNoDictionaryKeyBuffer.getShort(nonDictionaryKeyOffset);
+          firstNoDcitionaryLength =
+              firstNoDictionaryKeyBuffer.getShort(nonDictionaryKeyOffset + SHORT_SIZE_IN_BYTES);
+          secondNodeDictionaryLength =
+              secondNoDictionaryKeyBuffer.getShort(nonDictionaryKeyOffset + SHORT_SIZE_IN_BYTES);
+          compareResult = ByteUtil.UnsafeComparer.INSTANCE
+              .compareTo(first.getNoDictionaryKeys(), actualOffset, firstNoDcitionaryLength,
+                  second.getNoDictionaryKeys(), actualOffset, secondNodeDictionaryLength);
+          nonDictionaryKeyOffset += SHORT_SIZE_IN_BYTES;
+          processedNoDictionaryColumn--;
+        } else {
+          actualOffset = firstNoDictionaryKeyBuffer.getShort(nonDictionaryKeyOffset);
+          firstNoDcitionaryLength = first.getNoDictionaryKeys().length - actualOffset;
+          secondNodeDictionaryLength = second.getNoDictionaryKeys().length - actualOffset;
+          compareResult = ByteUtil.UnsafeComparer.INSTANCE
+              .compareTo(first.getNoDictionaryKeys(), actualOffset, firstNoDcitionaryLength,
+                  second.getNoDictionaryKeys(), actualOffset, secondNodeDictionaryLength);
+        }
+      }
+      if (compareResult != 0) {
+        return compareResult;
+      }
+    }
+
+    return 0;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeNode.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeNode.java
new file mode 100644
index 0000000..6b624c6
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeNode.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.impl.btree;
+
+import org.apache.carbondata.core.carbon.datastore.DataRefNode;
+import org.apache.carbondata.core.carbon.datastore.IndexKey;
+
+/**
+ * Interface for btree node
+ */
+public interface BTreeNode extends DataRefNode {
+
+  /**
+   * below method will return the one node indexes
+   *
+   * @return node entry array
+   */
+  IndexKey[] getNodeKeys();
+
+  /**
+   * to check whether node in a btree is a leaf node or not
+   *
+   * @return leaf node or not
+   */
+  boolean isLeafNode();
+
+  /**
+   * below method will be used to set the children of intermediate node
+   *
+   * @param children array
+   */
+  void setChildren(BTreeNode[] children);
+
+  /**
+   * below method will used to set the next node
+   *
+   * @param nextNode
+   */
+  void setNextNode(BTreeNode nextNode);
+
+  /**
+   * Below method is to get the children based on index
+   *
+   * @param index children index
+   * @return btree node
+   */
+  BTreeNode getChild(int index);
+
+  /**
+   * below method to set the node entry
+   *
+   * @param key node entry
+   */
+  void setKey(IndexKey key);
+}


Mime
View raw message