carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chenliang...@apache.org
Subject [37/52] [partial] incubator-carbondata git commit: move core package
Date Mon, 16 Jan 2017 14:53:14 GMT
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeByteMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeByteMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeByteMeasureChunkStore.java
new file mode 100644
index 0000000..6100dae
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeByteMeasureChunkStore.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datastore.chunk.store.impl.unsafe;
+
+import org.apache.carbondata.core.memory.CarbonUnsafe;
+import org.apache.carbondata.core.memory.MemoryAllocatorFactory;
+
+/**
+ * Responsible for storing Byte array data to memory. memory can be on heap or
+ * offheap based on the user configuration using unsafe interface
+ */
+public class UnsafeByteMeasureChunkStore extends UnsafeAbstractMeasureDataChunkStore<byte[]> {
+
+  public UnsafeByteMeasureChunkStore(int numberOfRows) {
+    super(numberOfRows);
+  }
+
+  /**
+   * Below method will be used to put byte array data to memory
+   *
+   * @param data
+   */
+  @Override public void putData(byte[] data) {
+    assert (!this.isMemoryOccupied);
+    this.dataPageMemoryBlock =
+        MemoryAllocatorFactory.INSATANCE.getMemoryAllocator().allocate(data.length);
+    // copy the data to memory
+    CarbonUnsafe.unsafe
+        .copyMemory(data, CarbonUnsafe.BYTE_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(),
+            dataPageMemoryBlock.getBaseOffset(), dataPageMemoryBlock.size());
+    this.isMemoryOccupied = true;
+  }
+
+  /**
+   * to get the byte value
+   *
+   * @param index
+   * @return byte value based on index
+   */
+  @Override public byte getByte(int index) {
+    return CarbonUnsafe.unsafe
+        .getByte(dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + index);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeDoubleMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeDoubleMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeDoubleMeasureChunkStore.java
new file mode 100644
index 0000000..5b888ed
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeDoubleMeasureChunkStore.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datastore.chunk.store.impl.unsafe;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.memory.CarbonUnsafe;
+import org.apache.carbondata.core.memory.MemoryAllocatorFactory;
+
+/**
+ * Responsible for storing double array data to memory. memory can be on heap or
+ * offheap based on the user configuration using unsafe interface
+ */
+public class UnsafeDoubleMeasureChunkStore extends UnsafeAbstractMeasureDataChunkStore<double[]> {
+
+  public UnsafeDoubleMeasureChunkStore(int numberOfRows) {
+    super(numberOfRows);
+  }
+
+  /**
+   * Below method will be used to put double array data to memory
+   *
+   * @param data
+   */
+  @Override public void putData(double[] data) {
+    assert (!this.isMemoryOccupied);
+    this.dataPageMemoryBlock = MemoryAllocatorFactory.INSATANCE.getMemoryAllocator()
+        .allocate(data.length * CarbonCommonConstants.DOUBLE_SIZE_IN_BYTE);
+    // copy the data to memory
+    CarbonUnsafe.unsafe
+        .copyMemory(data, CarbonUnsafe.DOUBLE_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(),
+            dataPageMemoryBlock.getBaseOffset(), dataPageMemoryBlock.size());
+    this.isMemoryOccupied = true;
+  }
+
+  /**
+   * to get the double value
+   *
+   * @param index
+   * @return double value based on index
+   */
+  @Override public double getDouble(int index) {
+    return CarbonUnsafe.unsafe.getDouble(dataPageMemoryBlock.getBaseObject(),
+        dataPageMemoryBlock.getBaseOffset() + (index * CarbonCommonConstants.DOUBLE_SIZE_IN_BYTE));
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
new file mode 100644
index 0000000..6a518d1
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.datastore.chunk.store.impl.unsafe;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.memory.CarbonUnsafe;
+
+/**
+ * Below class is responsible to store fixed length dimension data chunk in
+ * memory Memory occupied can be on heap or offheap using unsafe interface
+ */
+public class UnsafeFixedLengthDimensionDataChunkStore
+    extends UnsafeAbstractDimensionDataChunkStore {
+
+  /**
+   * Size of each value
+   */
+  private int columnValueSize;
+
+  /**
+   * Constructor
+   *
+   * @param columnValueSize value of each column
+   * @param isInvertedIdex  is inverted index present
+   * @param numberOfRows    total number of rows
+   */
+  public UnsafeFixedLengthDimensionDataChunkStore(long totalDataSize, int columnValueSize,
+      boolean isInvertedIdex, int numberOfRows) {
+    super(totalDataSize, isInvertedIdex, numberOfRows);
+    this.columnValueSize = columnValueSize;
+  }
+
+  /**
+   * Below method will be used to get the row based inverted index
+   *
+   * @param rowId Inverted index
+   */
+  @Override public byte[] getRow(int rowId) {
+    // if column was explicitly sorted we need to get the rowid based inverted index reverse
+    if (isExplicitSorted) {
+      rowId = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+          dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + (rowId
+              * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+    }
+    // creating a row
+    byte[] data = new byte[columnValueSize];
+    //copy the row from memory block based on offset
+    // offset position will be index * each column value length
+    CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(),
+        dataPageMemoryBlock.getBaseOffset() + (rowId * columnValueSize), data,
+        CarbonUnsafe.BYTE_ARRAY_OFFSET, columnValueSize);
+    return data;
+  }
+
+  /**
+   * Below method will be used to get the surrogate key of the based on the row
+   * id passed
+   *
+   * @param rowId row id
+   * @return surrogate key
+   */
+  @Override public int getSurrogate(int index) {
+    // if column was explicitly sorted we need to get the rowid based inverted index reverse
+    if (isExplicitSorted) {
+      index = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+          dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + (index
+              * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+    }
+    // below part is to convert the byte array to surrogate value
+    int startOffsetOfData = index * columnValueSize;
+    int surrogate = 0;
+    for (int i = 0; i < columnValueSize; i++) {
+      surrogate <<= 8;
+      surrogate ^= CarbonUnsafe.unsafe.getByte(dataPageMemoryBlock.getBaseObject(),
+          dataPageMemoryBlock.getBaseOffset() + startOffsetOfData) & 0xFF;
+      startOffsetOfData++;
+    }
+    return surrogate;
+  }
+
+  /**
+   * Below method will be used to fill the row values to buffer array
+   *
+   * @param rowId  row id of the data to be filled
+   * @param data   buffer in which data will be filled
+   * @param offset off the of the buffer
+   */
+  @Override public void fillRow(int rowId, byte[] buffer, int offset) {
+    // if column was explicitly sorted we need to get the rowid based inverted index reverse
+    if (isExplicitSorted) {
+      rowId = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+          dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + (rowId
+              * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+    }
+    //copy the row from memory block based on offset
+    // offset position will be index * each column value length
+    CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(),
+        dataPageMemoryBlock.getBaseOffset() + (rowId * columnValueSize), buffer,
+        CarbonUnsafe.BYTE_ARRAY_OFFSET + offset, columnValueSize);
+  }
+
+  /**
+   * @return size of each column value
+   */
+  @Override public int getColumnValueSize() {
+    return columnValueSize;
+  }
+
+  /**
+   * to compare the two byte array
+   *
+   * @param index        index of first byte array
+   * @param compareValue value of to be compared
+   * @return compare result
+   */
+  @Override public int compareTo(int index, byte[] compareValue) {
+    // based on index we need to calculate the actual position in memory block
+    index = index * columnValueSize;
+    int compareResult = 0;
+    for (int i = 0; i < compareValue.length; i++) {
+      compareResult = (CarbonUnsafe.unsafe
+          .getByte(dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + index)
+          & 0xff) - (compareValue[i] & 0xff);
+      if (compareResult != 0) {
+        break;
+      }
+      index++;
+    }
+    return compareResult;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeIntMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeIntMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeIntMeasureChunkStore.java
new file mode 100644
index 0000000..d6b8142
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeIntMeasureChunkStore.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datastore.chunk.store.impl.unsafe;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.memory.CarbonUnsafe;
+import org.apache.carbondata.core.memory.MemoryAllocatorFactory;
+
+/**
+ * Responsible for storing int array data to memory. memory can be on heap or
+ * offheap based on the user configuration using unsafe interface
+ */
+public class UnsafeIntMeasureChunkStore extends UnsafeAbstractMeasureDataChunkStore<int[]> {
+
+  public UnsafeIntMeasureChunkStore(int numberOfRows) {
+    super(numberOfRows);
+  }
+
+  /**
+   * Below method will be used to put int array data to memory
+   *
+   * @param data
+   */
+  @Override public void putData(int[] data) {
+    assert (!this.isMemoryOccupied);
+    this.dataPageMemoryBlock = MemoryAllocatorFactory.INSATANCE.getMemoryAllocator()
+        .allocate(data.length * CarbonCommonConstants.INT_SIZE_IN_BYTE);
+    // copy the data to memory
+    CarbonUnsafe.unsafe
+        .copyMemory(data, CarbonUnsafe.INT_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(),
+            dataPageMemoryBlock.getBaseOffset(), dataPageMemoryBlock.size());
+    this.isMemoryOccupied = true;
+  }
+
+  /**
+   * to get the int value
+   *
+   * @param index
+   * @return int value based on index
+   */
+  @Override public int getInt(int index) {
+    return CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+        dataPageMemoryBlock.getBaseOffset() + (index * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeLongMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeLongMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeLongMeasureChunkStore.java
new file mode 100644
index 0000000..da90a1e
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeLongMeasureChunkStore.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datastore.chunk.store.impl.unsafe;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.memory.CarbonUnsafe;
+import org.apache.carbondata.core.memory.MemoryAllocatorFactory;
+
+/**
+ * Responsible for storing long array data to memory. memory can be on heap or
+ * offheap based on the user configuration using unsafe interface
+ */
+public class UnsafeLongMeasureChunkStore extends UnsafeAbstractMeasureDataChunkStore<long[]> {
+
+  public UnsafeLongMeasureChunkStore(int numberOfRows) {
+    super(numberOfRows);
+  }
+
+  /**
+   * Below method will be used to put long array data to memory
+   *
+   * @param data
+   */
+  @Override public void putData(long[] data) {
+    assert (!this.isMemoryOccupied);
+    this.dataPageMemoryBlock = MemoryAllocatorFactory.INSATANCE.getMemoryAllocator()
+        .allocate(data.length * CarbonCommonConstants.LONG_SIZE_IN_BYTE);
+    // copy the data to memory
+    CarbonUnsafe.unsafe
+        .copyMemory(data, CarbonUnsafe.LONG_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(),
+            dataPageMemoryBlock.getBaseOffset(), dataPageMemoryBlock.size());
+    this.isMemoryOccupied = true;
+  }
+
+  /**
+   * to get the long value
+   *
+   * @param index
+   * @return long value based on index
+   */
+  @Override public long getLong(int index) {
+    return CarbonUnsafe.unsafe.getLong(dataPageMemoryBlock.getBaseObject(),
+        dataPageMemoryBlock.getBaseOffset() + (index * CarbonCommonConstants.LONG_SIZE_IN_BYTE));
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeShortMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeShortMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeShortMeasureChunkStore.java
new file mode 100644
index 0000000..7924099
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeShortMeasureChunkStore.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datastore.chunk.store.impl.unsafe;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.memory.CarbonUnsafe;
+import org.apache.carbondata.core.memory.MemoryAllocatorFactory;
+
+/**
+ * Responsible for storing short array data to memory. memory can be on heap or
+ * offheap based on the user configuration using unsafe interface
+ */
+public class UnsafeShortMeasureChunkStore extends UnsafeAbstractMeasureDataChunkStore<short[]> {
+
+  public UnsafeShortMeasureChunkStore(int numberOfRows) {
+    super(numberOfRows);
+  }
+
+  /**
+   * Below method will be used to put short array data to memory
+   *
+   * @param data
+   */
+  @Override public void putData(short[] data) {
+    assert (!this.isMemoryOccupied);
+    this.dataPageMemoryBlock = MemoryAllocatorFactory.INSATANCE.getMemoryAllocator()
+        .allocate(data.length * CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
+    // copy the data to memory
+    CarbonUnsafe.unsafe
+        .copyMemory(data, CarbonUnsafe.SHORT_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(),
+            dataPageMemoryBlock.getBaseOffset(), dataPageMemoryBlock.size());
+    this.isMemoryOccupied = true;
+  }
+
+  /**
+   * to get the short value
+   *
+   * @param index
+   * @return shot value based on index
+   */
+  @Override public short getShort(int index) {
+    return CarbonUnsafe.unsafe.getShort(dataPageMemoryBlock.getBaseObject(),
+        dataPageMemoryBlock.getBaseOffset() + (index * CarbonCommonConstants.SHORT_SIZE_IN_BYTE));
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
new file mode 100644
index 0000000..507ee2c
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.datastore.chunk.store.impl.unsafe;
+
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.memory.CarbonUnsafe;
+
+/**
+ * Below class is responsible to store variable length dimension data chunk in
+ * memory Memory occupied can be on heap or offheap using unsafe interface
+ */
+public class UnsafeVariableLengthDimesionDataChunkStore
+    extends UnsafeAbstractDimensionDataChunkStore {
+
+  /**
+   * total number of rows
+   */
+  private int numberOfRows;
+
+  /**
+   * pointers offsets
+   */
+  private long dataPointersOffsets;
+
+  public UnsafeVariableLengthDimesionDataChunkStore(long totalSize, boolean isInvertedIdex,
+      int numberOfRows) {
+    super(totalSize, isInvertedIdex, numberOfRows);
+    this.numberOfRows = numberOfRows;
+  }
+
+  /**
+   * Below method will be used to put the rows and its metadata in offheap
+   *
+   * @param invertedIndex        inverted index to be stored
+   * @param invertedIndexReverse inverted index reverse to be stored
+   * @param data                 data to be stored
+   */
+  @Override public void putArray(final int[] invertedIndex, final int[] invertedIndexReverse,
+      byte[] data) {
+    // first put the data, inverted index and reverse inverted index to memory
+    super.putArray(invertedIndex, invertedIndexReverse, data);
+    // position from where offsets will start
+    this.dataPointersOffsets = this.invertedIndexReverseOffset;
+    if (isExplicitSorted) {
+      this.dataPointersOffsets += numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE;
+    }
+    // As data is of variable length and data format is
+    // <length in short><data><length in short><data>
+    // we need to store offset of each data so data can be accessed directly
+    // for example:
+    //data = {0,5,1,2,3,4,5,0,6,0,1,2,3,4,5,0,2,8,9}
+    //so value stored in offset will be position of actual data
+    // [2,9,17]
+    // to store this value we need to get the actual data length + 2 bytes used for storing the
+    // length
+
+    // start position will be used to store the current data position
+    int startOffset = 0;
+    // position from where offsets will start
+    long pointerOffsets = this.dataPointersOffsets;
+    // as first position will be start from 2 byte as data is stored first in the memory block
+    // we need to skip first two bytes this is because first two bytes will be length of the data
+    // which we have to skip
+    CarbonUnsafe.unsafe.putInt(dataPageMemoryBlock.getBaseObject(),
+        dataPageMemoryBlock.getBaseOffset() + pointerOffsets,
+        CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
+    // incrementing the pointers as first value is already filled and as we are storing as int
+    // we need to increment the 4 bytes to set the position of the next value to set
+    pointerOffsets += CarbonCommonConstants.INT_SIZE_IN_BYTE;
+    // creating a byte buffer which will wrap the length of the row
+    // using byte buffer as unsafe will return bytes in little-endian encoding
+    ByteBuffer buffer = ByteBuffer.allocate(CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
+    // store length of data
+    byte[] length = new byte[CarbonCommonConstants.SHORT_SIZE_IN_BYTE];
+    // as first offset is already stored, we need to start from the 2nd row in data array
+    for (int i = 1; i < numberOfRows; i++) {
+      // first copy the length of previous row
+      CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(),
+          dataPageMemoryBlock.getBaseOffset() + startOffset, length, CarbonUnsafe.BYTE_ARRAY_OFFSET,
+          CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
+      buffer.put(length);
+      buffer.flip();
+      // so current row position will be
+      // previous row length + 2 bytes used for storing previous row data
+      startOffset += CarbonCommonConstants.SHORT_SIZE_IN_BYTE + buffer.getShort();
+      // as same byte buffer is used to avoid creating many byte buffer for each row
+      // we need to clear the byte buffer
+      buffer.clear();
+      // now put the offset of current row, here we need to add 2 more bytes as current will
+      // also have length part so we have to skip length
+      CarbonUnsafe.unsafe.putInt(dataPageMemoryBlock.getBaseObject(),
+          dataPageMemoryBlock.getBaseOffset() + pointerOffsets,
+          startOffset + CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
+      // incrementing the pointers as first value is already filled and as we are storing as int
+      // we need to increment the 4 bytes to set the position of the next value to set
+      pointerOffsets += CarbonCommonConstants.INT_SIZE_IN_BYTE;
+    }
+
+  }
+
+  /**
+   * Below method will be used to get the row based on row id passed
+   *
+   * @param index
+   * @return row
+   */
+  @Override public byte[] getRow(int rowId) {
+    byte[] data = null;
+    try {
+      // if column was explicitly sorted we need to get the rowid based inverted index reverse
+      if (isExplicitSorted) {
+        rowId = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+            dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + (rowId
+                * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+      }
+      // now to get the row from memory block we need to do following thing
+      // 1. first get the current offset
+      // 2. if it's not a last row- get the next row offset
+      // Subtract the current row offset + 2 bytes(to skip the data length) with next row offset
+      // else subtract the current row offset + 2 bytes(to skip the data length)
+      // with complete data length
+      int currentDataOffset = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+          dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + (rowId
+              * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+      short length = 0;
+      // calculating the length of data
+      if (rowId < numberOfRows - 1) {
+        int OffsetOfNextdata = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+            dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + ((rowId + 1)
+                * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+        length = (short) (OffsetOfNextdata - (currentDataOffset
+            + CarbonCommonConstants.SHORT_SIZE_IN_BYTE));
+      } else {
+        // for last record we need to subtract with data length
+        length = (short) (this.dataLength - currentDataOffset);
+      }
+      data = new byte[length];
+      CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(),
+          dataPageMemoryBlock.getBaseOffset() + currentDataOffset, data,
+          CarbonUnsafe.BYTE_ARRAY_OFFSET, length);
+    } catch (Throwable t) {
+      System.out.println();
+    }
+    return data;
+  }
+
+  /**
+   * to compare the two byte array
+   *
+   * @param index        index of first byte array
+   * @param compareValue value of to be compared
+   * @return compare result
+   */
+  @Override public int compareTo(int index, byte[] compareValue) {
+    // now to get the row from memory block we need to do following thing
+    // 1. first get the current offset
+    // 2. if it's not a last row- get the next row offset
+    // Subtract the current row offset + 2 bytes(to skip the data length) with next row offset
+    // else subtract the current row offset
+    // with complete data length get the offset of set of data
+    int currentDataOffset = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+        dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + (index
+            * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+    short length = 0;
+    // calculating the length of data
+    if (index < numberOfRows - 1) {
+      int OffsetOfNextdata = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+          dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + ((index + 1)
+              * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+      length = (short) (OffsetOfNextdata - (currentDataOffset
+          + CarbonCommonConstants.SHORT_SIZE_IN_BYTE));
+    } else {
+      // for last record we need to subtract with data length
+      length = (short) (this.dataLength - currentDataOffset);
+    }
+    // as this class handles this variable length data, so filter value can be
+    // smaller or bigger than than actual data, so we need to take the smaller length
+    int compareResult = 0;
+    int compareLength = length < compareValue.length ? length : compareValue.length;
+    for (int i = 0; i < compareLength; i++) {
+      compareResult = (CarbonUnsafe.unsafe.getByte(dataPageMemoryBlock.getBaseObject(),
+          dataPageMemoryBlock.getBaseOffset() + currentDataOffset) & 0xff) - (compareValue[i]
+          & 0xff);
+      // if compare result is not equal we can break
+      if (compareResult != 0) {
+        break;
+      }
+      // increment the offset by one as comparison is done byte by byte
+      currentDataOffset++;
+    }
+    return compareResult;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java
new file mode 100644
index 0000000..b6155fb
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.ByteUtil;
+
+public class BlockIndexerStorageForInt implements IndexStorage<int[]> {
+  private boolean alreadySorted;
+
+  private int[] dataAfterComp;
+
+  private int[] indexMap;
+
+  private byte[][] keyBlock;
+
+  private int[] dataIndexMap;
+
+  private int totalSize;
+
+  public BlockIndexerStorageForInt(byte[][] keyBlock, boolean compressData, boolean isNoDictionary,
+      boolean isSortRequired) {
+    ColumnWithIntIndex[] columnWithIndexs = createColumnWithIndexArray(keyBlock, isNoDictionary);
+    if (isSortRequired) {
+      Arrays.sort(columnWithIndexs);
+    }
+    compressMyOwnWay(extractDataAndReturnIndexes(columnWithIndexs, keyBlock));
+    if (compressData) {
+      compressDataMyOwnWay(columnWithIndexs);
+    }
+  }
+
+  /**
+   * Create an object with each column array and respective index
+   *
+   * @return
+   */
+  private ColumnWithIntIndex[] createColumnWithIndexArray(byte[][] keyBlock,
+      boolean isNoDictionary) {
+    ColumnWithIntIndex[] columnWithIndexs;
+    if (isNoDictionary) {
+      columnWithIndexs = new ColumnWithIntIndexForHighCard[keyBlock.length];
+      for (int i = 0; i < columnWithIndexs.length; i++) {
+        columnWithIndexs[i] = new ColumnWithIntIndexForHighCard(keyBlock[i], i);
+      }
+
+    } else {
+      columnWithIndexs = new ColumnWithIntIndex[keyBlock.length];
+      for (int i = 0; i < columnWithIndexs.length; i++) {
+        columnWithIndexs[i] = new ColumnWithIntIndex(keyBlock[i], i);
+      }
+    }
+
+    return columnWithIndexs;
+  }
+
+  private int[] extractDataAndReturnIndexes(ColumnWithIntIndex[] columnWithIndexs,
+      byte[][] keyBlock) {
+    int[] indexes = new int[columnWithIndexs.length];
+    for (int i = 0; i < indexes.length; i++) {
+      indexes[i] = columnWithIndexs[i].getIndex();
+      keyBlock[i] = columnWithIndexs[i].getColumn();
+    }
+    this.keyBlock = keyBlock;
+    return indexes;
+  }
+
+  /**
+   * It compresses depends up on the sequence numbers.
+   * [1,2,3,4,6,8,10,11,12,13] is translated to [1,4,6,8,10,13] and [0,6]. In
+   * first array the start and end of sequential numbers and second array
+   * keeps the indexes of where sequential numbers starts. If there is no
+   * sequential numbers then the same array it returns with empty second
+   * array.
+   *
+   * @param indexes
+   */
+  public void compressMyOwnWay(int[] indexes) {
+    List<Integer> list = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+    List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+    int k = 0;
+    int i = 1;
+    for (; i < indexes.length; i++) {
+      if (indexes[i] - indexes[i - 1] == 1) {
+        k++;
+      } else {
+        if (k > 0) {
+          map.add((list.size()));
+          list.add(indexes[i - k - 1]);
+          list.add(indexes[i - 1]);
+        } else {
+          list.add(indexes[i - 1]);
+        }
+        k = 0;
+      }
+    }
+    if (k > 0) {
+      map.add((list.size()));
+      list.add(indexes[i - k - 1]);
+      list.add(indexes[i - 1]);
+    } else {
+      list.add(indexes[i - 1]);
+    }
+    dataAfterComp = convertToArray(list);
+    if (indexes.length == dataAfterComp.length) {
+      indexMap = new int[0];
+    } else {
+      indexMap = convertToArray(map);
+    }
+    if (dataAfterComp.length == 2 && indexMap.length == 1) {
+      alreadySorted = true;
+    }
+  }
+
+  private int[] convertToArray(List<Integer> list) {
+    int[] shortArray = new int[list.size()];
+    for (int i = 0; i < shortArray.length; i++) {
+      shortArray[i] = list.get(i);
+    }
+    return shortArray;
+  }
+
+  /**
+   * @return the alreadySorted
+   */
+  public boolean isAlreadySorted() {
+    return alreadySorted;
+  }
+
+  /**
+   * @return the dataAfterComp
+   */
+  public int[] getDataAfterComp() {
+    return dataAfterComp;
+  }
+
+  /**
+   * @return the indexMap
+   */
+  public int[] getIndexMap() {
+    return indexMap;
+  }
+
+  /**
+   * @return the keyBlock
+   */
+  public byte[][] getKeyBlock() {
+    return keyBlock;
+  }
+
+  private void compressDataMyOwnWay(ColumnWithIntIndex[] indexes) {
+    byte[] prvKey = indexes[0].getColumn();
+    List<ColumnWithIntIndex> list =
+        new ArrayList<ColumnWithIntIndex>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+    list.add(indexes[0]);
+    int counter = 1;
+    int start = 0;
+    List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+    for (int i = 1; i < indexes.length; i++) {
+      if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, indexes[i].getColumn()) != 0) {
+        prvKey = indexes[i].getColumn();
+        list.add(indexes[i]);
+        map.add(start);
+        map.add(counter);
+        start += counter;
+        counter = 1;
+        continue;
+      }
+      counter++;
+    }
+    map.add(start);
+    map.add(counter);
+    this.keyBlock = convertToKeyArray(list);
+    if (indexes.length == keyBlock.length) {
+      dataIndexMap = new int[0];
+    } else {
+      dataIndexMap = convertToArray(map);
+    }
+  }
+
+  private byte[][] convertToKeyArray(List<ColumnWithIntIndex> list) {
+    byte[][] shortArray = new byte[list.size()][];
+    for (int i = 0; i < shortArray.length; i++) {
+      shortArray[i] = list.get(i).getColumn();
+      totalSize += shortArray[i].length;
+    }
+    return shortArray;
+  }
+
+  @Override public int[] getDataIndexMap() {
+    return dataIndexMap;
+  }
+
+  @Override public int getTotalSize() {
+    return totalSize;
+  }
+
+  @Override public byte[] getMin() {
+    return keyBlock[0];
+  }
+
+  @Override public byte[] getMax() {
+    return keyBlock[keyBlock.length - 1];
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
new file mode 100644
index 0000000..d7b980e
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.ByteUtil;
+
+public class BlockIndexerStorageForNoInvertedIndex implements IndexStorage<int[]> {
+  private byte[][] keyBlock;
+  private byte[][] sortedBlock;
+  private int totalSize;
+  private int[] dataIndexMap;
+
+  public BlockIndexerStorageForNoInvertedIndex(byte[][] keyBlockInput, boolean compressData,
+      boolean isNoDictionary) {
+    // without invertedindex but can be RLE
+    if (compressData) {
+      // with RLE
+      byte[] prvKey = keyBlockInput[0];
+      List<byte[]> list = new ArrayList<byte[]>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+      list.add(keyBlockInput[0]);
+      int counter = 1;
+      int start = 0;
+      List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+      int length = keyBlockInput.length;
+      for(int i = 1; i < length; i++) {
+        if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, keyBlockInput[i]) != 0) {
+          prvKey = keyBlockInput[i];
+          list.add(keyBlockInput[i]);
+          map.add(start);
+          map.add(counter);
+          start += counter;
+          counter = 1;
+          continue;
+        }
+        counter++;
+      }
+      map.add(start);
+      map.add(counter);
+      this.keyBlock = convertToKeyArray(list);
+      if (keyBlockInput.length == this.keyBlock.length) {
+        dataIndexMap = new int[0];
+      } else {
+        dataIndexMap = convertToArray(map);
+      }
+    } else {
+      this.keyBlock = keyBlockInput;
+      dataIndexMap = new int[0];
+    }
+
+    this.sortedBlock = new byte[keyBlock.length][];
+    System.arraycopy(keyBlock, 0, sortedBlock, 0, keyBlock.length);
+    if (isNoDictionary) {
+      Arrays.sort(sortedBlock, new Comparator<byte[]>() {
+        @Override
+        public int compare(byte[] col1, byte[] col2) {
+          return ByteUtil.UnsafeComparer.INSTANCE
+              .compareTo(col1, 2, col1.length - 2, col2, 2, col2.length - 2);
+        }
+      });
+    } else {
+      Arrays.sort(sortedBlock, new Comparator<byte[]>() {
+        @Override
+        public int compare(byte[] col1, byte[] col2) {
+          return ByteUtil.UnsafeComparer.INSTANCE.compareTo(col1, col2);
+        }
+      });
+    }
+
+  }
+
+  private int[] convertToArray(List<Integer> list) {
+    int[] shortArray = new int[list.size()];
+    for(int i = 0; i < shortArray.length; i++) {
+      shortArray[i] = list.get(i);
+    }
+    return shortArray;
+  }
+
+  private byte[][] convertToKeyArray(List<byte[]> list) {
+    byte[][] shortArray = new byte[list.size()][];
+    for (int i = 0; i < shortArray.length; i++) {
+      shortArray[i] = list.get(i);
+      totalSize += shortArray[i].length;
+    }
+    return shortArray;
+  }
+
+  @Override
+  public int[] getDataIndexMap() {
+    return dataIndexMap;
+  }
+
+  @Override
+  public int getTotalSize() {
+    return totalSize;
+  }
+
+  @Override
+  public boolean isAlreadySorted() {
+    return true;
+  }
+
+  /**
+   * no use
+   * @return
+   */
+  @Override
+  public int[] getDataAfterComp() {
+    return new int[0];
+  }
+
+  /**
+   * no use
+   * @return
+   */
+  @Override
+  public int[] getIndexMap() {
+    return new int[0];
+  }
+
+  /**
+   * @return the keyBlock
+   */
+  public byte[][] getKeyBlock() {
+    return keyBlock;
+  }
+
+  @Override public byte[] getMin() {
+    return sortedBlock[0];
+  }
+
+  @Override public byte[] getMax() {
+    return sortedBlock[sortedBlock.length - 1];
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java
new file mode 100644
index 0000000..4d4c19d
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.datastore.columnar;
+
+public class ColumnGroupModel {
+
+  /**
+   * number of columns in columnar block
+   */
+  private int[] columnSplit;
+
+  /**
+   * total number of columns
+   */
+  private int noOfColumnsStore;
+
+  /**
+   * whether given index is columnar or not
+   * true: columnar
+   * false: row block
+   */
+  private boolean[] columnarStore;
+
+  /**
+   * column groups
+   * e.g
+   * {{0,1,2},3,4,{5,6}}
+   */
+  private int[][] columnGroups;
+
+  /**
+   * return columnSplit
+   *
+   * @return
+   */
+  public int[] getColumnSplit() {
+    return columnSplit;
+  }
+
+  /**
+   * set columnSplit
+   *
+   * @param split
+   */
+  public void setColumnSplit(int[] split) {
+    this.columnSplit = split;
+  }
+
+  /**
+   * @return no of columnar block
+   */
+  public int getNoOfColumnStore() {
+    return this.noOfColumnsStore;
+  }
+
+  /**
+   * set no of columnar block
+   *
+   * @param noOfColumnsStore
+   */
+  public void setNoOfColumnStore(int noOfColumnsStore) {
+    this.noOfColumnsStore = noOfColumnsStore;
+  }
+
+  /**
+   * it's an identifier for row block or single column block
+   *
+   * @param columnarStore
+   */
+  public void setColumnarStore(boolean[] columnarStore) {
+    this.columnarStore = columnarStore;
+  }
+
+  /**
+   * set column groups
+   *
+   * @param columnGroups
+   */
+  public void setColumnGroup(int[][] columnGroups) {
+    this.columnGroups = columnGroups;
+  }
+
+  /**
+   * check if given column group is columnar
+   *
+   * @param colGroup
+   * @return true if given block is columnar
+   */
+  public boolean isColumnar(int colGroup) {
+    return columnarStore[colGroup];
+  }
+
+  /**
+   * @return columngroups
+   */
+  public int[][] getColumnGroup() {
+    return this.columnGroups;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndex.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndex.java
new file mode 100644
index 0000000..4b35a7f
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndex.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+import java.util.Arrays;
+
+import org.apache.carbondata.core.util.ByteUtil;
+
+public class ColumnWithIntIndex implements Comparable<ColumnWithIntIndex> {
+  protected byte[] column;
+
+  private int index;
+
+  public ColumnWithIntIndex(byte[] column, int index) {
+    this.column = column;
+    this.index = index;
+  }
+
+  public ColumnWithIntIndex() {
+  }
+
+  /**
+   * @return the column
+   */
+  public byte[] getColumn() {
+    return column;
+  }
+
+  /**
+   * @param column the column to set
+   */
+  public void setColumn(byte[] column) {
+    this.column = column;
+  }
+
+  /**
+   * @return the index
+   */
+  public int getIndex() {
+    return index;
+  }
+
+  /**
+   * @param index the index to set
+   */
+  public void setIndex(int index) {
+    this.index = index;
+  }
+
+  @Override public int compareTo(ColumnWithIntIndex o) {
+    return ByteUtil.UnsafeComparer.INSTANCE.compareTo(column, o.column);
+  }
+
+  @Override public boolean equals(Object obj) {
+    if(obj == null || getClass() != obj.getClass()) {
+      return false;
+    }
+    ColumnWithIntIndex o = (ColumnWithIntIndex)obj;
+    return Arrays.equals(column, o.column) && index == o.index;
+  }
+
+  @Override public int hashCode() {
+    return Arrays.hashCode(column) + index;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndexForHighCard.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndexForHighCard.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndexForHighCard.java
new file mode 100644
index 0000000..a6eb6c4
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndexForHighCard.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+import java.util.Arrays;
+
+import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
+
+public class ColumnWithIntIndexForHighCard extends ColumnWithIntIndex
+    implements Comparable<ColumnWithIntIndex> {
+
+  public ColumnWithIntIndexForHighCard(byte[] column, int index) {
+    super(column, index);
+  }
+
+  @Override public int compareTo(ColumnWithIntIndex o) {
+    return UnsafeComparer.INSTANCE
+        .compareTo(column, 2, column.length - 2, o.column, 2, o.column.length - 2);
+  }
+
+  @Override public boolean equals(Object obj) {
+    if(obj == null || getClass() != obj.getClass()) {
+      return false;
+    }
+    ColumnWithIntIndexForHighCard o = (ColumnWithIntIndexForHighCard)obj;
+    return Arrays.equals(column, o.column) && getIndex() == o.getIndex();
+  }
+
+  @Override public int hashCode() {
+    return Arrays.hashCode(column) + getIndex();
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreDataHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreDataHolder.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreDataHolder.java
new file mode 100644
index 0000000..6c3acf9
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreDataHolder.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+import java.nio.ByteBuffer;
+
+public class ColumnarKeyStoreDataHolder {
+  private byte[] keyblockData;
+  private ColumnarKeyStoreMetadata columnarKeyStoreMetadata;
+
+  public ColumnarKeyStoreDataHolder(final byte[] keyblockData,
+      final ColumnarKeyStoreMetadata columnarKeyStoreMetadata) {
+    this.keyblockData = keyblockData;
+    this.columnarKeyStoreMetadata = columnarKeyStoreMetadata;
+  }
+
+  public ColumnarKeyStoreDataHolder(final ColumnarKeyStoreMetadata columnarKeyStoreMetadata) {
+    this.columnarKeyStoreMetadata = columnarKeyStoreMetadata;
+  }
+
+  public int getSurrogateKey(int columnIndex) {
+    byte[] actual = new byte[4];
+    int startIndex;
+    if (null != columnarKeyStoreMetadata.getColumnReverseIndex()) {
+      startIndex =
+          columnarKeyStoreMetadata.getColumnReverseIndex()[columnIndex] * columnarKeyStoreMetadata
+              .getEachRowSize();
+    } else {
+      startIndex = columnIndex * columnarKeyStoreMetadata.getEachRowSize();
+    }
+    int destPos = 4 - columnarKeyStoreMetadata.getEachRowSize();
+    System.arraycopy(keyblockData, startIndex, actual, destPos,
+        columnarKeyStoreMetadata.getEachRowSize());
+    return ByteBuffer.wrap(actual).getInt();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreMetadata.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreMetadata.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreMetadata.java
new file mode 100644
index 0000000..706708f
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreMetadata.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+class ColumnarKeyStoreMetadata {
+
+  private int[] columnReverseIndex;
+
+  private int eachRowSize;
+
+  ColumnarKeyStoreMetadata(int eachRowSize) {
+    this.eachRowSize = eachRowSize;
+  }
+
+  /**
+   * @return the eachRowSize
+   */
+  int getEachRowSize() {
+    return eachRowSize;
+  }
+
+  /**
+   * @return the columnReverseIndex
+   */
+  int[] getColumnReverseIndex() {
+    return columnReverseIndex;
+  }
+
+  /**
+   * @param columnReverseIndex the columnReverseIndex to set
+   */
+  void setColumnReverseIndex(int[] columnReverseIndex) {
+    this.columnReverseIndex = columnReverseIndex;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java
new file mode 100644
index 0000000..ac8f6b7
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+public interface IndexStorage<T> {
+  boolean isAlreadySorted();
+
+  T getDataAfterComp();
+
+  T getIndexMap();
+
+  byte[][] getKeyBlock();
+
+  T getDataIndexMap();
+
+  int getTotalSize();
+
+  /**
+   * @return min value of block
+   */
+  byte[] getMin();
+
+  /**
+   * @return max value of block
+   */
+  byte[] getMax();
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastore/columnar/UnBlockIndexer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/UnBlockIndexer.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/UnBlockIndexer.java
new file mode 100644
index 0000000..4d4eac9
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/UnBlockIndexer.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+import java.util.Arrays;
+
+public final class UnBlockIndexer {
+
+  private UnBlockIndexer() {
+
+  }
+
+  public static int[] uncompressIndex(int[] indexData, int[] indexMap) {
+    int actualSize = indexData.length;
+    for (int i = 0; i < indexMap.length; i++) {
+      actualSize += indexData[indexMap[i] + 1] - indexData[indexMap[i]] - 1;
+    }
+    int[] indexes = new int[actualSize];
+    int k = 0;
+    for (int i = 0; i < indexData.length; i++) {
+      int index = Arrays.binarySearch(indexMap, i);
+      if (index > -1) {
+        for (int j = indexData[indexMap[index]]; j <= indexData[indexMap[index] + 1]; j++) {
+          indexes[k] = j;
+          k++;
+        }
+        i++;
+      } else {
+        indexes[k] = indexData[i];
+        k++;
+      }
+    }
+    return indexes;
+  }
+
+  public static byte[] uncompressData(byte[] data, int[] index, int keyLen) {
+    if (index.length < 1) {
+      return data;
+    }
+    int numberOfCopy = 0;
+    int actualSize = 0;
+    int srcPos = 0;
+    int destPos = 0;
+    for (int i = 1; i < index.length; i += 2) {
+      actualSize += index[i];
+    }
+    byte[] uncompressedData = new byte[actualSize * keyLen];
+    int picIndex = 0;
+    for (int i = 0; i < data.length; i += keyLen) {
+      numberOfCopy = index[picIndex * 2 + 1];
+      picIndex++;
+      for (int j = 0; j < numberOfCopy; j++) {
+        System.arraycopy(data, srcPos, uncompressedData, destPos, keyLen);
+        destPos += keyLen;
+      }
+      srcPos += keyLen;
+    }
+    return uncompressedData;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastore/compression/Compressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/Compressor.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/Compressor.java
new file mode 100644
index 0000000..d9803a0
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/Compressor.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression;
+
+public interface Compressor {
+
+  byte[] compressByte(byte[] unCompInput);
+
+  byte[] unCompressByte(byte[] compInput);
+
+  byte[] unCompressByte(byte[] compInput, int offset, int length);
+
+  byte[] compressShort(short[] unCompInput);
+
+  short[] unCompressShort(byte[] compInput);
+
+  short[] unCompressShort(byte[] compInput, int offset, int lenght);
+
+  byte[] compressInt(int[] unCompInput);
+
+  int[] unCompressInt(byte[] compInput);
+
+  int[] unCompressInt(byte[] compInput, int offset, int length);
+
+  byte[] compressLong(long[] unCompInput);
+
+  long[] unCompressLong(byte[] compInput);
+
+  long[] unCompressLong(byte[] compInput, int offset, int length);
+
+  byte[] compressFloat(float[] unCompInput);
+
+  float[] unCompressFloat(byte[] compInput);
+
+  float[] unCompressFloat(byte[] compInput, int offset, int length);
+
+  byte[] compressDouble(double[] unCompInput);
+
+  double[] unCompressDouble(byte[] compInput);
+
+  double[] unCompressDouble(byte[] compInput, int offset, int length);
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastore/compression/CompressorFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/CompressorFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/CompressorFactory.java
new file mode 100644
index 0000000..022ac97
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/CompressorFactory.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.CarbonProperties;
+
+public class CompressorFactory {
+
+  private static final CompressorFactory COMPRESSOR_FACTORY = new CompressorFactory();
+
+  private final Compressor compressor;
+
+  private CompressorFactory() {
+    String compressorType = CarbonProperties.getInstance()
+        .getProperty(CarbonCommonConstants.COMPRESSOR, CarbonCommonConstants.DEFAULT_COMPRESSOR);
+    switch (compressorType) {
+      case "snappy":
+        compressor = new SnappyCompressor();
+        break;
+      default:
+        throw new RuntimeException(
+            "Invalid compressor type provided! Please provide valid compressor type");
+    }
+  }
+
+  public static CompressorFactory getInstance() {
+    return COMPRESSOR_FACTORY;
+  }
+
+  public Compressor getCompressor() {
+    return compressor;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastore/compression/MeasureMetaDataModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/MeasureMetaDataModel.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/MeasureMetaDataModel.java
new file mode 100644
index 0000000..1ded77d
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/MeasureMetaDataModel.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression;
+
+public class MeasureMetaDataModel {
+  /**
+   * maxValue
+   */
+  private Object[] maxValue;
+
+  /**
+   * minValue
+   */
+  private Object[] minValue;
+
+  /**
+   * mantissa
+   */
+  private int[] mantissa;
+
+  /**
+   * measureCount
+   */
+  private int measureCount;
+
+  /**
+   * uniqueValue
+   */
+  private Object[] uniqueValue;
+
+  /**
+   * type
+   */
+  private char[] type;
+
+  /**
+   * dataTypeSelected
+   */
+  private byte[] dataTypeSelected;
+
+  public MeasureMetaDataModel(Object[] minValue, Object[] maxValue, int[] mantissa,
+      int measureCount, Object[] uniqueValue, char[] type, byte[] dataTypeSelected) {
+    this.minValue = minValue;
+    this.maxValue = maxValue;
+    this.mantissa = mantissa;
+    this.measureCount = measureCount;
+    this.uniqueValue = uniqueValue;
+    this.type = type;
+    this.dataTypeSelected = dataTypeSelected;
+  }
+
+  /**
+   * get Max value
+   *
+   * @return
+   */
+  public Object[] getMaxValue() {
+    return maxValue;
+  }
+
+  /**
+   * getMinValue
+   *
+   * @return
+   */
+  public Object[] getMinValue() {
+    return minValue;
+  }
+
+  /**
+   * getMantissa
+   *
+   * @return
+   */
+  public int[] getMantissa() {
+    return mantissa;
+  }
+
+  /**
+   * getMeasureCount
+   *
+   * @return
+   */
+  public int getMeasureCount() {
+    return measureCount;
+  }
+
+  /**
+   * getUniqueValue
+   *
+   * @return
+   */
+  public Object[] getUniqueValue() {
+    return uniqueValue;
+  }
+
+  /**
+   * @return the type
+   */
+  public char[] getType() {
+    return type;
+  }
+
+  /**
+   * @return the dataTypeSelected
+   */
+  public byte[] getDataTypeSelected() {
+    return dataTypeSelected;
+  }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java
new file mode 100644
index 0000000..7a5c670
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression;
+
+import org.apache.carbondata.core.metadata.ValueEncoderMeta;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+
+// Used in read path for decompression preparation
+public class ReaderCompressModel {
+  private ValueEncoderMeta valueEncoderMeta;
+
+  private ValueCompressionUtil.DataType convertedDataType;
+
+  private ValueCompressionHolder valueHolder;
+
+  public void setValueEncoderMeta(ValueEncoderMeta valueEncoderMeta) {
+    this.valueEncoderMeta = valueEncoderMeta;
+  }
+
+  public ValueCompressionUtil.DataType getConvertedDataType() {
+    return convertedDataType;
+  }
+
+  public void setConvertedDataType(ValueCompressionUtil.DataType convertedDataType) {
+    this.convertedDataType = convertedDataType;
+  }
+
+  public Object getMaxValue() {
+    return valueEncoderMeta.getMaxValue();
+  }
+
+  public int getMantissa() {
+    return valueEncoderMeta.getMantissa();
+  }
+
+  public ValueCompressionHolder getValueCompressionHolder() {
+    return valueHolder;
+  }
+
+  public void setValueCompressionHolder(ValueCompressionHolder valueHolder) {
+    this.valueHolder = valueHolder;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastore/compression/SnappyCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/SnappyCompressor.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/SnappyCompressor.java
new file mode 100644
index 0000000..dd78bb1
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/SnappyCompressor.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+
+import org.xerial.snappy.Snappy;
+import org.xerial.snappy.SnappyNative;
+
+public class SnappyCompressor implements Compressor {
+
+  private static final LogService LOGGER =
+      LogServiceFactory.getLogService(SnappyCompressor.class.getName());
+
+  private final SnappyNative snappyNative;
+
+  public SnappyCompressor() {
+    Snappy snappy = new Snappy();
+    Field privateField = null;
+    try {
+      privateField = snappy.getClass().getDeclaredField("impl");
+    } catch (NoSuchFieldException | SecurityException e) {
+      throw new RuntimeException(e);
+    }
+    privateField.setAccessible(true);
+    try {
+      snappyNative = (SnappyNative) privateField.get(snappy);
+    } catch (IllegalArgumentException | IllegalAccessException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Override public byte[] compressByte(byte[] unCompInput) {
+    try {
+      return Snappy.rawCompress(unCompInput, unCompInput.length);
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+      return null;
+    }
+  }
+
+  @Override public byte[] unCompressByte(byte[] compInput) {
+    try {
+      return Snappy.uncompress(compInput);
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+    }
+    return compInput;
+  }
+
+  @Override public byte[] unCompressByte(byte[] compInput, int offset, int length) {
+    int uncompressedLength = 0;
+    byte[] data = null;
+    try {
+      uncompressedLength = Snappy.uncompressedLength(compInput, offset, length);
+      data = new byte[uncompressedLength];
+      Snappy.uncompress(compInput, offset, length, data, 0);
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+    }
+    return data;
+  }
+
+  @Override public byte[] compressShort(short[] unCompInput) {
+    try {
+      return Snappy.compress(unCompInput);
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+      return null;
+    }
+  }
+
+  @Override public short[] unCompressShort(byte[] compInput) {
+    try {
+      return Snappy.uncompressShortArray(compInput);
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+    }
+    return null;
+  }
+
+  @Override public short[] unCompressShort(byte[] compInput, int offset, int lenght) {
+    try {
+      return Snappy.uncompressShortArray(compInput, offset, lenght);
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+    }
+    return null;
+  }
+
+  @Override public byte[] compressInt(int[] unCompInput) {
+    try {
+      return Snappy.compress(unCompInput);
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+      return null;
+    }
+  }
+
+  @Override public int[] unCompressInt(byte[] compInput) {
+    try {
+      return Snappy.uncompressIntArray(compInput);
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+    }
+    return null;
+  }
+
+  @Override public int[] unCompressInt(byte[] compInput, int offset, int length) {
+    try {
+      return Snappy.uncompressIntArray(compInput, offset, length);
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+    }
+    return null;
+  }
+
+  @Override public byte[] compressLong(long[] unCompInput) {
+    try {
+      return Snappy.compress(unCompInput);
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+      return null;
+    }
+  }
+
+  @Override public long[] unCompressLong(byte[] compInput) {
+    try {
+      return Snappy.uncompressLongArray(compInput);
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+    }
+    return null;
+  }
+
+  @Override public long[] unCompressLong(byte[] compInput, int offset, int length) {
+    try {
+      return Snappy.uncompressLongArray(compInput, offset, length);
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+    }
+    return null;
+  }
+
+  @Override public byte[] compressFloat(float[] unCompInput) {
+    try {
+      return Snappy.compress(unCompInput);
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+      return null;
+    }
+  }
+
+  @Override public float[] unCompressFloat(byte[] compInput) {
+    try {
+      return Snappy.uncompressFloatArray(compInput);
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+    }
+    return null;
+  }
+
+  @Override public float[] unCompressFloat(byte[] compInput, int offset, int length) {
+    try {
+      return Snappy.uncompressFloatArray(compInput, offset, length);
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+    }
+    return null;
+  }
+
+  @Override public byte[] compressDouble(double[] unCompInput) {
+    try {
+      return Snappy.compress(unCompInput);
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+      return null;
+    }
+  }
+
+  @Override public double[] unCompressDouble(byte[] compInput) {
+    try {
+      return Snappy.uncompressDoubleArray(compInput);
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+    }
+    return null;
+  }
+
+  @Override public double[] unCompressDouble(byte[] compInput, int offset, int length) {
+    try {
+      int uncompressedLength = Snappy.uncompressedLength(compInput, offset, length);
+      double[] result = new double[uncompressedLength / 8];
+      snappyNative.rawUncompress(compInput, offset, length, result, 0);
+      return result;
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+    }
+    return null;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ce09aaaf/core/src/main/java/org/apache/carbondata/core/datastore/compression/ValueCompressionHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/ValueCompressionHolder.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/ValueCompressionHolder.java
new file mode 100644
index 0000000..8fed2ad
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/ValueCompressionHolder.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression;
+
+import java.math.BigDecimal;
+
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+
+
+/**
+ * ValueCompressionHolder is the base class for handling
+ * compression / decompression of the measure data chunk
+ */
+public abstract class ValueCompressionHolder<T> {
+
+  /**
+   * compressedValue
+   */
+  protected byte[] compressedValue;
+
+  /**
+   * @param compressor the compressor used to decompress the data
+   * @param dataType data type of the data
+   * @param data compressed data
+   */
+  public void unCompress(Compressor compressor, DataType dataType, byte[] data,
+      int offset, int length) {
+    switch (dataType) {
+      case DATA_BYTE:
+        setValue((T)compressor.unCompressByte(data, offset, length));
+        break;
+      case DATA_SHORT:
+        setValue((T)compressor.unCompressShort(data, offset, length));
+        break;
+      case DATA_INT:
+        setValue((T)compressor.unCompressInt(data, offset, length));
+        break;
+      case DATA_LONG:
+      case DATA_BIGINT:
+        setValue((T)compressor.unCompressLong(data, offset, length));
+        break;
+      case DATA_FLOAT:
+        setValue((T)compressor.unCompressFloat(data, offset, length));
+        break;
+      default:
+        setValue((T)compressor.unCompressDouble(data, offset, length));
+        break;
+    }
+  }
+
+  /**
+   * @param compressor the compressor used to compress the data
+   * @param dataType data type of the data
+   * @param data original data
+   */
+  public byte[] compress(Compressor compressor, DataType dataType, Object data) {
+    switch (dataType) {
+      case DATA_BYTE:
+        return compressor.compressByte((byte[])data);
+      case DATA_SHORT:
+        return compressor.compressShort((short[])data);
+      case DATA_INT:
+        return compressor.compressInt((int[])data);
+      case DATA_LONG:
+      case DATA_BIGINT:
+        return compressor.compressLong((long[])data);
+      case DATA_FLOAT:
+        return compressor.compressFloat((float[])data);
+      case DATA_DOUBLE:
+      default:
+        return compressor.compressDouble((double[])data);
+    }
+  }
+
+  public abstract void setValue(T value);
+
+  public abstract T getValue();
+
+  public abstract void setValueInBytes(byte[] value);
+
+  public abstract void compress();
+
+  public abstract void uncompress(DataType dataType, byte[] compressData, int offset,
+      int length, int decimal, Object maxValueObject);
+
+  public byte[] getCompressedData() { return compressedValue; }
+
+  public abstract long getLongValue(int index);
+
+  public abstract double getDoubleValue(int index);
+
+  public abstract BigDecimal getBigDecimalValue(int index);
+
+  public abstract void freeMemory();
+
+
+}


Mime
View raw message