carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [50/57] [abbrv] incubator-carbondata git commit: fix style
Date Sat, 14 Jan 2017 07:57:59 GMT
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java
deleted file mode 100644
index e47cf42..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.impl.data.compressed;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.NodeMeasureDataStore;
-import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
-import org.apache.carbondata.core.datastorage.dataholder.CarbonWriteDataHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-
-public abstract class AbstractHeavyCompressedDoubleArrayDataStore
-    implements NodeMeasureDataStore //NodeMeasureDataStore<double[]>
-{
-
-  private LogService LOGGER =
-      LogServiceFactory.getLogService(AbstractHeavyCompressedDoubleArrayDataStore.class.getName());
-
-  /**
-   * values.
-   */
-  protected ValueCompressionHolder[] values;
-
-  /**
-   * compressionModel.
-   */
-  protected WriterCompressModel compressionModel;
-
-  /**
-   * type
-   */
-  private char[] type;
-
-  /**
-   * AbstractHeavyCompressedDoubleArrayDataStore constructor.
-   *
-   * @param compressionModel
-   */
-  public AbstractHeavyCompressedDoubleArrayDataStore(WriterCompressModel compressionModel) {
-    this.compressionModel = compressionModel;
-    if (null != compressionModel) {
-      this.type = compressionModel.getType();
-      values =
-          new ValueCompressionHolder[compressionModel.getValueCompressionHolder().length];
-    }
-  }
-
-  // this method first invokes encoding routine to encode the data chunk,
-  // followed by invoking compression routine for preparing the data chunk for writing.
-  @Override public byte[][] getWritableMeasureDataArray(CarbonWriteDataHolder[] dataHolder) {
-    byte[][] returnValue = new byte[values.length][];
-    for (int i = 0; i < compressionModel.getValueCompressionHolder().length; i++) {
-      values[i] = compressionModel.getValueCompressionHolder()[i];
-      if (type[i] != CarbonCommonConstants.BYTE_VALUE_MEASURE) {
-        // first perform encoding of the data chunk
-        values[i].setValue(
-            ValueCompressionUtil.getValueCompressor(compressionModel.getCompressionFinders()[i])
-                .getCompressedValues(compressionModel.getCompressionFinders()[i], dataHolder[i],
-                    compressionModel.getMaxValue()[i],
-                    compressionModel.getMantissa()[i]));
-      } else {
-        values[i].setValue(dataHolder[i].getWritableByteArrayValues());
-      }
-      values[i].compress();
-      returnValue[i] = values[i].getCompressedData();
-    }
-
-    return returnValue;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java
deleted file mode 100644
index ea42beb..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.impl.data.compressed;
-
-import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
-
-public class HeavyCompressedDoubleArrayDataInMemoryStore
-    extends AbstractHeavyCompressedDoubleArrayDataStore {
-
-  public HeavyCompressedDoubleArrayDataInMemoryStore(WriterCompressModel compressionModel) {
-    super(compressionModel);
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
index 9d4287a..7acfb39 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
@@ -20,7 +20,6 @@ package org.apache.carbondata.core.datastore;
 
 import java.io.IOException;
 
-import org.apache.carbondata.core.datastorage.FileHolder;
 import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
 import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
 import org.apache.carbondata.core.update.data.BlockletLevelDeleteDeltaDataCache;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java b/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java
new file mode 100644
index 0000000..95ec433
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore;
+
+import java.io.IOException;
+
+public interface FileHolder {
+  /**
+   * This method will be used to read the byte array from file based on offset
+   * and length(number of bytes) need to read
+   *
+   * @param filePath fully qualified file path
+   * @param offset   reading start position,
+   * @param length   number of bytes to be read
+   * @return read byte array
+   */
+  byte[] readByteArray(String filePath, long offset, int length) throws IOException;
+
+  /**
+   * This method will be used to read the byte array from file based on length(number of bytes)
+   *
+   * @param filePath fully qualified file path
+   * @param length   number of bytes to be read
+   * @return read byte array
+   */
+  byte[] readByteArray(String filePath, int length) throws IOException;
+
+  /**
+   * This method will be used to read int from file from postion(offset), here
+   * length will be always 4 bacause int byte size if 4
+   *
+   * @param filePath fully qualified file path
+   * @param offset   reading start position,
+   * @return read int
+   */
+  int readInt(String filePath, long offset) throws IOException;
+
+  /**
+   * This method will be used to read long from file from postion(offset), here
+   * length will be always 8 bacause int byte size is 8
+   *
+   * @param filePath fully qualified file path
+   * @param offset   reading start position,
+   * @return read long
+   */
+  long readLong(String filePath, long offset) throws IOException;
+
+  /**
+   * This method will be used to read int from file from postion(offset), here
+   * length will be always 4 bacause int byte size if 4
+   *
+   * @param filePath fully qualified file path
+   * @return read int
+   */
+  int readInt(String filePath) throws IOException;
+
+  /**
+   * This method will be used to read long value from file from postion(offset), here
+   * length will be always 8 because long byte size if 4
+   *
+   * @param filePath fully qualified file path
+   * @param offset   reading start position,
+   * @return read long
+   */
+  long readDouble(String filePath, long offset) throws IOException;
+
+  /**
+   * This method will be used to close all the streams currently present in the cache
+   */
+  void finish() throws IOException;
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/MeasureDataWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/MeasureDataWrapper.java b/core/src/main/java/org/apache/carbondata/core/datastore/MeasureDataWrapper.java
new file mode 100644
index 0000000..ff6ed34
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/MeasureDataWrapper.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore;
+
+import org.apache.carbondata.core.datastore.dataholder.CarbonReadDataHolder;
+
+/**
+ * MeasureDataWrapper, interface.
+ */
+public interface MeasureDataWrapper {
+  CarbonReadDataHolder[] getValues();
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/NodeMeasureDataStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/NodeMeasureDataStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/NodeMeasureDataStore.java
new file mode 100644
index 0000000..c4e5d15
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/NodeMeasureDataStore.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore;
+
+import org.apache.carbondata.core.datastore.dataholder.CarbonWriteDataHolder;
+
+public interface NodeMeasureDataStore {
+  /**
+   * This method will be used to get the writable key array.
+   * writable measure data array will hold below information:
+   * <size of measure data array><measure data array>
+   * total length will be 4 bytes for size + measure data array length
+   *
+   * @return writable array (compressed or normal)
+   */
+  byte[][] getWritableMeasureDataArray(CarbonWriteDataHolder[] dataHolderArray);
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
index dde8e76..f8c4374 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
@@ -29,11 +29,10 @@ import java.util.Map.Entry;
 import java.util.Set;
 
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.columnar.ColumnGroupModel;
+import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel;
 import org.apache.carbondata.core.keygenerator.KeyGenerator;
 import org.apache.carbondata.core.keygenerator.columnar.ColumnarSplitter;
-import org.apache.carbondata.core.keygenerator.columnar.impl
-    .MultiDimKeyVarLengthVariableSplitGenerator;
+import org.apache.carbondata.core.keygenerator.columnar.impl.MultiDimKeyVarLengthVariableSplitGenerator;
 import org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator;
 import org.apache.carbondata.core.metadata.DataType;
 import org.apache.carbondata.core.metadata.Encoding;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
index c7ce3db..8774579 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
@@ -23,7 +23,7 @@ import java.util.HashMap;
 import java.util.Map;
 
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
 import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
 import org.apache.carbondata.core.path.CarbonTablePath;
 import org.apache.carbondata.core.path.CarbonTablePath.DataFileUtil;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java
index ce09305..3c08f45 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java
@@ -18,7 +18,7 @@
  */
 package org.apache.carbondata.core.datastore.chunk;
 
-import org.apache.carbondata.core.datastorage.dataholder.CarbonReadDataHolder;
+import org.apache.carbondata.core.datastore.dataholder.CarbonReadDataHolder;
 import org.apache.carbondata.core.metadata.blocklet.datachunk.PresenceMeta;
 
 /**

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
index 6edc977..0730607 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
@@ -20,8 +20,7 @@ package org.apache.carbondata.core.datastore.chunk.impl;
 
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory
-    .DimensionStoreType;
+import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType;
 import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
 import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
 import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
index 549d6d8..342c392 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
@@ -22,8 +22,7 @@ import java.util.Arrays;
 
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory
-    .DimensionStoreType;
+import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType;
 import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
 import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
 import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java
index c83f2d7..3d4e3bf 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java
@@ -18,14 +18,10 @@
  */
 package org.apache.carbondata.core.datastore.chunk.reader;
 
-import org.apache.carbondata.core.datastore.chunk.reader.dimension
-    .v1.CompressedDimensionChunkFileBasedReaderV1;
-import org.apache.carbondata.core.datastore.chunk.reader.dimension
-    .v2.CompressedDimensionChunkFileBasedReaderV2;
-import org.apache.carbondata.core.datastore.chunk.reader.measure
-    .v1.CompressedMeasureChunkFileBasedReaderV1;
-import org.apache.carbondata.core.datastore.chunk.reader.measure
-    .v2.CompressedMeasureChunkFileBasedReaderV2;
+import org.apache.carbondata.core.datastore.chunk.reader.dimension.v1.CompressedDimensionChunkFileBasedReaderV1;
+import org.apache.carbondata.core.datastore.chunk.reader.dimension.v2.CompressedDimensionChunkFileBasedReaderV2;
+import org.apache.carbondata.core.datastore.chunk.reader.measure.v1.CompressedMeasureChunkFileBasedReaderV1;
+import org.apache.carbondata.core.datastore.chunk.reader.measure.v2.CompressedMeasureChunkFileBasedReaderV2;
 import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
 import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java
index 92dcb60..4981241 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java
@@ -20,7 +20,7 @@ package org.apache.carbondata.core.datastore.chunk.reader;
 
 import java.io.IOException;
 
-import org.apache.carbondata.core.datastorage.FileHolder;
+import org.apache.carbondata.core.datastore.FileHolder;
 import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
 
 /**

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java
index 94004d9..a706d71 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java
@@ -20,7 +20,7 @@ package org.apache.carbondata.core.datastore.chunk.reader;
 
 import java.io.IOException;
 
-import org.apache.carbondata.core.datastorage.FileHolder;
+import org.apache.carbondata.core.datastore.FileHolder;
 import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
 
 /**

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java
index 4cc24d9..2de673a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java
@@ -19,9 +19,9 @@
 package org.apache.carbondata.core.datastore.chunk.reader.dimension;
 
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
 import org.apache.carbondata.core.datastore.chunk.reader.DimensionColumnChunkReader;
+import org.apache.carbondata.core.datastore.compression.Compressor;
+import org.apache.carbondata.core.datastore.compression.CompressorFactory;
 import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor;
 import org.apache.carbondata.core.util.CarbonProperties;
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
index dd487d9..a8d15fe 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
@@ -21,13 +21,13 @@ package org.apache.carbondata.core.datastore.chunk.reader.dimension.v1;
 import java.io.IOException;
 import java.util.List;
 
-import org.apache.carbondata.core.datastorage.FileHolder;
-import org.apache.carbondata.core.datastorage.columnar.UnBlockIndexer;
+import org.apache.carbondata.core.datastore.FileHolder;
 import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
 import org.apache.carbondata.core.datastore.chunk.impl.ColumnGroupDimensionDataChunk;
 import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
 import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk;
 import org.apache.carbondata.core.datastore.chunk.reader.dimension.AbstractChunkReader;
+import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer;
 import org.apache.carbondata.core.metadata.Encoding;
 import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
 import org.apache.carbondata.core.metadata.blocklet.datachunk.DataChunk;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
index 3bddf4e..ae5c9b4 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
@@ -21,13 +21,13 @@ package org.apache.carbondata.core.datastore.chunk.reader.dimension.v2;
 import java.io.IOException;
 import java.util.List;
 
-import org.apache.carbondata.core.datastorage.FileHolder;
-import org.apache.carbondata.core.datastorage.columnar.UnBlockIndexer;
+import org.apache.carbondata.core.datastore.FileHolder;
 import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
 import org.apache.carbondata.core.datastore.chunk.impl.ColumnGroupDimensionDataChunk;
 import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
 import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk;
 import org.apache.carbondata.core.datastore.chunk.reader.dimension.AbstractChunkReader;
+import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer;
 import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
 import org.apache.carbondata.core.util.CarbonUtil;
 import org.apache.carbondata.format.DataChunk2;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
index 48cb244..1f3fec0 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
@@ -21,12 +21,12 @@ package org.apache.carbondata.core.datastore.chunk.reader.measure.v1;
 import java.io.IOException;
 import java.util.List;
 
-import org.apache.carbondata.core.datastorage.FileHolder;
-import org.apache.carbondata.core.datastorage.compression.ReaderCompressModel;
-import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastorage.dataholder.CarbonReadDataHolder;
+import org.apache.carbondata.core.datastore.FileHolder;
 import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
 import org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReader;
+import org.apache.carbondata.core.datastore.compression.ReaderCompressModel;
+import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastore.dataholder.CarbonReadDataHolder;
 import org.apache.carbondata.core.metadata.ValueEncoderMeta;
 import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
 import org.apache.carbondata.core.metadata.blocklet.datachunk.DataChunk;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
index 3325299..2731163 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
@@ -23,13 +23,13 @@ import java.util.ArrayList;
 import java.util.BitSet;
 import java.util.List;
 
-import org.apache.carbondata.core.datastorage.FileHolder;
-import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
-import org.apache.carbondata.core.datastorage.dataholder.CarbonReadDataHolder;
+import org.apache.carbondata.core.datastore.FileHolder;
 import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
 import org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReader;
+import org.apache.carbondata.core.datastore.compression.CompressorFactory;
+import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastore.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastore.dataholder.CarbonReadDataHolder;
 import org.apache.carbondata.core.metadata.ValueEncoderMeta;
 import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
 import org.apache.carbondata.core.metadata.blocklet.datachunk.PresenceMeta;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionChunkStoreFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionChunkStoreFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionChunkStoreFactory.java
index df054c5..e1ae9b3 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionChunkStoreFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionChunkStoreFactory.java
@@ -17,14 +17,10 @@
 package org.apache.carbondata.core.datastore.chunk.store;
 
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.chunk.store.impl.safe
-    .SafeFixedLengthDimensionDataChunkStore;
-import org.apache.carbondata.core.datastore.chunk.store.impl.safe
-    .SafeVariableLengthDimensionDataChunkStore;
-import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe
-    .UnsafeFixedLengthDimensionDataChunkStore;
-import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe
-    .UnsafeVariableLengthDimesionDataChunkStore;
+import org.apache.carbondata.core.datastore.chunk.store.impl.safe.SafeFixedLengthDimensionDataChunkStore;
+import org.apache.carbondata.core.datastore.chunk.store.impl.safe.SafeVariableLengthDimensionDataChunkStore;
+import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeFixedLengthDimensionDataChunkStore;
+import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeVariableLengthDimesionDataChunkStore;
 import org.apache.carbondata.core.util.CarbonProperties;
 
 /**

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java
new file mode 100644
index 0000000..b6155fb
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.ByteUtil;
+
+public class BlockIndexerStorageForInt implements IndexStorage<int[]> {
+  private boolean alreadySorted;
+
+  private int[] dataAfterComp;
+
+  private int[] indexMap;
+
+  private byte[][] keyBlock;
+
+  private int[] dataIndexMap;
+
+  private int totalSize;
+
+  public BlockIndexerStorageForInt(byte[][] keyBlock, boolean compressData, boolean isNoDictionary,
+      boolean isSortRequired) {
+    ColumnWithIntIndex[] columnWithIndexs = createColumnWithIndexArray(keyBlock, isNoDictionary);
+    if (isSortRequired) {
+      Arrays.sort(columnWithIndexs);
+    }
+    compressMyOwnWay(extractDataAndReturnIndexes(columnWithIndexs, keyBlock));
+    if (compressData) {
+      compressDataMyOwnWay(columnWithIndexs);
+    }
+  }
+
+  /**
+   * Create an object with each column array and respective index
+   *
+   * @return
+   */
+  private ColumnWithIntIndex[] createColumnWithIndexArray(byte[][] keyBlock,
+      boolean isNoDictionary) {
+    ColumnWithIntIndex[] columnWithIndexs;
+    if (isNoDictionary) {
+      columnWithIndexs = new ColumnWithIntIndexForHighCard[keyBlock.length];
+      for (int i = 0; i < columnWithIndexs.length; i++) {
+        columnWithIndexs[i] = new ColumnWithIntIndexForHighCard(keyBlock[i], i);
+      }
+
+    } else {
+      columnWithIndexs = new ColumnWithIntIndex[keyBlock.length];
+      for (int i = 0; i < columnWithIndexs.length; i++) {
+        columnWithIndexs[i] = new ColumnWithIntIndex(keyBlock[i], i);
+      }
+    }
+
+    return columnWithIndexs;
+  }
+
+  private int[] extractDataAndReturnIndexes(ColumnWithIntIndex[] columnWithIndexs,
+      byte[][] keyBlock) {
+    int[] indexes = new int[columnWithIndexs.length];
+    for (int i = 0; i < indexes.length; i++) {
+      indexes[i] = columnWithIndexs[i].getIndex();
+      keyBlock[i] = columnWithIndexs[i].getColumn();
+    }
+    this.keyBlock = keyBlock;
+    return indexes;
+  }
+
+  /**
+   * It compresses depends up on the sequence numbers.
+   * [1,2,3,4,6,8,10,11,12,13] is translated to [1,4,6,8,10,13] and [0,6]. In
+   * first array the start and end of sequential numbers and second array
+   * keeps the indexes of where sequential numbers starts. If there is no
+   * sequential numbers then the same array it returns with empty second
+   * array.
+   *
+   * @param indexes
+   */
+  public void compressMyOwnWay(int[] indexes) {
+    List<Integer> list = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+    List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+    int k = 0;
+    int i = 1;
+    for (; i < indexes.length; i++) {
+      if (indexes[i] - indexes[i - 1] == 1) {
+        k++;
+      } else {
+        if (k > 0) {
+          map.add((list.size()));
+          list.add(indexes[i - k - 1]);
+          list.add(indexes[i - 1]);
+        } else {
+          list.add(indexes[i - 1]);
+        }
+        k = 0;
+      }
+    }
+    if (k > 0) {
+      map.add((list.size()));
+      list.add(indexes[i - k - 1]);
+      list.add(indexes[i - 1]);
+    } else {
+      list.add(indexes[i - 1]);
+    }
+    dataAfterComp = convertToArray(list);
+    if (indexes.length == dataAfterComp.length) {
+      indexMap = new int[0];
+    } else {
+      indexMap = convertToArray(map);
+    }
+    if (dataAfterComp.length == 2 && indexMap.length == 1) {
+      alreadySorted = true;
+    }
+  }
+
+  private int[] convertToArray(List<Integer> list) {
+    int[] shortArray = new int[list.size()];
+    for (int i = 0; i < shortArray.length; i++) {
+      shortArray[i] = list.get(i);
+    }
+    return shortArray;
+  }
+
+  /**
+   * @return the alreadySorted
+   */
+  public boolean isAlreadySorted() {
+    return alreadySorted;
+  }
+
+  /**
+   * @return the dataAfterComp
+   */
+  public int[] getDataAfterComp() {
+    return dataAfterComp;
+  }
+
+  /**
+   * @return the indexMap
+   */
+  public int[] getIndexMap() {
+    return indexMap;
+  }
+
+  /**
+   * @return the keyBlock
+   */
+  public byte[][] getKeyBlock() {
+    return keyBlock;
+  }
+
+  private void compressDataMyOwnWay(ColumnWithIntIndex[] indexes) {
+    byte[] prvKey = indexes[0].getColumn();
+    List<ColumnWithIntIndex> list =
+        new ArrayList<ColumnWithIntIndex>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+    list.add(indexes[0]);
+    int counter = 1;
+    int start = 0;
+    List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+    for (int i = 1; i < indexes.length; i++) {
+      if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, indexes[i].getColumn()) != 0) {
+        prvKey = indexes[i].getColumn();
+        list.add(indexes[i]);
+        map.add(start);
+        map.add(counter);
+        start += counter;
+        counter = 1;
+        continue;
+      }
+      counter++;
+    }
+    map.add(start);
+    map.add(counter);
+    this.keyBlock = convertToKeyArray(list);
+    if (indexes.length == keyBlock.length) {
+      dataIndexMap = new int[0];
+    } else {
+      dataIndexMap = convertToArray(map);
+    }
+  }
+
+  private byte[][] convertToKeyArray(List<ColumnWithIntIndex> list) {
+    byte[][] shortArray = new byte[list.size()][];
+    for (int i = 0; i < shortArray.length; i++) {
+      shortArray[i] = list.get(i).getColumn();
+      totalSize += shortArray[i].length;
+    }
+    return shortArray;
+  }
+
+  @Override public int[] getDataIndexMap() {
+    return dataIndexMap;
+  }
+
+  @Override public int getTotalSize() {
+    return totalSize;
+  }
+
+  @Override public byte[] getMin() {
+    return keyBlock[0];
+  }
+
+  @Override public byte[] getMax() {
+    return keyBlock[keyBlock.length - 1];
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
new file mode 100644
index 0000000..d7b980e
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.ByteUtil;
+
+public class BlockIndexerStorageForNoInvertedIndex implements IndexStorage<int[]> {
+  private byte[][] keyBlock;
+  private byte[][] sortedBlock;
+  private int totalSize;
+  private int[] dataIndexMap;
+
+  public BlockIndexerStorageForNoInvertedIndex(byte[][] keyBlockInput, boolean compressData,
+      boolean isNoDictionary) {
+    // without invertedindex but can be RLE
+    if (compressData) {
+      // with RLE
+      byte[] prvKey = keyBlockInput[0];
+      List<byte[]> list = new ArrayList<byte[]>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+      list.add(keyBlockInput[0]);
+      int counter = 1;
+      int start = 0;
+      List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+      int length = keyBlockInput.length;
+      for(int i = 1; i < length; i++) {
+        if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, keyBlockInput[i]) != 0) {
+          prvKey = keyBlockInput[i];
+          list.add(keyBlockInput[i]);
+          map.add(start);
+          map.add(counter);
+          start += counter;
+          counter = 1;
+          continue;
+        }
+        counter++;
+      }
+      map.add(start);
+      map.add(counter);
+      this.keyBlock = convertToKeyArray(list);
+      if (keyBlockInput.length == this.keyBlock.length) {
+        dataIndexMap = new int[0];
+      } else {
+        dataIndexMap = convertToArray(map);
+      }
+    } else {
+      this.keyBlock = keyBlockInput;
+      dataIndexMap = new int[0];
+    }
+
+    this.sortedBlock = new byte[keyBlock.length][];
+    System.arraycopy(keyBlock, 0, sortedBlock, 0, keyBlock.length);
+    if (isNoDictionary) {
+      Arrays.sort(sortedBlock, new Comparator<byte[]>() {
+        @Override
+        public int compare(byte[] col1, byte[] col2) {
+          return ByteUtil.UnsafeComparer.INSTANCE
+              .compareTo(col1, 2, col1.length - 2, col2, 2, col2.length - 2);
+        }
+      });
+    } else {
+      Arrays.sort(sortedBlock, new Comparator<byte[]>() {
+        @Override
+        public int compare(byte[] col1, byte[] col2) {
+          return ByteUtil.UnsafeComparer.INSTANCE.compareTo(col1, col2);
+        }
+      });
+    }
+
+  }
+
+  private int[] convertToArray(List<Integer> list) {
+    int[] shortArray = new int[list.size()];
+    for(int i = 0; i < shortArray.length; i++) {
+      shortArray[i] = list.get(i);
+    }
+    return shortArray;
+  }
+
+  private byte[][] convertToKeyArray(List<byte[]> list) {
+    byte[][] shortArray = new byte[list.size()][];
+    for (int i = 0; i < shortArray.length; i++) {
+      shortArray[i] = list.get(i);
+      totalSize += shortArray[i].length;
+    }
+    return shortArray;
+  }
+
+  @Override
+  public int[] getDataIndexMap() {
+    return dataIndexMap;
+  }
+
+  @Override
+  public int getTotalSize() {
+    return totalSize;
+  }
+
+  @Override
+  public boolean isAlreadySorted() {
+    return true;
+  }
+
+  /**
+   * no use
+   * @return
+   */
+  @Override
+  public int[] getDataAfterComp() {
+    return new int[0];
+  }
+
+  /**
+   * no use
+   * @return
+   */
+  @Override
+  public int[] getIndexMap() {
+    return new int[0];
+  }
+
+  /**
+   * @return the keyBlock
+   */
+  public byte[][] getKeyBlock() {
+    return keyBlock;
+  }
+
+  @Override public byte[] getMin() {
+    return sortedBlock[0];
+  }
+
+  @Override public byte[] getMax() {
+    return sortedBlock[sortedBlock.length - 1];
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java
new file mode 100644
index 0000000..4d4c19d
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.datastore.columnar;
+
+public class ColumnGroupModel {
+
+  /**
+   * number of columns in columnar block
+   */
+  private int[] columnSplit;
+
+  /**
+   * total number of columns
+   */
+  private int noOfColumnsStore;
+
+  /**
+   * whether given index is columnar or not
+   * true: columnar
+   * false: row block
+   */
+  private boolean[] columnarStore;
+
+  /**
+   * column groups
+   * e.g
+   * {{0,1,2},3,4,{5,6}}
+   */
+  private int[][] columnGroups;
+
+  /**
+   * return columnSplit
+   *
+   * @return
+   */
+  public int[] getColumnSplit() {
+    return columnSplit;
+  }
+
+  /**
+   * set columnSplit
+   *
+   * @param split
+   */
+  public void setColumnSplit(int[] split) {
+    this.columnSplit = split;
+  }
+
+  /**
+   * @return no of columnar block
+   */
+  public int getNoOfColumnStore() {
+    return this.noOfColumnsStore;
+  }
+
+  /**
+   * set no of columnar block
+   *
+   * @param noOfColumnsStore
+   */
+  public void setNoOfColumnStore(int noOfColumnsStore) {
+    this.noOfColumnsStore = noOfColumnsStore;
+  }
+
+  /**
+   * it's an identifier for row block or single column block
+   *
+   * @param columnarStore
+   */
+  public void setColumnarStore(boolean[] columnarStore) {
+    this.columnarStore = columnarStore;
+  }
+
+  /**
+   * set column groups
+   *
+   * @param columnGroups
+   */
+  public void setColumnGroup(int[][] columnGroups) {
+    this.columnGroups = columnGroups;
+  }
+
+  /**
+   * check if given column group is columnar
+   *
+   * @param colGroup
+   * @return true if given block is columnar
+   */
+  public boolean isColumnar(int colGroup) {
+    return columnarStore[colGroup];
+  }
+
+  /**
+   * @return columngroups
+   */
+  public int[][] getColumnGroup() {
+    return this.columnGroups;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndex.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndex.java
new file mode 100644
index 0000000..4b35a7f
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndex.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+import java.util.Arrays;
+
+import org.apache.carbondata.core.util.ByteUtil;
+
+public class ColumnWithIntIndex implements Comparable<ColumnWithIntIndex> {
+  protected byte[] column;
+
+  private int index;
+
+  public ColumnWithIntIndex(byte[] column, int index) {
+    this.column = column;
+    this.index = index;
+  }
+
+  public ColumnWithIntIndex() {
+  }
+
+  /**
+   * @return the column
+   */
+  public byte[] getColumn() {
+    return column;
+  }
+
+  /**
+   * @param column the column to set
+   */
+  public void setColumn(byte[] column) {
+    this.column = column;
+  }
+
+  /**
+   * @return the index
+   */
+  public int getIndex() {
+    return index;
+  }
+
+  /**
+   * @param index the index to set
+   */
+  public void setIndex(int index) {
+    this.index = index;
+  }
+
+  @Override public int compareTo(ColumnWithIntIndex o) {
+    return ByteUtil.UnsafeComparer.INSTANCE.compareTo(column, o.column);
+  }
+
+  @Override public boolean equals(Object obj) {
+    if(obj == null || getClass() != obj.getClass()) {
+      return false;
+    }
+    ColumnWithIntIndex o = (ColumnWithIntIndex)obj;
+    return Arrays.equals(column, o.column) && index == o.index;
+  }
+
+  @Override public int hashCode() {
+    return Arrays.hashCode(column) + index;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndexForHighCard.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndexForHighCard.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndexForHighCard.java
new file mode 100644
index 0000000..a6eb6c4
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndexForHighCard.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+import java.util.Arrays;
+
+import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
+
+public class ColumnWithIntIndexForHighCard extends ColumnWithIntIndex
+    implements Comparable<ColumnWithIntIndex> {
+
+  public ColumnWithIntIndexForHighCard(byte[] column, int index) {
+    super(column, index);
+  }
+
+  @Override public int compareTo(ColumnWithIntIndex o) {
+    return UnsafeComparer.INSTANCE
+        .compareTo(column, 2, column.length - 2, o.column, 2, o.column.length - 2);
+  }
+
+  @Override public boolean equals(Object obj) {
+    if(obj == null || getClass() != obj.getClass()) {
+      return false;
+    }
+    ColumnWithIntIndexForHighCard o = (ColumnWithIntIndexForHighCard)obj;
+    return Arrays.equals(column, o.column) && getIndex() == o.getIndex();
+  }
+
+  @Override public int hashCode() {
+    return Arrays.hashCode(column) + getIndex();
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreDataHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreDataHolder.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreDataHolder.java
new file mode 100644
index 0000000..6c3acf9
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreDataHolder.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+import java.nio.ByteBuffer;
+
+public class ColumnarKeyStoreDataHolder {
+  private byte[] keyblockData;
+  private ColumnarKeyStoreMetadata columnarKeyStoreMetadata;
+
+  public ColumnarKeyStoreDataHolder(final byte[] keyblockData,
+      final ColumnarKeyStoreMetadata columnarKeyStoreMetadata) {
+    this.keyblockData = keyblockData;
+    this.columnarKeyStoreMetadata = columnarKeyStoreMetadata;
+  }
+
+  public ColumnarKeyStoreDataHolder(final ColumnarKeyStoreMetadata columnarKeyStoreMetadata) {
+    this.columnarKeyStoreMetadata = columnarKeyStoreMetadata;
+  }
+
+  public int getSurrogateKey(int columnIndex) {
+    byte[] actual = new byte[4];
+    int startIndex;
+    if (null != columnarKeyStoreMetadata.getColumnReverseIndex()) {
+      startIndex =
+          columnarKeyStoreMetadata.getColumnReverseIndex()[columnIndex] * columnarKeyStoreMetadata
+              .getEachRowSize();
+    } else {
+      startIndex = columnIndex * columnarKeyStoreMetadata.getEachRowSize();
+    }
+    int destPos = 4 - columnarKeyStoreMetadata.getEachRowSize();
+    System.arraycopy(keyblockData, startIndex, actual, destPos,
+        columnarKeyStoreMetadata.getEachRowSize());
+    return ByteBuffer.wrap(actual).getInt();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreMetadata.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreMetadata.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreMetadata.java
new file mode 100644
index 0000000..706708f
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreMetadata.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+class ColumnarKeyStoreMetadata {
+
+  private int[] columnReverseIndex;
+
+  private int eachRowSize;
+
+  ColumnarKeyStoreMetadata(int eachRowSize) {
+    this.eachRowSize = eachRowSize;
+  }
+
+  /**
+   * @return the eachRowSize
+   */
+  int getEachRowSize() {
+    return eachRowSize;
+  }
+
+  /**
+   * @return the columnReverseIndex
+   */
+  int[] getColumnReverseIndex() {
+    return columnReverseIndex;
+  }
+
+  /**
+   * @param columnReverseIndex the columnReverseIndex to set
+   */
+  void setColumnReverseIndex(int[] columnReverseIndex) {
+    this.columnReverseIndex = columnReverseIndex;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java
new file mode 100644
index 0000000..ac8f6b7
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+public interface IndexStorage<T> {
+  boolean isAlreadySorted();
+
+  T getDataAfterComp();
+
+  T getIndexMap();
+
+  byte[][] getKeyBlock();
+
+  T getDataIndexMap();
+
+  int getTotalSize();
+
+  /**
+   * @return min value of block
+   */
+  byte[] getMin();
+
+  /**
+   * @return max value of block
+   */
+  byte[] getMax();
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/UnBlockIndexer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/UnBlockIndexer.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/UnBlockIndexer.java
new file mode 100644
index 0000000..4d4eac9
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/UnBlockIndexer.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+import java.util.Arrays;
+
+public final class UnBlockIndexer {
+
+  private UnBlockIndexer() {
+
+  }
+
+  public static int[] uncompressIndex(int[] indexData, int[] indexMap) {
+    int actualSize = indexData.length;
+    for (int i = 0; i < indexMap.length; i++) {
+      actualSize += indexData[indexMap[i] + 1] - indexData[indexMap[i]] - 1;
+    }
+    int[] indexes = new int[actualSize];
+    int k = 0;
+    for (int i = 0; i < indexData.length; i++) {
+      int index = Arrays.binarySearch(indexMap, i);
+      if (index > -1) {
+        for (int j = indexData[indexMap[index]]; j <= indexData[indexMap[index] + 1]; j++) {
+          indexes[k] = j;
+          k++;
+        }
+        i++;
+      } else {
+        indexes[k] = indexData[i];
+        k++;
+      }
+    }
+    return indexes;
+  }
+
+  public static byte[] uncompressData(byte[] data, int[] index, int keyLen) {
+    if (index.length < 1) {
+      return data;
+    }
+    int numberOfCopy = 0;
+    int actualSize = 0;
+    int srcPos = 0;
+    int destPos = 0;
+    for (int i = 1; i < index.length; i += 2) {
+      actualSize += index[i];
+    }
+    byte[] uncompressedData = new byte[actualSize * keyLen];
+    int picIndex = 0;
+    for (int i = 0; i < data.length; i += keyLen) {
+      numberOfCopy = index[picIndex * 2 + 1];
+      picIndex++;
+      for (int j = 0; j < numberOfCopy; j++) {
+        System.arraycopy(data, srcPos, uncompressedData, destPos, keyLen);
+        destPos += keyLen;
+      }
+      srcPos += keyLen;
+    }
+    return uncompressedData;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/Compressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/Compressor.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/Compressor.java
new file mode 100644
index 0000000..d9803a0
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/Compressor.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression;
+
+public interface Compressor {
+
+  byte[] compressByte(byte[] unCompInput);
+
+  byte[] unCompressByte(byte[] compInput);
+
+  byte[] unCompressByte(byte[] compInput, int offset, int length);
+
+  byte[] compressShort(short[] unCompInput);
+
+  short[] unCompressShort(byte[] compInput);
+
+  short[] unCompressShort(byte[] compInput, int offset, int lenght);
+
+  byte[] compressInt(int[] unCompInput);
+
+  int[] unCompressInt(byte[] compInput);
+
+  int[] unCompressInt(byte[] compInput, int offset, int length);
+
+  byte[] compressLong(long[] unCompInput);
+
+  long[] unCompressLong(byte[] compInput);
+
+  long[] unCompressLong(byte[] compInput, int offset, int length);
+
+  byte[] compressFloat(float[] unCompInput);
+
+  float[] unCompressFloat(byte[] compInput);
+
+  float[] unCompressFloat(byte[] compInput, int offset, int length);
+
+  byte[] compressDouble(double[] unCompInput);
+
+  double[] unCompressDouble(byte[] compInput);
+
+  double[] unCompressDouble(byte[] compInput, int offset, int length);
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/CompressorFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/CompressorFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/CompressorFactory.java
new file mode 100644
index 0000000..022ac97
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/CompressorFactory.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.CarbonProperties;
+
+public class CompressorFactory {
+
+  private static final CompressorFactory COMPRESSOR_FACTORY = new CompressorFactory();
+
+  private final Compressor compressor;
+
+  private CompressorFactory() {
+    String compressorType = CarbonProperties.getInstance()
+        .getProperty(CarbonCommonConstants.COMPRESSOR, CarbonCommonConstants.DEFAULT_COMPRESSOR);
+    switch (compressorType) {
+      case "snappy":
+        compressor = new SnappyCompressor();
+        break;
+      default:
+        throw new RuntimeException(
+            "Invalid compressor type provided! Please provide valid compressor type");
+    }
+  }
+
+  public static CompressorFactory getInstance() {
+    return COMPRESSOR_FACTORY;
+  }
+
+  public Compressor getCompressor() {
+    return compressor;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/MeasureMetaDataModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/MeasureMetaDataModel.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/MeasureMetaDataModel.java
new file mode 100644
index 0000000..1ded77d
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/MeasureMetaDataModel.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression;
+
+public class MeasureMetaDataModel {
+  /**
+   * maxValue
+   */
+  private Object[] maxValue;
+
+  /**
+   * minValue
+   */
+  private Object[] minValue;
+
+  /**
+   * mantissa
+   */
+  private int[] mantissa;
+
+  /**
+   * measureCount
+   */
+  private int measureCount;
+
+  /**
+   * uniqueValue
+   */
+  private Object[] uniqueValue;
+
+  /**
+   * type
+   */
+  private char[] type;
+
+  /**
+   * dataTypeSelected
+   */
+  private byte[] dataTypeSelected;
+
+  public MeasureMetaDataModel(Object[] minValue, Object[] maxValue, int[] mantissa,
+      int measureCount, Object[] uniqueValue, char[] type, byte[] dataTypeSelected) {
+    this.minValue = minValue;
+    this.maxValue = maxValue;
+    this.mantissa = mantissa;
+    this.measureCount = measureCount;
+    this.uniqueValue = uniqueValue;
+    this.type = type;
+    this.dataTypeSelected = dataTypeSelected;
+  }
+
+  /**
+   * get Max value
+   *
+   * @return
+   */
+  public Object[] getMaxValue() {
+    return maxValue;
+  }
+
+  /**
+   * getMinValue
+   *
+   * @return
+   */
+  public Object[] getMinValue() {
+    return minValue;
+  }
+
+  /**
+   * getMantissa
+   *
+   * @return
+   */
+  public int[] getMantissa() {
+    return mantissa;
+  }
+
+  /**
+   * getMeasureCount
+   *
+   * @return
+   */
+  public int getMeasureCount() {
+    return measureCount;
+  }
+
+  /**
+   * getUniqueValue
+   *
+   * @return
+   */
+  public Object[] getUniqueValue() {
+    return uniqueValue;
+  }
+
+  /**
+   * @return the type
+   */
+  public char[] getType() {
+    return type;
+  }
+
+  /**
+   * @return the dataTypeSelected
+   */
+  public byte[] getDataTypeSelected() {
+    return dataTypeSelected;
+  }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java
new file mode 100644
index 0000000..7a5c670
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression;
+
+import org.apache.carbondata.core.metadata.ValueEncoderMeta;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+
+// Used in read path for decompression preparation
+public class ReaderCompressModel {
+  private ValueEncoderMeta valueEncoderMeta;
+
+  private ValueCompressionUtil.DataType convertedDataType;
+
+  private ValueCompressionHolder valueHolder;
+
+  public void setValueEncoderMeta(ValueEncoderMeta valueEncoderMeta) {
+    this.valueEncoderMeta = valueEncoderMeta;
+  }
+
+  public ValueCompressionUtil.DataType getConvertedDataType() {
+    return convertedDataType;
+  }
+
+  public void setConvertedDataType(ValueCompressionUtil.DataType convertedDataType) {
+    this.convertedDataType = convertedDataType;
+  }
+
+  public Object getMaxValue() {
+    return valueEncoderMeta.getMaxValue();
+  }
+
+  public int getMantissa() {
+    return valueEncoderMeta.getMantissa();
+  }
+
+  public ValueCompressionHolder getValueCompressionHolder() {
+    return valueHolder;
+  }
+
+  public void setValueCompressionHolder(ValueCompressionHolder valueHolder) {
+    this.valueHolder = valueHolder;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/SnappyCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/SnappyCompressor.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/SnappyCompressor.java
new file mode 100644
index 0000000..dd78bb1
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/SnappyCompressor.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+
+import org.xerial.snappy.Snappy;
+import org.xerial.snappy.SnappyNative;
+
+public class SnappyCompressor implements Compressor {
+
+  private static final LogService LOGGER =
+      LogServiceFactory.getLogService(SnappyCompressor.class.getName());
+
+  private final SnappyNative snappyNative;
+
+  public SnappyCompressor() {
+    Snappy snappy = new Snappy();
+    Field privateField = null;
+    try {
+      privateField = snappy.getClass().getDeclaredField("impl");
+    } catch (NoSuchFieldException | SecurityException e) {
+      throw new RuntimeException(e);
+    }
+    privateField.setAccessible(true);
+    try {
+      snappyNative = (SnappyNative) privateField.get(snappy);
+    } catch (IllegalArgumentException | IllegalAccessException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Override public byte[] compressByte(byte[] unCompInput) {
+    try {
+      return Snappy.rawCompress(unCompInput, unCompInput.length);
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+      return null;
+    }
+  }
+
+  @Override public byte[] unCompressByte(byte[] compInput) {
+    try {
+      return Snappy.uncompress(compInput);
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+    }
+    return compInput;
+  }
+
+  @Override public byte[] unCompressByte(byte[] compInput, int offset, int length) {
+    int uncompressedLength = 0;
+    byte[] data = null;
+    try {
+      uncompressedLength = Snappy.uncompressedLength(compInput, offset, length);
+      data = new byte[uncompressedLength];
+      Snappy.uncompress(compInput, offset, length, data, 0);
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+    }
+    return data;
+  }
+
+  @Override public byte[] compressShort(short[] unCompInput) {
+    try {
+      return Snappy.compress(unCompInput);
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+      return null;
+    }
+  }
+
+  @Override public short[] unCompressShort(byte[] compInput) {
+    try {
+      return Snappy.uncompressShortArray(compInput);
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+    }
+    return null;
+  }
+
+  @Override public short[] unCompressShort(byte[] compInput, int offset, int lenght) {
+    try {
+      return Snappy.uncompressShortArray(compInput, offset, lenght);
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+    }
+    return null;
+  }
+
+  @Override public byte[] compressInt(int[] unCompInput) {
+    try {
+      return Snappy.compress(unCompInput);
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+      return null;
+    }
+  }
+
+  @Override public int[] unCompressInt(byte[] compInput) {
+    try {
+      return Snappy.uncompressIntArray(compInput);
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+    }
+    return null;
+  }
+
+  @Override public int[] unCompressInt(byte[] compInput, int offset, int length) {
+    try {
+      return Snappy.uncompressIntArray(compInput, offset, length);
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+    }
+    return null;
+  }
+
+  @Override public byte[] compressLong(long[] unCompInput) {
+    try {
+      return Snappy.compress(unCompInput);
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+      return null;
+    }
+  }
+
+  @Override public long[] unCompressLong(byte[] compInput) {
+    try {
+      return Snappy.uncompressLongArray(compInput);
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+    }
+    return null;
+  }
+
+  @Override public long[] unCompressLong(byte[] compInput, int offset, int length) {
+    try {
+      return Snappy.uncompressLongArray(compInput, offset, length);
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+    }
+    return null;
+  }
+
+  @Override public byte[] compressFloat(float[] unCompInput) {
+    try {
+      return Snappy.compress(unCompInput);
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+      return null;
+    }
+  }
+
+  @Override public float[] unCompressFloat(byte[] compInput) {
+    try {
+      return Snappy.uncompressFloatArray(compInput);
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+    }
+    return null;
+  }
+
+  @Override public float[] unCompressFloat(byte[] compInput, int offset, int length) {
+    try {
+      return Snappy.uncompressFloatArray(compInput, offset, length);
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+    }
+    return null;
+  }
+
+  @Override public byte[] compressDouble(double[] unCompInput) {
+    try {
+      return Snappy.compress(unCompInput);
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+      return null;
+    }
+  }
+
+  @Override public double[] unCompressDouble(byte[] compInput) {
+    try {
+      return Snappy.uncompressDoubleArray(compInput);
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+    }
+    return null;
+  }
+
+  @Override public double[] unCompressDouble(byte[] compInput, int offset, int length) {
+    try {
+      int uncompressedLength = Snappy.uncompressedLength(compInput, offset, length);
+      double[] result = new double[uncompressedLength / 8];
+      snappyNative.rawUncompress(compInput, offset, length, result, 0);
+      return result;
+    } catch (IOException e) {
+      LOGGER.error(e, e.getMessage());
+    }
+    return null;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/ValueCompressionHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/ValueCompressionHolder.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/ValueCompressionHolder.java
new file mode 100644
index 0000000..8fed2ad
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/ValueCompressionHolder.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression;
+
+import java.math.BigDecimal;
+
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+
+
+/**
+ * ValueCompressionHolder is the base class for handling
+ * compression / decompression of the measure data chunk
+ */
+public abstract class ValueCompressionHolder<T> {
+
+  /**
+   * compressedValue
+   */
+  protected byte[] compressedValue;
+
+  /**
+   * @param compressor the compressor used to decompress the data
+   * @param dataType data type of the data
+   * @param data compressed data
+   */
+  public void unCompress(Compressor compressor, DataType dataType, byte[] data,
+      int offset, int length) {
+    switch (dataType) {
+      case DATA_BYTE:
+        setValue((T)compressor.unCompressByte(data, offset, length));
+        break;
+      case DATA_SHORT:
+        setValue((T)compressor.unCompressShort(data, offset, length));
+        break;
+      case DATA_INT:
+        setValue((T)compressor.unCompressInt(data, offset, length));
+        break;
+      case DATA_LONG:
+      case DATA_BIGINT:
+        setValue((T)compressor.unCompressLong(data, offset, length));
+        break;
+      case DATA_FLOAT:
+        setValue((T)compressor.unCompressFloat(data, offset, length));
+        break;
+      default:
+        setValue((T)compressor.unCompressDouble(data, offset, length));
+        break;
+    }
+  }
+
+  /**
+   * @param compressor the compressor used to compress the data
+   * @param dataType data type of the data
+   * @param data original data
+   */
+  public byte[] compress(Compressor compressor, DataType dataType, Object data) {
+    switch (dataType) {
+      case DATA_BYTE:
+        return compressor.compressByte((byte[])data);
+      case DATA_SHORT:
+        return compressor.compressShort((short[])data);
+      case DATA_INT:
+        return compressor.compressInt((int[])data);
+      case DATA_LONG:
+      case DATA_BIGINT:
+        return compressor.compressLong((long[])data);
+      case DATA_FLOAT:
+        return compressor.compressFloat((float[])data);
+      case DATA_DOUBLE:
+      default:
+        return compressor.compressDouble((double[])data);
+    }
+  }
+
+  public abstract void setValue(T value);
+
+  public abstract T getValue();
+
+  public abstract void setValueInBytes(byte[] value);
+
+  public abstract void compress();
+
+  public abstract void uncompress(DataType dataType, byte[] compressData, int offset,
+      int length, int decimal, Object maxValueObject);
+
+  public byte[] getCompressedData() { return compressedValue; }
+
+  public abstract long getLongValue(int index);
+
+  public abstract double getDoubleValue(int index);
+
+  public abstract BigDecimal getBigDecimalValue(int index);
+
+  public abstract void freeMemory();
+
+
+}



Mime
View raw message