Repository: incubator-carbondata
Updated Branches:
refs/heads/master b7f3be7e0 -> bc8265c9a
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
index be270e4..b3647a8 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
@@ -23,25 +23,17 @@ import mockit.Mock;
import mockit.MockUp;
import org.apache.carbondata.core.carbon.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.carbon.metadata.blocklet.index.*;
-import org.apache.carbondata.core.carbon.metadata.blocklet.index.BlockletIndex;
import org.apache.carbondata.core.carbon.metadata.index.BlockIndexInfo;
-import org.apache.carbondata.core.carbon.metadata.schema.table.column.*;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel;
+import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
import org.apache.carbondata.core.metadata.BlockletInfoColumnar;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
import org.apache.carbondata.format.*;
-import org.apache.carbondata.format.BlockletBTreeIndex;
import org.apache.carbondata.format.BlockletMinMaxIndex;
import org.apache.carbondata.format.ColumnSchema;
import org.junit.BeforeClass;
import org.junit.Test;
-import java.io.ByteArrayInputStream;
-import java.io.ObjectInputStream;
import java.nio.ByteBuffer;
-import java.nio.MappedByteBuffer;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.HashSet;
@@ -49,7 +41,6 @@ import java.util.List;
import java.util.Set;
import static junit.framework.TestCase.*;
-import static org.apache.carbondata.core.util.CarbonMetadataUtil.convertBlockletInfo;
import static org.apache.carbondata.core.util.CarbonMetadataUtil.getIndexHeader;
import static org.apache.carbondata.core.util.CarbonMetadataUtil.convertFileFooter;
import static org.apache.carbondata.core.util.CarbonMetadataUtil.getBlockIndexInfo;
@@ -119,7 +110,7 @@ public class CarbonMetadataUtilTest {
blockletInfoList.add(blockletInfo);
ValueEncoderMeta valueEncoderMeta = new ValueEncoderMeta();
- valueEncoderMeta.setDecimal(5);
+ valueEncoderMeta.setMantissa(5);
valueEncoderMeta.setMinValue(objMinArr);
valueEncoderMeta.setMaxValue(objMaxArr);
valueEncoderMeta.setUniqueValue(lngObj);
@@ -196,13 +187,13 @@ public class CarbonMetadataUtilTest {
integerList.add(new Integer("1"));
integerList.add(new Integer("2"));
- ValueCompressionModel valueCompressionModel = new ValueCompressionModel();
- valueCompressionModel.setMaxValue(objMaxArr);
- valueCompressionModel.setMinValue(objMinArr);
- valueCompressionModel.setDataTypeSelected(byteArr);
- valueCompressionModel.setDecimal(intArr);
- valueCompressionModel.setType(charArr);
- valueCompressionModel.setUniqueValue(objMinArr);
+ WriterCompressModel writerCompressModel = new WriterCompressModel();
+ writerCompressModel.setMaxValue(objMaxArr);
+ writerCompressModel.setMinValue(objMinArr);
+ writerCompressModel.setDataTypeSelected(byteArr);
+ writerCompressModel.setMantissa(intArr);
+ writerCompressModel.setType(charArr);
+ writerCompressModel.setUniqueValue(objMinArr);
BlockletInfoColumnar blockletInfoColumnar = new BlockletInfoColumnar();
@@ -226,7 +217,7 @@ public class CarbonMetadataUtilTest {
blockletInfoColumnar.setMeasureLength(intArr);
blockletInfoColumnar.setMeasureOffset(longArr);
blockletInfoColumnar.setMeasureNullValueIndex(bitSetArr);
- blockletInfoColumnar.setCompressionModel(valueCompressionModel);
+ blockletInfoColumnar.setCompressionModel(writerCompressModel);
BlockletInfoColumnar blockletInfoColumnar1 = new BlockletInfoColumnar();
blockletInfoColumnar1.setColumnMaxData(maxByteArr);
@@ -242,7 +233,7 @@ public class CarbonMetadataUtilTest {
blockletInfoColumnar1.setMeasureLength(intArr);
blockletInfoColumnar1.setMeasureOffset(longArr);
blockletInfoColumnar1.setMeasureNullValueIndex(bitSetArr);
- blockletInfoColumnar1.setCompressionModel(valueCompressionModel);
+ blockletInfoColumnar1.setCompressionModel(writerCompressModel);
blockletInfoColumnar1.setColGrpBlocks(boolArr);
List<BlockletInfoColumnar> blockletInfoColumnarList = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
index 869c8cc..14d9248 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
@@ -33,11 +33,10 @@ import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonDime
import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.carbon.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.datastorage.store.columnar.ColumnGroupModel;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel;
+import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
import org.apache.carbondata.core.datastorage.store.filesystem.LocalCarbonFile;
import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
-import org.apache.carbondata.scan.executor.infos.KeyStructureInfo;
import org.apache.carbondata.scan.model.QueryDimension;
import org.apache.hadoop.security.UserGroupInformation;
import org.junit.AfterClass;
@@ -701,9 +700,9 @@ public class CarbonUtilTest {
valueEncoderMetas.add(valueEncoderMeta);
dataChunk.setValueEncoderMeta(valueEncoderMetas);
dataChunkList.add(dataChunk);
- ValueCompressionModel valueCompressionModel =
+ WriterCompressModel writerCompressModel =
CarbonUtil.getValueCompressionModel(dataChunkList.get(0).getValueEncoderMeta());
- assertEquals(1, valueCompressionModel.getMaxValue().length);
+ assertEquals(1, writerCompressModel.getMaxValue().length);
}
@Test public void testToGetDictionaryChunkSize() {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java
index 25d1426..a1bb6f0 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java
@@ -19,9 +19,33 @@
package org.apache.carbondata.core.util;
import org.apache.carbondata.core.datastorage.store.compression.MeasureMetaDataModel;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel;
+import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
-import org.apache.carbondata.core.datastorage.store.compression.type.*;
+import org.apache.carbondata.core.datastorage.store.compression.decimal.UnCompressMaxMinByte;
+import org.apache.carbondata.core.datastorage.store.compression.decimal.UnCompressMaxMinDefault;
+import org.apache.carbondata.core.datastorage.store.compression.decimal.UnCompressMaxMinFloat;
+import org.apache.carbondata.core.datastorage.store.compression.decimal.UnCompressMaxMinInt;
+import org.apache.carbondata.core.datastorage.store.compression.decimal.UnCompressMaxMinLong;
+import org.apache.carbondata.core.datastorage.store.compression.decimal.UnCompressMaxMinShort;
+import org.apache.carbondata.core.datastorage.store.compression.nondecimal.UnCompressNonDecimalByte;
+import org.apache.carbondata.core.datastorage.store.compression.nondecimal
+ .UnCompressNonDecimalDefault;
+import org.apache.carbondata.core.datastorage.store.compression.nondecimal.UnCompressNonDecimalFloat;
+import org.apache.carbondata.core.datastorage.store.compression.nondecimal.UnCompressNonDecimalInt;
+import org.apache.carbondata.core.datastorage.store.compression.nondecimal.UnCompressNonDecimalLong;
+import org.apache.carbondata.core.datastorage.store.compression.nondecimal.UnCompressNonDecimalMaxMinByte;
+import org.apache.carbondata.core.datastorage.store.compression.nondecimal.UnCompressNonDecimalMaxMinDefault;
+import org.apache.carbondata.core.datastorage.store.compression.nondecimal.UnCompressNonDecimalMaxMinFloat;
+import org.apache.carbondata.core.datastorage.store.compression.nondecimal.UnCompressNonDecimalMaxMinInt;
+import org.apache.carbondata.core.datastorage.store.compression.nondecimal.UnCompressNonDecimalMaxMinLong;
+import org.apache.carbondata.core.datastorage.store.compression.nondecimal.UnCompressNonDecimalMaxMinShort;
+import org.apache.carbondata.core.datastorage.store.compression.nondecimal.UnCompressNonDecimalShort;
+import org.apache.carbondata.core.datastorage.store.compression.none.UnCompressNoneByte;
+import org.apache.carbondata.core.datastorage.store.compression.none.UnCompressNoneDefault;
+import org.apache.carbondata.core.datastorage.store.compression.none.UnCompressNoneFloat;
+import org.apache.carbondata.core.datastorage.store.compression.none.UnCompressNoneInt;
+import org.apache.carbondata.core.datastorage.store.compression.none.UnCompressNoneLong;
+import org.apache.carbondata.core.datastorage.store.compression.none.UnCompressNoneShort;
import org.junit.Test;
@@ -45,7 +69,7 @@ public class ValueCompressionUtilTest {
@Test public void testToGetCompressedValuesWithCompressionTypeMin_MaxForDataInt() {
double[] values = { 25, 12, 22 };
int[] result = (int[]) ValueCompressionUtil
- .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.MAX_MIN, values,
+ .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_DOUBLE, values,
DataType.DATA_INT, 22, 0);
int[] expectedResult = { -3, 10, 0 };
for (int i = 0; i < result.length; i++) {
@@ -56,7 +80,7 @@ public class ValueCompressionUtilTest {
@Test public void testToGetCompressedValuesWithCompressionTypeMin_MaxForDataByte() {
double[] values = { 20, 21, 22 };
byte[] result = (byte[]) ValueCompressionUtil
- .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.MAX_MIN, values,
+ .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_DOUBLE, values,
DataType.DATA_BYTE, 22, 0);
byte[] expectedResult = { 2, 1, 0 };
for (int i = 0; i < result.length; i++) {
@@ -67,7 +91,7 @@ public class ValueCompressionUtilTest {
@Test public void testToGetCompressedValuesWithCompressionTypeMin_MaxForDataShort() {
double[] values = { 200, 21, 22 };
short[] result = (short[]) ValueCompressionUtil
- .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.MAX_MIN, values,
+ .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_DOUBLE, values,
DataType.DATA_SHORT, 22, 0);
short[] expectedResult = { -178, 1, 0 };
for (int i = 0; i < result.length; i++) {
@@ -78,7 +102,7 @@ public class ValueCompressionUtilTest {
@Test public void testToGetCompressedValuesWithCompressionTypeMin_MaxForDataLong() {
double[] values = { 2000, 2100, 2002 };
long[] result = (long[]) ValueCompressionUtil
- .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.MAX_MIN, values,
+ .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_DOUBLE, values,
DataType.DATA_LONG, 2125, 0);
long[] expectedResult = { 125, 25, 123 };
for (int i = 0; i < result.length; i++) {
@@ -89,7 +113,7 @@ public class ValueCompressionUtilTest {
@Test public void testToGetCompressedValuesWithCompressionTypeMin_MaxForDataFloat() {
double[] values = { 20.121, 21.223, 22.345 };
float[] result = (float[]) ValueCompressionUtil
- .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.MAX_MIN, values,
+ .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_DOUBLE, values,
DataType.DATA_FLOAT, 22.345, 3);
float[] expectedResult = { 2.224f, 1.122f, 0f };
for (int i = 0; i < result.length; i++) {
@@ -100,7 +124,7 @@ public class ValueCompressionUtilTest {
@Test public void testToGetCompressedValuesWithCompressionTypeMin_MaxForDataDouble() {
double[] values = { 20.121, 21.223, 22.345 };
double[] result = (double[]) ValueCompressionUtil
- .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.MAX_MIN, values,
+ .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_DOUBLE, values,
DataType.DATA_DOUBLE, 102.345, 3);
double[] expectedResult = { 82.224, 81.122, 80.0 };
for (int i = 0; i < result.length; i++) {
@@ -111,7 +135,7 @@ public class ValueCompressionUtilTest {
@Test public void testToGetCompressedValuesWithCompressionTypeNoneForBigInt() {
double[] values = { 20.121, 21.223, 22.345 };
long[] result = (long[]) ValueCompressionUtil
- .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.NONE, values,
+ .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, values,
DataType.DATA_BIGINT, 22, 0);
long[] expectedResult = { 20, 21, 22 };
for (int i = 0; i < result.length; i++) {
@@ -122,7 +146,7 @@ public class ValueCompressionUtilTest {
@Test public void testToGetCompressedValuesWithCompressionTypeNoneForDataByte() {
double[] values = { 20, 21, 22 };
byte[] result = (byte[]) ValueCompressionUtil
- .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.NONE, values, DataType.DATA_BYTE,
+ .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, values, DataType.DATA_BYTE,
22, 0);
byte[] expectedResult = { 20, 21, 22 };
for (int i = 0; i < result.length; i++) {
@@ -133,7 +157,7 @@ public class ValueCompressionUtilTest {
@Test public void testToGetCompressedValuesWithCompressionTypeNoneForDataShort() {
double[] values = { 200000, 21, 22 };
short[] result = (short[]) ValueCompressionUtil
- .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.NONE, values,
+ .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, values,
DataType.DATA_SHORT, 22, 0);
short[] expectedResult = { 3392, 21, 22 };
for (int i = 0; i < result.length; i++) {
@@ -144,7 +168,7 @@ public class ValueCompressionUtilTest {
@Test public void testToGetCompressedValuesWithCompressionTypeNoneForDataInt() {
double[] values = { 20, 21, 22 };
int[] result = (int[]) ValueCompressionUtil
- .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.NONE, values, DataType.DATA_INT,
+ .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, values, DataType.DATA_INT,
22, 0);
int[] expectedResult = { 20, 21, 22 };
for (int i = 0; i < result.length; i++) {
@@ -155,7 +179,7 @@ public class ValueCompressionUtilTest {
@Test public void testToGetCompressedValuesWithCompressionTypeNoneForDataLong() {
double[] values = { 20, 21, 22 };
long[] result = (long[]) ValueCompressionUtil
- .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.NONE, values, DataType.DATA_LONG,
+ .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, values, DataType.DATA_LONG,
22, 0);
long[] expectedResult = { 20, 21, 22 };
for (int i = 0; i < result.length; i++) {
@@ -166,7 +190,7 @@ public class ValueCompressionUtilTest {
@Test public void testToGetCompressedValuesWithCompressionTypeNoneForDataFloat() {
double[] values = { 20.121, 21.223, 22.345 };
float[] result = (float[]) ValueCompressionUtil
- .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.NONE, values,
+ .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, values,
DataType.DATA_FLOAT, 22, 3);
float[] expectedResult = { 20.121f, 21.223f, 22.345f };
for (int i = 0; i < result.length; i++) {
@@ -177,7 +201,7 @@ public class ValueCompressionUtilTest {
@Test public void testToGetCompressedValuesWithCompressionTypeNoneForDataDouble() {
double[] values = { 20.121, 21.223, 22.345 };
double[] result = (double[]) ValueCompressionUtil
- .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.NONE, values,
+ .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, values,
DataType.DATA_DOUBLE, 22, 3);
double[] expectedResult = { 20.121, 21.223, 22.345 };
for (int i = 0; i < result.length; i++) {
@@ -188,7 +212,7 @@ public class ValueCompressionUtilTest {
@Test public void testToGetCompressedValuesWithCompressionTypeNon_DecimalForFloat() {
double[] values = { 20.1, 21.2, 22.3 };
float[] result = (float[]) ValueCompressionUtil
- .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.NON_DECIMAL_CONVERT, values,
+ .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.BIGINT, values,
DataType.DATA_FLOAT, 22, 1);
float[] expectedResult = { 201f, 212f, 223f };
for (int i = 0; i < result.length; i++) {
@@ -199,7 +223,7 @@ public class ValueCompressionUtilTest {
@Test public void testToGetCompressedValuesWithCompressionTypeNon_DecimalForByte() {
double[] values = { 20.1, 21.2, 22.3 };
byte[] result = (byte[]) ValueCompressionUtil
- .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.NON_DECIMAL_CONVERT, values,
+ .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.BIGINT, values,
DataType.DATA_BYTE, 22, 1);
byte[] expectedResult = { -55, -44, -33 };
for (int i = 0; i < result.length; i++) {
@@ -210,7 +234,7 @@ public class ValueCompressionUtilTest {
@Test public void testToGetCompressedValuesWithCompressionTypeNon_DecimalForShort() {
double[] values = { 20.1, 21.2, 22.3 };
short[] result = (short[]) ValueCompressionUtil
- .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.NON_DECIMAL_CONVERT, values,
+ .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.BIGINT, values,
DataType.DATA_SHORT, 22, 1);
short[] expectedResult = { 201, 212, 223 };
for (int i = 0; i < result.length; i++) {
@@ -221,7 +245,7 @@ public class ValueCompressionUtilTest {
@Test public void testToGetCompressedValuesWithCompressionTypeNon_DecimalForInt() {
double[] values = { 20.1, 21.2, 22.3 };
int[] result = (int[]) ValueCompressionUtil
- .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.NON_DECIMAL_CONVERT, values,
+ .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.BIGINT, values,
DataType.DATA_INT, 22, 1);
int[] expectedResult = { 201, 212, 223 };
for (int i = 0; i < result.length; i++) {
@@ -232,7 +256,7 @@ public class ValueCompressionUtilTest {
@Test public void testToGetCompressedValuesWithCompressionTypeNon_DecimalForLong() {
double[] values = { 20.1, 21.2, 22.3 };
long[] result = (long[]) ValueCompressionUtil
- .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.NON_DECIMAL_CONVERT, values,
+ .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.BIGINT, values,
DataType.DATA_LONG, 22, 1);
long[] expectedResult = { 201, 212, 223 };
for (int i = 0; i < result.length; i++) {
@@ -243,7 +267,7 @@ public class ValueCompressionUtilTest {
@Test public void testToGetCompressedValuesWithCompressionTypeNon_DecimalForDouble() {
double[] values = { 20.1, 21.2, 22.3 };
double[] result = (double[]) ValueCompressionUtil
- .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.NON_DECIMAL_CONVERT, values,
+ .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.BIGINT, values,
DataType.DATA_DOUBLE, 22, 1);
double[] expectedResult = { 201, 212, 223 };
for (int i = 0; i < result.length; i++) {
@@ -254,7 +278,7 @@ public class ValueCompressionUtilTest {
@Test public void testToGetCompressedValuesWithCompressionTypeMax_Min_NDCForByte() {
double[] values = { 20, 21, 22 };
byte[] result = (byte[]) ValueCompressionUtil
- .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.MAX_MIN_NDC, values,
+ .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_NON_DECIMAL, values,
DataType.DATA_BYTE, 22, 1);
byte[] expectedResult = { 20, 10, 0 };
for (int i = 0; i < result.length; i++) {
@@ -265,7 +289,7 @@ public class ValueCompressionUtilTest {
@Test public void testToGetCompressedValuesWithCompressionTypeMax_Min_NDCForInt() {
double[] values = { 20, 21, 22 };
int[] result = (int[]) ValueCompressionUtil
- .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.MAX_MIN_NDC, values,
+ .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_NON_DECIMAL, values,
DataType.DATA_INT, 22, 1);
int[] expectedResult = { 20, 10, 0 };
for (int i = 0; i < result.length; i++) {
@@ -276,7 +300,7 @@ public class ValueCompressionUtilTest {
@Test public void testToGetCompressedValuesWithCompressionTypeMax_Min_NDCForDouble() {
double[] values = { 20, 21, 22 };
double[] result = (double[]) ValueCompressionUtil
- .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.MAX_MIN_NDC, values,
+ .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_NON_DECIMAL, values,
DataType.DATA_DOUBLE, 22, 1);
double[] expectedResult = { 20, 10, 0 };
for (int i = 0; i < result.length; i++) {
@@ -287,7 +311,7 @@ public class ValueCompressionUtilTest {
@Test public void testToGetCompressedValuesWithCompressionTypeMax_Min_NDCForShort() {
double[] values = { 20000, 21, 22 };
short[] result = (short[]) ValueCompressionUtil
- .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.MAX_MIN_NDC, values,
+ .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_NON_DECIMAL, values,
DataType.DATA_SHORT, 22, 1);
short[] expectedResult = { -3172, 10, 0 };
for (int i = 0; i < result.length; i++) {
@@ -298,7 +322,7 @@ public class ValueCompressionUtilTest {
@Test public void testToGetCompressedValuesWithCompressionTypeMax_Min_NDCForLong() {
double[] values = { 20, 21, 22 };
long[] result = (long[]) ValueCompressionUtil
- .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.MAX_MIN_NDC, values,
+ .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_NON_DECIMAL, values,
DataType.DATA_LONG, 22, 1);
long[] expectedResult = { 20, 10, 0 };
for (int i = 0; i < result.length; i++) {
@@ -309,7 +333,7 @@ public class ValueCompressionUtilTest {
@Test public void testToGetCompressedValuesWithCompressionTypeMax_Min_NDCForFloat() {
double[] values = { 20, 21, 22 };
float[] result = (float[]) ValueCompressionUtil
- .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.MAX_MIN_NDC, values,
+ .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_NON_DECIMAL, values,
DataType.DATA_FLOAT, 22, 1);
float[] expectedResult = { 20f, 10f, 0f };
for (int i = 0; i < result.length; i++) {
@@ -319,151 +343,151 @@ public class ValueCompressionUtilTest {
@Test public void testToUnCompressNone() {
ValueCompressonHolder.UnCompressValue result =
- ValueCompressionUtil.unCompressNone(DataType.DATA_BIGINT, DataType.DATA_BIGINT);
+ ValueCompressionUtil.getUnCompressNone(DataType.DATA_BIGINT, DataType.DATA_BIGINT);
assertEquals(result.getClass(), UnCompressNoneDefault.class);
}
@Test public void testToUnCompressNoneForByte() {
ValueCompressonHolder.UnCompressValue result =
- ValueCompressionUtil.unCompressNone(DataType.DATA_BYTE, DataType.DATA_FLOAT);
+ ValueCompressionUtil.getUnCompressNone(DataType.DATA_BYTE, DataType.DATA_FLOAT);
assertEquals(result.getClass(), UnCompressNoneByte.class);
}
@Test public void testToUnCompressNoneForLong() {
ValueCompressonHolder.UnCompressValue result =
- ValueCompressionUtil.unCompressNone(DataType.DATA_LONG, DataType.DATA_FLOAT);
+ ValueCompressionUtil.getUnCompressNone(DataType.DATA_LONG, DataType.DATA_FLOAT);
assertEquals(result.getClass(), UnCompressNoneLong.class);
}
@Test public void testToUnCompressNoneForShort() {
ValueCompressonHolder.UnCompressValue result =
- ValueCompressionUtil.unCompressNone(DataType.DATA_SHORT, DataType.DATA_FLOAT);
+ ValueCompressionUtil.getUnCompressNone(DataType.DATA_SHORT, DataType.DATA_FLOAT);
assertEquals(result.getClass(), UnCompressNoneShort.class);
}
@Test public void testToUnCompressNoneForFloat() {
ValueCompressonHolder.UnCompressValue result =
- ValueCompressionUtil.unCompressNone(DataType.DATA_FLOAT, DataType.DATA_FLOAT);
+ ValueCompressionUtil.getUnCompressNone(DataType.DATA_FLOAT, DataType.DATA_FLOAT);
assertEquals(result.getClass(), UnCompressNoneFloat.class);
}
@Test public void testToUnCompressNoneForInt() {
ValueCompressonHolder.UnCompressValue result =
- ValueCompressionUtil.unCompressNone(DataType.DATA_INT, DataType.DATA_FLOAT);
+ ValueCompressionUtil.getUnCompressNone(DataType.DATA_INT, DataType.DATA_FLOAT);
assertEquals(result.getClass(), UnCompressNoneInt.class);
}
@Test public void testToUnCompressNoneForDouble() {
ValueCompressonHolder.UnCompressValue result =
- ValueCompressionUtil.unCompressNone(DataType.DATA_DOUBLE, DataType.DATA_FLOAT);
+ ValueCompressionUtil.getUnCompressNone(DataType.DATA_DOUBLE, DataType.DATA_FLOAT);
assertEquals(result.getClass(), UnCompressNoneDefault.class);
}
@Test public void testToUnCompressMaxMinForDouble() {
ValueCompressonHolder.UnCompressValue result =
- ValueCompressionUtil.unCompressMaxMin(DataType.DATA_DOUBLE, null);
+ ValueCompressionUtil.getUnCompressDecimalMaxMin(DataType.DATA_DOUBLE, null);
assertEquals(result.getClass(), UnCompressMaxMinDefault.class);
}
@Test public void testToUnCompressMaxMinForInt() {
ValueCompressonHolder.UnCompressValue result =
- ValueCompressionUtil.unCompressMaxMin(DataType.DATA_INT, null);
+ ValueCompressionUtil.getUnCompressDecimalMaxMin(DataType.DATA_INT, null);
assertEquals(result.getClass(), UnCompressMaxMinInt.class);
}
@Test public void testToUnCompressMaxMinForFloat() {
ValueCompressonHolder.UnCompressValue result =
- ValueCompressionUtil.unCompressMaxMin(DataType.DATA_FLOAT, null);
+ ValueCompressionUtil.getUnCompressDecimalMaxMin(DataType.DATA_FLOAT, null);
assertEquals(result.getClass(), UnCompressMaxMinFloat.class);
}
@Test public void testToUnCompressMaxMinForLong() {
ValueCompressonHolder.UnCompressValue result =
- ValueCompressionUtil.unCompressMaxMin(DataType.DATA_LONG, null);
+ ValueCompressionUtil.getUnCompressDecimalMaxMin(DataType.DATA_LONG, null);
assertEquals(result.getClass(), UnCompressMaxMinLong.class);
}
@Test public void testToUnCompressMaxMinForByte() {
ValueCompressonHolder.UnCompressValue result =
- ValueCompressionUtil.unCompressMaxMin(DataType.DATA_BYTE, null);
+ ValueCompressionUtil.getUnCompressDecimalMaxMin(DataType.DATA_BYTE, null);
assertEquals(result.getClass(), UnCompressMaxMinByte.class);
}
@Test public void testToUnCompressMaxMinForShort() {
ValueCompressonHolder.UnCompressValue result =
- ValueCompressionUtil.unCompressMaxMin(DataType.DATA_SHORT, null);
+ ValueCompressionUtil.getUnCompressDecimalMaxMin(DataType.DATA_SHORT, null);
assertEquals(result.getClass(), UnCompressMaxMinShort.class);
}
@Test public void testToUnCompressNonDecimalForDouble() {
ValueCompressonHolder.UnCompressValue result =
- ValueCompressionUtil.unCompressNonDecimal(DataType.DATA_DOUBLE, null);
+ ValueCompressionUtil.getUnCompressNonDecimal(DataType.DATA_DOUBLE);
assertEquals(result.getClass(), UnCompressNonDecimalDefault.class);
}
@Test public void testToUnCompressNonDecimalForInt() {
ValueCompressonHolder.UnCompressValue result =
- ValueCompressionUtil.unCompressNonDecimal(DataType.DATA_INT, null);
+ ValueCompressionUtil.getUnCompressNonDecimal(DataType.DATA_INT);
assertEquals(result.getClass(), UnCompressNonDecimalInt.class);
}
@Test public void testToUnCompressNonDecimalForFloat() {
ValueCompressonHolder.UnCompressValue result =
- ValueCompressionUtil.unCompressNonDecimal(DataType.DATA_FLOAT, null);
+ ValueCompressionUtil.getUnCompressNonDecimal(DataType.DATA_FLOAT);
assertEquals(result.getClass(), UnCompressNonDecimalFloat.class);
}
@Test public void testToUnCompressNonDecimalForLong() {
ValueCompressonHolder.UnCompressValue result =
- ValueCompressionUtil.unCompressNonDecimal(DataType.DATA_LONG, null);
+ ValueCompressionUtil.getUnCompressNonDecimal(DataType.DATA_LONG);
assertEquals(result.getClass(), UnCompressNonDecimalLong.class);
}
@Test public void testToUnCompressNonDecimalForByte() {
ValueCompressonHolder.UnCompressValue result =
- ValueCompressionUtil.unCompressNonDecimal(DataType.DATA_BYTE, null);
+ ValueCompressionUtil.getUnCompressNonDecimal(DataType.DATA_BYTE);
assertEquals(result.getClass(), UnCompressNonDecimalByte.class);
}
@Test public void testToUnCompressNonDecimalForShort() {
ValueCompressonHolder.UnCompressValue result =
- ValueCompressionUtil.unCompressNonDecimal(DataType.DATA_SHORT, null);
+ ValueCompressionUtil.getUnCompressNonDecimal(DataType.DATA_SHORT);
assertEquals(result.getClass(), UnCompressNonDecimalShort.class);
}
@Test public void testToUnCompressNonDecimalMaxMinForDouble() {
ValueCompressonHolder.UnCompressValue result =
- ValueCompressionUtil.unCompressNonDecimalMaxMin(DataType.DATA_DOUBLE, null);
+ ValueCompressionUtil.getUnCompressNonDecimalMaxMin(DataType.DATA_DOUBLE);
assertEquals(result.getClass(), UnCompressNonDecimalMaxMinDefault.class);
}
@Test public void testToUnCompressNonDecimalMaxMinForInt() {
ValueCompressonHolder.UnCompressValue result =
- ValueCompressionUtil.unCompressNonDecimalMaxMin(DataType.DATA_INT, null);
+ ValueCompressionUtil.getUnCompressNonDecimalMaxMin(DataType.DATA_INT);
assertEquals(result.getClass(), UnCompressNonDecimalMaxMinInt.class);
}
@Test public void testToUnCompressNonDecimalMaxMinForFloat() {
ValueCompressonHolder.UnCompressValue result =
- ValueCompressionUtil.unCompressNonDecimalMaxMin(DataType.DATA_FLOAT, null);
+ ValueCompressionUtil.getUnCompressNonDecimalMaxMin(DataType.DATA_FLOAT);
assertEquals(result.getClass(), UnCompressNonDecimalMaxMinFloat.class);
}
@Test public void testToUnCompressNonDecimalMaxMinForLong() {
ValueCompressonHolder.UnCompressValue result =
- ValueCompressionUtil.unCompressNonDecimalMaxMin(DataType.DATA_LONG, null);
+ ValueCompressionUtil.getUnCompressNonDecimalMaxMin(DataType.DATA_LONG);
assertEquals(result.getClass(), UnCompressNonDecimalMaxMinLong.class);
}
@Test public void testToUnCompressNonDecimalMaxMinForByte() {
ValueCompressonHolder.UnCompressValue result =
- ValueCompressionUtil.unCompressNonDecimalMaxMin(DataType.DATA_BYTE, null);
+ ValueCompressionUtil.getUnCompressNonDecimalMaxMin(DataType.DATA_BYTE);
assertEquals(result.getClass(), UnCompressNonDecimalMaxMinByte.class);
}
@Test public void testToUnCompressNonDecimalMaxMinForShort() {
ValueCompressonHolder.UnCompressValue result =
- ValueCompressionUtil.unCompressNonDecimalMaxMin(DataType.DATA_SHORT, null);
+ ValueCompressionUtil.getUnCompressNonDecimalMaxMin(DataType.DATA_SHORT);
assertEquals(result.getClass(), UnCompressNonDecimalMaxMinShort.class);
}
@@ -563,23 +587,23 @@ public class ValueCompressionUtilTest {
}
@Test public void testToGetValueCompressionModel() {
- Object[] maxValues = { 10, 20, 30l };
- Object[] minValues = { 1, 2, 3l };
+ Object[] maxValues = { 10l, 20l, 30l };
+ Object[] minValues = { 1l, 2l, 3l };
int[] decimalLength = { 0, 0, 0 };
Object[] uniqueValues = { 5, 3, 2l };
- char[] types = { 'c', 'b', 'l' };
+ char[] types = { 'l', 'l', 'l' };
byte[] dataTypeSelected = { 1, 2, 4 };
MeasureMetaDataModel measureMetaDataModel =
new MeasureMetaDataModel(maxValues, minValues, decimalLength, 3, uniqueValues, types,
dataTypeSelected);
- ValueCompressionModel valueCompressionModel =
- ValueCompressionUtil.getValueCompressionModel(measureMetaDataModel);
- assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.CUSTOM,
- valueCompressionModel.getCompType()[0]);
- assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.CUSTOM_BIGDECIMAL,
- valueCompressionModel.getCompType()[1]);
- assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.NONE,
- valueCompressionModel.getCompType()[2]);
+ WriterCompressModel writerCompressModel =
+ ValueCompressionUtil.getWriterCompressModel(measureMetaDataModel);
+ assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE,
+ writerCompressModel.getCompType()[0]);
+ assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE,
+ writerCompressModel.getCompType()[1]);
+ assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE,
+ writerCompressModel.getCompType()[2]);
}
@Test public void testToGetValueCompressionModelForDefaultAggregatorType() {
@@ -592,10 +616,10 @@ public class ValueCompressionUtilTest {
MeasureMetaDataModel measureMetaDataModel =
new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types,
dataTypeSelected);
- ValueCompressionModel valueCompressionModel =
- ValueCompressionUtil.getValueCompressionModel(measureMetaDataModel);
- assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.NONE,
- valueCompressionModel.getCompType()[0]);
+ WriterCompressModel writerCompressModel =
+ ValueCompressionUtil.getWriterCompressModel(measureMetaDataModel);
+ assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE,
+ writerCompressModel.getCompType()[0]);
}
@Test public void testToGetValueCompressionModelForShortAndByte() {
@@ -608,10 +632,10 @@ public class ValueCompressionUtilTest {
MeasureMetaDataModel measureMetaDataModel =
new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types,
dataTypeSelected);
- ValueCompressionModel valueCompressionModel =
- ValueCompressionUtil.getValueCompressionModel(measureMetaDataModel);
- assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.MAX_MIN,
- valueCompressionModel.getCompType()[0]);
+ WriterCompressModel writerCompressModel =
+ ValueCompressionUtil.getWriterCompressModel(measureMetaDataModel);
+ assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_DOUBLE,
+ writerCompressModel.getCompType()[0]);
}
@Test public void testToGetValueCompressionModelForIntAndShort() {
@@ -624,10 +648,10 @@ public class ValueCompressionUtilTest {
MeasureMetaDataModel measureMetaDataModel =
new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types,
dataTypeSelected);
- ValueCompressionModel valueCompressionModel =
- ValueCompressionUtil.getValueCompressionModel(measureMetaDataModel);
- assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.MAX_MIN,
- valueCompressionModel.getCompType()[0]);
+ WriterCompressModel writerCompressModel =
+ ValueCompressionUtil.getWriterCompressModel(measureMetaDataModel);
+ assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_DOUBLE,
+ writerCompressModel.getCompType()[0]);
}
@Test public void testToGetValueCompressionModelForByteAndInt() {
@@ -640,10 +664,10 @@ public class ValueCompressionUtilTest {
MeasureMetaDataModel measureMetaDataModel =
new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types,
dataTypeSelected);
- ValueCompressionModel valueCompressionModel =
- ValueCompressionUtil.getValueCompressionModel(measureMetaDataModel);
- assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.NONE,
- valueCompressionModel.getCompType()[0]);
+ WriterCompressModel writerCompressModel =
+ ValueCompressionUtil.getWriterCompressModel(measureMetaDataModel);
+ assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE,
+ writerCompressModel.getCompType()[0]);
}
@Test public void testToGetValueCompressionModelForByteAndIntAndDecimal1() {
@@ -656,10 +680,10 @@ public class ValueCompressionUtilTest {
MeasureMetaDataModel measureMetaDataModel =
new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types,
dataTypeSelected);
- ValueCompressionModel valueCompressionModel =
- ValueCompressionUtil.getValueCompressionModel(measureMetaDataModel);
- assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.MAX_MIN,
- valueCompressionModel.getCompType()[0]);
+ WriterCompressModel writerCompressModel =
+ ValueCompressionUtil.getWriterCompressModel(measureMetaDataModel);
+ assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_DOUBLE,
+ writerCompressModel.getCompType()[0]);
}
@Test public void testToGetValueCompressionModelForByteAndIntAndDataTypeSelected0() {
@@ -672,10 +696,10 @@ public class ValueCompressionUtilTest {
MeasureMetaDataModel measureMetaDataModel =
new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types,
dataTypeSelected);
- ValueCompressionModel valueCompressionModel =
- ValueCompressionUtil.getValueCompressionModel(measureMetaDataModel);
- assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.NON_DECIMAL_CONVERT,
- valueCompressionModel.getCompType()[0]);
+ WriterCompressModel writerCompressModel =
+ ValueCompressionUtil.getWriterCompressModel(measureMetaDataModel);
+ assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.BIGINT,
+ writerCompressModel.getCompType()[0]);
}
@Test public void testToGetValueCompressionModelForFloatAndDataTypeSelected1() {
@@ -688,10 +712,10 @@ public class ValueCompressionUtilTest {
MeasureMetaDataModel measureMetaDataModel =
new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types,
dataTypeSelected);
- ValueCompressionModel valueCompressionModel =
- ValueCompressionUtil.getValueCompressionModel(measureMetaDataModel);
- assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.NONE,
- valueCompressionModel.getCompType()[0]);
+ WriterCompressModel writerCompressModel =
+ ValueCompressionUtil.getWriterCompressModel(measureMetaDataModel);
+ assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE,
+ writerCompressModel.getCompType()[0]);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java b/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java
index b36233b..20a0790 100644
--- a/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java
@@ -30,7 +30,7 @@ import org.apache.carbondata.core.carbon.datastore.block.SegmentProperties;
import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
import org.apache.carbondata.core.carbon.metadata.encoder.Encoding;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel;
+import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
import org.apache.carbondata.core.metadata.BlockletInfoColumnar;
@@ -182,10 +182,10 @@ public class CarbonFooterWriterTest extends TestCase{
infoColumnar.setAggKeyBlock(new boolean[] { true, true, true, true });
infoColumnar.setColGrpBlocks(new boolean[] { false, false, false, false });
infoColumnar.setMeasureNullValueIndex(new BitSet[] {new BitSet(),new BitSet()});
- ValueCompressionModel compressionModel = new ValueCompressionModel();
+ WriterCompressModel compressionModel = new WriterCompressModel();
compressionModel.setMaxValue(new Object[] { 44d, 55d });
compressionModel.setMinValue(new Object[] { 0d, 0d });
- compressionModel.setDecimal(new int[] { 0, 0 });
+ compressionModel.setMantissa(new int[] { 0, 0 });
compressionModel.setType(new char[] { 'n', 'n' });
compressionModel.setUniqueValue(new Object[] { 0d, 0d });
compressionModel.setDataTypeSelected(new byte[2]);
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/format/src/main/thrift/schema.thrift
----------------------------------------------------------------------
diff --git a/format/src/main/thrift/schema.thrift b/format/src/main/thrift/schema.thrift
index 885beba..1df59a5 100644
--- a/format/src/main/thrift/schema.thrift
+++ b/format/src/main/thrift/schema.thrift
@@ -68,7 +68,7 @@ struct ColumnSchema{
6: required bool dimension; // Whether the column is a dimension or measure
7: optional i32 column_group_id; // The group ID for column used for row format columns, where in columns in each group are chunked together.
/**
- * Used when this column contains decimal data.
+ * Used when this column contains mantissa data.
*/
8: optional i32 scale;
9: optional i32 precision;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index d6cd075..39d0841 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -757,7 +757,7 @@ private[sql] case class DescribeCommandFormatted(
if (colPropStr.length() > 0) {
results ++= Seq((colPropStr, "", ""))
} else {
- results ++= Seq(("NONE", "", ""))
+ results ++= Seq(("ADAPTIVE", "", ""))
}
val dimension = carbonTable
.getDimensionByTableName(relation.tableMeta.carbonTableIdentifier.getTableName)
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/step-attributes.xml
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/step-attributes.xml b/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/step-attributes.xml
index 65fdd57..32b1452 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/step-attributes.xml
+++ b/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/step-attributes.xml
@@ -202,7 +202,7 @@
<parentid>FIELD</parentid>
</attribute>
<attribute id="FIELD_DECIMAL">
- <xmlcode>decimal</xmlcode>
+ <xmlcode>mantissa</xmlcode>
<repcode>field_decimal</repcode>
<description>CsvInputDialog.DecimalColumn.Column</description>
<tooltip/>
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
index c961700..0398cd7 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
@@ -51,7 +51,7 @@ import org.apache.carbondata.core.datastorage.store.columnar.BlockIndexerStorage
import org.apache.carbondata.core.datastorage.store.columnar.BlockIndexerStorageForNoInvertedIndex;
import org.apache.carbondata.core.datastorage.store.columnar.ColumnGroupModel;
import org.apache.carbondata.core.datastorage.store.columnar.IndexStorage;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel;
+import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
import org.apache.carbondata.core.datastorage.store.dataholder.CarbonWriteDataHolder;
import org.apache.carbondata.core.datastorage.util.StoreFactory;
import org.apache.carbondata.core.keygenerator.KeyGenException;
@@ -603,8 +603,8 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
if ((noDictionaryCount + complexColCount) > 0) {
noDictionaryValueHolder = noDictionaryKeyDataHolder.getByteArrayValues();
}
- ValueCompressionModel compressionModel = ValueCompressionUtil
- .getValueCompressionModel(max, min, decimal, uniqueValue, type, new byte[max.length]);
+ WriterCompressModel compressionModel = ValueCompressionUtil
+ .getWriterCompressModel(max, min, decimal, uniqueValue, type, new byte[max.length]);
byte[][] writableMeasureDataArray =
StoreFactory.createDataStore(compressionModel).getWritableMeasureDataArray(dataHolder)
.clone();
@@ -738,8 +738,8 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
if ((noDictionaryCount + complexColCount) > 0) {
noDictionaryValueHolder = noDictionaryKeyDataHolder.getNonDictByteArrayValues();
}
- ValueCompressionModel compressionModel = ValueCompressionUtil
- .getValueCompressionModel(max, min, decimal, uniqueValue, type, new byte[max.length]);
+ WriterCompressModel compressionModel = ValueCompressionUtil
+ .getWriterCompressModel(max, min, decimal, uniqueValue, type, new byte[max.length]);
byte[][] writableMeasureDataArray =
StoreFactory.createDataStore(compressionModel).getWritableMeasureDataArray(dataHolder)
.clone();
@@ -755,7 +755,7 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
// TODO remove after kettle flow is removed
private NodeHolder getNodeHolderObject(byte[][] dataHolderLocal, byte[][] byteArrayValues,
int entryCountLocal, byte[] startkeyLocal, byte[] endKeyLocal,
- ValueCompressionModel compressionModel, byte[][] noDictionaryData,
+ WriterCompressModel compressionModel, byte[][] noDictionaryData,
byte[] noDictionaryStartKey, byte[] noDictionaryEndKey)
throws CarbonDataWriterException {
byte[][][] noDictionaryColumnsData = null;
@@ -882,7 +882,7 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
private NodeHolder getNodeHolderObjectWithOutKettle(byte[][] dataHolderLocal,
byte[][] byteArrayValues, int entryCountLocal, byte[] startkeyLocal, byte[] endKeyLocal,
- ValueCompressionModel compressionModel, byte[][][] noDictionaryData,
+ WriterCompressModel compressionModel, byte[][][] noDictionaryData,
byte[][] noDictionaryStartKey, byte[][] noDictionaryEndKey)
throws CarbonDataWriterException {
byte[][][] noDictionaryColumnsData = null;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonFactDataWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonFactDataWriter.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonFactDataWriter.java
index 3e31901..5745e9f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonFactDataWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonFactDataWriter.java
@@ -20,7 +20,7 @@
package org.apache.carbondata.processing.store.writer;
import org.apache.carbondata.core.datastorage.store.columnar.IndexStorage;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel;
+import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
import org.apache.carbondata.processing.store.writer.exception.CarbonDataWriterException;
public interface CarbonFactDataWriter<T> {
@@ -40,7 +40,7 @@ public interface CarbonFactDataWriter<T> {
*/
NodeHolder buildDataNodeHolder(IndexStorage<T>[] keyStorageArray, byte[][] dataArray,
- int entryCount, byte[] startKey, byte[] endKey, ValueCompressionModel compressionModel,
+ int entryCount, byte[] startKey, byte[] endKey, WriterCompressModel compressionModel,
byte[] noDictionaryStartKey, byte[] noDictionaryEndKey) throws CarbonDataWriterException;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/processing/src/main/java/org/apache/carbondata/processing/store/writer/NodeHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/NodeHolder.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/NodeHolder.java
index a7d14f0..0864458 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/NodeHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/NodeHolder.java
@@ -21,7 +21,7 @@ package org.apache.carbondata.processing.store.writer;
import java.util.BitSet;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel;
+import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
public class NodeHolder {
/**
@@ -110,7 +110,7 @@ public class NodeHolder {
/**
* compression model for numbers data block.
*/
- private ValueCompressionModel compressionModel;
+ private WriterCompressModel compressionModel;
/**
* array of aggBlocks flag to identify the aggBlocks
@@ -382,11 +382,11 @@ public class NodeHolder {
this.columnMinData = columnMinData;
}
- public ValueCompressionModel getCompressionModel() {
+ public WriterCompressModel getCompressionModel() {
return compressionModel;
}
- public void setCompressionModel(ValueCompressionModel compressionModel) {
+ public void setCompressionModel(WriterCompressModel compressionModel) {
this.compressionModel = compressionModel;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/360edc8d/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
index 19e781d..8d53fe4 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
@@ -28,7 +28,7 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastorage.store.columnar.IndexStorage;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel;
+import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
import org.apache.carbondata.core.metadata.BlockletInfoColumnar;
import org.apache.carbondata.core.util.CarbonMetadataUtil;
import org.apache.carbondata.core.writer.CarbonFooterWriter;
@@ -50,7 +50,7 @@ public class CarbonFactDataWriterImplV1 extends AbstractFactDataWriter<int[]> {
@Override
public NodeHolder buildDataNodeHolder(IndexStorage<int[]>[] keyStorageArray, byte[][] dataArray,
- int entryCount, byte[] startKey, byte[] endKey, ValueCompressionModel compressionModel,
+ int entryCount, byte[] startKey, byte[] endKey, WriterCompressModel compressionModel,
byte[] noDictionaryStartKey, byte[] noDictionaryEndKey) throws CarbonDataWriterException {
// if there are no NO-Dictionary column present in the table then
// set the empty byte array
|