carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [48/50] [abbrv] incubator-carbondata git commit: fix sort_columns issue
Date Tue, 02 May 2017 14:13:12 GMT
fix sort_columns issue


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/0287e1e9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/0287e1e9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/0287e1e9

Branch: refs/heads/12-dev
Commit: 0287e1e94d3c86fad34cc4114b86a6c88a2e9cd2
Parents: a0d65f6
Author: QiangCai <qiangcai@qq.com>
Authored: Sat Apr 8 22:10:17 2017 +0530
Committer: jackylk <jacky.likun@huawei.com>
Committed: Tue May 2 09:43:27 2017 +0800

----------------------------------------------------------------------
 .../executer/RowLevelFilterExecuterImpl.java    |  3 ++-
 .../RowLevelRangeFilterResolverImpl.java        |  5 +++-
 .../apache/carbondata/core/util/ByteUtil.java   | 28 +++++++++++---------
 .../carbondata/core/util/DataTypeUtil.java      |  2 +-
 .../testsuite/sortcolumns/TestSortColumns.scala | 11 ++++++--
 .../CarbonRowDataWriterProcessorStepImpl.java   |  4 +--
 6 files changed, 33 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/0287e1e9/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index 8e7c95f..38204e9 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -254,7 +254,8 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
               memberBytes = null;
             }
             record[dimColumnEvaluatorInfo.getRowIndex()] = DataTypeUtil
-                .getDataBasedOnDataType(memberBytes, dimColumnEvaluatorInfo.getDimension());
+                .getDataBasedOnDataTypeForNoDictionaryColumn(memberBytes,
+                    dimColumnEvaluatorInfo.getDimension().getDataType());
           } else {
             continue;
           }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/0287e1e9/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
index 2a1af65..6d4e528 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
@@ -43,6 +43,7 @@ import org.apache.carbondata.core.scan.filter.intf.FilterExecuterType;
 import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
 import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
 import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
 
 public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverImpl {
 
@@ -147,7 +148,9 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm
           filterValuesList.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL.getBytes());
           continue;
         }
-        filterValuesList.add(result.getString().getBytes());
+        filterValuesList.add(DataTypeUtil
+            .getBytesBasedOnDataTypeForNoDictionaryColumn(result.getString(),
+                result.getDataType()));
       } catch (FilterIllegalMemberException e) {
         // Any invalid member while evaluation shall be ignored, system will log the
         // error only once since all rows the evaluation happens so inorder to avoid

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/0287e1e9/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
index f3f0fcd..70ddec9 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
@@ -404,6 +404,7 @@ public final class ByteUtil {
    * @return
    */
   public static byte[] toBytes(short val) {
+    val = (short)(val ^ Short.MIN_VALUE);
     byte[] b = new byte[SIZEOF_SHORT];
     b[1] = (byte) val;
     val >>= 8;
@@ -423,20 +424,21 @@ public final class ByteUtil {
     if (length != SIZEOF_SHORT || offset + length > bytes.length) {
       throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT);
     }
+    short n = 0;
     if (CarbonUnsafe.unsafe != null) {
       if (CarbonUnsafe.ISLITTLEENDIAN) {
-        return Short.reverseBytes(
+        n = Short.reverseBytes(
             CarbonUnsafe.unsafe.getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET));
       } else {
-        return CarbonUnsafe.unsafe.getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
+        n = CarbonUnsafe.unsafe.getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
       }
     } else {
-      short n = 0;
+
       n ^= bytes[offset] & 0xFF;
       n <<= 8;
       n ^= bytes[offset + 1] & 0xFF;
-      return n;
     }
+    return (short)(n ^ Short.MIN_VALUE);
   }
 
   /**
@@ -446,6 +448,7 @@ public final class ByteUtil {
    * @return
    */
   public static byte[] toBytes(int val) {
+    val = val ^ Integer.MIN_VALUE;
     byte[] b = new byte[4];
     for (int i = 3; i > 0; i--) {
       b[i] = (byte) val;
@@ -467,21 +470,21 @@ public final class ByteUtil {
     if (length != SIZEOF_INT || offset + length > bytes.length) {
       throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_INT);
     }
+    int n = 0;
     if (CarbonUnsafe.unsafe != null) {
       if (CarbonUnsafe.ISLITTLEENDIAN) {
-        return Integer.reverseBytes(
+        n = Integer.reverseBytes(
             CarbonUnsafe.unsafe.getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET));
       } else {
-        return CarbonUnsafe.unsafe.getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
+        n = CarbonUnsafe.unsafe.getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
       }
     } else {
-      int n = 0;
       for (int i = offset; i < (offset + length); i++) {
         n <<= 8;
         n ^= bytes[i] & 0xFF;
       }
-      return n;
     }
+    return n ^ Integer.MIN_VALUE;
   }
 
   /**
@@ -513,6 +516,7 @@ public final class ByteUtil {
    * @return
    */
   public static byte[] toBytes(long val) {
+    val = val ^ Long.MIN_VALUE;
     byte[] b = new byte[8];
     for (int i = 7; i > 0; i--) {
       b[i] = (byte) val;
@@ -529,21 +533,21 @@ public final class ByteUtil {
     if (length != SIZEOF_LONG || offset + length > bytes.length) {
       throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_LONG);
     }
+    long l = 0;
     if (CarbonUnsafe.unsafe != null) {
       if (CarbonUnsafe.ISLITTLEENDIAN) {
-        return Long.reverseBytes(
+        l = Long.reverseBytes(
             CarbonUnsafe.unsafe.getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET));
       } else {
-        return CarbonUnsafe.unsafe.getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
+        l = CarbonUnsafe.unsafe.getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
       }
     } else {
-      long l = 0;
       for (int i = offset; i < offset + length; i++) {
         l <<= 8;
         l ^= bytes[i] & 0xFF;
       }
-      return l;
     }
+    return l ^ Long.MIN_VALUE;
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/0287e1e9/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index 2e16b0f..80c9e72 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -324,7 +324,7 @@ public final class DataTypeUtil {
   }
 
   public static byte[] getBytesBasedOnDataTypeForNoDictionaryColumn(String dimensionValue,
-      DataType actualDataType) throws Throwable {
+      DataType actualDataType) {
     switch (actualDataType) {
       case STRING:
         return ByteUtil.toBytes(dimensionValue);

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/0287e1e9/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
index 88b0421..afdca21 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
@@ -220,13 +220,20 @@ class TestSortColumns extends QueryTest with BeforeAndAfterAll {
       sql("CREATE TABLE unsortedtable_offheap_inmemory (empno int, empname String, designation
String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname
String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization
int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
       sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_offheap_inmemory
OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
       checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno = 11"), sql("select
* from origintable1 where empno = 11"))
+      checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno < 15 order
by empno"), sql("select * from origintable1 where empno < 15 order by empno"))
+      checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno <= 15
order by empno"), sql("select * from origintable1 where empno <= 15 order by empno"))
+      checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno > 15 order
by empno"), sql("select * from origintable1 where empno > 15 order by empno"))
+      checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno >= 15
order by empno"), sql("select * from origintable1 where empno >= 15 order by empno"))
+      checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno <>
15 order by empno"), sql("select * from origintable1 where empno <> 15 order by empno"))
+      checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno in (15, 16,
17) order by empno"), sql("select * from origintable1 where empno in (15, 16, 17) order by
empno"))
+      checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno is null"),
sql("select * from origintable1 where empno is null order by empno"))
+      checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno is not null"),
sql("select * from origintable1 where empno is not null order by empno"))
       checkAnswer(sql("select * from unsortedtable_offheap_inmemory order by empno"), sql("select
* from origintable1 order by empno"))
     } finally {
       defaultLoadingProperties
     }
   }
-
-
+  
   override def afterAll = {
     dropTable
   }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/0287e1e9/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
index 929a09e..c50f335 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
@@ -156,9 +156,7 @@ public class CarbonRowDataWriterProcessorStepImpl extends AbstractDataLoadProces
   private void doExecute(Iterator<CarbonRowBatch> iterator, int partitionId, int iteratorIndex)
{
     String storeLocation = getStoreLocation(tableIdentifier, String.valueOf(partitionId));
     CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel
-        .createCarbonFactDataHandlerModel(configuration, storeLocation, partitionId, 0);
-    model.getCarbonDataFileAttributes()
-        .setFactTimeStamp(model.getCarbonDataFileAttributes().getFactTimeStamp() + iteratorIndex);
+        .createCarbonFactDataHandlerModel(configuration, storeLocation, partitionId, iteratorIndex);
     CarbonFactHandler dataHandler = null;
     boolean rowsNotExist = true;
     while (iterator.hasNext()) {


Mime
View raw message