carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [1/2] incubator-carbondata git commit: fix sort_columns issue
Date Tue, 11 Apr 2017 12:55:56 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/12-dev 47b2fdd90 -> 222a80762


fix sort_columns issue


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/14c0a8b4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/14c0a8b4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/14c0a8b4

Branch: refs/heads/12-dev
Commit: 14c0a8b412d52c5fd03050df8eeb61f9f72afc07
Parents: 47b2fdd
Author: QiangCai <qiangcai@qq.com>
Authored: Sat Apr 8 22:10:17 2017 +0530
Committer: QiangCai <qiangcai@qq.com>
Committed: Tue Apr 11 09:52:55 2017 +0530

----------------------------------------------------------------------
 .../executer/RowLevelFilterExecuterImpl.java    |  3 ++-
 .../RowLevelRangeFilterResolverImpl.java        |  5 +++-
 .../apache/carbondata/core/util/ByteUtil.java   | 28 +++++++++++---------
 .../carbondata/core/util/DataTypeUtil.java      |  2 +-
 .../testsuite/sortcolumns/TestSortColumns.scala | 11 ++++++--
 .../CarbonRowDataWriterProcessorStepImpl.java   |  4 +--
 6 files changed, 33 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/14c0a8b4/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index d92b7d4..22490ac 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -246,7 +246,8 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
               memberBytes = null;
             }
             record[dimColumnEvaluatorInfo.getRowIndex()] = DataTypeUtil
-                .getDataBasedOnDataType(memberBytes, dimColumnEvaluatorInfo.getDimension());
+                .getDataBasedOnDataTypeForNoDictionaryColumn(memberBytes,
+                    dimColumnEvaluatorInfo.getDimension().getDataType());
           } else {
             continue;
           }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/14c0a8b4/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
index 2a1af65..6d4e528 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
@@ -43,6 +43,7 @@ import org.apache.carbondata.core.scan.filter.intf.FilterExecuterType;
 import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
 import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
 import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
 
 public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverImpl {
 
@@ -147,7 +148,9 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm
           filterValuesList.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL.getBytes());
           continue;
         }
-        filterValuesList.add(result.getString().getBytes());
+        filterValuesList.add(DataTypeUtil
+            .getBytesBasedOnDataTypeForNoDictionaryColumn(result.getString(),
+                result.getDataType()));
       } catch (FilterIllegalMemberException e) {
         // Any invalid member while evaluation shall be ignored, system will log the
         // error only once since all rows the evaluation happens so inorder to avoid

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/14c0a8b4/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
index 84a8634..1aa2fa1 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
@@ -441,6 +441,7 @@ public final class ByteUtil {
    * @return
    */
   public static byte[] toBytes(short val) {
+    val = (short)(val ^ Short.MIN_VALUE);
     byte[] b = new byte[SIZEOF_SHORT];
     b[1] = (byte) val;
     val >>= 8;
@@ -460,20 +461,21 @@ public final class ByteUtil {
     if (length != SIZEOF_SHORT || offset + length > bytes.length) {
       throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT);
     }
+    short n = 0;
     if (CarbonUnsafe.unsafe != null) {
       if (CarbonUnsafe.ISLITTLEENDIAN) {
-        return Short.reverseBytes(
+        n = Short.reverseBytes(
             CarbonUnsafe.unsafe.getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET));
       } else {
-        return CarbonUnsafe.unsafe.getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
+        n = CarbonUnsafe.unsafe.getShort(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
       }
     } else {
-      short n = 0;
+
       n ^= bytes[offset] & 0xFF;
       n <<= 8;
       n ^= bytes[offset + 1] & 0xFF;
-      return n;
     }
+    return (short)(n ^ Short.MIN_VALUE);
   }
 
   /**
@@ -483,6 +485,7 @@ public final class ByteUtil {
    * @return
    */
   public static byte[] toBytes(int val) {
+    val = val ^ Integer.MIN_VALUE;
     byte[] b = new byte[4];
     for (int i = 3; i > 0; i--) {
       b[i] = (byte) val;
@@ -504,21 +507,21 @@ public final class ByteUtil {
     if (length != SIZEOF_INT || offset + length > bytes.length) {
       throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_INT);
     }
+    int n = 0;
     if (CarbonUnsafe.unsafe != null) {
       if (CarbonUnsafe.ISLITTLEENDIAN) {
-        return Integer.reverseBytes(
+        n = Integer.reverseBytes(
             CarbonUnsafe.unsafe.getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET));
       } else {
-        return CarbonUnsafe.unsafe.getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
+        n = CarbonUnsafe.unsafe.getInt(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
       }
     } else {
-      int n = 0;
       for (int i = offset; i < (offset + length); i++) {
         n <<= 8;
         n ^= bytes[i] & 0xFF;
       }
-      return n;
     }
+    return n ^ Integer.MIN_VALUE;
   }
 
   /**
@@ -550,6 +553,7 @@ public final class ByteUtil {
    * @return
    */
   public static byte[] toBytes(long val) {
+    val = val ^ Long.MIN_VALUE;
     byte[] b = new byte[8];
     for (int i = 7; i > 0; i--) {
       b[i] = (byte) val;
@@ -566,21 +570,21 @@ public final class ByteUtil {
     if (length != SIZEOF_LONG || offset + length > bytes.length) {
       throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_LONG);
     }
+    long l = 0;
     if (CarbonUnsafe.unsafe != null) {
       if (CarbonUnsafe.ISLITTLEENDIAN) {
-        return Long.reverseBytes(
+        l = Long.reverseBytes(
             CarbonUnsafe.unsafe.getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET));
       } else {
-        return CarbonUnsafe.unsafe.getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
+        l = CarbonUnsafe.unsafe.getLong(bytes, offset + CarbonUnsafe.BYTE_ARRAY_OFFSET);
       }
     } else {
-      long l = 0;
       for (int i = offset; i < offset + length; i++) {
         l <<= 8;
         l ^= bytes[i] & 0xFF;
       }
-      return l;
     }
+    return l ^ Long.MIN_VALUE;
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/14c0a8b4/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index 2e16b0f..80c9e72 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -324,7 +324,7 @@ public final class DataTypeUtil {
   }
 
   public static byte[] getBytesBasedOnDataTypeForNoDictionaryColumn(String dimensionValue,
-      DataType actualDataType) throws Throwable {
+      DataType actualDataType) {
     switch (actualDataType) {
       case STRING:
         return ByteUtil.toBytes(dimensionValue);

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/14c0a8b4/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
index 88b0421..afdca21 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/sortcolumns/TestSortColumns.scala
@@ -220,13 +220,20 @@ class TestSortColumns extends QueryTest with BeforeAndAfterAll {
       sql("CREATE TABLE unsortedtable_offheap_inmemory (empno int, empname String, designation
String, doj Timestamp, workgroupcategory int, workgroupcategoryname String, deptno int, deptname
String, projectcode int, projectjoindate Timestamp, projectenddate Timestamp,attendance int,utilization
int,salary int) STORED BY 'org.apache.carbondata.format' tblproperties('sort_columns'='')")
       sql(s"""LOAD DATA local inpath '$resourcesPath/data.csv' INTO TABLE unsortedtable_offheap_inmemory
OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')""")
       checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno = 11"), sql("select
* from origintable1 where empno = 11"))
+      checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno < 15 order
by empno"), sql("select * from origintable1 where empno < 15 order by empno"))
+      checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno <= 15
order by empno"), sql("select * from origintable1 where empno <= 15 order by empno"))
+      checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno > 15 order
by empno"), sql("select * from origintable1 where empno > 15 order by empno"))
+      checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno >= 15
order by empno"), sql("select * from origintable1 where empno >= 15 order by empno"))
+      checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno <>
15 order by empno"), sql("select * from origintable1 where empno <> 15 order by empno"))
+      checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno in (15, 16,
17) order by empno"), sql("select * from origintable1 where empno in (15, 16, 17) order by
empno"))
+      checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno is null"),
sql("select * from origintable1 where empno is null order by empno"))
+      checkAnswer(sql("select * from unsortedtable_offheap_inmemory where empno is not null"),
sql("select * from origintable1 where empno is not null order by empno"))
       checkAnswer(sql("select * from unsortedtable_offheap_inmemory order by empno"), sql("select
* from origintable1 order by empno"))
     } finally {
       defaultLoadingProperties
     }
   }
-
-
+  
   override def afterAll = {
     dropTable
   }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/14c0a8b4/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
index 929a09e..c50f335 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
@@ -156,9 +156,7 @@ public class CarbonRowDataWriterProcessorStepImpl extends AbstractDataLoadProces
   private void doExecute(Iterator<CarbonRowBatch> iterator, int partitionId, int iteratorIndex)
{
     String storeLocation = getStoreLocation(tableIdentifier, String.valueOf(partitionId));
     CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel
-        .createCarbonFactDataHandlerModel(configuration, storeLocation, partitionId, 0);
-    model.getCarbonDataFileAttributes()
-        .setFactTimeStamp(model.getCarbonDataFileAttributes().getFactTimeStamp() + iteratorIndex);
+        .createCarbonFactDataHandlerModel(configuration, storeLocation, partitionId, iteratorIndex);
     CarbonFactHandler dataHandler = null;
     boolean rowsNotExist = true;
     while (iterator.hasNext()) {


Mime
View raw message