carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [1/2] incubator-carbondata git commit: Fix load fail with unsafe enabled and with bigdecimal datatypes
Date Tue, 21 Feb 2017 01:18:20 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master a6c8d2a79 -> 716517619


Fix load fail with unsafe enabled and with bigdecimal datatypes


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/aca59cea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/aca59cea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/aca59cea

Branch: refs/heads/master
Commit: aca59cea38df7a5509d1f4c96817c2ef7cd7dd9a
Parents: a6c8d2a
Author: ravipesala <ravi.pesala@gmail.com>
Authored: Fri Feb 3 19:35:13 2017 +0530
Committer: jackylk <jacky.likun@huawei.com>
Committed: Tue Feb 21 09:06:12 2017 +0800

----------------------------------------------------------------------
 .../bigdecimal/TestDimensionWithDecimalDataType.scala | 14 ++++++++++----
 .../newflow/sort/unsafe/UnsafeCarbonRowPage.java      |  3 +--
 .../unsafe/holder/UnsafeSortTempFileChunkHolder.java  |  3 +--
 .../unsafe/merger/UnsafeIntermediateFileMerger.java   |  5 +----
 .../store/CarbonFactDataHandlerColumnar.java          |  1 +
 5 files changed, 14 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/aca59cea/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/bigdecimal/TestDimensionWithDecimalDataType.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/bigdecimal/TestDimensionWithDecimalDataType.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/bigdecimal/TestDimensionWithDecimalDataType.scala
index 743b39b..e267763 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/bigdecimal/TestDimensionWithDecimalDataType.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/bigdecimal/TestDimensionWithDecimalDataType.scala
@@ -39,10 +39,16 @@ class TestDimensionWithDecimalDataType extends QueryTest with BeforeAndAfterAll
     sql(s"LOAD DATA local inpath '$resourcesPath/decimalDataWithoutHeader.csv' INTO table
hiveTable")
   }
 
-//  test("test detail query on dimension column with decimal data type") {
-//    checkAnswer(sql("select salary from carbonTable order by salary"),
-//      sql("select salary from hiveTable order by salary"))
-//  }
+  test("test unsafe with bigdecimal") {
+    sql("drop table if exists unsafecarbonTable")
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT, "true")
+    sql("CREATE TABLE IF NOT EXISTS unsafecarbonTable (ID Int, date Timestamp, country String,
name String, phonetype String, serialname String, salary Decimal(19,2))STORED BY 'org.apache.carbondata.format'")
+    sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/decimalDataWithHeader.csv' into table unsafecarbonTable")
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.ENABLE_UNSAFE_SORT, "false")
+    sql("drop table if exists unsafecarbonTable")
+  }
 
   test("test aggregate query on dimension column with decimal data type") {
     checkAnswer(sql("select sum(salary) from carbonTable"),

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/aca59cea/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java
b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java
index 561ad38..60af5d8 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeCarbonRowPage.java
@@ -196,9 +196,8 @@ public class UnsafeCarbonRowPage {
           size += 2;
           CarbonUnsafe.unsafe.copyMemory(baseObject, address + size, bigDecimalInBytes,
               CarbonUnsafe.BYTE_ARRAY_OFFSET, bigDecimalInBytes.length);
-          BigDecimal val = DataTypeUtil.byteToBigDecimal(bigDecimalInBytes);
           size += bigDecimalInBytes.length;
-          rowToFill[dimensionSize + mesCount] = val;
+          rowToFill[dimensionSize + mesCount] = bigDecimalInBytes;
         }
       } else {
         rowToFill[dimensionSize + mesCount] = null;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/aca59cea/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
index 0c89821..60f259e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
@@ -34,7 +34,6 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.util.CarbonProperties;
 import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.DataTypeUtil;
 import org.apache.carbondata.processing.newflow.sort.unsafe.UnsafeCarbonRowPage;
 import org.apache.carbondata.processing.sortandgroupby.exception.CarbonSortKeyAndGroupByException;
 import org.apache.carbondata.processing.sortandgroupby.sortdata.NewRowComparator;
@@ -333,7 +332,7 @@ public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder
{
             short aShort = stream.readShort();
             byte[] bigDecimalInBytes = new byte[aShort];
             stream.readFully(bigDecimalInBytes);
-            row[dimensionCount + mesCount] = DataTypeUtil.byteToBigDecimal(bigDecimalInBytes);
+            row[dimensionCount + mesCount] = bigDecimalInBytes;
           }
         }
       }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/aca59cea/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateFileMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateFileMerger.java
b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateFileMerger.java
index c6b5e9a..ed8023b 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateFileMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateFileMerger.java
@@ -23,7 +23,6 @@ import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
-import java.math.BigDecimal;
 import java.nio.ByteBuffer;
 import java.util.AbstractQueue;
 import java.util.Arrays;
@@ -34,7 +33,6 @@ import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.DataTypeUtil;
 import org.apache.carbondata.processing.newflow.sort.unsafe.UnsafeCarbonRowPage;
 import org.apache.carbondata.processing.newflow.sort.unsafe.holder.SortTempChunkHolder;
 import org.apache.carbondata.processing.newflow.sort.unsafe.holder.UnsafeSortTempFileChunkHolder;
@@ -320,8 +318,7 @@ public class UnsafeIntermediateFileMerger implements Callable<Void>
{
           rowData.putLong(size, val);
           size += 8;
         } else if (aggType[mesCount] == CarbonCommonConstants.BIG_DECIMAL_MEASURE) {
-          BigDecimal val = (BigDecimal) value;
-          byte[] bigDecimalInBytes = DataTypeUtil.bigDecimalToByte(val);
+          byte[] bigDecimalInBytes = (byte[]) value;
           rowData.putShort(size, (short)bigDecimalInBytes.length);
           size += 2;
           for (int i = 0; i < bigDecimalInBytes.length; i++) {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/aca59cea/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
index 7c48cec..bf66700 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
@@ -1540,6 +1540,7 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler
{
         blockletDataHolder.put(nodeHolder, indexInNodeHolderArray);
         return null;
       } catch (Throwable throwable) {
+        LOGGER.error(throwable, "Error in producer");
         consumerExecutorService.shutdownNow();
         resetBlockletProcessingCount();
         throw new CarbonDataWriterException(throwable.getMessage(), throwable);


Mime
View raw message