carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vimal...@apache.org
Subject [1/2] incubator-carbondata git commit: Fix the bug that negtive data compress is not properly when datatype is Double
Date Wed, 31 Aug 2016 07:28:40 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 0d7fa0383 -> 3eec687fe


Fix the bug that negtive data compress is not properly when datatype is Double


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/3b1c8345
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/3b1c8345
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/3b1c8345

Branch: refs/heads/master
Commit: 3b1c8345ef2fd5d2ae3bcabcca64b3dbfacb1ccc
Parents: 0d7fa03
Author: Zhangshunyu <zhangshunyu@huawei.com>
Authored: Wed Aug 31 09:34:12 2016 +0800
Committer: Vimal Das Kammath <vkammath@VIMALDK-M01.vmware.com>
Committed: Wed Aug 31 12:50:47 2016 +0530

----------------------------------------------------------------------
 .../core/util/ValueCompressionUtil.java         | 26 ++++++-----
 .../ValueCompressionDataTypeTestCase.scala      | 48 ++++++++++++++++++++
 2 files changed, 63 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3b1c8345/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
b/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
index 33c7a72..d3448f1 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
@@ -78,18 +78,18 @@ public final class ValueCompressionUtil {
   private static DataType getDataType(double value, int decimal, byte dataTypeSelected) {
     DataType dataType = DataType.DATA_DOUBLE;
     if (decimal == 0) {
-      if (value < Byte.MAX_VALUE) {
+      if (value <= Byte.MAX_VALUE && value >= Byte.MIN_VALUE) {
         dataType = DataType.DATA_BYTE;
-      } else if (value < Short.MAX_VALUE) {
+      } else if (value <= Short.MAX_VALUE && value >= Short.MIN_VALUE) {
         dataType = DataType.DATA_SHORT;
-      } else if (value < Integer.MAX_VALUE) {
+      } else if (value <= Integer.MAX_VALUE && value >= Integer.MIN_VALUE)
{
         dataType = DataType.DATA_INT;
-      } else if (value < Long.MAX_VALUE) {
+      } else if (value <= Long.MAX_VALUE && value >= Long.MIN_VALUE) {
         dataType = DataType.DATA_LONG;
       }
     } else {
       if (dataTypeSelected == 1) {
-        if (value < Float.MAX_VALUE) {
+        if (value <= Float.MAX_VALUE && value >= Float.MIN_VALUE) {
           float floatValue = (float) value;
           if (floatValue - value != 0) {
             dataType = DataType.DATA_DOUBLE;
@@ -97,7 +97,7 @@ public final class ValueCompressionUtil {
           } else {
             dataType = DataType.DATA_FLOAT;
           }
-        } else if (value < Double.MAX_VALUE) {
+        } else if (value <= Double.MAX_VALUE && value >= Double.MIN_VALUE)
{
           dataType = DataType.DATA_DOUBLE;
         }
       }
@@ -154,28 +154,32 @@ public final class ValueCompressionUtil {
       default:
         break;
     }
+    //Here we should use the Max abs as max to getDatatype, let's say -1 and -10000000, -1
is max,
+    //but we can't use -1 to getDatatype, we should use -10000000.
+    double absMaxValue = Math.abs((double) maxValue) >= Math.abs((double) minValue) ?
+        (double) maxValue:(double) minValue;
     // None Decimal
     if (decimal == 0) {
-      if (getSize(getDataType((double) maxValue, decimal, dataTypeSelected)) > getSize(
+      if (getSize(getDataType(absMaxValue, decimal, dataTypeSelected)) > getSize(
           getDataType((double) maxValue - (double) minValue, decimal, dataTypeSelected)))
{
         return new CompressionFinder(COMPRESSION_TYPE.MAX_MIN, DataType.DATA_DOUBLE,
             getDataType((double) maxValue - (double) minValue, decimal, dataTypeSelected));
-      } else if (getSize(getDataType((double) maxValue, decimal, dataTypeSelected)) <
getSize(
+      } else if (getSize(getDataType(absMaxValue, decimal, dataTypeSelected)) < getSize(
               getDataType((double) maxValue - (double) minValue, decimal, dataTypeSelected)))
{
         return new CompressionFinder(COMPRESSION_TYPE.NONE, DataType.DATA_DOUBLE,
                 getDataType((double) maxValue - (double) minValue, decimal, dataTypeSelected));
       } else {
         return new CompressionFinder(COMPRESSION_TYPE.NONE, DataType.DATA_DOUBLE,
-            getDataType((double) maxValue, decimal, dataTypeSelected));
+            getDataType(absMaxValue, decimal, dataTypeSelected));
       }
     }
     // decimal
     else {
-      DataType actualDataType = getDataType((double) maxValue, decimal, dataTypeSelected);
+      DataType actualDataType = getDataType(absMaxValue, decimal, dataTypeSelected);
       DataType diffDataType =
           getDataType((double) maxValue - (double) minValue, decimal, dataTypeSelected);
       DataType maxNonDecDataType =
-          getDataType(Math.pow(10, decimal) * (double) maxValue, 0, dataTypeSelected);
+          getDataType(Math.pow(10, decimal) * absMaxValue, 0, dataTypeSelected);
       DataType diffNonDecDataType =
           getDataType(Math.pow(10, decimal) * ((double) maxValue - (double) minValue), 0,
               dataTypeSelected);

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3b1c8345/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ValueCompressionDataTypeTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ValueCompressionDataTypeTestCase.scala
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ValueCompressionDataTypeTestCase.scala
index 5375167..93f96d8 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ValueCompressionDataTypeTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ValueCompressionDataTypeTestCase.scala
@@ -81,6 +81,54 @@ class ValueCompressionDataTypeTestCase extends QueryTest with BeforeAndAfterAll
     }
   }
 
+  test("When the values of Double datatype are negative values") {
+    val tempFilePath = "./src/test/resources/temp/doubleISnegtive.csv"
+    try {
+      sql("drop table if exists doubleISnegtive")
+      sql("drop table if exists doubleISnegtive_hive")
+      sql("CREATE TABLE doubleISnegtive (name String, value double) STORED BY 'org.apache.carbondata.format'")
+      sql("CREATE TABLE doubleISnegtive_hive (name String, value double)row format delimited
fields terminated by ','")
+      val data ="a,-7489.7976000000\nb,-11234567489.797\nc,-11234567489.7\nd,-1.2\ne,-2\nf,-11234567489.7976000000\ng,-11234567489.7976000000"
+      writedata(tempFilePath, data)
+      sql(s"LOAD data local inpath '${tempFilePath}' into table doubleISnegtive options('fileheader'='name,value')")
+      sql(s"LOAD data local inpath '${tempFilePath}' into table doubleISnegtive_hive")
+
+      checkAnswer(sql("select * from doubleISnegtive"),
+        sql("select * from doubleISnegtive_hive"))
+    } catch{
+      case ex:Exception => ex.printStackTrace()
+        assert(false)
+    } finally {
+      sql("drop table if exists doubleISnegtive")
+      sql("drop table if exists doubleISnegtive_hive")
+      deleteFile(tempFilePath)
+    }
+  }
+
+  test("When the values of Double datatype have both postive and negative values") {
+    val tempFilePath = "./src/test/resources/temp/doublePAN.csv"
+    try {
+      sql("drop table if exists doublePAN")
+      sql("drop table if exists doublePAN_hive")
+      sql("CREATE TABLE doublePAN (name String, value double) STORED BY 'org.apache.carbondata.format'")
+      sql("CREATE TABLE doublePAN_hive (name String, value double)row format delimited fields
terminated by ','")
+      val data ="a,-7489.7976000000\nb,11234567489.797\nc,-11234567489.7\nd,-1.2\ne,2\nf,-11234567489.7976000000\ng,11234567489.7976000000"
+      writedata(tempFilePath, data)
+      sql(s"LOAD data local inpath '${tempFilePath}' into table doublePAN options('fileheader'='name,value')")
+      sql(s"LOAD data local inpath '${tempFilePath}' into table doublePAN_hive")
+
+      checkAnswer(sql("select * from doublePAN"),
+        sql("select * from doublePAN_hive"))
+    } catch{
+      case ex:Exception => ex.printStackTrace()
+        assert(false)
+    } finally {
+      sql("drop table if exists doublePAN")
+      sql("drop table if exists doublePAN_hive")
+      deleteFile(tempFilePath)
+    }
+  }
+
   def writedata(filePath: String, data: String) = {
     val dis = FileFactory.getDataOutputStream(filePath, FileFactory.getFileType(filePath))
     dis.writeBytes(data.toString())


Mime
View raw message