carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject carbondata git commit: [CARBONDATA-2683][32K] fix data convertion problem for Varchar
Date Fri, 06 Jul 2018 06:43:33 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master 5159abfc3 -> 3e88858a2


[CARBONDATA-2683][32K] fix data convertion problem for Varchar

Spark uses org.apache.spark.unsafe.types.UTF8String for string datatype internally.
In carbon, varchar datatype should do the same convertion as string datatype. Or it may throw
exception

This closes #2438


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/3e88858a
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/3e88858a
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/3e88858a

Branch: refs/heads/master
Commit: 3e88858a2145102b78f44d7a744ec4a34823f29e
Parents: 5159abf
Author: Manhua <kevinjmh@qq.com>
Authored: Tue Jul 3 16:07:10 2018 +0800
Committer: Jacky Li <jacky.likun@qq.com>
Committed: Fri Jul 6 14:43:18 2018 +0800

----------------------------------------------------------------------
 .../carbondata/core/util/DataTypeUtil.java      | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/3e88858a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index ff41b9b..0306c02 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -323,8 +323,6 @@ public final class DataTypeUtil {
       DataType actualDataType, String dateFormat) {
     if (actualDataType == DataTypes.BOOLEAN) {
       return ByteUtil.toBytes(BooleanConvert.parseBoolean(dimensionValue));
-    } else if (actualDataType == DataTypes.STRING) {
-      return ByteUtil.toBytes(dimensionValue);
     } else if (actualDataType == DataTypes.SHORT) {
       return ByteUtil.toBytes(Short.parseShort(dimensionValue));
     } else if (actualDataType == DataTypes.INT) {
@@ -351,6 +349,7 @@ public final class DataTypeUtil {
         throw new NumberFormatException(e.getMessage());
       }
     } else {
+      // Default action for String/Varchar
       return ByteUtil.toBytes(dimensionValue);
     }
   }
@@ -359,8 +358,6 @@ public final class DataTypeUtil {
       DataType actualDataType, String dateFormat) {
     if (actualDataType == DataTypes.BOOLEAN) {
       return BooleanConvert.parseBoolean(dimensionValue);
-    } else if (actualDataType == DataTypes.STRING) {
-      return converter.convertFromStringToUTF8String(dimensionValue);
     } else if (actualDataType == DataTypes.SHORT) {
       return Short.parseShort(dimensionValue);
     } else if (actualDataType == DataTypes.INT) {
@@ -387,6 +384,7 @@ public final class DataTypeUtil {
         throw new NumberFormatException(e.getMessage());
       }
     } else {
+      // Default action for String/Varchar
       return converter.convertFromStringToUTF8String(dimensionValue);
     }
   }
@@ -402,8 +400,6 @@ public final class DataTypeUtil {
     }
     if (actualDataType == DataTypes.BOOLEAN) {
       return ByteUtil.toBytes((Boolean) dimensionValue);
-    } else if (actualDataType == DataTypes.STRING) {
-      return ByteUtil.toBytes(dimensionValue.toString());
     } else if (actualDataType == DataTypes.SHORT) {
       return ByteUtil.toBytes((Short) dimensionValue);
     } else if (actualDataType == DataTypes.INT) {
@@ -413,6 +409,7 @@ public final class DataTypeUtil {
     } else if (actualDataType == DataTypes.TIMESTAMP) {
       return ByteUtil.toBytes((Long)dimensionValue);
     } else {
+      // Default action for String/Varchar
       return ByteUtil.toBytes(dimensionValue.toString());
     }
   }
@@ -423,7 +420,9 @@ public final class DataTypeUtil {
    * @return
    */
   public static boolean isFixedSizeDataType(DataType dataType) {
-    if (dataType == DataTypes.STRING || DataTypes.isDecimal(dataType)) {
+    if (dataType == DataTypes.STRING ||
+        dataType == DataTypes.VARCHAR ||
+        DataTypes.isDecimal(dataType)) {
       return false;
     } else {
       return true;
@@ -447,8 +446,6 @@ public final class DataTypeUtil {
     try {
       if (actualDataType == DataTypes.BOOLEAN) {
         return ByteUtil.toBoolean(dataInBytes);
-      } else if (actualDataType == DataTypes.STRING) {
-        return getDataTypeConverter().convertFromByteToUTF8String(dataInBytes);
       } else if (actualDataType == DataTypes.SHORT) {
         // for non string type no dictionary column empty byte array is empty value
         // so no need to parse
@@ -480,9 +477,10 @@ public final class DataTypeUtil {
         if (isEmptyByteArray(dataInBytes)) {
           return null;
         }
-        return converter.convertFromBigDecimalToDecimal(byteToBigDecimal(dataInBytes));
+        return getDataTypeConverter().convertFromBigDecimalToDecimal(byteToBigDecimal(dataInBytes));
       } else {
-        return ByteUtil.toString(dataInBytes, 0, dataInBytes.length);
+        // Default action for String/Varchar
+        return getDataTypeConverter().convertFromByteToUTF8String(dataInBytes);
       }
     } catch (Throwable ex) {
       String data = new String(dataInBytes, CarbonCommonConstants.DEFAULT_CHARSET_CLASS);


Mime
View raw message