carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chenliang...@apache.org
Subject [1/2] incubator-carbondata git commit: Supported tinyint datatype in carbondata
Date Tue, 28 Jun 2016 21:23:45 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 02fcb1969 -> 634bdeefc


Supported tinyint datatype in carbondata


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/a7437571
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/a7437571
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/a7437571

Branch: refs/heads/master
Commit: a74375719a5c09249bb742bf7faef33b02bcf252
Parents: 63d3284
Author: ravipesala <ravi.pesala@gmail.com>
Authored: Tue Jun 28 21:53:58 2016 +0530
Committer: ravipesala <ravi.pesala@gmail.com>
Committed: Tue Jun 28 21:53:58 2016 +0530

----------------------------------------------------------------------
 .../cache/dictionary/ColumnDictionaryInfo.java  |  2 +
 .../ThriftWrapperSchemaConverterImpl.java       |  4 ++
 .../core/carbon/metadata/datatype/DataType.java |  1 +
 .../core/constants/CarbonCommonConstants.java   |  1 +
 .../core/util/DataFileFooterConverter.java      |  2 +
 .../org/carbondata/core/util/DataTypeUtil.java  |  4 ++
 .../sortindex/CarbonDictionarySortModel.java    |  1 +
 .../carbondata/scan/expression/DataType.java    |  3 +-
 .../scan/expression/ExpressionResult.java       | 59 +++++++++++++++++++-
 .../expression/arithmetic/AddExpression.java    |  3 +
 .../expression/arithmetic/DivideExpression.java |  3 +
 .../arithmetic/MultiplyExpression.java          |  3 +
 .../arithmetic/SubstractExpression.java         |  3 +
 .../conditional/EqualToExpression.java          |  3 +
 .../GreaterThanEqualToExpression.java           |  3 +
 .../conditional/GreaterThanExpression.java      |  3 +
 .../expression/conditional/InExpression.java    |  3 +
 .../conditional/LessThanEqualToExpression.java  |  3 +
 .../conditional/LessThanExpression.java         |  3 +
 .../conditional/NotEqualsExpression.java        |  3 +
 .../expression/conditional/NotInExpression.java |  3 +
 .../org/carbondata/scan/filter/FilterUtil.java  |  4 +-
 .../org/carbondata/scan/util/DataTypeUtil.java  | 54 +-----------------
 format/src/main/thrift/schema.thrift            | 11 ++--
 .../spark/sql/CarbonDatasourceRelation.scala    |  1 +
 .../spark/sql/CarbonDictionaryDecoder.scala     |  1 +
 .../execution/command/carbonTableSchema.scala   |  2 +
 .../spark/sql/hive/CarbonMetastoreCatalog.scala |  1 +
 .../scala/org/carbondata/spark/package.scala    |  2 +-
 .../carbondata/spark/util/CarbonScalaUtil.scala | 46 +--------------
 30 files changed, 130 insertions(+), 105 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java
b/core/src/main/java/org/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java
index c9dff58..5c39eaa 100644
--- a/core/src/main/java/org/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java
+++ b/core/src/main/java/org/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java
@@ -223,6 +223,8 @@ public class ColumnDictionaryInfo extends AbstractColumnDictionaryInfo
{
       DataType dataType) {
     try {
       switch (dataType) {
+        case SHORT:
+          return Short.compare((Short.parseShort(dictionaryVal)), (Short.parseShort(memberVal)));
         case INT:
           return Integer.compare((Integer.parseInt(dictionaryVal)), (Integer.parseInt(memberVal)));
         case DOUBLE:

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java
b/core/src/main/java/org/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java
index 249d2a5..51b269a 100644
--- a/core/src/main/java/org/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java
+++ b/core/src/main/java/org/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java
@@ -123,6 +123,8 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter
{
         return org.carbondata.format.DataType.STRING;
       case INT:
         return org.carbondata.format.DataType.INT;
+      case SHORT:
+        return org.carbondata.format.DataType.SHORT;
       case LONG:
         return org.carbondata.format.DataType.LONG;
       case DOUBLE:
@@ -285,6 +287,8 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter
{
         return DataType.STRING;
       case INT:
         return DataType.INT;
+      case SHORT:
+        return DataType.SHORT;
       case LONG:
         return DataType.LONG;
       case DOUBLE:

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/core/carbon/metadata/datatype/DataType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/metadata/datatype/DataType.java
b/core/src/main/java/org/carbondata/core/carbon/metadata/datatype/DataType.java
index 7586022..9f2be8a 100644
--- a/core/src/main/java/org/carbondata/core/carbon/metadata/datatype/DataType.java
+++ b/core/src/main/java/org/carbondata/core/carbon/metadata/datatype/DataType.java
@@ -22,6 +22,7 @@ public enum DataType {
 
   STRING,
   INT,
+  SHORT,
   LONG,
   DOUBLE,
   BOOLEAN,

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
index 13fce9f..96b0b39 100644
--- a/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
@@ -707,6 +707,7 @@ public final class CarbonCommonConstants {
   public static final String COLUMNAR = "columnar";
 
   public static final String INTEGER = "Integer";
+  public static final String SHORT = "Short";
   public static final String NUMERIC = "Numeric";
   public static final String TIMESTAMP = "Timestamp";
   public static final String ARRAY = "ARRAY";

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/core/util/DataFileFooterConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/util/DataFileFooterConverter.java b/core/src/main/java/org/carbondata/core/util/DataFileFooterConverter.java
index 6165731..6641c18 100644
--- a/core/src/main/java/org/carbondata/core/util/DataFileFooterConverter.java
+++ b/core/src/main/java/org/carbondata/core/util/DataFileFooterConverter.java
@@ -301,6 +301,8 @@ class DataFileFooterConverter {
     switch (dataTypeThrift) {
       case STRING:
         return DataType.STRING;
+      case SHORT:
+        return DataType.SHORT;
       case INT:
         return DataType.INT;
       case LONG:

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/carbondata/core/util/DataTypeUtil.java
index d3cd120..68bca6c 100644
--- a/core/src/main/java/org/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/carbondata/core/util/DataTypeUtil.java
@@ -120,6 +120,9 @@ public final class DataTypeUtil {
       case "INT":
         dataType = DataType.INT;
         break;
+      case "SHORT":
+        dataType = DataType.SHORT;
+        break;
       case "LONG":
         dataType = DataType.LONG;
         break;
@@ -158,6 +161,7 @@ public final class DataTypeUtil {
     try {
       switch (actualDataType) {
         case INT:
+        case SHORT:
         case LONG:
         case DOUBLE:
         case DECIMAL:

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/core/writer/sortindex/CarbonDictionarySortModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/writer/sortindex/CarbonDictionarySortModel.java
b/core/src/main/java/org/carbondata/core/writer/sortindex/CarbonDictionarySortModel.java
index e518107..8e66cc7 100644
--- a/core/src/main/java/org/carbondata/core/writer/sortindex/CarbonDictionarySortModel.java
+++ b/core/src/main/java/org/carbondata/core/writer/sortindex/CarbonDictionarySortModel.java
@@ -64,6 +64,7 @@ public class CarbonDictionarySortModel implements Comparable<CarbonDictionarySor
    */
   @Override public int compareTo(CarbonDictionarySortModel o) {
     switch (dataType) {
+      case SHORT:
       case INT:
       case LONG:
       case DOUBLE:

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/scan/expression/DataType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/expression/DataType.java b/core/src/main/java/org/carbondata/scan/expression/DataType.java
index fc9d0bf..062fad1 100644
--- a/core/src/main/java/org/carbondata/scan/expression/DataType.java
+++ b/core/src/main/java/org/carbondata/scan/expression/DataType.java
@@ -21,7 +21,8 @@ package org.carbondata.scan.expression;
 
 public enum DataType {
   StringType(0), DateType(1), TimestampType(2), BooleanType(1), IntegerType(3), FloatType(
-      4), LongType(5), DoubleType(6), NullType(7), DecimalType(8), ArrayType(9), StructType(10);
+      4), LongType(5), DoubleType(6), NullType(7), DecimalType(8), ArrayType(9), StructType(
+      10), ShortType(11);
   private int presedenceOrder;
 
   private DataType(int value) {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/scan/expression/ExpressionResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/expression/ExpressionResult.java b/core/src/main/java/org/carbondata/scan/expression/ExpressionResult.java
index decbdc0..a86052c 100644
--- a/core/src/main/java/org/carbondata/scan/expression/ExpressionResult.java
+++ b/core/src/main/java/org/carbondata/scan/expression/ExpressionResult.java
@@ -73,6 +73,8 @@ public class ExpressionResult implements Comparable<ExpressionResult>
{
             throw new FilterUnsupportedException(e);
           }
 
+        case ShortType:
+          return ((Short) value).intValue();
         case IntegerType:
         case DoubleType:
 
@@ -100,6 +102,48 @@ public class ExpressionResult implements Comparable<ExpressionResult>
{
     }
   }
 
+  public Short getShort() throws FilterUnsupportedException {
+    if (value == null) {
+      return null;
+    }
+    try {
+      switch (this.getDataType()) {
+        case StringType:
+          try {
+            return Short.parseShort(value.toString());
+          } catch (NumberFormatException e) {
+            throw new FilterUnsupportedException(e);
+          }
+        case ShortType:
+        case IntegerType:
+        case DoubleType:
+
+          if (value instanceof Double) {
+            return ((Double) value).shortValue();
+          } else if (value instanceof Integer) {
+            return ((Integer) value).shortValue();
+          }
+          return (Short) value;
+
+        case TimestampType:
+
+          if (value instanceof Timestamp) {
+            return (short) (((Timestamp) value).getTime() % 1000);
+          } else {
+            return (Short) value;
+          }
+
+        default:
+          throw new FilterUnsupportedException(
+              "Cannot convert" + this.getDataType().name() + " to integer type value");
+      }
+
+    } catch (ClassCastException e) {
+      throw new FilterUnsupportedException(
+          "Cannot convert" + this.getDataType().name() + " to Integer type value");
+    }
+  }
+
   public String getString() {
     if (value == null) {
       return null;
@@ -132,7 +176,8 @@ public class ExpressionResult implements Comparable<ExpressionResult>
{
           } catch (NumberFormatException e) {
             throw new FilterUnsupportedException(e);
           }
-
+        case ShortType:
+          return ((Short) value).doubleValue();
         case IntegerType:
           return ((Integer) value).doubleValue();
         case LongType:
@@ -168,7 +213,8 @@ public class ExpressionResult implements Comparable<ExpressionResult>
{
           } catch (NumberFormatException e) {
             throw new FilterUnsupportedException(e);
           }
-
+        case ShortType:
+          return ((Short) value).longValue();
         case IntegerType:
           return (Long) value;
         case LongType:
@@ -205,7 +251,8 @@ public class ExpressionResult implements Comparable<ExpressionResult>
{
           } catch (NumberFormatException e) {
             throw new FilterUnsupportedException(e);
           }
-
+        case ShortType:
+          return new BigDecimal((short) value);
         case IntegerType:
           return new BigDecimal((int) value);
         case LongType:
@@ -249,6 +296,8 @@ public class ExpressionResult implements Comparable<ExpressionResult>
{
             throw new FilterUnsupportedException(
                 "Cannot convert" + this.getDataType().name() + " to Time/Long type value");
           }
+        case ShortType:
+          return ((Short) value).longValue();
         case IntegerType:
         case LongType:
           return (Long) value;
@@ -354,6 +403,9 @@ public class ExpressionResult implements Comparable<ExpressionResult>
{
         case StringType:
           result = this.getString().equals(objToCompare.getString());
           break;
+        case ShortType:
+          result = this.getShort().equals(objToCompare.getShort());
+          break;
         case IntegerType:
           result = this.getInt().equals(objToCompare.getInt());
           break;
@@ -381,6 +433,7 @@ public class ExpressionResult implements Comparable<ExpressionResult>
{
   @Override public int compareTo(ExpressionResult o) {
     try {
       switch (o.dataType) {
+        case ShortType:
         case IntegerType:
         case LongType:
         case DoubleType:

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/scan/expression/arithmetic/AddExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/expression/arithmetic/AddExpression.java
b/core/src/main/java/org/carbondata/scan/expression/arithmetic/AddExpression.java
index e7ce159..62bab78 100644
--- a/core/src/main/java/org/carbondata/scan/expression/arithmetic/AddExpression.java
+++ b/core/src/main/java/org/carbondata/scan/expression/arithmetic/AddExpression.java
@@ -55,6 +55,9 @@ public class AddExpression extends BinaryArithmeticExpression {
       case DoubleType:
         addExprRightRes.set(DataType.DoubleType, val1.getDouble() + val2.getDouble());
         break;
+      case ShortType:
+        addExprRightRes.set(DataType.ShortType, val1.getShort() + val2.getShort());
+        break;
       case IntegerType:
         addExprRightRes.set(DataType.IntegerType, val1.getInt() + val2.getInt());
         break;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/scan/expression/arithmetic/DivideExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/expression/arithmetic/DivideExpression.java
b/core/src/main/java/org/carbondata/scan/expression/arithmetic/DivideExpression.java
index f4df604..d98b301 100644
--- a/core/src/main/java/org/carbondata/scan/expression/arithmetic/DivideExpression.java
+++ b/core/src/main/java/org/carbondata/scan/expression/arithmetic/DivideExpression.java
@@ -54,6 +54,9 @@ public class DivideExpression extends BinaryArithmeticExpression {
       case DoubleType:
         divideExprRightRes.set(DataType.DoubleType, val1.getDouble() / val2.getDouble());
         break;
+      case ShortType:
+        divideExprRightRes.set(DataType.ShortType, val1.getShort() / val2.getShort());
+        break;
       case IntegerType:
         divideExprRightRes.set(DataType.IntegerType, val1.getInt() / val2.getInt());
         break;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/scan/expression/arithmetic/MultiplyExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/expression/arithmetic/MultiplyExpression.java
b/core/src/main/java/org/carbondata/scan/expression/arithmetic/MultiplyExpression.java
index 3347250..7f8d8aa 100644
--- a/core/src/main/java/org/carbondata/scan/expression/arithmetic/MultiplyExpression.java
+++ b/core/src/main/java/org/carbondata/scan/expression/arithmetic/MultiplyExpression.java
@@ -55,6 +55,9 @@ public class MultiplyExpression extends BinaryArithmeticExpression {
       case DoubleType:
         multiplyExprRightRes.set(DataType.DoubleType, val1.getDouble() * val2.getDouble());
         break;
+      case ShortType:
+        multiplyExprRightRes.set(DataType.ShortType, val1.getShort() * val2.getShort());
+        break;
       case IntegerType:
         multiplyExprRightRes.set(DataType.IntegerType, val1.getInt() * val2.getInt());
         break;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/scan/expression/arithmetic/SubstractExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/expression/arithmetic/SubstractExpression.java
b/core/src/main/java/org/carbondata/scan/expression/arithmetic/SubstractExpression.java
index 35fec1c..0e7ab2e 100644
--- a/core/src/main/java/org/carbondata/scan/expression/arithmetic/SubstractExpression.java
+++ b/core/src/main/java/org/carbondata/scan/expression/arithmetic/SubstractExpression.java
@@ -55,6 +55,9 @@ public class SubstractExpression extends BinaryArithmeticExpression {
       case DoubleType:
         subtractExprRightRes.set(DataType.DoubleType, val1.getDouble() - val2.getDouble());
         break;
+      case ShortType:
+        subtractExprRightRes.set(DataType.ShortType, val1.getShort() - val2.getShort());
+        break;
       case IntegerType:
         subtractExprRightRes.set(DataType.IntegerType, val1.getInt() - val2.getInt());
         break;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/scan/expression/conditional/EqualToExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/expression/conditional/EqualToExpression.java
b/core/src/main/java/org/carbondata/scan/expression/conditional/EqualToExpression.java
index 7b2a32d..80ad32e 100644
--- a/core/src/main/java/org/carbondata/scan/expression/conditional/EqualToExpression.java
+++ b/core/src/main/java/org/carbondata/scan/expression/conditional/EqualToExpression.java
@@ -61,6 +61,9 @@ public class EqualToExpression extends BinaryConditionalExpression {
       case StringType:
         result = val1.getString().equals(val2.getString());
         break;
+      case ShortType:
+        result = val1.getShort().equals(val2.getShort());
+        break;
       case IntegerType:
         result = val1.getInt().equals(val2.getInt());
         break;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/scan/expression/conditional/GreaterThanEqualToExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/expression/conditional/GreaterThanEqualToExpression.java
b/core/src/main/java/org/carbondata/scan/expression/conditional/GreaterThanEqualToExpression.java
index bb00467..4da578b 100644
--- a/core/src/main/java/org/carbondata/scan/expression/conditional/GreaterThanEqualToExpression.java
+++ b/core/src/main/java/org/carbondata/scan/expression/conditional/GreaterThanEqualToExpression.java
@@ -52,6 +52,9 @@ public class GreaterThanEqualToExpression extends BinaryConditionalExpression
{
       case StringType:
         result = elRes.getString().compareTo(erRes.getString()) >= 0;
         break;
+      case ShortType:
+        result = elRes.getShort() >= (erRes.getShort());
+        break;
       case IntegerType:
         result = elRes.getInt() >= (erRes.getInt());
         break;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/scan/expression/conditional/GreaterThanExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/expression/conditional/GreaterThanExpression.java
b/core/src/main/java/org/carbondata/scan/expression/conditional/GreaterThanExpression.java
index f1bde04..5c3bd56 100644
--- a/core/src/main/java/org/carbondata/scan/expression/conditional/GreaterThanExpression.java
+++ b/core/src/main/java/org/carbondata/scan/expression/conditional/GreaterThanExpression.java
@@ -58,6 +58,9 @@ public class GreaterThanExpression extends BinaryConditionalExpression {
       case DoubleType:
         result = exprLeftRes.getDouble() > (exprRightRes.getDouble());
         break;
+      case ShortType:
+        result = exprLeftRes.getShort() > (exprRightRes.getShort());
+        break;
       case IntegerType:
         result = exprLeftRes.getInt() > (exprRightRes.getInt());
         break;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/scan/expression/conditional/InExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/expression/conditional/InExpression.java
b/core/src/main/java/org/carbondata/scan/expression/conditional/InExpression.java
index 6574d3e..ddb35af 100644
--- a/core/src/main/java/org/carbondata/scan/expression/conditional/InExpression.java
+++ b/core/src/main/java/org/carbondata/scan/expression/conditional/InExpression.java
@@ -59,6 +59,9 @@ public class InExpression extends BinaryConditionalExpression {
             case StringType:
               val = new ExpressionResult(val.getDataType(), expressionResVal.getString());
               break;
+            case ShortType:
+              val = new ExpressionResult(val.getDataType(), expressionResVal.getShort());
+              break;
             case IntegerType:
               val = new ExpressionResult(val.getDataType(), expressionResVal.getInt());
               break;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/scan/expression/conditional/LessThanEqualToExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/expression/conditional/LessThanEqualToExpression.java
b/core/src/main/java/org/carbondata/scan/expression/conditional/LessThanEqualToExpression.java
index f194157..c8aa6a7 100644
--- a/core/src/main/java/org/carbondata/scan/expression/conditional/LessThanEqualToExpression.java
+++ b/core/src/main/java/org/carbondata/scan/expression/conditional/LessThanEqualToExpression.java
@@ -53,6 +53,9 @@ public class LessThanEqualToExpression extends BinaryConditionalExpression
{
       case StringType:
         result = elRes.getString().compareTo(erRes.getString()) <= 0;
         break;
+      case ShortType:
+        result = elRes.getShort() <= (erRes.getShort());
+        break;
       case IntegerType:
         result = elRes.getInt() <= (erRes.getInt());
         break;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/scan/expression/conditional/LessThanExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/expression/conditional/LessThanExpression.java
b/core/src/main/java/org/carbondata/scan/expression/conditional/LessThanExpression.java
index b55d0cc..7e6580a 100644
--- a/core/src/main/java/org/carbondata/scan/expression/conditional/LessThanExpression.java
+++ b/core/src/main/java/org/carbondata/scan/expression/conditional/LessThanExpression.java
@@ -56,6 +56,9 @@ public class LessThanExpression extends BinaryConditionalExpression {
       case StringType:
         result = elRes.getString().compareTo(erRes.getString()) < 0;
         break;
+      case ShortType:
+        result = elRes.getShort() < (erRes.getShort());
+        break;
       case IntegerType:
         result = elRes.getInt() < (erRes.getInt());
         break;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/scan/expression/conditional/NotEqualsExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/expression/conditional/NotEqualsExpression.java
b/core/src/main/java/org/carbondata/scan/expression/conditional/NotEqualsExpression.java
index df40f7f..4f31c07 100644
--- a/core/src/main/java/org/carbondata/scan/expression/conditional/NotEqualsExpression.java
+++ b/core/src/main/java/org/carbondata/scan/expression/conditional/NotEqualsExpression.java
@@ -60,6 +60,9 @@ public class NotEqualsExpression extends BinaryConditionalExpression {
       case StringType:
         result = !val1.getString().equals(val2.getString());
         break;
+      case ShortType:
+        result = val1.getShort().shortValue() != val2.getShort().shortValue();
+        break;
       case IntegerType:
         result = val1.getInt().intValue() != val2.getInt().intValue();
         break;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/scan/expression/conditional/NotInExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/expression/conditional/NotInExpression.java
b/core/src/main/java/org/carbondata/scan/expression/conditional/NotInExpression.java
index 5777c4d..caa722a 100644
--- a/core/src/main/java/org/carbondata/scan/expression/conditional/NotInExpression.java
+++ b/core/src/main/java/org/carbondata/scan/expression/conditional/NotInExpression.java
@@ -59,6 +59,9 @@ public class NotInExpression extends BinaryConditionalExpression {
             case StringType:
               val = new ExpressionResult(val.getDataType(), exprResVal.getString());
               break;
+            case ShortType:
+              val = new ExpressionResult(val.getDataType(), exprResVal.getShort());
+              break;
             case IntegerType:
               val = new ExpressionResult(val.getDataType(), exprResVal.getInt());
               break;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/filter/FilterUtil.java b/core/src/main/java/org/carbondata/scan/filter/FilterUtil.java
index 802682b..2f394f7 100644
--- a/core/src/main/java/org/carbondata/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/carbondata/scan/filter/FilterUtil.java
@@ -1012,8 +1012,9 @@ public final class FilterUtil {
       DataType dataType) {
     try {
       switch (dataType) {
+        case SHORT:
+          return Short.compare((Short.parseShort(dictionaryVal)), (Short.parseShort(memberVal)));
         case INT:
-
           return Integer.compare((Integer.parseInt(dictionaryVal)), (Integer.parseInt(memberVal)));
         case DOUBLE:
           return Double
@@ -1095,6 +1096,7 @@ public final class FilterUtil {
       String filterMember2, org.carbondata.scan.expression.DataType dataType) {
     try {
       switch (dataType) {
+        case ShortType:
         case IntegerType:
         case LongType:
         case DoubleType:

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/core/src/main/java/org/carbondata/scan/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/scan/util/DataTypeUtil.java b/core/src/main/java/org/carbondata/scan/util/DataTypeUtil.java
index 8f301ca..5654b1d 100644
--- a/core/src/main/java/org/carbondata/scan/util/DataTypeUtil.java
+++ b/core/src/main/java/org/carbondata/scan/util/DataTypeUtil.java
@@ -18,7 +18,6 @@
  */
 package org.carbondata.scan.util;
 
-import java.math.BigDecimal;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.Date;
@@ -52,30 +51,20 @@ public class DataTypeUtil {
    */
   public static Object getDataBasedOnDataType(String data, DataType actualDataType) {
 
-    if (null == data) {
+    if (null == data || data.isEmpty()) {
       return null;
     }
     try {
       switch (actualDataType) {
         case INT:
-          if (data.isEmpty()) {
-            return null;
-          }
           return Integer.parseInt(data);
+        case SHORT:
+          return Short.parseShort(data);
         case DOUBLE:
-          if (data.isEmpty()) {
-            return null;
-          }
           return Double.parseDouble(data);
         case LONG:
-          if (data.isEmpty()) {
-            return null;
-          }
           return Long.parseLong(data);
         case TIMESTAMP:
-          if (data.isEmpty()) {
-            return null;
-          }
           SimpleDateFormat parser = new SimpleDateFormat(CarbonProperties.getInstance()
               .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
                   CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
@@ -88,9 +77,6 @@ public class DataTypeUtil {
             return null;
           }
         case DECIMAL:
-          if (data.isEmpty()) {
-            return null;
-          }
           java.math.BigDecimal javaDecVal = new java.math.BigDecimal(data);
           scala.math.BigDecimal scalaDecVal = new scala.math.BigDecimal(javaDecVal);
           org.apache.spark.sql.types.Decimal decConverter =
@@ -138,38 +124,4 @@ public class DataTypeUtil {
 
   }
 
-  public static int compareBasedOnDatatYpe(Object data1, Object data2, DataType dataType)
{
-    switch (dataType) {
-      case INT:
-        return ((Integer) data1).compareTo((Integer) data2);
-      case LONG:
-      case TIMESTAMP:
-        return ((Long) data1).compareTo((Long) data2);
-      case DOUBLE:
-        return ((Double) data1).compareTo((Double) data2);
-      case DECIMAL:
-        return ((BigDecimal) data1).compareTo((BigDecimal) data2);
-      default:
-        return ((String) data1).compareTo((String) data2);
-    }
-  }
-
-  /**
-   * below method is to check whether data type is of numeric type or not
-   *
-   * @param dataType data type
-   * @return true if numeric data type
-   */
-  public boolean isNumericDatatype(DataType dataType) {
-    switch (dataType) {
-      case INT:
-      case LONG:
-      case DOUBLE:
-      case DECIMAL:
-        return true;
-      default:
-        return false;
-    }
-  }
-
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/format/src/main/thrift/schema.thrift
----------------------------------------------------------------------
diff --git a/format/src/main/thrift/schema.thrift b/format/src/main/thrift/schema.thrift
index 8e12361..9244dac 100644
--- a/format/src/main/thrift/schema.thrift
+++ b/format/src/main/thrift/schema.thrift
@@ -27,11 +27,12 @@ namespace java org.carbondata.format
 */
 enum DataType {
 	STRING = 0,
-	INT = 1,
-	LONG = 2,
-	DOUBLE = 3,
-	DECIMAL = 4,
-	TIMESTAMP = 5,
+	SHORT = 1,
+	INT = 2,
+	LONG = 3,
+	DOUBLE = 4,
+	DECIMAL = 5,
+	TIMESTAMP = 6,
 	ARRAY = 20,
 	STRUCT = 21,
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
index e14f86a..bbc9f25 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
@@ -242,6 +242,7 @@ case class CarbonRelation(
         metaData.carbonTable.getMeasureByName(factTable, x.getColName).getDataType.toString
           .toLowerCase match {
           case "int" => "double"
+          case "short" => "double"
           case "decimal" => "decimal(" + x.getPrecision + "," + x.getScale + ")"
           case others => others
         }),

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
index c2e1564..05149eb 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
@@ -92,6 +92,7 @@ case class CarbonDictionaryDecoder(
       relation: CarbonRelation): types.DataType = {
     carbonDimension.getDataType match {
       case DataType.STRING => StringType
+      case DataType.SHORT => ShortType
       case DataType.INT => IntegerType
       case DataType.LONG => LongType
       case DataType.DOUBLE => DoubleType

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 476f25d..cb94f72 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -379,6 +379,8 @@ class TableNewProcessor(cm: tableModel, sqlContext: SQLContext) {
       case "String" => DataType.STRING
       case "int" => DataType.INT
       case "Integer" => DataType.INT
+      case "tinyint" => DataType.SHORT
+      case "short" => DataType.SHORT
       case "Long" => DataType.LONG
       case "BigInt" => DataType.LONG
       case "Numeric" => DataType.DOUBLE

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
index dc44f28..d87e132 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastoreCatalog.scala
@@ -707,6 +707,7 @@ object CarbonMetastoreTypes extends RegexParsers {
       "float" ^^^ FloatType |
       "int" ^^^ IntegerType |
       "tinyint" ^^^ ShortType |
+      "short" ^^^ ShortType |
       "double" ^^^ DoubleType |
       "long" ^^^ LongType |
       "binary" ^^^ BinaryType |

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/integration/spark/src/main/scala/org/carbondata/spark/package.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/package.scala b/integration/spark/src/main/scala/org/carbondata/spark/package.scala
index 2745229..60c8880 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/package.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/package.scala
@@ -81,7 +81,7 @@ package object spark {
         case StringType => CarbonType.STRING.name
         case IntegerType => CarbonType.INT.name
         case ByteType => CarbonType.INT.name
-        case ShortType => CarbonType.INT.name
+        case ShortType => CarbonType.SHORT.name
         case LongType => CarbonType.LONG.name
         case FloatType => CarbonType.DOUBLE.name
         case DoubleType => CarbonType.DOUBLE.name

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a7437571/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
b/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
index 593aadb..0930887 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/util/CarbonScalaUtil.scala
@@ -35,6 +35,7 @@ object CarbonScalaUtil {
       dataType: org.apache.spark.sql.types.DataType): CarbonDataType = {
     dataType match {
       case StringType => CarbonDataType.StringType
+      case ShortType => CarbonDataType.ShortType
       case IntegerType => CarbonDataType.IntegerType
       case LongType => CarbonDataType.LongType
       case DoubleType => CarbonDataType.DoubleType
@@ -54,7 +55,7 @@ object CarbonScalaUtil {
       case CarbonCommonConstants.STRING_TYPE => CarbonCommonConstants.STRING
       case CarbonCommonConstants.INTEGER_TYPE => CarbonCommonConstants.INTEGER
       case CarbonCommonConstants.BYTE_TYPE => CarbonCommonConstants.INTEGER
-      case CarbonCommonConstants.SHORT_TYPE => CarbonCommonConstants.INTEGER
+      case CarbonCommonConstants.SHORT_TYPE => CarbonCommonConstants.SHORT
       case CarbonCommonConstants.LONG_TYPE => CarbonCommonConstants.NUMERIC
       case CarbonCommonConstants.DOUBLE_TYPE => CarbonCommonConstants.NUMERIC
       case CarbonCommonConstants.FLOAT_TYPE => CarbonCommonConstants.NUMERIC
@@ -66,36 +67,10 @@ object CarbonScalaUtil {
     }
   }
 
-  def convertSparkColumnToCarbonLevel(field: (String, String)): Seq[Level] = {
-    field._2 match {
-      case CarbonCommonConstants.STRING_TYPE => Seq(
-        Level(field._1, field._1, Int.MaxValue, CarbonCommonConstants.STRING))
-      case CarbonCommonConstants.INTEGER_TYPE => Seq(
-        Level(field._1, field._1, Int.MaxValue, CarbonCommonConstants.INTEGER))
-      case CarbonCommonConstants.BYTE_TYPE => Seq(
-        Level(field._1, field._1, Int.MaxValue, CarbonCommonConstants.INTEGER))
-      case CarbonCommonConstants.SHORT_TYPE => Seq(
-        Level(field._1, field._1, Int.MaxValue, CarbonCommonConstants.INTEGER))
-      case CarbonCommonConstants.LONG_TYPE => Seq(
-        Level(field._1, field._1, Int.MaxValue, CarbonCommonConstants.NUMERIC))
-      case CarbonCommonConstants.DOUBLE_TYPE => Seq(
-        Level(field._1, field._1, Int.MaxValue, CarbonCommonConstants.NUMERIC))
-      case CarbonCommonConstants.FLOAT_TYPE => Seq(
-        Level(field._1, field._1, Int.MaxValue, CarbonCommonConstants.NUMERIC))
-      case CarbonCommonConstants.DECIMAL_TYPE => Seq(
-        Level(field._1, field._1, Int.MaxValue, CarbonCommonConstants.NUMERIC))
-      case CarbonCommonConstants.DATE_TYPE => Seq(
-        Level(field._1, field._1, Int.MaxValue, CarbonCommonConstants.STRING))
-      case CarbonCommonConstants.BOOLEAN_TYPE => Seq(
-        Level(field._1, field._1, Int.MaxValue, CarbonCommonConstants.STRING))
-      case CarbonCommonConstants.TIMESTAMP_TYPE => Seq(
-        Level(field._1, field._1, Int.MaxValue, CarbonCommonConstants.TIMESTAMP))
-    }
-  }
-
   def convertCarbonToSparkDataType(dataType: DataType): types.DataType = {
     dataType match {
       case DataType.STRING => StringType
+      case DataType.SHORT => ShortType
       case DataType.INT => IntegerType
       case DataType.LONG => LongType
       case DataType.DOUBLE => DoubleType
@@ -105,21 +80,6 @@ object CarbonScalaUtil {
     }
   }
 
-  def convertValueToSparkDataType(value: Any,
-      dataType: org.apache.spark.sql.types.DataType): Any = {
-    dataType match {
-      case StringType => value.toString
-      case IntegerType => value.toString.toInt
-      case LongType => value.toString.toLong
-      case DoubleType => value.toString.toDouble
-      case FloatType => value.toString.toFloat
-      case _ => value.toString.toDouble
-    }
-  }
-
-
-  case class TransformHolder(rdd: Any, mataData: CarbonMetaData)
-
   object CarbonSparkUtil {
 
     def createSparkMeta(carbonTable: CarbonTable): CarbonMetaData = {



Mime
View raw message