carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [2/2] carbondata git commit: [CARBONDATA-1662] Make ArrayType and StructType contain child DataType
Date Fri, 10 Nov 2017 14:09:07 GMT
[CARBONDATA-1662] Make ArrayType and StructType contain child DataType

StructType and ArrayType should be class that have nested children.

This closes #1429


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/933e30cc
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/933e30cc
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/933e30cc

Branch: refs/heads/master
Commit: 933e30ccc74d499a6323f328349c9a71ba0c44e3
Parents: a91c6e4
Author: Jacky Li <jacky.likun@qq.com>
Authored: Mon Nov 6 11:13:40 2017 +0800
Committer: ravipesala <ravi.pesala@gmail.com>
Committed: Fri Nov 10 19:38:49 2017 +0530

----------------------------------------------------------------------
 .../core/datastore/block/SegmentProperties.java |   3 +-
 .../blockletindex/BlockletDataMap.java          |   3 +-
 .../core/indexstore/row/DataMapRowImpl.java     |   2 +-
 .../ThriftWrapperSchemaConverterImpl.java       |   8 +-
 .../core/metadata/datatype/ArrayType.java       |  12 +-
 .../core/metadata/datatype/BooleanType.java     |   4 +-
 .../core/metadata/datatype/ByteArrayType.java   |   4 +-
 .../core/metadata/datatype/ByteType.java        |   4 +-
 .../core/metadata/datatype/DataTypes.java       | 110 ++++++++++++++-----
 .../core/metadata/datatype/DateType.java        |   4 +-
 .../core/metadata/datatype/DoubleType.java      |   4 +-
 .../core/metadata/datatype/FloatType.java       |   5 +-
 .../core/metadata/datatype/IntType.java         |   4 +-
 .../core/metadata/datatype/LongType.java        |   4 +-
 .../core/metadata/datatype/MapType.java         |  14 +--
 .../core/metadata/datatype/NullType.java        |   4 +-
 .../core/metadata/datatype/ShortIntType.java    |   4 +-
 .../core/metadata/datatype/ShortType.java       |   4 +-
 .../core/metadata/datatype/StringType.java      |   4 +-
 .../core/metadata/datatype/StructField.java     |  42 +++++++
 .../core/metadata/datatype/StructType.java      |  18 +--
 .../core/metadata/datatype/TimestampType.java   |   4 +-
 .../executor/impl/AbstractQueryExecutor.java    |   4 +-
 .../core/scan/executor/util/QueryUtil.java      |   8 +-
 .../expression/RangeExpressionEvaluator.java    |  28 +----
 .../scan/filter/FilterExpressionProcessor.java  |  29 ++---
 .../executer/RowLevelFilterExecuterImpl.java    |   6 +-
 .../resolver/ConditionalFilterResolverImpl.java |   7 +-
 .../util/AbstractDataFileFooterConverter.java   |   4 +-
 .../DictionaryColumnUniqueIdentifierTest.java   |   8 +-
 .../datastore/block/SegmentPropertiesTest.java  |   2 +-
 .../block/SegmentPropertiesTestUtil.java        |   2 +-
 ...ctDictionaryKeyGeneratorFactoryUnitTest.java |   2 +-
 .../ThriftWrapperSchemaConverterImplTest.java   |   6 +-
 .../table/CarbonTableWithComplexTypesTest.java  |   4 +-
 .../carbondata/core/util/CarbonUtilTest.java    |   4 +-
 .../CarbonDictionarySortInfoPreparatorTest.java |   4 +-
 .../streaming/CarbonStreamInputFormat.java      |   8 +-
 .../streaming/CarbonStreamRecordReader.java     |  32 +++---
 .../streaming/CarbonStreamRecordWriter.java     |  21 ++--
 .../hive/CarbonDictionaryDecodeReadSupport.java |   4 +-
 .../carbondata/spark/util/CarbonScalaUtil.scala |  16 ++-
 .../spark/util/DataTypeConverterUtil.scala      |  14 ++-
 .../spark/util/GlobalDictionaryUtil.scala       |  37 +++----
 .../spark/sql/CarbonDictionaryDecoder.scala     |  12 +-
 .../converter/impl/FieldEncoderFactory.java     |   6 +-
 .../loading/parser/CarbonParserFactory.java     |   6 +-
 .../util/CarbonDataProcessorUtil.java           |   6 +-
 48 files changed, 305 insertions(+), 240 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
index c93b771..5f82e87 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
@@ -378,8 +378,7 @@ public class SegmentProperties {
         }
         // as complex type will be stored at last so once complex type started all the dimension
         // will be added to complex type
-        else if (isComplexDimensionStarted || CarbonUtil.hasDataType(columnSchema.getDataType(),
-            new DataType[] { DataTypes.ARRAY, DataTypes.STRUCT })) {
+        else if (isComplexDimensionStarted || columnSchema.getDataType().isComplexType()) {
           cardinalityIndexForComplexDimensionColumn.add(tableOrdinal);
           carbonDimension =
               new CarbonDimension(columnSchema, dimensonOrdinal++, -1, -1, ++complexTypeOrdinal);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
index d5bd695..7829034 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
@@ -284,7 +284,8 @@ public class BlockletDataMap implements DataMap, Cacheable {
           mapSchemas[i] = new DataMapSchema.FixedDataMapSchema(DataTypes.BYTE_ARRAY, minMaxLen[i]);
         }
       }
-      DataMapSchema mapSchema = new DataMapSchema.StructDataMapSchema(DataTypes.STRUCT, mapSchemas);
+      DataMapSchema mapSchema = new DataMapSchema.StructDataMapSchema(
+          DataTypes.createDefaultStructType(), mapSchemas);
       indexSchemas.add(mapSchema);
     }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/indexstore/row/DataMapRowImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/row/DataMapRowImpl.java b/core/src/main/java/org/apache/carbondata/core/indexstore/row/DataMapRowImpl.java
index bc55e74..032b29e 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/row/DataMapRowImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/row/DataMapRowImpl.java
@@ -99,7 +99,7 @@ public class DataMapRowImpl extends DataMapRow {
   }
 
   @Override public void setRow(DataMapRow row, int ordinal) {
-    assert (schemas[ordinal].getDataType() == DataTypes.STRUCT);
+    assert (DataTypes.isStructType(schemas[ordinal].getDataType()));
     data[ordinal] = row;
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
index 70a6e63..adcac7d 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
@@ -145,9 +145,9 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
       return org.apache.carbondata.format.DataType.DATE;
     } else if (dataType.getId() == DataTypes.TIMESTAMP.getId()) {
       return org.apache.carbondata.format.DataType.TIMESTAMP;
-    } else if (dataType.getId() == DataTypes.ARRAY.getId()) {
+    } else if (DataTypes.isArrayType(dataType)) {
       return org.apache.carbondata.format.DataType.ARRAY;
-    } else if (dataType.getId() == DataTypes.STRUCT.getId()) {
+    } else if (DataTypes.isStructType(dataType)) {
       return org.apache.carbondata.format.DataType.STRUCT;
     } else {
       return org.apache.carbondata.format.DataType.STRING;
@@ -392,9 +392,9 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter {
       case DATE:
         return DataTypes.DATE;
       case ARRAY:
-        return DataTypes.ARRAY;
+        return DataTypes.createDefaultArrayType();
       case STRUCT:
-        return DataTypes.STRUCT;
+        return DataTypes.createDefaultStructType();
       default:
         return DataTypes.STRING;
     }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java
index 241d0f6..c30e21c 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ArrayType.java
@@ -19,15 +19,11 @@ package org.apache.carbondata.core.metadata.datatype;
 
 public class ArrayType extends DataType {
 
-  public static final DataType ARRAY = new ArrayType(DataTypes.ARRAY_TYPE_ID, 9, "ARRAY", -1);
+  private DataType elementType;
 
-  private ArrayType(int id, int precedenceOrder, String name, int sizeInBytes) {
-    super(id, precedenceOrder, name, sizeInBytes);
-  }
-
-  // this function is needed to ensure singleton pattern while supporting java serialization
-  private Object readResolve() {
-    return DataTypes.ARRAY;
+  ArrayType(DataType elementType) {
+    super(DataTypes.ARRAY_TYPE_ID, 9, "ARRAY", -1);
+    this.elementType = elementType;
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/BooleanType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/BooleanType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/BooleanType.java
index 36cb84f..0ce0412 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/BooleanType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/BooleanType.java
@@ -17,9 +17,9 @@
 
 package org.apache.carbondata.core.metadata.datatype;
 
-public class BooleanType extends DataType {
+class BooleanType extends DataType {
 
-  public static final DataType BOOLEAN =
+  static final DataType BOOLEAN =
       new BooleanType(DataTypes.BOOLEAN_TYPE_ID, 1, "BOOLEAN", 1);
 
   private BooleanType(int id, int precedenceOrder, String name, int sizeInBytes) {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ByteArrayType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ByteArrayType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ByteArrayType.java
index 1f2872d..9cb33ed 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ByteArrayType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ByteArrayType.java
@@ -17,9 +17,9 @@
 
 package org.apache.carbondata.core.metadata.datatype;
 
-public class ByteArrayType extends DataType {
+class ByteArrayType extends DataType {
 
-  public static final DataType BYTE_ARRAY =
+  static final DataType BYTE_ARRAY =
       new ByteArrayType(DataTypes.BYTE_ARRAY_TYPE_ID, 13, "BYTE_ARRAY", -1);
 
   private ByteArrayType(int id, int precedenceOrder, String name, int sizeInBytes) {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ByteType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ByteType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ByteType.java
index 11138d2..e1b39a3 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ByteType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ByteType.java
@@ -17,9 +17,9 @@
 
 package org.apache.carbondata.core.metadata.datatype;
 
-public class ByteType extends DataType {
+class ByteType extends DataType {
 
-  public static final DataType BYTE = new ByteType(DataTypes.BYTE_TYPE_ID, 12, "BYTE", 1);
+  static final DataType BYTE = new ByteType(DataTypes.BYTE_TYPE_ID, 12, "BYTE", 1);
 
   private ByteType(int id, int precedenceOrder, String name, int sizeInBytes) {
     super(id, precedenceOrder, name, sizeInBytes);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypes.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypes.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypes.java
index 8686583..e96917f 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypes.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypes.java
@@ -17,6 +17,9 @@
 
 package org.apache.carbondata.core.metadata.datatype;
 
+import java.util.ArrayList;
+import java.util.List;
+
 /**
  * Holds all singleton object for all data type used in carbon
  */
@@ -44,28 +47,25 @@ public class DataTypes {
   // Only for internal use for backward compatability. It is only used for V1 version
   public static final DataType LEGACY_LONG = LegacyLongType.LEGACY_LONG;
 
-  public static final DataType ARRAY = ArrayType.ARRAY;
-  public static final DataType STRUCT = StructType.STRUCT;
-  public static final DataType MAP = MapType.MAP;
-
-  public static final int STRING_TYPE_ID = 0;
-  public static final int DATE_TYPE_ID = 1;
-  public static final int TIMESTAMP_TYPE_ID = 2;
-  public static final int BOOLEAN_TYPE_ID = 3;
-  public static final int SHORT_TYPE_ID = 4;
-  public static final int INT_TYPE_ID = 5;
-  public static final int FLOAT_TYPE_ID = 6;
-  public static final int LONG_TYPE_ID = 7;
-  public static final int DOUBLE_TYPE_ID = 8;
-  public static final int NULL_TYPE_ID = 9;
-  public static final int BYTE_TYPE_ID = 10;
-  public static final int BYTE_ARRAY_TYPE_ID = 11;
-  public static final int SHORT_INT_TYPE_ID = 12;
-  public static final int LEGACY_LONG_TYPE_ID = 13;
-  public static final int DECIMAL_TYPE_ID = 20;
-  public static final int ARRAY_TYPE_ID = 21;
-  public static final int STRUCT_TYPE_ID = 22;
-  public static final int MAP_TYPE_ID = 23;
+  // these IDs are used within this package only
+  static final int STRING_TYPE_ID = 0;
+  static final int DATE_TYPE_ID = 1;
+  static final int TIMESTAMP_TYPE_ID = 2;
+  static final int BOOLEAN_TYPE_ID = 3;
+  static final int SHORT_TYPE_ID = 4;
+  static final int INT_TYPE_ID = 5;
+  static final int FLOAT_TYPE_ID = 6;
+  static final int LONG_TYPE_ID = 7;
+  static final int DOUBLE_TYPE_ID = 8;
+  static final int NULL_TYPE_ID = 9;
+  static final int BYTE_TYPE_ID = 10;
+  static final int BYTE_ARRAY_TYPE_ID = 11;
+  static final int SHORT_INT_TYPE_ID = 12;
+  static final int LEGACY_LONG_TYPE_ID = 13;
+  static final int DECIMAL_TYPE_ID = 20;
+  static final int ARRAY_TYPE_ID = 21;
+  static final int STRUCT_TYPE_ID = 22;
+  static final int MAP_TYPE_ID = 23;
 
   /**
    * create a DataType instance from uniqueId of the DataType
@@ -99,12 +99,12 @@ public class DataTypes {
       return NULL;
     } else if (id == DECIMAL_TYPE_ID) {
       return createDefaultDecimalType();
-    } else if (id == ARRAY.getId()) {
-      return ARRAY;
-    } else if (id == STRUCT.getId()) {
-      return STRUCT;
-    } else if (id == MAP.getId()) {
-      return MAP;
+    } else if (id == ARRAY_TYPE_ID) {
+      return createDefaultArrayType();
+    } else if (id == STRUCT_TYPE_ID) {
+      return createDefaultStructType();
+    } else if (id == MAP_TYPE_ID) {
+      return createDefaultMapType();
     } else if (id == BYTE_ARRAY.getId()) {
       return BYTE_ARRAY;
     } else {
@@ -130,4 +130,58 @@ public class DataTypes {
     return dataType.getId() == DECIMAL_TYPE_ID;
   }
 
+  /**
+   * create array type with specified element type
+   */
+  public static ArrayType createArrayType(DataType elementType) {
+    return new ArrayType(elementType);
+  }
+
+  /**
+   * create a array type object with no child
+   */
+  public static ArrayType createDefaultArrayType() {
+    return new ArrayType(STRING);
+  }
+
+  public static boolean isArrayType(DataType dataType) {
+    return dataType.getId() == ARRAY_TYPE_ID;
+  }
+
+  /**
+   * create struct type with specified fields
+   */
+  public static StructType createStructType(List<StructField> fields) {
+    return new StructType(fields);
+  }
+
+  /**
+   * create a struct type object with no field
+   */
+  public static StructType createDefaultStructType() {
+    return new StructType(new ArrayList<StructField>());
+  }
+
+  public static boolean isStructType(DataType dataType) {
+    return dataType.getId() == STRUCT_TYPE_ID;
+  }
+
+  /**
+   * create map type with specified key type and value type
+   */
+  public static MapType createMapType(DataType keyType, DataType valueType) {
+    return new MapType(keyType, valueType);
+  }
+
+  /**
+   * create a map type object with no child
+   */
+  public static MapType createDefaultMapType() {
+    return new MapType(STRING, STRING);
+  }
+
+  public static boolean isMapType(DataType dataType) {
+    return dataType.getId() == MAP_TYPE_ID;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DateType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DateType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DateType.java
index 2b930c3..efc9799 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DateType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DateType.java
@@ -17,9 +17,9 @@
 
 package org.apache.carbondata.core.metadata.datatype;
 
-public class DateType extends DataType {
+class DateType extends DataType {
 
-  public static final DataType DATE = new DateType(DataTypes.DATE_TYPE_ID, 1, "DATE", -1);
+  static final DataType DATE = new DateType(DataTypes.DATE_TYPE_ID, 1, "DATE", -1);
 
   private DateType(int id, int precedenceOrder, String name, int sizeInBytes) {
     super(id, precedenceOrder, name, sizeInBytes);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DoubleType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DoubleType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DoubleType.java
index 59a4cc2..5e032a0 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DoubleType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DoubleType.java
@@ -17,9 +17,9 @@
 
 package org.apache.carbondata.core.metadata.datatype;
 
-public class DoubleType extends DataType {
+class DoubleType extends DataType {
 
-  public static final DataType DOUBLE = new DoubleType(DataTypes.DOUBLE_TYPE_ID, 6, "DOUBLE", 8);
+  static final DataType DOUBLE = new DoubleType(DataTypes.DOUBLE_TYPE_ID, 6, "DOUBLE", 8);
 
   private DoubleType(int id, int precedenceOrder, String name, int sizeInBytes) {
     super(id, precedenceOrder, name, sizeInBytes);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/FloatType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/FloatType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/FloatType.java
index 1c0bfb6..9f600f9 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/FloatType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/FloatType.java
@@ -17,8 +17,9 @@
 
 package org.apache.carbondata.core.metadata.datatype;
 
-public class FloatType extends DataType {
-  public static final DataType FLOAT = new FloatType(DataTypes.FLOAT_TYPE_ID, 4, "FLOAT", 4);
+class FloatType extends DataType {
+
+  static final DataType FLOAT = new FloatType(DataTypes.FLOAT_TYPE_ID, 4, "FLOAT", 4);
 
   private FloatType(int id, int precedenceOrder, String name, int sizeInBytes) {
     super(id, precedenceOrder, name, sizeInBytes);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/IntType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/IntType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/IntType.java
index 4324d9e..308a53b 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/IntType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/IntType.java
@@ -17,9 +17,9 @@
 
 package org.apache.carbondata.core.metadata.datatype;
 
-public class IntType extends DataType {
+class IntType extends DataType {
 
-  public static final DataType INT = new IntType(DataTypes.INT_TYPE_ID, 3, "INT", 4);
+  static final DataType INT = new IntType(DataTypes.INT_TYPE_ID, 3, "INT", 4);
 
   private IntType(int id, int precedenceOrder, String name, int sizeInBytes) {
     super(id, precedenceOrder, name, sizeInBytes);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/LongType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/LongType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/LongType.java
index 3d05fd9..3bc1a4b 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/LongType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/LongType.java
@@ -17,9 +17,9 @@
 
 package org.apache.carbondata.core.metadata.datatype;
 
-public class LongType extends DataType {
+class LongType extends DataType {
 
-  public static final DataType LONG = new LongType(DataTypes.LONG_TYPE_ID, 5, "LONG", 8);
+  static final DataType LONG = new LongType(DataTypes.LONG_TYPE_ID, 5, "LONG", 8);
 
   private LongType(int id, int precedenceOrder, String name, int sizeInBytes) {
     super(id, precedenceOrder, name, sizeInBytes);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/MapType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/MapType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/MapType.java
index a360243..69d49b8 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/MapType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/MapType.java
@@ -19,15 +19,13 @@ package org.apache.carbondata.core.metadata.datatype;
 
 public class MapType extends DataType {
 
-  public static final DataType MAP = new MapType(DataTypes.MAP_TYPE_ID, 11, "MAP", -1);
+  private DataType keyType;
+  private DataType valueType;
 
-  private MapType(int id, int precedenceOrder, String name, int sizeInBytes) {
-    super(id, precedenceOrder, name, sizeInBytes);
-  }
-
-  // this function is needed to ensure singleton pattern while supporting java serialization
-  private Object readResolve() {
-    return DataTypes.MAP;
+  MapType(DataType keyType, DataType valueType) {
+    super(DataTypes.MAP_TYPE_ID, 11, "MAP", -1);
+    this.keyType = keyType;
+    this.valueType = valueType;
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/NullType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/NullType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/NullType.java
index 43ffe41..81248c5 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/NullType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/NullType.java
@@ -17,9 +17,9 @@
 
 package org.apache.carbondata.core.metadata.datatype;
 
-public class NullType extends DataType {
+class NullType extends DataType {
 
-  public static final DataType NULL = new NullType(DataTypes.NULL_TYPE_ID, 7, "NULL", 1);
+  static final DataType NULL = new NullType(DataTypes.NULL_TYPE_ID, 7, "NULL", 1);
 
   private NullType(int id, int precedenceOrder, String name, int sizeInBytes) {
     super(id, precedenceOrder, name, sizeInBytes);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ShortIntType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ShortIntType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ShortIntType.java
index 5d57748..24c90d5 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ShortIntType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ShortIntType.java
@@ -17,9 +17,9 @@
 
 package org.apache.carbondata.core.metadata.datatype;
 
-public class ShortIntType extends DataType {
+class ShortIntType extends DataType {
 
-  public static final DataType SHORT_INT =
+  static final DataType SHORT_INT =
       new ShortIntType(DataTypes.SHORT_INT_TYPE_ID, 14, "SHORT_INT", 3);
 
   private ShortIntType(int id, int precedenceOrder, String name, int sizeInBytes) {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ShortType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ShortType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ShortType.java
index 0085227..4bf5563 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ShortType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/ShortType.java
@@ -17,9 +17,9 @@
 
 package org.apache.carbondata.core.metadata.datatype;
 
-public class ShortType extends DataType {
+class ShortType extends DataType {
 
-  public static final DataType SHORT = new ShortType(DataTypes.SHORT_TYPE_ID, 2, "SHORT", 2);
+  static final DataType SHORT = new ShortType(DataTypes.SHORT_TYPE_ID, 2, "SHORT", 2);
 
   private ShortType(int id, int precedenceOrder, String name, int sizeInBytes) {
     super(id, precedenceOrder, name, sizeInBytes);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StringType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StringType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StringType.java
index 5c95480..18c505a 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StringType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StringType.java
@@ -17,8 +17,8 @@
 
 package org.apache.carbondata.core.metadata.datatype;
 
-public class StringType extends DataType {
-  public static final DataType STRING = new StringType(DataTypes.STRING_TYPE_ID, 0, "STRING", -1);
+class StringType extends DataType {
+  static final DataType STRING = new StringType(DataTypes.STRING_TYPE_ID, 0, "STRING", -1);
 
   private StringType(int id, int precedenceOrder, String name, int sizeInBytes) {
     super(id, precedenceOrder, name, sizeInBytes);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructField.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructField.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructField.java
new file mode 100644
index 0000000..efdc8e2
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructField.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.metadata.datatype;
+
+import java.io.Serializable;
+
+public class StructField implements Serializable {
+
+  private static final long serialVersionUID = 3271726L;
+
+  private String fieldName;
+
+  private DataType dataType;
+
+  public StructField(String fieldName, DataType dataType) {
+    this.fieldName = fieldName;
+    this.dataType = dataType;
+  }
+
+  public DataType getDataType() {
+    return dataType;
+  }
+
+  public String getFieldName() {
+    return fieldName;
+  }
+}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructType.java
index 081f952..6417f37 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/StructType.java
@@ -17,21 +17,23 @@
 
 package org.apache.carbondata.core.metadata.datatype;
 
-public class StructType extends DataType {
+import java.util.List;
 
-  public static final DataType STRUCT = new StructType(DataTypes.STRUCT_TYPE_ID, 10, "STRUCT", -1);
+class StructType extends DataType {
 
-  private StructType(int id, int precedenceOrder, String name, int sizeInBytes) {
-    super(id, precedenceOrder, name, sizeInBytes);
-  }
+  private List<StructField> fields;
 
-  // this function is needed to ensure singleton pattern while supporting java serialization
-  private Object readResolve() {
-    return DataTypes.STRUCT;
+  StructType(List<StructField> fields) {
+    super(DataTypes.STRUCT_TYPE_ID, 10, "STRUCT", -1);
+    this.fields = fields;
   }
 
   @Override
   public boolean isComplexType() {
     return true;
   }
+
+  public List<StructField> getFields() {
+    return fields;
+  }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/metadata/datatype/TimestampType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/TimestampType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/TimestampType.java
index 287c457..42a6855 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/TimestampType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/TimestampType.java
@@ -17,8 +17,8 @@
 
 package org.apache.carbondata.core.metadata.datatype;
 
-public class TimestampType extends DataType {
-  public static final TimestampType TIMESTAMP =
+class TimestampType extends DataType {
+  static final TimestampType TIMESTAMP =
       new TimestampType(DataTypes.TIMESTAMP_TYPE_ID, 2, "TIMESTAMP", -1);
 
   private TimestampType(int id, int precedenceOrder, String name, int sizeInBytes) {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
index 56ed575..251a06b 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
@@ -49,7 +49,6 @@ import org.apache.carbondata.core.keygenerator.KeyGenerator;
 import org.apache.carbondata.core.memory.UnsafeMemoryManager;
 import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
 import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
 import org.apache.carbondata.core.metadata.encoder.Encoding;
 import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
 import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
@@ -478,8 +477,7 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
   private int[] getComplexDimensionParentBlockIndexes(List<QueryDimension> queryDimensions) {
     List<Integer> parentBlockIndexList = new ArrayList<Integer>();
     for (QueryDimension queryDimension : queryDimensions) {
-      if (CarbonUtil.hasDataType(queryDimension.getDimension().getDataType(),
-          new DataType[] { DataTypes.ARRAY, DataTypes.STRUCT, DataTypes.MAP })) {
+      if (queryDimension.getDimension().getDataType().isComplexType()) {
         parentBlockIndexList.add(queryDimension.getDimension().getOrdinal());
       }
     }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
index 6aab8a0..b090e59 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
@@ -763,10 +763,10 @@ public class QueryUtil {
       int[] eachComplexColumnValueSize, Map<String, Dictionary> columnIdToDictionaryMap) {
     int parentBlockIndex = dimensionToBlockIndexMap.get(dimension.getOrdinal());
     GenericQueryType parentQueryType;
-    if (dimension.getDataType() == DataTypes.ARRAY) {
+    if (DataTypes.isArrayType(dimension.getDataType())) {
       parentQueryType =
           new ArrayQueryType(dimension.getColName(), dimension.getColName(), parentBlockIndex);
-    } else if (dimension.getDataType() == DataTypes.STRUCT) {
+    } else if (DataTypes.isStructType(dimension.getDataType())) {
       parentQueryType =
           new StructQueryType(dimension.getColName(), dimension.getColName(),
               dimensionToBlockIndexMap.get(dimension.getOrdinal()));
@@ -784,11 +784,11 @@ public class QueryUtil {
       CarbonDimension dimension, GenericQueryType parentQueryType) {
     for (int i = 0; i < dimension.getNumberOfChild(); i++) {
       DataType dataType = dimension.getListOfChildDimensions().get(i).getDataType();
-      if (dataType == DataTypes.ARRAY) {
+      if (DataTypes.isArrayType(dataType)) {
         parentQueryType.addChildren(
             new ArrayQueryType(dimension.getListOfChildDimensions().get(i).getColName(),
                 dimension.getColName(), ++parentBlockIndex));
-      } else if (dataType == DataTypes.STRUCT) {
+      } else if (DataTypes.isStructType(dataType)) {
         parentQueryType.addChildren(
             new StructQueryType(dimension.getListOfChildDimensions().get(i).getColName(),
                 dimension.getColName(), ++parentBlockIndex));

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/scan/expression/RangeExpressionEvaluator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/RangeExpressionEvaluator.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/RangeExpressionEvaluator.java
index 9d418ee..e7cc1c3 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/RangeExpressionEvaluator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/RangeExpressionEvaluator.java
@@ -27,7 +27,6 @@ import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
 import org.apache.carbondata.core.scan.expression.conditional.GreaterThanEqualToExpression;
 import org.apache.carbondata.core.scan.expression.conditional.GreaterThanExpression;
 import org.apache.carbondata.core.scan.expression.conditional.LessThanEqualToExpression;
@@ -259,13 +258,9 @@ public class RangeExpressionEvaluator {
    * @return
    */
   private boolean isLessThanGreaterThanExp(Expression expr) {
-    if ((expr instanceof LessThanEqualToExpression) || (expr instanceof LessThanExpression)
+    return (expr instanceof LessThanEqualToExpression) || (expr instanceof LessThanExpression)
         || (expr instanceof GreaterThanEqualToExpression)
-        || (expr instanceof GreaterThanExpression)) {
-      return true;
-    } else {
-      return false;
-    }
+        || (expr instanceof GreaterThanExpression);
   }
 
   /**
@@ -277,15 +272,8 @@ public class RangeExpressionEvaluator {
   private boolean eligibleForRangeExpConv(Expression expChild) {
     for (Expression exp : expChild.getChildren()) {
       if (exp instanceof ColumnExpression) {
-        if (((ColumnExpression) exp).isDimension() == false) {
-          return false;
-        }
-        if ((((ColumnExpression) exp).getDimension().getDataType() == DataTypes.ARRAY) || (
-            ((ColumnExpression) exp).getDimension().getDataType() == DataTypes.STRUCT)) {
-          return false;
-        } else {
-          return true;
-        }
+        return ((ColumnExpression) exp).isDimension() &&
+            ! (((ColumnExpression) exp).getDimension().getDataType().isComplexType());
       }
     }
     return false;
@@ -384,13 +372,9 @@ public class RangeExpressionEvaluator {
    * @return
    */
   private boolean matchExpType(ExpressionType src, ExpressionType tar) {
-    if ((((src == LESSTHAN) || (src == LESSTHAN_EQUALTO)) && ((tar == GREATERTHAN) || (tar
+    return (((src == LESSTHAN) || (src == LESSTHAN_EQUALTO)) && ((tar == GREATERTHAN) || (tar
         == GREATERTHAN_EQUALTO))) || (((src == GREATERTHAN) || (src == GREATERTHAN_EQUALTO)) && (
-        (tar == LESSTHAN) || (tar == LESSTHAN_EQUALTO)))) {
-      return true;
-    } else {
-      return false;
-    }
+        (tar == LESSTHAN) || (tar == LESSTHAN_EQUALTO)));
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
index 68787e3..e77d58e 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
@@ -30,9 +30,9 @@ import org.apache.carbondata.core.datastore.block.AbstractIndex;
 import org.apache.carbondata.core.datastore.impl.btree.BTreeDataRefNodeFinder;
 import org.apache.carbondata.core.keygenerator.KeyGenException;
 import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.datatype.DataTypes;
 import org.apache.carbondata.core.metadata.encoder.Encoding;
 import org.apache.carbondata.core.metadata.schema.PartitionInfo;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
 import org.apache.carbondata.core.scan.expression.BinaryExpression;
 import org.apache.carbondata.core.scan.expression.ColumnExpression;
 import org.apache.carbondata.core.scan.expression.Expression;
@@ -402,13 +402,9 @@ public class FilterExpressionProcessor implements FilterProcessor {
         return new TrueConditionalResolverImpl(expression, false, false, tableIdentifier);
       case EQUALS:
         currentCondExpression = (BinaryConditionalExpression) expression;
-        if (currentCondExpression.isSingleColumn()
-            && currentCondExpression.getColumnList().get(0).getCarbonColumn().getDataType()
-            != DataTypes.ARRAY
-            && currentCondExpression.getColumnList().get(0).getCarbonColumn().getDataType()
-            != DataTypes.STRUCT) {
-
-          if (currentCondExpression.getColumnList().get(0).getCarbonColumn().isMeasure()) {
+        CarbonColumn column = currentCondExpression.getColumnList().get(0).getCarbonColumn();
+        if (currentCondExpression.isSingleColumn() && ! column.getDataType().isComplexType()) {
+          if (column.isMeasure()) {
             if (FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getLeft())
                 && FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getRight()) || (
                 FilterUtil.checkIfRightExpressionRequireEvaluation(currentCondExpression.getRight())
@@ -463,13 +459,9 @@ public class FilterExpressionProcessor implements FilterProcessor {
             tableIdentifier, false);
       case NOT_EQUALS:
         currentCondExpression = (BinaryConditionalExpression) expression;
-        if (currentCondExpression.isSingleColumn()
-            && currentCondExpression.getColumnList().get(0).getCarbonColumn().getDataType()
-            != DataTypes.ARRAY
-            && currentCondExpression.getColumnList().get(0).getCarbonColumn().getDataType()
-            != DataTypes.STRUCT) {
-
-          if (currentCondExpression.getColumnList().get(0).getCarbonColumn().isMeasure()) {
+        column = currentCondExpression.getColumnList().get(0).getCarbonColumn();
+        if (currentCondExpression.isSingleColumn() && ! column.getDataType().isComplexType()) {
+          if (column.isMeasure()) {
             if (FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getLeft())
                 && FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getRight()) || (
                 FilterUtil.checkIfRightExpressionRequireEvaluation(currentCondExpression.getRight())
@@ -522,11 +514,8 @@ public class FilterExpressionProcessor implements FilterProcessor {
       default:
         if (expression instanceof ConditionalExpression) {
           condExpression = (ConditionalExpression) expression;
-          if (condExpression.isSingleColumn()
-              && condExpression.getColumnList().get(0).getCarbonColumn().getDataType()
-              != DataTypes.ARRAY
-              && condExpression.getColumnList().get(0).getCarbonColumn().getDataType()
-              != DataTypes.STRUCT) {
+          column = condExpression.getColumnList().get(0).getCarbonColumn();
+          if (condExpression.isSingleColumn() && ! column.isComplex()) {
             condExpression = (ConditionalExpression) expression;
             if ((condExpression.getColumnList().get(0).getCarbonColumn()
                 .hasEncoding(Encoding.DICTIONARY) && !condExpression.getColumnList().get(0)

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index 777f564..224a69f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -297,8 +297,7 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
             getDimensionDefaultValue(dimColumnEvaluatorInfo);
         continue;
       }
-      if (dimColumnEvaluatorInfo.getDimension().getDataType() != DataTypes.ARRAY
-          && dimColumnEvaluatorInfo.getDimension().getDataType() != DataTypes.STRUCT) {
+      if (!dimColumnEvaluatorInfo.getDimension().getDataType().isComplexType()) {
         if (!dimColumnEvaluatorInfo.isDimensionExistsInCurrentSilce()) {
           record[dimColumnEvaluatorInfo.getRowIndex()] =
               dimColumnEvaluatorInfo.getDimension().getDefaultValue();
@@ -534,8 +533,7 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
   @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
     for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) {
       DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = dimColEvaluatorInfoList.get(i);
-      if (dimColumnEvaluatorInfo.getDimension().getDataType() != DataTypes.ARRAY
-          && dimColumnEvaluatorInfo.getDimension().getDataType() != DataTypes.STRUCT) {
+      if (!dimColumnEvaluatorInfo.getDimension().getDataType().isComplexType()) {
         if (null == blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[i]]) {
           blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[i]] =
               blockChunkHolder.getDataBlock()

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
index bd78e08..d966802 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
@@ -167,11 +167,8 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf {
             metadata);
 
       } else if ((null != columnList.get(0).getDimension()) && (
-          columnList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY) && !(
-              columnList.get(0).getDimension().getDataType()
-                  == org.apache.carbondata.core.metadata.datatype.DataTypes.STRUCT
-                  || columnList.get(0).getDimension().getDataType()
-                  == org.apache.carbondata.core.metadata.datatype.DataTypes.ARRAY))) {
+          columnList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY) &&
+              ! columnList.get(0).getDimension().getDataType().isComplexType())) {
         dimColResolvedFilterInfo.setFilterValues(FilterUtil
             .getFilterListForAllValues(absoluteTableIdentifier, exp, columnList.get(0),
                 isIncludeFilter, tableProvider));

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
index aca2fd6..ea8bcb2 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
@@ -381,9 +381,9 @@ public abstract class AbstractDataFileFooterConverter {
       case TIMESTAMP:
         return DataTypes.TIMESTAMP;
       case ARRAY:
-        return DataTypes.ARRAY;
+        return DataTypes.createDefaultArrayType();
       case STRUCT:
-        return DataTypes.STRUCT;
+        return DataTypes.createDefaultStructType();
       default:
         return DataTypes.STRING;
     }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifierTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifierTest.java b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifierTest.java
index ec9a19a..bd8b6bc 100644
--- a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifierTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifierTest.java
@@ -49,17 +49,17 @@ public class DictionaryColumnUniqueIdentifierTest {
     ColumnIdentifier columnIdentifier2 = new ColumnIdentifier("1", properties, DataTypes.INT);
     dictionaryColumnUniqueIdentifier1 =
         new DictionaryColumnUniqueIdentifier(carbonTableIdentifier1, columnIdentifier,
-            DataTypes.MAP, null);
+            DataTypes.STRING, null);
     dictionaryColumnUniqueIdentifier2 =
         new DictionaryColumnUniqueIdentifier(carbonTableIdentifier2, columnIdentifier2,
-            DataTypes.MAP, null);
+            DataTypes.STRING, null);
     dictionaryColumnUniqueIdentifier3 =
         new DictionaryColumnUniqueIdentifier(carbonTableIdentifier2, columnIdentifier,
-            DataTypes.MAP, null);
+            DataTypes.STRING, null);
   }
 
   @Test public void testToGetDataType() {
-    assertEquals(dictionaryColumnUniqueIdentifier1.getDataType(), DataTypes.MAP);
+    assertEquals(dictionaryColumnUniqueIdentifier1.getDataType(), DataTypes.STRING);
   }
 
   @Test public void testForEqualsWithDifferentObjectsWithDifferentColumnIdentifier() {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTest.java
index 2020dd8..9c43553 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTest.java
@@ -296,7 +296,7 @@ public class SegmentPropertiesTest extends TestCase {
     dimColumn.setColumnar(true);
     dimColumn.setColumnName("IMEI5");
     dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
-    dimColumn.setDataType(DataTypes.ARRAY);
+    dimColumn.setDataType(DataTypes.createDefaultArrayType());
     dimColumn.setDimensionColumn(true);
     List<Encoding> encodeList =
         new ArrayList<Encoding>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTestUtil.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTestUtil.java b/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTestUtil.java
index 1f9c712..9572627 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTestUtil.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/block/SegmentPropertiesTestUtil.java
@@ -179,7 +179,7 @@ public class SegmentPropertiesTestUtil {
     dimColumn.setColumnar(true);
     dimColumn.setColumnName("IMEI5");
     dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
-    dimColumn.setDataType(DataTypes.ARRAY);
+    dimColumn.setDataType(DataTypes.createDefaultArrayType());
     dimColumn.setDimensionColumn(true);
     List<Encoding> encodeList =
         new ArrayList<Encoding>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactoryUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactoryUnitTest.java b/core/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactoryUnitTest.java
index c077eeb..2b73b86 100644
--- a/core/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactoryUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactoryUnitTest.java
@@ -37,7 +37,7 @@ public class DirectDictionaryKeyGeneratorFactoryUnitTest {
 
   @Test public void testGetDirectDictionaryGeneratorReturnNull() throws Exception {
     DirectDictionaryGenerator result =
-        DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(DataTypes.ARRAY);
+        DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(DataTypes.STRING);
     Assert.assertNull(result);
   }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java b/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
index 42c0ad6..b8e4eea 100644
--- a/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImplTest.java
@@ -23,8 +23,6 @@ import java.util.Map;
 
 import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.metadata.datatype.DataTypes;
-import org.apache.carbondata.core.metadata.datatype.DecimalType;
-import org.apache.carbondata.core.metadata.datatype.IntType;
 import org.apache.carbondata.core.metadata.encoder.Encoding;
 import org.apache.carbondata.core.metadata.schema.SchemaEvolution;
 import org.apache.carbondata.core.metadata.schema.SchemaEvolutionEntry;
@@ -1025,7 +1023,7 @@ public class ThriftWrapperSchemaConverterImplTest {
       }
 
       @Mock public DataType getDataType() {
-        return DataTypes.ARRAY;
+        return DataTypes.createDefaultArrayType();
       }
 
       @Mock public String getColumnName() {
@@ -1099,7 +1097,7 @@ public class ThriftWrapperSchemaConverterImplTest {
       }
 
       @Mock public DataType getDataType() {
-        return DataTypes.STRUCT;
+        return DataTypes.createDefaultStructType();
       }
 
       @Mock public String getColumnName() {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableWithComplexTypesTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableWithComplexTypesTest.java b/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableWithComplexTypesTest.java
index 4979038..0aed719 100644
--- a/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableWithComplexTypesTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/CarbonTableWithComplexTypesTest.java
@@ -83,7 +83,7 @@ public class CarbonTableWithComplexTypesTest extends TestCase {
     structColumn.setColumnar(true);
     structColumn.setColumnName("mobile");
     structColumn.setColumnUniqueId(UUID.randomUUID().toString());
-    structColumn.setDataType(DataTypes.STRUCT);
+    structColumn.setDataType(DataTypes.createDefaultStructType());
     structColumn.setDimensionColumn(true);
     structColumn.setEncodingList(encodeList);
     structColumn.setNumberOfChild(2);
@@ -103,7 +103,7 @@ public class CarbonTableWithComplexTypesTest extends TestCase {
     arrayColumn.setColumnar(true);
     arrayColumn.setColumnName("mobile.val");
     arrayColumn.setColumnUniqueId(UUID.randomUUID().toString());
-    arrayColumn.setDataType(DataTypes.ARRAY);
+    arrayColumn.setDataType(DataTypes.createDefaultArrayType());
     arrayColumn.setDimensionColumn(true);
     arrayColumn.setEncodingList(encodeList);
     arrayColumn.setNumberOfChild(1);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
index 56c59e1..f9b5ec8 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
@@ -515,7 +515,7 @@ public class CarbonUtilTest {
   }
 
   @Test public void testForHasComplexDataTypes() {
-    assertTrue(DataTypes.ARRAY.isComplexType());
+    assertTrue(DataTypes.createDefaultArrayType().isComplexType());
     assertTrue(!DataTypes.DATE.isComplexType());
   }
 
@@ -581,7 +581,7 @@ public class CarbonUtilTest {
     column1.setDimension(new CarbonDimension(column1Schema, 1, 1, 1, 1));
 
     column2Schema.setColumnName("Column2");
-    column2Schema.setDataType(DataTypes.ARRAY);
+    column2Schema.setDataType(DataTypes.createDefaultArrayType());
     column2.setDimension(new CarbonDimension(column2Schema, 1, 1, 1, 1));
 
     QueryDimension[] queryDimensions = { column1, column2 };

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparatorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparatorTest.java b/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparatorTest.java
index cff1291..0775a3d 100644
--- a/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparatorTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparatorTest.java
@@ -68,7 +68,7 @@ public class CarbonDictionarySortInfoPreparatorTest {
     };
 
     CarbonDictionarySortInfo carbonDictionarySortInfo = carbonDictionarySortInfoPreparator
-        .getDictionarySortInfo(newDistinctValues, dictionary, DataTypes.ARRAY);
+        .getDictionarySortInfo(newDistinctValues, dictionary, DataTypes.STRING);
     int expectedGetSortIndexValue = 1;
     int expectedGetSortInvertedIndexLength = 3;
     int actualGetSortIndexValue = carbonDictionarySortInfo.getSortIndex().get(0);
@@ -87,7 +87,7 @@ public class CarbonDictionarySortInfoPreparatorTest {
     newDistinctValues.add("xyz");
     Dictionary dictionary = null;
     CarbonDictionarySortInfo carbonDictionarySortInfo = carbonDictionarySortInfoPreparator
-        .getDictionarySortInfo(newDistinctValues, dictionary, DataTypes.ARRAY);
+        .getDictionarySortInfo(newDistinctValues, dictionary, DataTypes.createDefaultArrayType());
     int expectedGetSortIndexValue = 1;
     int expectedGetSortInvertedIndexLength = 2;
     int actualGetSortIndexValue = carbonDictionarySortInfo.getSortIndex().get(0);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamInputFormat.java
index b10bc8b..eb07f7e 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamInputFormat.java
@@ -59,10 +59,10 @@ public class CarbonStreamInputFormat extends FileInputFormat<Void, Object> {
     GenericQueryType[] queryTypes = new GenericQueryType[carbonColumns.length];
     for (int i = 0; i < carbonColumns.length; i++) {
       if (carbonColumns[i].isComplex()) {
-        if (carbonColumns[i].getDataType() == DataTypes.ARRAY) {
+        if (DataTypes.isArrayType(carbonColumns[i].getDataType())) {
           queryTypes[i] = new ArrayQueryType(carbonColumns[i].getColName(),
               carbonColumns[i].getColName(), i);
-        } else if (carbonColumns[i].getDataType() == DataTypes.STRUCT) {
+        } else if (DataTypes.isStructType(carbonColumns[i].getDataType())) {
           queryTypes[i] = new StructQueryType(carbonColumns[i].getColName(),
               carbonColumns[i].getColName(), i);
         } else {
@@ -84,11 +84,11 @@ public class CarbonStreamInputFormat extends FileInputFormat<Void, Object> {
       CarbonDimension child = dimension.getListOfChildDimensions().get(i);
       DataType dataType = child.getDataType();
       GenericQueryType queryType = null;
-      if (dataType == DataTypes.ARRAY) {
+      if (DataTypes.isArrayType(dataType)) {
         queryType =
             new ArrayQueryType(child.getColName(), dimension.getColName(), ++parentBlockIndex);
 
-      } else if (dataType == DataTypes.STRUCT) {
+      } else if (DataTypes.isStructType(dataType)) {
         queryType =
             new StructQueryType(child.getColName(), dimension.getColName(), ++parentBlockIndex);
         parentQueryType.addChildren(queryType);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java
index 1ff0fa7..4e3e6cf 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordReader.java
@@ -35,6 +35,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.datastore.block.SegmentProperties;
 import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
 import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.metadata.datatype.DataTypes;
 import org.apache.carbondata.core.metadata.encoder.Encoding;
 import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
@@ -75,12 +76,9 @@ import org.apache.spark.sql.catalyst.expressions.GenericInternalRow;
 import org.apache.spark.sql.execution.vectorized.ColumnVector;
 import org.apache.spark.sql.execution.vectorized.ColumnarBatch;
 import org.apache.spark.sql.types.CalendarIntervalType;
-import org.apache.spark.sql.types.DataType;
-import org.apache.spark.sql.types.DateType;
 import org.apache.spark.sql.types.Decimal;
 import org.apache.spark.sql.types.DecimalType;
 import org.apache.spark.sql.types.StructType;
-import org.apache.spark.sql.types.TimestampType;
 import org.apache.spark.unsafe.types.CalendarInterval;
 import org.apache.spark.unsafe.types.UTF8String;
 
@@ -95,7 +93,7 @@ public class CarbonStreamRecordReader extends RecordReader<Void, Object> {
   private CarbonTable carbonTable;
   private CarbonColumn[] storageColumns;
   private boolean[] isRequired;
-  private int[] measureDataTypes;
+  private DataType[] measureDataTypes;
   private int dimensionCount;
   private int measureCount;
 
@@ -171,9 +169,9 @@ public class CarbonStreamRecordReader extends RecordReader<Void, Object> {
             .getDirectDictionaryGenerator(storageColumns[i].getDataType());
       }
     }
-    measureDataTypes = new int[measureCount];
+    measureDataTypes = new DataType[measureCount];
     for (int i = 0; i < measureCount; i++) {
-      measureDataTypes[i] = storageColumns[dimensionCount + i].getDataType().getId();
+      measureDataTypes[i] = storageColumns[dimensionCount + i].getDataType();
     }
 
     // decode data
@@ -518,7 +516,7 @@ public class CarbonStreamRecordReader extends RecordReader<Void, Object> {
       }
     }
     // measure
-    int dataType;
+    DataType dataType;
     for (int msrCount = 0; msrCount < measureCount; msrCount++, colCount++) {
       if (nullBitSet.get(colCount)) {
         if (isFilterRequired[colCount]) {
@@ -529,7 +527,7 @@ public class CarbonStreamRecordReader extends RecordReader<Void, Object> {
         }
       } else {
         dataType = measureDataTypes[msrCount];
-        if (dataType == DataTypes.BOOLEAN_TYPE_ID) {
+        if (dataType == DataTypes.BOOLEAN) {
           if (isRequired[colCount]) {
             boolean v = input.readBoolean();
             if (isFilterRequired[colCount]) {
@@ -541,7 +539,7 @@ public class CarbonStreamRecordReader extends RecordReader<Void, Object> {
           } else {
             input.skipBytes(1);
           }
-        } else if (dataType == DataTypes.SHORT_TYPE_ID) {
+        } else if (dataType == DataTypes.SHORT) {
           if (isRequired[colCount]) {
             short v = input.readShort();
             if (isFilterRequired[colCount]) {
@@ -553,7 +551,7 @@ public class CarbonStreamRecordReader extends RecordReader<Void, Object> {
           } else {
             input.skipBytes(2);
           }
-        } else if (dataType == DataTypes.INT_TYPE_ID) {
+        } else if (dataType == DataTypes.INT) {
           if (isRequired[colCount]) {
             int v = input.readInt();
             if (isFilterRequired[colCount]) {
@@ -565,7 +563,7 @@ public class CarbonStreamRecordReader extends RecordReader<Void, Object> {
           } else {
             input.skipBytes(4);
           }
-        } else if (dataType == DataTypes.LONG_TYPE_ID) {
+        } else if (dataType == DataTypes.LONG) {
           if (isRequired[colCount]) {
             long v = input.readLong();
             if (isFilterRequired[colCount]) {
@@ -577,7 +575,7 @@ public class CarbonStreamRecordReader extends RecordReader<Void, Object> {
           } else {
             input.skipBytes(8);
           }
-        } else if (dataType == DataTypes.DOUBLE_TYPE_ID) {
+        } else if (dataType == DataTypes.DOUBLE) {
           if (isRequired[colCount]) {
             double v = input.readDouble();
             if (isFilterRequired[colCount]) {
@@ -589,7 +587,7 @@ public class CarbonStreamRecordReader extends RecordReader<Void, Object> {
           } else {
             input.skipBytes(8);
           }
-        } else if (dataType == DataTypes.DECIMAL_TYPE_ID) {
+        } else if (DataTypes.isDecimal(dataType)) {
           int len = input.readShort();
           if (isRequired[colCount]) {
             BigDecimal v = DataTypeUtil.byteToBigDecimal(input.readBytes(len));
@@ -611,7 +609,7 @@ public class CarbonStreamRecordReader extends RecordReader<Void, Object> {
     for (int i = 0; i < projection.length; i++) {
       Object value = outputValues[i];
       ColumnVector col = columnarBatch.column(i);
-      DataType t = col.dataType();
+      org.apache.spark.sql.types.DataType t = col.dataType();
       if (null == value) {
         col.putNull(rowId);
       } else {
@@ -632,7 +630,7 @@ public class CarbonStreamRecordReader extends RecordReader<Void, Object> {
         } else if (t == org.apache.spark.sql.types.DataTypes.StringType) {
           UTF8String v = (UTF8String) value;
           col.putByteArray(rowId, v.getBytes());
-        } else if (t instanceof DecimalType) {
+        } else if (t instanceof org.apache.spark.sql.types.DecimalType) {
           DecimalType dt = (DecimalType)t;
           Decimal d = (Decimal) value;
           if (dt.precision() <= Decimal.MAX_INT_DIGITS()) {
@@ -648,9 +646,9 @@ public class CarbonStreamRecordReader extends RecordReader<Void, Object> {
           CalendarInterval c = (CalendarInterval) value;
           col.getChildColumn(0).putInt(rowId, c.months);
           col.getChildColumn(1).putLong(rowId, c.microseconds);
-        } else if (t instanceof DateType) {
+        } else if (t instanceof org.apache.spark.sql.types.DateType) {
           col.putInt(rowId, (int) value);
-        } else if (t instanceof TimestampType) {
+        } else if (t instanceof org.apache.spark.sql.types.TimestampType) {
           col.putLong(rowId, (long) value);
         }
       }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
index 8d7a2e3..3c4b55c 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/streaming/CarbonStreamRecordWriter.java
@@ -30,6 +30,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
 import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
 import org.apache.carbondata.core.datastore.impl.FileFactory;
 import org.apache.carbondata.core.datastore.row.CarbonRow;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.metadata.datatype.DataTypes;
 import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
 import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
@@ -83,7 +84,7 @@ public class CarbonStreamRecordWriter extends RecordWriter<Void, Object> {
   private boolean[] isNoDictionaryDimensionColumn;
   private int dimensionWithComplexCount;
   private int measureCount;
-  private int[] measureDataTypes;
+  private DataType[] measureDataTypes;
   private StreamBlockletWriter output = null;
 
   // data write
@@ -129,10 +130,10 @@ public class CarbonStreamRecordWriter extends RecordWriter<Void, Object> {
     dimensionWithComplexCount = configuration.getDimensionCount();
     measureCount = configuration.getMeasureCount();
     dataFields = configuration.getDataFields();
-    measureDataTypes = new int[measureCount];
+    measureDataTypes = new DataType[measureCount];
     for (int i = 0; i < measureCount; i++) {
       measureDataTypes[i] =
-          dataFields[dimensionWithComplexCount + i].getColumn().getDataType().getId();
+          dataFields[dimensionWithComplexCount + i].getColumn().getDataType();
     }
 
     // initialize parser and converter
@@ -211,22 +212,22 @@ public class CarbonStreamRecordWriter extends RecordWriter<Void, Object> {
       }
     }
     // measure
-    int dataType;
+    DataType dataType;
     for (int msrCount = 0; msrCount < measureCount; msrCount++) {
       columnValue = currentRow.getObject(dimCount + msrCount);
       if (null != columnValue) {
         dataType = measureDataTypes[msrCount];
-        if (dataType == DataTypes.BOOLEAN_TYPE_ID) {
+        if (dataType == DataTypes.BOOLEAN) {
           output.writeBoolean((boolean) columnValue);
-        } else if (dataType == DataTypes.SHORT_TYPE_ID) {
+        } else if (dataType == DataTypes.SHORT) {
           output.writeShort((short) columnValue);
-        } else if (dataType == DataTypes.INT_TYPE_ID) {
+        } else if (dataType == DataTypes.INT) {
           output.writeInt((int) columnValue);
-        } else if (dataType == DataTypes.LONG_TYPE_ID) {
+        } else if (dataType == DataTypes.LONG) {
           output.writeLong((long) columnValue);
-        } else if (dataType == DataTypes.DOUBLE_TYPE_ID) {
+        } else if (dataType == DataTypes.DOUBLE) {
           output.writeDouble((double) columnValue);
-        } else if (dataType == DataTypes.DECIMAL_TYPE_ID) {
+        } else if (DataTypes.isDecimal(dataType)) {
           BigDecimal val = (BigDecimal) columnValue;
           byte[] bigDecimalInBytes = DataTypeUtil.bigDecimalToByte(val);
           output.writeShort(bigDecimalInBytes.length);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
----------------------------------------------------------------------
diff --git a/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java b/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
index 05ad93a..2a19271 100644
--- a/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
+++ b/integration/hive/src/main/java/org/apache/carbondata/hive/CarbonDictionaryDecodeReadSupport.java
@@ -137,9 +137,9 @@ public class CarbonDictionaryDecodeReadSupport<T> implements CarbonReadSupport<T
    */
   private Writable createWritableObject(Object obj, CarbonColumn carbonColumn) throws IOException {
     DataType dataType = carbonColumn.getDataType();
-    if (dataType == DataTypes.STRUCT) {
+    if (DataTypes.isStructType(dataType)) {
       return createStruct(obj, carbonColumn);
-    } else if (dataType == DataTypes.ARRAY) {
+    } else if (DataTypes.isArrayType(dataType)) {
       return createArray(obj, carbonColumn);
     } else {
       return createWritablePrimitive(obj, carbonColumn);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
index 2043ecf..7abb737 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
@@ -19,13 +19,14 @@ package org.apache.carbondata.spark.util
 
 import java.nio.charset.Charset
 import java.text.SimpleDateFormat
+import java.util
 
 import org.apache.spark.sql._
 import org.apache.spark.sql.execution.command.DataTypeInfo
 import org.apache.spark.sql.types._
 
 import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.metadata.datatype.{DataType => CarbonDataType, DataTypes => CarbonDataTypes, DecimalType => CarbonDecimalType}
+import org.apache.carbondata.core.metadata.datatype.{DataType => CarbonDataType, DataTypes => CarbonDataTypes, DecimalType => CarbonDecimalType, StructField => CarbonStructField}
 import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn
 
 object CarbonScalaUtil {
@@ -40,8 +41,17 @@ object CarbonScalaUtil {
       case DateType => CarbonDataTypes.DATE
       case BooleanType => CarbonDataTypes.BOOLEAN
       case TimestampType => CarbonDataTypes.TIMESTAMP
-      case ArrayType(_, _) => CarbonDataTypes.ARRAY
-      case StructType(_) => CarbonDataTypes.STRUCT
+      case ArrayType(elementType, _) =>
+        CarbonDataTypes.createArrayType(CarbonScalaUtil.convertSparkToCarbonDataType(elementType))
+      case StructType(fields) =>
+        val carbonFields = new util.ArrayList[CarbonStructField]
+        fields.map { field =>
+          carbonFields.add(
+            new CarbonStructField(
+              field.name,
+              CarbonScalaUtil.convertSparkToCarbonDataType(field.dataType)))
+        }
+        CarbonDataTypes.createStructType(carbonFields)
       case NullType => CarbonDataTypes.NULL
       case decimal: DecimalType =>
         CarbonDataTypes.createDecimalType(decimal.precision, decimal.scale)

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
index 0460e30..6cf7298 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataTypeConverterUtil.scala
@@ -41,8 +41,8 @@ object DataTypeConverterUtil {
       case FIXED_DECIMAL(_, _) => DataTypes.createDefaultDecimalType
       case "timestamp" => DataTypes.TIMESTAMP
       case "date" => DataTypes.DATE
-      case "array" => DataTypes.ARRAY
-      case "struct" => DataTypes.STRUCT
+      case "array" => DataTypes.createDefaultArrayType
+      case "struct" => DataTypes.createDefaultStructType
       case _ => convertToCarbonTypeForSpark2(dataType)
     }
   }
@@ -66,9 +66,9 @@ object DataTypeConverterUtil {
       case "datetype" => DataTypes.DATE
       case others =>
         if (others != null && others.startsWith("arraytype")) {
-          DataTypes.ARRAY
+          DataTypes.createDefaultArrayType()
         } else if (others != null && others.startsWith("structtype")) {
-          DataTypes.STRUCT
+          DataTypes.createDefaultStructType()
         } else if (others != null && others.startsWith("char")) {
           DataTypes.STRING
         } else if (others != null && others.startsWith("varchar")) {
@@ -82,6 +82,10 @@ object DataTypeConverterUtil {
   def convertToString(dataType: DataType): String = {
     if (DataTypes.isDecimal(dataType)) {
       "decimal"
+    } else if (DataTypes.isArrayType(dataType)) {
+      "array"
+    } else if (DataTypes.isStructType(dataType)) {
+      "struct"
     } else {
       dataType match {
         case DataTypes.BOOLEAN => "boolean"
@@ -93,8 +97,6 @@ object DataTypeConverterUtil {
         case DataTypes.FLOAT => "double"
         case DataTypes.TIMESTAMP => "timestamp"
         case DataTypes.DATE => "date"
-        case DataTypes.ARRAY => "array"
-        case DataTypes.STRUCT => "struct"
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
index ddc4763..840e8ae 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
@@ -259,17 +259,16 @@ object GlobalDictionaryUtil {
       case None =>
         None
       case Some(dim) =>
-        dim.getDataType match {
-          case DataTypes.ARRAY =>
-            val arrDim = ArrayParser(dim, format)
-            generateParserForChildrenDimension(dim, format, mapColumnValuesWithId, arrDim)
-            Some(arrDim)
-          case DataTypes.STRUCT =>
-            val stuDim = StructParser(dim, format)
-            generateParserForChildrenDimension(dim, format, mapColumnValuesWithId, stuDim)
-            Some(stuDim)
-          case _ =>
-            Some(PrimitiveParser(dim, mapColumnValuesWithId.get(dim.getColumnId)))
+        if (DataTypes.isArrayType(dim.getDataType)) {
+          val arrDim = ArrayParser(dim, format)
+          generateParserForChildrenDimension(dim, format, mapColumnValuesWithId, arrDim)
+          Some(arrDim)
+        } else if (DataTypes.isStructType(dim.getDataType)) {
+          val stuDim = StructParser(dim, format)
+          generateParserForChildrenDimension(dim, format, mapColumnValuesWithId, stuDim)
+          Some(stuDim)
+        } else {
+          Some(PrimitiveParser(dim, mapColumnValuesWithId.get(dim.getColumnId)))
         }
     }
   }
@@ -477,14 +476,14 @@ object GlobalDictionaryUtil {
       val children = preDictDimension.getListOfChildDimensions.asScala.toArray
       // for Array, user set ArrayFiled: path, while ArrayField has a child Array.val
       val currentColName = {
-        preDictDimension.getDataType match {
-          case DataTypes.ARRAY =>
-            if (children(0).isComplex) {
-              "val." + colName.substring(middleDimName.length + 1)
-            } else {
-              "val"
-            }
-          case _ => colName.substring(middleDimName.length + 1)
+        if (DataTypes.isArrayType(preDictDimension.getDataType)) {
+          if (children(0).isComplex) {
+            "val." + colName.substring(middleDimName.length + 1)
+          } else {
+            "val"
+          }
+        } else {
+          colName.substring(middleDimName.length + 1)
         }
       }
       setPredefineDict(carbonLoadModel, children, table, currentColName,

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
index 91c07de..18f76d1 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
@@ -392,6 +392,12 @@ object CarbonDictionaryDecoder {
       } else {
         DecimalType(precision, scale)
       }
+    } else if (CarbonDataTypes.isArrayType(carbonDimension.getDataType)) {
+      CarbonMetastoreTypes
+        .toDataType(s"array<${ relation.getArrayChildren(carbonDimension.getColName) }>")
+    } else if (CarbonDataTypes.isStructType(carbonDimension.getDataType)) {
+      CarbonMetastoreTypes
+        .toDataType(s"struct<${ relation.getStructChildren(carbonDimension.getColName) }>")
     } else {
       carbonDimension.getDataType match {
         case CarbonDataTypes.STRING => StringType
@@ -402,12 +408,6 @@ object CarbonDictionaryDecoder {
         case CarbonDataTypes.BOOLEAN => BooleanType
         case CarbonDataTypes.TIMESTAMP => TimestampType
         case CarbonDataTypes.DATE => DateType
-        case CarbonDataTypes.STRUCT =>
-          CarbonMetastoreTypes
-            .toDataType(s"struct<${ relation.getStructChildren(carbonDimension.getColName) }>")
-        case CarbonDataTypes.ARRAY =>
-          CarbonMetastoreTypes
-            .toDataType(s"array<${ relation.getArrayChildren(carbonDimension.getColName) }>")
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/933e30cc/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/FieldEncoderFactory.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/FieldEncoderFactory.java b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/FieldEncoderFactory.java
index 193d3e6..5dbd177 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/FieldEncoderFactory.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/converter/impl/FieldEncoderFactory.java
@@ -111,7 +111,7 @@ public class FieldEncoderFactory {
       CarbonTableIdentifier carbonTableIdentifier, DictionaryClient client, Boolean useOnePass,
       String storePath, Map<Object, Integer> localCache) {
     DataType dataType = carbonColumn.getDataType();
-    if (dataType == DataTypes.ARRAY) {
+    if (DataTypes.isArrayType(dataType)) {
       List<CarbonDimension> listOfChildDimensions =
           ((CarbonDimension) carbonColumn).getListOfChildDimensions();
       // Create array parser with complex delimiter
@@ -123,7 +123,7 @@ public class FieldEncoderFactory {
                 client, useOnePass, storePath, localCache));
       }
       return arrayDataType;
-    } else if (dataType == DataTypes.STRUCT) {
+    } else if (DataTypes.isStructType(dataType)) {
       List<CarbonDimension> dimensions =
           ((CarbonDimension) carbonColumn).getListOfChildDimensions();
       // Create struct parser with complex delimiter
@@ -135,7 +135,7 @@ public class FieldEncoderFactory {
                 client, useOnePass, storePath, localCache));
       }
       return structDataType;
-    } else if (dataType == DataTypes.MAP) {
+    } else if (DataTypes.isMapType(dataType)) {
       throw new UnsupportedOperationException("Complex type Map is not supported yet");
     } else {
       return new PrimitiveDataType(carbonColumn.getColName(), parentName,


Mime
View raw message