carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject carbondata git commit: [CARBONDATA-1978] Handled preaggregate issues with hive metastore
Date Fri, 12 Jan 2018 18:15:59 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master 6d77fdff4 -> df278312c


[CARBONDATA-1978] Handled preaggregate issues with hive metastore

During creation of TableInfo from hivemetastore the DataMapSchemas and the columns DataTypes
are not converted to the appropriate child classes due to which data types not supported exception
is thrown

This closes #1758


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/df278312
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/df278312
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/df278312

Branch: refs/heads/master
Commit: df278312c32ee9e377eced1f7d70fcec9c9248ba
Parents: 6d77fdf
Author: kunal642 <kunalkapoor642@gmail.com>
Authored: Wed Jan 3 19:17:53 2018 +0530
Committer: ravipesala <ravi.pesala@gmail.com>
Committed: Fri Jan 12 23:45:46 2018 +0530

----------------------------------------------------------------------
 .../ThriftWrapperSchemaConverterImpl.java       |  5 +-
 .../core/metadata/schema/table/CarbonTable.java | 34 +++++++++++
 .../carbondata/core/util/DataTypeUtil.java      | 61 ++++++++++++++++++--
 .../dictionary/client/DictionaryClientTest.java |  4 ++
 ...ncrementalColumnDictionaryGeneratorTest.java |  3 +
 .../ServerDictionaryGeneratorTest.java          |  3 +
 .../generator/TableDictionaryGeneratorTest.java |  3 +
 core/src/test/resources/sampleCSV.csv           |  1 -
 .../preaggregate/TestPreAggregateLoad.scala     |  9 +++
 .../testsuite/datamap/TestDataMapCommand.scala  | 22 +++++++
 10 files changed, 137 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/df278312/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
index 2b8cfa5..e9c5505 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/converter/ThriftWrapperSchemaConverterImpl.java
@@ -32,7 +32,6 @@ import org.apache.carbondata.core.metadata.schema.SchemaEvolutionEntry;
 import org.apache.carbondata.core.metadata.schema.partition.PartitionType;
 import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
 import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
-import org.apache.carbondata.core.metadata.schema.table.DataMapSchemaFactory;
 import org.apache.carbondata.core.metadata.schema.table.RelationIdentifier;
 import org.apache.carbondata.core.metadata.schema.table.TableInfo;
 import org.apache.carbondata.core.metadata.schema.table.TableSchema;
@@ -646,8 +645,8 @@ public class ThriftWrapperSchemaConverterImpl implements SchemaConverter
{
 
   @Override public DataMapSchema fromExternalToWrapperDataMapSchema(
       org.apache.carbondata.format.DataMapSchema thriftDataMapSchema) {
-    DataMapSchema childSchema = DataMapSchemaFactory.INSTANCE
-        .getDataMapSchema(thriftDataMapSchema.getDataMapName(), thriftDataMapSchema.getClassName());
+    DataMapSchema childSchema = new DataMapSchema(thriftDataMapSchema.getDataMapName(),
+        thriftDataMapSchema.getClassName());
     childSchema.setProperties(thriftDataMapSchema.getProperties());
     if (null != thriftDataMapSchema.getRelationIdentifire()) {
       RelationIdentifier relationIdentifier =

http://git-wip-us.apache.org/repos/asf/carbondata/blob/df278312/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index 5e07fc0..e27b126 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -34,6 +34,7 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonImplicitDim
 import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
 import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
 import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
 import org.apache.carbondata.core.util.path.CarbonTablePath;
 
 /**
@@ -133,10 +134,43 @@ public class CarbonTable implements Serializable {
   }
 
   /**
+   * During creation of TableInfo from hivemetastore the DataMapSchemas and the columns
+   * DataTypes are not converted to the appropriate child classes.
+   *
+   * This method will cast the same to the appropriate classes
+   *
+   * @param tableInfo
+   */
+  private static void updateTableInfo(TableInfo tableInfo) {
+    List<DataMapSchema> dataMapSchemas = new ArrayList<>();
+    for (DataMapSchema dataMapSchema : tableInfo.getDataMapSchemaList()) {
+      DataMapSchema newDataMapSchema = DataMapSchemaFactory.INSTANCE
+          .getDataMapSchema(dataMapSchema.getDataMapName(), dataMapSchema.getClassName());
+      newDataMapSchema.setChildSchema(dataMapSchema.getChildSchema());
+      newDataMapSchema.setProperties(dataMapSchema.getProperties());
+      newDataMapSchema.setRelationIdentifier(dataMapSchema.getRelationIdentifier());
+      dataMapSchemas.add(newDataMapSchema);
+    }
+    tableInfo.setDataMapSchemaList(dataMapSchemas);
+    for (ColumnSchema columnSchema : tableInfo.getFactTable().getListOfColumns()) {
+      columnSchema.setDataType(DataTypeUtil.valueOf(columnSchema.getDataType(),
+          columnSchema.getPrecision(), columnSchema.getScale()));
+    }
+    if (tableInfo.getFactTable().getBucketingInfo() != null) {
+      for (ColumnSchema columnSchema : tableInfo.getFactTable()
+          .getBucketingInfo().getListOfColumns()) {
+        columnSchema.setDataType(DataTypeUtil.valueOf(columnSchema.getDataType(),
+            columnSchema.getPrecision(), columnSchema.getScale()));
+      }
+    }
+  }
+
+  /**
    * @param tableInfo
    */
   public static CarbonTable buildFromTableInfo(TableInfo tableInfo) {
     CarbonTable table = new CarbonTable();
+    updateTableInfo(tableInfo);
     table.tableInfo = tableInfo;
     table.blockSize = tableInfo.getTableBlockSizeInMB();
     table.tableLastUpdatedTime = tableInfo.getLastUpdatedTime();

http://git-wip-us.apache.org/repos/asf/carbondata/blob/df278312/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index 5d188b5..6d224cf 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -735,10 +735,6 @@ public final class DataTypeUtil {
     return converter;
   }
 
-  /**
-   * @param name datatype string extracted from the json data
-   * @return returns the datatype based on the input string from json to deserialize the
tableInfo
-   */
   public static DataType valueOf(String name) {
     if (DataTypes.STRING.getName().equalsIgnoreCase(name)) {
       return DataTypes.STRING;
@@ -770,12 +766,69 @@ public final class DataTypeUtil {
       return DataTypes.BYTE_ARRAY;
     } else if (DataTypes.BYTE_ARRAY.getName().equalsIgnoreCase(name)) {
       return DataTypes.BYTE_ARRAY;
+    } else if (name.equalsIgnoreCase("decimal")) {
+      return DataTypes.createDefaultDecimalType();
+    } else if (name.equalsIgnoreCase("array")) {
+      return DataTypes.createDefaultArrayType();
+    } else if (name.equalsIgnoreCase("struct")) {
+      return DataTypes.createDefaultStructType();
+    } else if (name.equalsIgnoreCase("map")) {
+      return DataTypes.createDefaultMapType();
     } else {
       throw new RuntimeException("create DataType with invalid name: " + name);
     }
   }
 
   /**
+   * @param dataType extracted from the json data
+   * @return returns the datatype based on the input string from json to deserialize the
tableInfo
+   */
+  public static DataType valueOf(DataType dataType, int precision, int scale) {
+    if (DataTypes.STRING.getName().equalsIgnoreCase(dataType.getName())) {
+      return DataTypes.STRING;
+    } else if (DataTypes.DATE.getName().equalsIgnoreCase(dataType.getName())) {
+      return DataTypes.DATE;
+    } else if (DataTypes.TIMESTAMP.getName().equalsIgnoreCase(dataType.getName())) {
+      return DataTypes.TIMESTAMP;
+    } else if (DataTypes.BOOLEAN.getName().equalsIgnoreCase(dataType.getName())) {
+      return DataTypes.BOOLEAN;
+    } else if (DataTypes.BYTE.getName().equalsIgnoreCase(dataType.getName())) {
+      return DataTypes.BYTE;
+    } else if (DataTypes.SHORT.getName().equalsIgnoreCase(dataType.getName())) {
+      return DataTypes.SHORT;
+    } else if (DataTypes.SHORT_INT.getName().equalsIgnoreCase(dataType.getName())) {
+      return DataTypes.SHORT_INT;
+    } else if (DataTypes.INT.getName().equalsIgnoreCase(dataType.getName())) {
+      return DataTypes.INT;
+    } else if (DataTypes.LONG.getName().equalsIgnoreCase(dataType.getName())) {
+      return DataTypes.LONG;
+    } else if (DataTypes.LEGACY_LONG.getName().equalsIgnoreCase(dataType.getName())) {
+      return DataTypes.LEGACY_LONG;
+    } else if (DataTypes.FLOAT.getName().equalsIgnoreCase(dataType.getName())) {
+      return DataTypes.FLOAT;
+    } else if (DataTypes.DOUBLE.getName().equalsIgnoreCase(dataType.getName())) {
+      return DataTypes.DOUBLE;
+    } else if (DataTypes.NULL.getName().equalsIgnoreCase(dataType.getName())) {
+      return DataTypes.NULL;
+    } else if (DataTypes.BYTE_ARRAY.getName().equalsIgnoreCase(dataType.getName())) {
+      return DataTypes.BYTE_ARRAY;
+    } else if (DataTypes.BYTE_ARRAY.getName().equalsIgnoreCase(dataType.getName())) {
+      return DataTypes.BYTE_ARRAY;
+    } else if (dataType.getName().equalsIgnoreCase("decimal")) {
+      return DataTypes.createDecimalType(precision, scale);
+    } else if (dataType.getName().equalsIgnoreCase("array")) {
+      return DataTypes.createDefaultArrayType();
+    } else if (dataType.getName().equalsIgnoreCase("struct")) {
+      return DataTypes.createDefaultStructType();
+    } else if (dataType.getName().equalsIgnoreCase("map")) {
+      return DataTypes.createDefaultMapType();
+    } else {
+      throw new RuntimeException(
+          "create DataType with invalid dataType.getName(): " + dataType.getName());
+    }
+  }
+
+  /**
    * Method to type case the data based on modified data type. This method will used for
    * retrieving the data after change in data type restructure operation
    *

http://git-wip-us.apache.org/repos/asf/carbondata/blob/df278312/core/src/test/java/org/apache/carbondata/core/dictionary/client/DictionaryClientTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/dictionary/client/DictionaryClientTest.java
b/core/src/test/java/org/apache/carbondata/core/dictionary/client/DictionaryClientTest.java
index dbd29e5..4847177 100644
--- a/core/src/test/java/org/apache/carbondata/core/dictionary/client/DictionaryClientTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/dictionary/client/DictionaryClientTest.java
@@ -28,6 +28,8 @@ import org.apache.carbondata.core.dictionary.generator.key.DictionaryMessageType
 import org.apache.carbondata.core.dictionary.server.DictionaryServer;
 import org.apache.carbondata.core.dictionary.server.NonSecureDictionaryServer;
 import org.apache.carbondata.core.metadata.CarbonMetadata;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
 import org.apache.carbondata.core.metadata.encoder.Encoding;
 import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
 import org.apache.carbondata.core.metadata.schema.table.TableInfo;
@@ -68,11 +70,13 @@ public class DictionaryClientTest {
     empColumnSchema.setColumnName("empNameCol");
     empColumnSchema.setColumnUniqueId("empNameCol");
     empColumnSchema.setDimensionColumn(true);
+    empColumnSchema.setDataType(DataTypes.STRING);
     empColumnSchema.setEncodingList(Arrays.asList(Encoding.DICTIONARY));
     empDimension = new CarbonDimension(empColumnSchema, 0, 0, 0, 0, 0);
 
     ageColumnSchema = new ColumnSchema();
     ageColumnSchema.setColumnName("ageNameCol");
+    ageColumnSchema.setDataType(DataTypes.SHORT_INT);
     ageColumnSchema.setColumnUniqueId("ageNameCol");
     ageColumnSchema.setDimensionColumn(true);
     ageColumnSchema.setEncodingList(Arrays.asList(Encoding.DICTIONARY));

http://git-wip-us.apache.org/repos/asf/carbondata/blob/df278312/core/src/test/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGeneratorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGeneratorTest.java
b/core/src/test/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGeneratorTest.java
index 414a350..2d31fa5 100644
--- a/core/src/test/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGeneratorTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGeneratorTest.java
@@ -22,6 +22,7 @@ import java.util.Arrays;
 
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.metadata.CarbonMetadata;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
 import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
 import org.apache.carbondata.core.metadata.schema.table.TableInfo;
 import org.apache.carbondata.core.metadata.schema.table.TableSchema;
@@ -50,6 +51,7 @@ public class IncrementalColumnDictionaryGeneratorTest {
         .addProperty(CarbonCommonConstants.CARBON_MAX_DRIVER_LRU_CACHE_SIZE, "10");
     ColumnSchema columnSchema = new ColumnSchema();
     columnSchema.setColumnName("empName");
+    columnSchema.setDataType(DataTypes.STRING);
     TableSchema tableSchema = new TableSchema();
     tableSchema.setTableName("TestTable");
     tableSchema.setListOfColumns(Arrays.asList(columnSchema));
@@ -131,6 +133,7 @@ public class IncrementalColumnDictionaryGeneratorTest {
     //Create required column schema
     ColumnSchema columnSchema = new ColumnSchema();
     columnSchema.setColumnName("empNameCol");
+    columnSchema.setDataType(DataTypes.STRING);
     columnSchema.setColumnUniqueId("empNameCol");
     CarbonDimension carbonDimension = new CarbonDimension(columnSchema, 0, 0, 0, 0, 0);
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/df278312/core/src/test/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGeneratorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGeneratorTest.java
b/core/src/test/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGeneratorTest.java
index 95d73a9..2564ead 100644
--- a/core/src/test/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGeneratorTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGeneratorTest.java
@@ -23,6 +23,7 @@ import java.util.Arrays;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.dictionary.generator.key.DictionaryMessage;
 import org.apache.carbondata.core.metadata.CarbonMetadata;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
 import org.apache.carbondata.core.metadata.encoder.Encoding;
 import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
 import org.apache.carbondata.core.metadata.schema.table.TableInfo;
@@ -60,6 +61,7 @@ public class ServerDictionaryGeneratorTest {
     // Create two column schemas and dimensions for the table
     empColumnSchema = new ColumnSchema();
     empColumnSchema.setColumnName("empNameCol");
+    empColumnSchema.setDataType(DataTypes.STRING);
     empColumnSchema.setColumnUniqueId("empNameCol");
     empColumnSchema.setDimensionColumn(true);
     empColumnSchema.setEncodingList(Arrays.asList(Encoding.DICTIONARY));
@@ -68,6 +70,7 @@ public class ServerDictionaryGeneratorTest {
     ageColumnSchema = new ColumnSchema();
     ageColumnSchema.setColumnName("empNameCol");
     ageColumnSchema.setColumnUniqueId("empNameCol");
+    ageColumnSchema.setDataType(DataTypes.SHORT_INT);
     ageColumnSchema.setDimensionColumn(true);
     ageColumnSchema.setEncodingList(Arrays.asList(Encoding.DICTIONARY));
     ageDimension = new CarbonDimension(ageColumnSchema, 0, 0, 0, 0, 0);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/df278312/core/src/test/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGeneratorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGeneratorTest.java
b/core/src/test/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGeneratorTest.java
index 1ea77f8..e40bdad 100644
--- a/core/src/test/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGeneratorTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGeneratorTest.java
@@ -23,6 +23,7 @@ import java.util.Arrays;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.dictionary.generator.key.DictionaryMessage;
 import org.apache.carbondata.core.metadata.CarbonMetadata;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
 import org.apache.carbondata.core.metadata.encoder.Encoding;
 import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
 import org.apache.carbondata.core.metadata.schema.table.TableInfo;
@@ -62,10 +63,12 @@ public class TableDictionaryGeneratorTest {
     empColumnSchema.setColumnName("empNameCol");
     empColumnSchema.setColumnUniqueId("empNameCol");
     empColumnSchema.setDimensionColumn(true);
+    empColumnSchema.setDataType(DataTypes.STRING);
     empColumnSchema.setEncodingList(Arrays.asList(Encoding.DICTIONARY));
     empDimension = new CarbonDimension(empColumnSchema, 0, 0, 0, 0, 0);
 
     ageColumnSchema = new ColumnSchema();
+    ageColumnSchema.setDataType(DataTypes.SHORT_INT);
     ageColumnSchema.setColumnName("ageNameCol");
     ageColumnSchema.setColumnUniqueId("ageNameCol");
     ageColumnSchema.setDimensionColumn(true);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/df278312/core/src/test/resources/sampleCSV.csv
----------------------------------------------------------------------
diff --git a/core/src/test/resources/sampleCSV.csv b/core/src/test/resources/sampleCSV.csv
deleted file mode 100644
index 79dfd50..0000000
--- a/core/src/test/resources/sampleCSV.csv
+++ /dev/null
@@ -1 +0,0 @@
-id,name
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/carbondata/blob/df278312/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateLoad.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateLoad.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateLoad.scala
index fb5f81d..3d0e9b9 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateLoad.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateLoad.scala
@@ -168,6 +168,15 @@ class TestPreAggregateLoad extends QueryTest with BeforeAndAfterAll {
   }
 
   test("test to check if exception is thrown for direct load on pre-aggregate table") {
+    sql("drop table if exists maintable")
+    sql(
+      """
+        | CREATE TABLE maintable(id int, name string, city string, age int)
+        | STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('dictionary_include'='id')
+      """.stripMargin)
+    sql(
+      s"""create datamap preagg_sum on table maintable using 'preaggregate' as select id,sum(age)
from maintable group by id"""
+        .stripMargin)
     assert(intercept[RuntimeException] {
       sql(s"insert into maintable_preagg_sum values(1, 30)")
     }.getMessage.equalsIgnoreCase("Cannot insert/load data directly into pre-aggregate table"))

http://git-wip-us.apache.org/repos/asf/carbondata/blob/df278312/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
index c38e6cf..d0a342b 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
@@ -17,6 +17,7 @@
 
 package org.apache.carbondata.spark.testsuite.datamap
 
+import org.apache.spark.sql.Row
 import org.apache.spark.sql.test.util.QueryTest
 import org.scalatest.BeforeAndAfterAll
 
@@ -26,6 +27,8 @@ import org.apache.carbondata.core.util.CarbonProperties
 
 class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
 
+  val testData = s"$resourcesPath/sample.csv"
+
   override def beforeAll {
     sql("drop table if exists datamaptest")
     sql("drop table if exists datamapshowtest")
@@ -188,8 +191,27 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
     checkExistence(frame, true, "datamap2", "(NA)", "new.class")
   }
 
+  test("test if preaggregate load is successfull for hivemetastore") {
+    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
"true")
+    sql("DROP TABLE IF EXISTS maintable")
+    sql(
+      """
+        | CREATE TABLE maintable(id int, name string, city string, age int)
+        | STORED BY 'org.apache.carbondata.format'
+      """.stripMargin)
+    sql(
+      s"""create datamap preagg_sum on table maintable using 'preaggregate' as select id,sum(age)
from maintable group by id"""
+        .stripMargin)
+    sql(s"LOAD DATA LOCAL INPATH '$testData' into table maintable")
+    checkAnswer(sql(s"select * from maintable_preagg_sum"),
+      Seq(Row(1, 31), Row(2, 27), Row(3, 70), Row(4, 55)))
+  }
+
 
   override def afterAll {
+    sql("DROP TABLE IF EXISTS maintable")
+    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE,
+      CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE_DEFAULT)
     sql("drop table if exists datamaptest")
     sql("drop table if exists datamapshowtest")
   }


Mime
View raw message