carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [1/2] incubator-carbondata git commit: supported date type for dictionary_exclude
Date Tue, 17 Jan 2017 20:05:03 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 050d05118 -> 33c9a8165


supported date type for dictionary_exclude

removed java style errors

added validation for both date and timestamp

added test case for validation of both timestamp and date

refactor test cases

removed code from unnecessary file

rebased the code

added validation in table creator as well


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/80eb1fdb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/80eb1fdb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/80eb1fdb

Branch: refs/heads/master
Commit: 80eb1fdb48f31da46c1d1adde4e6b84e4ee366f6
Parents: 050d051
Author: anubhav100 <anubhav.tarar@knoldus.in>
Authored: Mon Jan 16 16:04:43 2017 +0530
Committer: ravipesala <ravi.pesala@gmail.com>
Committed: Wed Jan 18 01:32:40 2017 +0530

----------------------------------------------------------------------
 .../allqueries/TestQueryWithoutDataLoad.scala   |  2 +-
 .../spark/testsuite/datetype/DateTypeTest.scala | 73 ++++++++++++++++++++
 .../spark/sql/catalyst/CarbonDDLSqlParser.scala |  8 +--
 .../org/apache/spark/sql/TableCreator.scala     |  8 +--
 4 files changed, 82 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/80eb1fdb/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/TestQueryWithoutDataLoad.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/TestQueryWithoutDataLoad.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/TestQueryWithoutDataLoad.scala
index 3036ffc..8fddf84 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/TestQueryWithoutDataLoad.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/TestQueryWithoutDataLoad.scala
@@ -31,7 +31,7 @@ class TestQueryWithoutDataLoad extends QueryTest with BeforeAndAfterAll
{
     sql("""
         CREATE TABLE no_load(imei string, age int, productdate timestamp, gamePointId double)
         STORED BY 'org.apache.carbondata.format'
-        TBLPROPERTIES('DICTIONARY_EXCLUDE'='productdate', 'DICTIONARY_INCLUDE'='gamePointId')
+        TBLPROPERTIES('DICTIONARY_INCLUDE'='gamePointId')
       """)
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/80eb1fdb/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datetype/DateTypeTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datetype/DateTypeTest.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datetype/DateTypeTest.scala
new file mode 100644
index 0000000..53b138e
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datetype/DateTypeTest.scala
@@ -0,0 +1,73 @@
+package org.apache.carbondata.spark.testsuite.datetype
+
+import org.apache.spark.sql.common.util.QueryTest
+
+import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
+
+
+class DateTypeTest extends QueryTest {
+
+
+  test("must throw exception for date data type in dictionary_exclude") {
+    try {
+      sql(
+        "create table if not exists Carbon_automation_testdate (imei string,doj Date," +
+        "deviceInformationId int,MAC string,deviceColor string,device_backColor string,modelId
" +
+        "string,marketName string,AMSize string,ROMSize string,CUPAudit string,CPIClocked
string," +
+        "series string,productionDate timestamp,bomCode string,internalModels string, " +
+        "deliveryTime string, channelsId string, channelsName string , deliveryAreaId string,
" +
+        "deliveryCountry string, deliveryProvince string, deliveryCity string,deliveryDistrict
" +
+        "string, deliveryStreet string, oxSingleNumber string, ActiveCheckTime string, "
+
+        "ActiveAreaId string, ActiveCountry string, ActiveProvince string, Activecity string,
" +
+        "ActiveDistrict string, ActiveStreet string, ActiveOperatorId string, Active_releaseId
" +
+        "string, Active_EMUIVersion string, Active_operaSysVersion string, Active_BacVerNumber
" +
+        "string, Active_BacFlashVer string, Active_webUIVersion string, Active_webUITypeCarrVer
" +
+        "string,Active_webTypeDataVerNumber string, Active_operatorsVersion string, " +
+        "Active_phonePADPartitionedVersions string, Latest_YEAR int, Latest_MONTH int, Latest_DAY"
+
+        " int, Latest_HOUR string, Latest_areaId string, Latest_country string, Latest_province
" +
+        "string, Latest_city string, Latest_district string, Latest_street string, " +
+        "Latest_releaseId string, Latest_EMUIVersion string, Latest_operaSysVersion string,
" +
+        "Latest_BacVerNumber string, Latest_BacFlashVer string, Latest_webUIVersion string,
" +
+        "Latest_webUITypeCarrVer string, Latest_webTypeDataVerNumber string, " +
+        "Latest_operatorsVersion string, Latest_phonePADPartitionedVersions string, " +
+        "Latest_operatorId string, gamePointDescription string, gamePointId int,contractNumber
" +
+        "int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('DICTIONARY_EXCLUDE'='doj')")
+
+      assert(false)
+    }
+    catch {
+      case exception: MalformedCarbonCommandException => assert(true)
+    }
+  }
+  test("must throw exception for timestamp data type in dictionary_exclude") {
+    try {
+      sql(
+        "create table if not exists Carbon_automation_testtimestamp (imei string,doj timestamp,"
+
+        "deviceInformationId int,MAC string,deviceColor string,device_backColor string,modelId
" +
+        "string,marketName string,AMSize string,ROMSize string,CUPAudit string,CPIClocked
string," +
+        "series string,productionDate timestamp,bomCode string,internalModels string, " +
+        "deliveryTime string, channelsId string, channelsName string , deliveryAreaId string,
" +
+        "deliveryCountry string, deliveryProvince string, deliveryCity string,deliveryDistrict
" +
+        "string, deliveryStreet string, oxSingleNumber string, ActiveCheckTime string, "
+
+        "ActiveAreaId string, ActiveCountry string, ActiveProvince string, Activecity string,
" +
+        "ActiveDistrict string, ActiveStreet string, ActiveOperatorId string, Active_releaseId
" +
+        "string, Active_EMUIVersion string, Active_operaSysVersion string, Active_BacVerNumber
" +
+        "string, Active_BacFlashVer string, Active_webUIVersion string, Active_webUITypeCarrVer
" +
+        "string,Active_webTypeDataVerNumber string, Active_operatorsVersion string, " +
+        "Active_phonePADPartitionedVersions string, Latest_YEAR int, Latest_MONTH int, Latest_DAY"
+
+        " int, Latest_HOUR string, Latest_areaId string, Latest_country string, Latest_province
" +
+        "string, Latest_city string, Latest_district string, Latest_street string, " +
+        "Latest_releaseId string, Latest_EMUIVersion string, Latest_operaSysVersion string,
" +
+        "Latest_BacVerNumber string, Latest_BacFlashVer string, Latest_webUIVersion string,
" +
+        "Latest_webUITypeCarrVer string, Latest_webTypeDataVerNumber string, " +
+        "Latest_operatorsVersion string, Latest_phonePADPartitionedVersions string, " +
+        "Latest_operatorId string, gamePointDescription string, gamePointId int,contractNumber
" +
+        "int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES('DICTIONARY_EXCLUDE'='doj')")
+
+      assert(false)
+    }
+    catch {
+      case exception: MalformedCarbonCommandException => assert(true)
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/80eb1fdb/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 047cc97..166534f 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -503,7 +503,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser
{
               val errormsg = "DICTIONARY_EXCLUDE is unsupported for complex datatype column:
" +
                              dictExcludeCol
               throw new MalformedCarbonCommandException(errormsg)
-            } else if (!isStringAndTimestampColDictionaryExclude(dataType)) {
+            } else if (!isDataTypeSupportedForDictionary_Exclude(dataType)) {
               val errorMsg = "DICTIONARY_EXCLUDE is unsupported for " + dataType.toLowerCase()
+
                              " data type column: " + dictExcludeCol
               throw new MalformedCarbonCommandException(errorMsg)
@@ -588,10 +588,10 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser
{
   }
 
   /**
-   * detects whether double or decimal column is part of dictionary_exclude
+   * detects whether datatype is part of dictionary_exclude
    */
-  def isStringAndTimestampColDictionaryExclude(columnDataType: String): Boolean = {
-    val dataTypes = Array("string", "timestamp")
+  def isDataTypeSupportedForDictionary_Exclude(columnDataType: String): Boolean = {
+    val dataTypes = Array("string")
     dataTypes.exists(x => x.equalsIgnoreCase(columnDataType))
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/80eb1fdb/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
index 91ab138..f8bdcf8 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
@@ -37,9 +37,9 @@ object TableCreator {
     dimensionType.exists(x => x.equalsIgnoreCase(dimensionDataType))
   }
 
-  // detects whether double or decimal column is part of dictionary_exclude
-  def isStringAndTimestampColDictionaryExclude(columnDataType: String): Boolean = {
-    val dataTypes = Array("string", "timestamp", "date", "stringtype", "timestamptype", "datetype")
+  // detects whether datatype is part of dictionary_exclude
+  def isDataTypeSupportedForDictionary_Exclude(columnDataType: String): Boolean = {
+    val dataTypes = Array("string")
     dataTypes.exists(x => x.equalsIgnoreCase(columnDataType))
   }
 
@@ -76,7 +76,7 @@ object TableCreator {
               val errormsg = "DICTIONARY_EXCLUDE is unsupported for complex datatype column:
" +
                 dictExcludeCol
               throw new MalformedCarbonCommandException(errormsg)
-            } else if (!isStringAndTimestampColDictionaryExclude(dataType)) {
+            } else if (!isDataTypeSupportedForDictionary_Exclude(dataType)) {
               val errorMsg = "DICTIONARY_EXCLUDE is unsupported for " + dataType.toLowerCase()
+
                 " data type column: " + dictExcludeCol
               throw new MalformedCarbonCommandException(errorMsg)


Mime
View raw message