carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chenliang...@apache.org
Subject [04/50] [abbrv] carbondata git commit: move testcase and fix
Date Thu, 06 Jul 2017 14:41:29 GMT
move testcase and fix


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/a8b67261
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/a8b67261
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/a8b67261

Branch: refs/heads/branch-1.1
Commit: a8b672610ef845fffec5f6f9062e51ad040bcad4
Parents: 959e851
Author: jackylk <jacky.likun@huawei.com>
Authored: Mon May 22 22:08:58 2017 +0800
Committer: ravipesala <ravi.pesala@gmail.com>
Committed: Thu Jun 15 12:57:13 2017 +0530

----------------------------------------------------------------------
 .../impl/DictionaryBasedResultCollector.java    |   1 +
 .../complexType/TestComplexTypeQuery.scala      | 289 +++++++++++++++++++
 .../complexType/TestComplexTypeQuery.scala      | 289 -------------------
 3 files changed, 290 insertions(+), 289 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/a8b67261/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
index e5e4b3c..b784f94 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
@@ -147,6 +147,7 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
       row[order[i]] = comlexDimensionInfoMap.get(queryDimensions[i].getDimension().getOrdinal())
           .getDataBasedOnDataTypeFromSurrogates(
               ByteBuffer.wrap(complexTypeKeyArray[complexTypeColumnIndex++]));
+      dictionaryColumnIndex++;
     } else {
       row[order[i]] = surrogateResult[actualIndexInSurrogateKey[dictionaryColumnIndex++]];
     }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/a8b67261/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
new file mode 100644
index 0000000..c2c15eb
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.integration.spark.testsuite.complexType
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.common.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+/**
+ * Test class of creating and loading for carbon table with double
+ *
+ */
+class TestComplexTypeQuery extends QueryTest with BeforeAndAfterAll {
+
+  override def beforeAll: Unit = {
+    sql("drop table if exists complexcarbontable")
+    sql("drop table if exists complexhivetable")
+    sql("drop table if exists complex_filter")
+    sql("drop table if exists structusingstructCarbon")
+    sql("drop table if exists structusingstructHive")
+    sql("drop table if exists structusingarraycarbon")
+    sql("drop table if exists structusingarrayhive")
+    sql(
+      "create table complexcarbontable(deviceInformationId int, channelsId string, ROMSize
" +
+      "string, ROMName String, purchasedate string, mobile struct<imei:string, imsi:string>,
MAC " +
+      "array<string>, locationinfo array<struct<ActiveAreaId:int, ActiveCountry:string,
" +
+      "ActiveProvince:string, Activecity:string, ActiveDistrict:string, ActiveStreet:string>>,
" +
+      "proddate struct<productionDate:string,activeDeactivedate:array<string>>,
gamePointId " +
+      "double,contractNumber double)  STORED BY 'org.apache.carbondata.format'  TBLPROPERTIES
" +
+      "('DICTIONARY_INCLUDE'='deviceInformationId', 'DICTIONARY_EXCLUDE'='channelsId'," +
+      "'COLUMN_GROUP'='(ROMSize,ROMName)')")
+    sql("LOAD DATA local inpath '" + resourcesPath +
+        "/complextypesample.csv' INTO table complexcarbontable  OPTIONS('DELIMITER'=',',
" +
+        "'QUOTECHAR'='\"', 'FILEHEADER'='deviceInformationId,channelsId,ROMSize,ROMName,"
+
+        "purchasedate,mobile,MAC,locationinfo,proddate,gamePointId,contractNumber', " +
+        "'COMPLEX_DELIMITER_LEVEL_1'='$', 'COMPLEX_DELIMITER_LEVEL_2'=':')");
+    sql(
+      "create table complexhivetable(deviceInformationId int, channelsId string, ROMSize
string, " +
+      "ROMName String, purchasedate string, mobile struct<imei:string, imsi:string>,
MAC " +
+      "array<string>, locationinfo array<struct<ActiveAreaId:int, ActiveCountry:string,
" +
+      "ActiveProvince:string, Activecity:string, ActiveDistrict:string, ActiveStreet:string>>,
" +
+      "proddate struct<productionDate:string,activeDeactivedate:array<string>>,
gamePointId " +
+      "double,contractNumber double)row format delimited fields terminated by ',' collection
" +
+      "items terminated by '$' map keys terminated by ':'")
+    sql(s"LOAD DATA local inpath '$resourcesPath/complextypesample.csv' INTO table " +
+        s"complexhivetable")
+    sql(
+      "create table complex_filter(test1 int, test2 array<String>,test3 array<bigint>,test4
" +
+      "array<int>,test5 array<decimal>,test6 array<timestamp>,test7 array<double>)
STORED BY 'org" +
+      ".apache.carbondata.format'")
+    sql("LOAD DATA INPATH '" + resourcesPath +
+        "/array1.csv'  INTO TABLE complex_filter options ('DELIMITER'=',', 'QUOTECHAR'='\"',
" +
+        "'COMPLEX_DELIMITER_LEVEL_1'='$', 'FILEHEADER'= 'test1,test2,test3,test4,test5,test6,"
+
+        "test7')")
+      ()
+
+    sql(
+      "create table structusingarraycarbon (MAC struct<MAC1:array<string>," +
+      "ActiveCountry:array<string>>) STORED BY 'org.apache.carbondata.format'");
+    sql("LOAD DATA local INPATH '" + resourcesPath +
+        "/struct_all.csv' INTO table structusingarraycarbon options ('DELIMITER'=',', " +
+        "'QUOTECHAR'='\"', 'FILEHEADER'='MAC','COMPLEX_DELIMITER_LEVEL_1'='$'," +
+        "'COMPLEX_DELIMITER_LEVEL_2'='&')")
+    sql(
+      "create table structusingarrayhive (MAC struct<MAC1:array<string>," +
+      "ActiveCountry:array<string>>)row format delimited fields terminated by ','
collection " +
+      "items terminated by '$' map keys terminated by '&'");
+    sql("LOAD DATA local INPATH '" + resourcesPath +
+        "/struct_all.csv' INTO table structusingarrayhive")
+
+    sql(
+      "create table structusingstructCarbon(name struct<middlename:string, " +
+      "othernames:struct<firstname:string,lastname:string>,age:int> ) STORED BY
'org.apache" +
+      ".carbondata.format'")
+    sql("LOAD DATA local INPATH '" + resourcesPath +
+        "/structusingstruct.csv' INTO table structusingstructCarbon options ('DELIMITER'=',',
" +
+        "'QUOTECHAR'='\"', 'FILEHEADER'='name','COMPLEX_DELIMITER_LEVEL_1'='$'," +
+        "'COMPLEX_DELIMITER_LEVEL_2'='&')")
+    sql(
+      "create table structusingstructhive(name struct<middlename:string, " +
+      "othernames:struct<firstname:string,lastname:string>,age:int> )row format
delimited fields " +
+      "terminated by ',' collection items terminated by '$' map keys terminated by '&'")
+    sql("LOAD DATA local INPATH '" + resourcesPath +
+        "/structusingstruct.csv' INTO table structusingstructhive")
+
+  }
+
+  test("test for create table with complex type") {
+    try {
+      sql("drop table if exists carbon_table")
+      sql(
+        ("CREATE TABLE CARBON_TABLE(stringField string,complexData array<string>)stored
by " +
+         "'CARBONDATA' ")
+          .stripMargin)
+      assert(true)
+    }
+    catch {
+      case exception: Exception => assert(false)
+    }
+  }
+
+  test(
+    "Test ^ * special character data loading for complex types") {
+    sql(
+      "create table complexcarbonwithspecialchardelimeter(deviceInformationId int, channelsId
" +
+      "string, ROMSize string, ROMName String, purchasedate string, mobile struct<imei:string,
" +
+      "imsi:string>, MAC array<string>, locationinfo array<struct<ActiveAreaId:int,
" +
+      "ActiveCountry:string, ActiveProvince:string, Activecity:string, ActiveDistrict:string,
" +
+      "ActiveStreet:string>>, proddate struct<productionDate:string," +
+      "activeDeactivedate:array<string>>, gamePointId double,contractNumber double)
 STORED BY " +
+      "'org.apache.carbondata.format'  TBLPROPERTIES ('DICTIONARY_INCLUDE'='deviceInformationId',"
+
+      " 'DICTIONARY_EXCLUDE'='channelsId','COLUMN_GROUP'='(ROMSize,ROMName)')");
+    sql("LOAD DATA local inpath '" + resourcesPath +
+        "/complextypespecialchardelimiter.csv' INTO table complexcarbonwithspecialchardelimeter
 " +
+        "OPTIONS('DELIMITER'=',', 'QUOTECHAR'='\"', 'FILEHEADER'='deviceInformationId,channelsId,"
+
+        "ROMSize,ROMName,purchasedate,mobile,MAC,locationinfo,proddate,gamePointId," +
+        "contractNumber', 'COMPLEX_DELIMITER_LEVEL_1'='^', 'COMPLEX_DELIMITER_LEVEL_2'='*')");
+    sql(
+      "create table complexhivewithspecialchardelimeter(deviceInformationId int, channelsId
" +
+      "string, ROMSize string, ROMName String, purchasedate string, mobile struct<imei:string,
" +
+      "imsi:string>, MAC array<string>, locationinfo array<struct<ActiveAreaId:int,
" +
+      "ActiveCountry:string, ActiveProvince:string, Activecity:string, ActiveDistrict:string,
" +
+      "ActiveStreet:string>>, proddate struct<productionDate:string," +
+      "activeDeactivedate:array<string>>, gamePointId double,contractNumber double)row
format " +
+      "delimited fields terminated by ',' collection items terminated by '^' map keys terminated
" +
+      "by '*'")
+    sql("LOAD DATA local inpath '" + resourcesPath +
+        "/complextypespecialchardelimiter.csv' INTO table complexhivewithspecialchardelimeter");
+    checkAnswer(sql("select * from complexcarbonwithspecialchardelimeter"),
+      sql("select * from complexhivewithspecialchardelimeter"))
+    sql("drop table if exists complexcarbonwithspecialchardelimeter")
+    sql("drop table if exists complexhivewithspecialchardelimeter")
+  }
+
+  test("complex filter set1") {
+    checkAnswer(
+      sql("select test3[1] from complex_filter where test4[1] not like'%1%' order by test1"),
+      Seq(Row(5678), Row(1234))
+    )
+  }
+  test("complex filter set2") {
+    checkAnswer(
+      sql("select test2[0] from complex_filter  where  test3[0] like '%1234%'"),
+      Seq(Row("hello"))
+    )
+  }
+  test("select * from structusingarraycarbon") {
+    checkAnswer(sql("select * from structusingarraycarbon"),
+      sql("select * from structusingarrayhive"))
+  }
+
+  test("select * from structusingstructCarbon") {
+    checkAnswer(sql("select * from structusingstructCarbon"),
+      sql("select * from structusingstructhive"))
+  }
+
+  test("select * from complexcarbontable") {
+    checkAnswer(sql("select * from complexcarbontable"),
+      sql("select * from complexhivetable"))
+  }
+
+  test("select mobile, proddate, deviceInformationId  from complexcarbontable") {
+    checkAnswer(sql("select mobile, proddate, deviceInformationId  from complexcarbontable"),
+      sql("select mobile, proddate, deviceInformationId  from complexhivetable"))
+  }
+
+  test("select mobile, MAC, deviceInformationId, purchasedate from complexcarbontable") {
+    checkAnswer(sql("select mobile, MAC, deviceInformationId, purchasedate from " +
+                    "complexcarbontable"),
+      sql("select mobile, MAC, deviceInformationId, purchasedate from complexhivetable"))
+  }
+
+  test("select mobile, ROMSize, deviceInformationId from complexcarbontable") {
+    checkAnswer(sql("select mobile, ROMSize, deviceInformationId from complexcarbontable"),
+      sql("select mobile, ROMSize, deviceInformationId from complexhivetable"))
+  }
+
+  test("select locationinfo, purchasedate, deviceInformationId from complexcarbontable")
{
+    checkAnswer(sql("select locationinfo, purchasedate, deviceInformationId from " +
+                    "complexcarbontable"),
+      sql("select locationinfo, purchasedate, deviceInformationId from complexhivetable"))
+  }
+  test("select locationinfo, ROMName, purchasedate, deviceinformationId from complexcarbontable")
{
+    checkAnswer(sql(
+      "select locationinfo, ROMName, purchasedate, deviceinformationId from complexcarbontable"),
+      sql("select locationinfo, ROMName, purchasedate, deviceinformationId from complexhivetable"))
+  }
+  test("select MAC from complexcarbontable where MAC[0] = 'MAC1'") {
+    checkAnswer(sql("select MAC from complexcarbontable where MAC[0] = 'MAC1'"),
+      sql("select MAC from complexhivetable where MAC[0] = 'MAC1'"))
+  }
+  test("select mobile from complexcarbontable where mobile.imei like '1AA%'") {
+    checkAnswer(sql("select mobile from complexcarbontable where mobile.imei like '1AA%'"),
+      sql("select mobile from complexhivetable where mobile.imei like '1AA%'"))
+  }
+
+
+  test(
+    "select locationinfo from complexcarbontable where locationinfo[0].ActiveAreaId >
2 AND " +
+    "locationinfo[0].ActiveAreaId < 7") {
+    checkAnswer(sql(
+      "select locationinfo from complexcarbontable where locationinfo[0].ActiveAreaId >
2 AND " +
+      "locationinfo[0].ActiveAreaId < 7"),
+      sql(
+        "select locationinfo from complexhivetable where locationinfo[0].ActiveAreaId >
2 AND " +
+        "locationinfo[0].ActiveAreaId < 7"))
+  }
+  test(
+    "select locationinfo from complexcarbontable where locationinfo[0].ActiveAreaId >=
2 AND " +
+    "locationinfo[0].ActiveAreaId <= 7") {
+    checkAnswer(sql(
+      "select locationinfo from complexcarbontable where locationinfo[0].ActiveAreaId >=
2 AND " +
+      "locationinfo[0].ActiveAreaId <= 7"),
+      sql(
+        "select locationinfo from complexhivetable where locationinfo[0].ActiveAreaId >=
2 AND " +
+        "locationinfo[0].ActiveAreaId <= 7"))
+  }
+  test(
+    "select locationinfo from complexcarbontable where (locationinfo[0].ActiveAreaId +5 )>
6 AND " +
+    "(locationinfo[0].ActiveAreaId+10) < 20") {
+    checkAnswer(sql(
+      "select locationinfo from complexcarbontable where (locationinfo[0].ActiveAreaId +5
)> 6 " +
+      "AND (locationinfo[0].ActiveAreaId+10) < 20"),
+      sql(
+        "select locationinfo from complexhivetable where (locationinfo[0].ActiveAreaId +5
)> 6 " +
+        "AND (locationinfo[0].ActiveAreaId+10) < 20"))
+  }
+  test("select count(mobile),channelsId from complexcarbontable group by mobile,channelsId")
{
+    checkAnswer(sql(
+      "select count(mobile),channelsId from complexcarbontable group by mobile,channelsId"),
+      sql("select count(mobile),channelsId from complexhivetable group by mobile,channelsId"))
+  }
+
+  test(
+    "select count(mobile),channelsId from complexcarbontable group by mobile,channelsId order
by " +
+    "channelsId") {
+    checkAnswer(sql(
+      "select count(mobile),channelsId from complexcarbontable group by mobile,channelsId
order " +
+      "by channelsId"),
+      sql(
+        "select count(mobile),channelsId from complexhivetable group by mobile,channelsId
order " +
+        "by channelsId"))
+  }
+  test(
+    "select count(mobile),channelsId from complexcarbontable group by mobile,channelsId order
by " +
+    "channelsId limit 10") {
+    checkAnswer(sql(
+      "select count(mobile),channelsId from complexcarbontable group by mobile,channelsId
order " +
+      "by channelsId limit 10"),
+      sql(
+        "select count(mobile),channelsId from complexhivetable group by mobile,channelsId
order " +
+        "by channelsId limit 10"))
+  }
+  test(
+    "select count(mobile),channelsId from complexcarbontable where MAC[0] = 'MAC1'  group
by " +
+    "mobile,channelsId order by channelsId limit 10") {
+    checkAnswer(sql(
+      "select count(mobile),channelsId from complexcarbontable where MAC[0] = 'MAC1'  group
by " +
+      "mobile,channelsId order by channelsId limit 10"),
+      sql(
+        "select count(mobile),channelsId from complexhivetable where MAC[0] = 'MAC1'  group
by " +
+        "mobile,channelsId order by channelsId limit 10"))
+  }
+
+  override def afterAll {
+    sql("drop table if exists complexcarbontable")
+    sql("drop table if exists complexhivetable")
+    sql("drop table if exists structusingstructCarbon")
+    sql("drop table if exists structusingstructHive")
+    sql("drop table if exists structusingarraycarbon")
+    sql("drop table if exists structusingarrayhive")
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/a8b67261/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
deleted file mode 100644
index c2c15eb..0000000
--- a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
+++ /dev/null
@@ -1,289 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.integration.spark.testsuite.complexType
-
-import org.apache.spark.sql.Row
-import org.apache.spark.sql.common.util.QueryTest
-import org.scalatest.BeforeAndAfterAll
-
-/**
- * Test class of creating and loading for carbon table with double
- *
- */
-class TestComplexTypeQuery extends QueryTest with BeforeAndAfterAll {
-
-  override def beforeAll: Unit = {
-    sql("drop table if exists complexcarbontable")
-    sql("drop table if exists complexhivetable")
-    sql("drop table if exists complex_filter")
-    sql("drop table if exists structusingstructCarbon")
-    sql("drop table if exists structusingstructHive")
-    sql("drop table if exists structusingarraycarbon")
-    sql("drop table if exists structusingarrayhive")
-    sql(
-      "create table complexcarbontable(deviceInformationId int, channelsId string, ROMSize
" +
-      "string, ROMName String, purchasedate string, mobile struct<imei:string, imsi:string>,
MAC " +
-      "array<string>, locationinfo array<struct<ActiveAreaId:int, ActiveCountry:string,
" +
-      "ActiveProvince:string, Activecity:string, ActiveDistrict:string, ActiveStreet:string>>,
" +
-      "proddate struct<productionDate:string,activeDeactivedate:array<string>>,
gamePointId " +
-      "double,contractNumber double)  STORED BY 'org.apache.carbondata.format'  TBLPROPERTIES
" +
-      "('DICTIONARY_INCLUDE'='deviceInformationId', 'DICTIONARY_EXCLUDE'='channelsId'," +
-      "'COLUMN_GROUP'='(ROMSize,ROMName)')")
-    sql("LOAD DATA local inpath '" + resourcesPath +
-        "/complextypesample.csv' INTO table complexcarbontable  OPTIONS('DELIMITER'=',',
" +
-        "'QUOTECHAR'='\"', 'FILEHEADER'='deviceInformationId,channelsId,ROMSize,ROMName,"
+
-        "purchasedate,mobile,MAC,locationinfo,proddate,gamePointId,contractNumber', " +
-        "'COMPLEX_DELIMITER_LEVEL_1'='$', 'COMPLEX_DELIMITER_LEVEL_2'=':')");
-    sql(
-      "create table complexhivetable(deviceInformationId int, channelsId string, ROMSize
string, " +
-      "ROMName String, purchasedate string, mobile struct<imei:string, imsi:string>,
MAC " +
-      "array<string>, locationinfo array<struct<ActiveAreaId:int, ActiveCountry:string,
" +
-      "ActiveProvince:string, Activecity:string, ActiveDistrict:string, ActiveStreet:string>>,
" +
-      "proddate struct<productionDate:string,activeDeactivedate:array<string>>,
gamePointId " +
-      "double,contractNumber double)row format delimited fields terminated by ',' collection
" +
-      "items terminated by '$' map keys terminated by ':'")
-    sql(s"LOAD DATA local inpath '$resourcesPath/complextypesample.csv' INTO table " +
-        s"complexhivetable")
-    sql(
-      "create table complex_filter(test1 int, test2 array<String>,test3 array<bigint>,test4
" +
-      "array<int>,test5 array<decimal>,test6 array<timestamp>,test7 array<double>)
STORED BY 'org" +
-      ".apache.carbondata.format'")
-    sql("LOAD DATA INPATH '" + resourcesPath +
-        "/array1.csv'  INTO TABLE complex_filter options ('DELIMITER'=',', 'QUOTECHAR'='\"',
" +
-        "'COMPLEX_DELIMITER_LEVEL_1'='$', 'FILEHEADER'= 'test1,test2,test3,test4,test5,test6,"
+
-        "test7')")
-      ()
-
-    sql(
-      "create table structusingarraycarbon (MAC struct<MAC1:array<string>," +
-      "ActiveCountry:array<string>>) STORED BY 'org.apache.carbondata.format'");
-    sql("LOAD DATA local INPATH '" + resourcesPath +
-        "/struct_all.csv' INTO table structusingarraycarbon options ('DELIMITER'=',', " +
-        "'QUOTECHAR'='\"', 'FILEHEADER'='MAC','COMPLEX_DELIMITER_LEVEL_1'='$'," +
-        "'COMPLEX_DELIMITER_LEVEL_2'='&')")
-    sql(
-      "create table structusingarrayhive (MAC struct<MAC1:array<string>," +
-      "ActiveCountry:array<string>>)row format delimited fields terminated by ','
collection " +
-      "items terminated by '$' map keys terminated by '&'");
-    sql("LOAD DATA local INPATH '" + resourcesPath +
-        "/struct_all.csv' INTO table structusingarrayhive")
-
-    sql(
-      "create table structusingstructCarbon(name struct<middlename:string, " +
-      "othernames:struct<firstname:string,lastname:string>,age:int> ) STORED BY
'org.apache" +
-      ".carbondata.format'")
-    sql("LOAD DATA local INPATH '" + resourcesPath +
-        "/structusingstruct.csv' INTO table structusingstructCarbon options ('DELIMITER'=',',
" +
-        "'QUOTECHAR'='\"', 'FILEHEADER'='name','COMPLEX_DELIMITER_LEVEL_1'='$'," +
-        "'COMPLEX_DELIMITER_LEVEL_2'='&')")
-    sql(
-      "create table structusingstructhive(name struct<middlename:string, " +
-      "othernames:struct<firstname:string,lastname:string>,age:int> )row format
delimited fields " +
-      "terminated by ',' collection items terminated by '$' map keys terminated by '&'")
-    sql("LOAD DATA local INPATH '" + resourcesPath +
-        "/structusingstruct.csv' INTO table structusingstructhive")
-
-  }
-
-  test("test for create table with complex type") {
-    try {
-      sql("drop table if exists carbon_table")
-      sql(
-        ("CREATE TABLE CARBON_TABLE(stringField string,complexData array<string>)stored
by " +
-         "'CARBONDATA' ")
-          .stripMargin)
-      assert(true)
-    }
-    catch {
-      case exception: Exception => assert(false)
-    }
-  }
-
-  test(
-    "Test ^ * special character data loading for complex types") {
-    sql(
-      "create table complexcarbonwithspecialchardelimeter(deviceInformationId int, channelsId
" +
-      "string, ROMSize string, ROMName String, purchasedate string, mobile struct<imei:string,
" +
-      "imsi:string>, MAC array<string>, locationinfo array<struct<ActiveAreaId:int,
" +
-      "ActiveCountry:string, ActiveProvince:string, Activecity:string, ActiveDistrict:string,
" +
-      "ActiveStreet:string>>, proddate struct<productionDate:string," +
-      "activeDeactivedate:array<string>>, gamePointId double,contractNumber double)
 STORED BY " +
-      "'org.apache.carbondata.format'  TBLPROPERTIES ('DICTIONARY_INCLUDE'='deviceInformationId',"
+
-      " 'DICTIONARY_EXCLUDE'='channelsId','COLUMN_GROUP'='(ROMSize,ROMName)')");
-    sql("LOAD DATA local inpath '" + resourcesPath +
-        "/complextypespecialchardelimiter.csv' INTO table complexcarbonwithspecialchardelimeter
 " +
-        "OPTIONS('DELIMITER'=',', 'QUOTECHAR'='\"', 'FILEHEADER'='deviceInformationId,channelsId,"
+
-        "ROMSize,ROMName,purchasedate,mobile,MAC,locationinfo,proddate,gamePointId," +
-        "contractNumber', 'COMPLEX_DELIMITER_LEVEL_1'='^', 'COMPLEX_DELIMITER_LEVEL_2'='*')");
-    sql(
-      "create table complexhivewithspecialchardelimeter(deviceInformationId int, channelsId
" +
-      "string, ROMSize string, ROMName String, purchasedate string, mobile struct<imei:string,
" +
-      "imsi:string>, MAC array<string>, locationinfo array<struct<ActiveAreaId:int,
" +
-      "ActiveCountry:string, ActiveProvince:string, Activecity:string, ActiveDistrict:string,
" +
-      "ActiveStreet:string>>, proddate struct<productionDate:string," +
-      "activeDeactivedate:array<string>>, gamePointId double,contractNumber double)row
format " +
-      "delimited fields terminated by ',' collection items terminated by '^' map keys terminated
" +
-      "by '*'")
-    sql("LOAD DATA local inpath '" + resourcesPath +
-        "/complextypespecialchardelimiter.csv' INTO table complexhivewithspecialchardelimeter");
-    checkAnswer(sql("select * from complexcarbonwithspecialchardelimeter"),
-      sql("select * from complexhivewithspecialchardelimeter"))
-    sql("drop table if exists complexcarbonwithspecialchardelimeter")
-    sql("drop table if exists complexhivewithspecialchardelimeter")
-  }
-
-  test("complex filter set1") {
-    checkAnswer(
-      sql("select test3[1] from complex_filter where test4[1] not like'%1%' order by test1"),
-      Seq(Row(5678), Row(1234))
-    )
-  }
-  test("complex filter set2") {
-    checkAnswer(
-      sql("select test2[0] from complex_filter  where  test3[0] like '%1234%'"),
-      Seq(Row("hello"))
-    )
-  }
-  test("select * from structusingarraycarbon") {
-    checkAnswer(sql("select * from structusingarraycarbon"),
-      sql("select * from structusingarrayhive"))
-  }
-
-  test("select * from structusingstructCarbon") {
-    checkAnswer(sql("select * from structusingstructCarbon"),
-      sql("select * from structusingstructhive"))
-  }
-
-  test("select * from complexcarbontable") {
-    checkAnswer(sql("select * from complexcarbontable"),
-      sql("select * from complexhivetable"))
-  }
-
-  test("select mobile, proddate, deviceInformationId  from complexcarbontable") {
-    checkAnswer(sql("select mobile, proddate, deviceInformationId  from complexcarbontable"),
-      sql("select mobile, proddate, deviceInformationId  from complexhivetable"))
-  }
-
-  test("select mobile, MAC, deviceInformationId, purchasedate from complexcarbontable") {
-    checkAnswer(sql("select mobile, MAC, deviceInformationId, purchasedate from " +
-                    "complexcarbontable"),
-      sql("select mobile, MAC, deviceInformationId, purchasedate from complexhivetable"))
-  }
-
-  test("select mobile, ROMSize, deviceInformationId from complexcarbontable") {
-    checkAnswer(sql("select mobile, ROMSize, deviceInformationId from complexcarbontable"),
-      sql("select mobile, ROMSize, deviceInformationId from complexhivetable"))
-  }
-
-  test("select locationinfo, purchasedate, deviceInformationId from complexcarbontable")
{
-    checkAnswer(sql("select locationinfo, purchasedate, deviceInformationId from " +
-                    "complexcarbontable"),
-      sql("select locationinfo, purchasedate, deviceInformationId from complexhivetable"))
-  }
-  test("select locationinfo, ROMName, purchasedate, deviceinformationId from complexcarbontable")
{
-    checkAnswer(sql(
-      "select locationinfo, ROMName, purchasedate, deviceinformationId from complexcarbontable"),
-      sql("select locationinfo, ROMName, purchasedate, deviceinformationId from complexhivetable"))
-  }
-  test("select MAC from complexcarbontable where MAC[0] = 'MAC1'") {
-    checkAnswer(sql("select MAC from complexcarbontable where MAC[0] = 'MAC1'"),
-      sql("select MAC from complexhivetable where MAC[0] = 'MAC1'"))
-  }
-  test("select mobile from complexcarbontable where mobile.imei like '1AA%'") {
-    checkAnswer(sql("select mobile from complexcarbontable where mobile.imei like '1AA%'"),
-      sql("select mobile from complexhivetable where mobile.imei like '1AA%'"))
-  }
-
-
-  test(
-    "select locationinfo from complexcarbontable where locationinfo[0].ActiveAreaId >
2 AND " +
-    "locationinfo[0].ActiveAreaId < 7") {
-    checkAnswer(sql(
-      "select locationinfo from complexcarbontable where locationinfo[0].ActiveAreaId >
2 AND " +
-      "locationinfo[0].ActiveAreaId < 7"),
-      sql(
-        "select locationinfo from complexhivetable where locationinfo[0].ActiveAreaId >
2 AND " +
-        "locationinfo[0].ActiveAreaId < 7"))
-  }
-  test(
-    "select locationinfo from complexcarbontable where locationinfo[0].ActiveAreaId >=
2 AND " +
-    "locationinfo[0].ActiveAreaId <= 7") {
-    checkAnswer(sql(
-      "select locationinfo from complexcarbontable where locationinfo[0].ActiveAreaId >=
2 AND " +
-      "locationinfo[0].ActiveAreaId <= 7"),
-      sql(
-        "select locationinfo from complexhivetable where locationinfo[0].ActiveAreaId >=
2 AND " +
-        "locationinfo[0].ActiveAreaId <= 7"))
-  }
-  test(
-    "select locationinfo from complexcarbontable where (locationinfo[0].ActiveAreaId +5 )>
6 AND " +
-    "(locationinfo[0].ActiveAreaId+10) < 20") {
-    checkAnswer(sql(
-      "select locationinfo from complexcarbontable where (locationinfo[0].ActiveAreaId +5
)> 6 " +
-      "AND (locationinfo[0].ActiveAreaId+10) < 20"),
-      sql(
-        "select locationinfo from complexhivetable where (locationinfo[0].ActiveAreaId +5
)> 6 " +
-        "AND (locationinfo[0].ActiveAreaId+10) < 20"))
-  }
-  test("select count(mobile),channelsId from complexcarbontable group by mobile,channelsId")
{
-    checkAnswer(sql(
-      "select count(mobile),channelsId from complexcarbontable group by mobile,channelsId"),
-      sql("select count(mobile),channelsId from complexhivetable group by mobile,channelsId"))
-  }
-
-  test(
-    "select count(mobile),channelsId from complexcarbontable group by mobile,channelsId order
by " +
-    "channelsId") {
-    checkAnswer(sql(
-      "select count(mobile),channelsId from complexcarbontable group by mobile,channelsId
order " +
-      "by channelsId"),
-      sql(
-        "select count(mobile),channelsId from complexhivetable group by mobile,channelsId
order " +
-        "by channelsId"))
-  }
-  test(
-    "select count(mobile),channelsId from complexcarbontable group by mobile,channelsId order
by " +
-    "channelsId limit 10") {
-    checkAnswer(sql(
-      "select count(mobile),channelsId from complexcarbontable group by mobile,channelsId
order " +
-      "by channelsId limit 10"),
-      sql(
-        "select count(mobile),channelsId from complexhivetable group by mobile,channelsId
order " +
-        "by channelsId limit 10"))
-  }
-  test(
-    "select count(mobile),channelsId from complexcarbontable where MAC[0] = 'MAC1'  group
by " +
-    "mobile,channelsId order by channelsId limit 10") {
-    checkAnswer(sql(
-      "select count(mobile),channelsId from complexcarbontable where MAC[0] = 'MAC1'  group
by " +
-      "mobile,channelsId order by channelsId limit 10"),
-      sql(
-        "select count(mobile),channelsId from complexhivetable where MAC[0] = 'MAC1'  group
by " +
-        "mobile,channelsId order by channelsId limit 10"))
-  }
-
-  override def afterAll {
-    sql("drop table if exists complexcarbontable")
-    sql("drop table if exists complexhivetable")
-    sql("drop table if exists structusingstructCarbon")
-    sql("drop table if exists structusingstructHive")
-    sql("drop table if exists structusingarraycarbon")
-    sql("drop table if exists structusingarrayhive")
-
-  }
-}


Mime
View raw message