carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject carbondata git commit: [CARBONDATA-2866] Block schema in external table
Date Wed, 05 Sep 2018 12:43:22 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master b611a861a -> af2c469bb


[CARBONDATA-2866] Block schema in external table

This PR blocks schema in CREATE EXTERNAL TABLE

This closes #2645


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/af2c469b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/af2c469b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/af2c469b

Branch: refs/heads/master
Commit: af2c469bb8cd9e9fd87c7e6cc9f3f8fb66c578ed
Parents: b611a86
Author: Jacky Li <jacky.likun@qq.com>
Authored: Fri Aug 17 22:04:27 2018 +0800
Committer: ravipesala <ravi.pesala@gmail.com>
Committed: Wed Sep 5 18:13:13 2018 +0530

----------------------------------------------------------------------
 .../createTable/TestCreateExternalTable.scala   | 19 +++++++++++
 .../TestNonTransactionalCarbonTable.scala       | 36 +++++++++++---------
 ...ansactionalCarbonTableWithAvroDataType.scala | 28 +++++++--------
 .../org/apache/spark/sql/CarbonSource.scala     |  9 ++---
 .../sql/parser/CarbonSparkSqlParserUtil.scala   | 18 +++++++---
 5 files changed, 68 insertions(+), 42 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/af2c469b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala
index 6fb24c7..56f038e 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala
@@ -90,6 +90,25 @@ class TestCreateExternalTable extends QueryTest with BeforeAndAfterAll
{
     }
   }
 
+  test("create external table with specified schema") {
+    assert(new File(originDataPath).exists())
+    sql("DROP TABLE IF EXISTS source")
+    val ex = intercept[AnalysisException] {
+      sql(
+        s"""
+           |CREATE EXTERNAL TABLE source (key INT)
+           |STORED BY 'carbondata'
+           |LOCATION '$storeLocation/origin'
+     """.stripMargin)
+    }
+    assert(ex.message.contains("Schema may not be specified for external table"))
+
+    sql("DROP TABLE IF EXISTS source")
+
+    // DROP TABLE should not delete data
+    assert(new File(originDataPath).exists())
+  }
+
   test("create external table with empty folder") {
     val exception = intercept[AnalysisException] {
       sql(

http://git-wip-us.apache.org/repos/asf/carbondata/blob/af2c469b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
index 6406a66..87d5622 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
@@ -278,16 +278,18 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll
{
 
     // with partition
     sql(
-      s"""CREATE EXTERNAL TABLE sdkOutputTable(name string) PARTITIONED BY (age int) STORED
BY
-         |'carbondata' LOCATION
-         |'$writerPath' """.stripMargin)
+      s"""
+         | CREATE EXTERNAL TABLE sdkOutputTable
+         | STORED BY 'carbondata'
+         | LOCATION '$writerPath'
+      """.stripMargin)
 
     sql("drop table if exists t1")
     sql("create table if not exists t1 (name string, age int, height double) STORED BY 'org.apache.carbondata.format'")
-    var i =0;
+    var i =0
     while (i<50){
       sql (s"""insert into t1 values ("aaaaa", 12, 20)""").show(200,false)
-      i = i+1;
+      i = i+1
     }
     checkAnswer(sql("select count(*) from t1"),Seq(Row(50)))
     val one = Future {
@@ -400,7 +402,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll
{
 
     // with partition
     sql(
-      s"""CREATE EXTERNAL TABLE sdkOutputTable(name string) PARTITIONED BY (age int) STORED
BY
+      s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
          |'carbondata' LOCATION
          |'$writerPath' """.stripMargin)
 
@@ -413,7 +415,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll
{
     sql("DROP TABLE IF EXISTS sdkOutputTable")
     // with partition
     sql(
-      s"""CREATE EXTERNAL TABLE sdkOutputTable(name string) PARTITIONED BY (age int) STORED
BY
+      s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
          |'carbondata' LOCATION
          |'$writerPath' """.stripMargin)
 
@@ -425,7 +427,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll
{
     sql("DROP TABLE IF EXISTS sdkOutputTable")
     // with partition
     sql(
-      s"""CREATE EXTERNAL TABLE sdkOutputTable(name string) PARTITIONED BY (age int) STORED
BY
+      s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
          |'carbondata' LOCATION
          |'$writerPath' """.stripMargin)
 
@@ -437,7 +439,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll
{
 
     // with partition
     sql(
-      s"""CREATE EXTERNAL TABLE sdkOutputTable(name string) PARTITIONED BY (age int) STORED
BY
+      s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
          |'carbondata' LOCATION
          |'$writerPath' """.stripMargin)
 
@@ -464,7 +466,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll
{
 
     // with partition
     sql(
-      s"""CREATE EXTERNAL TABLE sdkOutputTable(name string) PARTITIONED BY (age int) STORED
BY
+      s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
          |'carbondata' LOCATION
          |'$writerPath' """.stripMargin)
 
@@ -487,7 +489,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll
{
 
     // with partition
     sql(
-      s"""CREATE EXTERNAL TABLE sdkOutputTable(name string) PARTITIONED BY (age int) STORED
BY
+      s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
          |'carbondata' LOCATION
          |'$writerPath' """.stripMargin)
 
@@ -520,7 +522,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll
{
 
     // with partition
     sql(
-      s"""CREATE EXTERNAL TABLE sdkOutputTable(name string) PARTITIONED BY (age int) STORED
BY
+      s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
          |'carbondata' LOCATION
          |'$writerPath' """.stripMargin)
 
@@ -558,7 +560,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll
{
 
     // with partition
     sql(
-      s"""CREATE EXTERNAL TABLE sdkOutputTable(name string) PARTITIONED BY (age int) STORED
BY
+      s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
          |'carbondata' LOCATION
          |'$writerPath' """.stripMargin)
 
@@ -737,7 +739,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll
{
 
     // with schema
     sql(
-      s"""CREATE EXTERNAL TABLE sdkOutputTable(age int) STORED BY
+      s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
          |'carbondata' LOCATION
          |'$writerPath' """.stripMargin)
 
@@ -2260,7 +2262,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll
{
     writer.write(record)
     writer.close()
     sql(
-      s"""CREATE EXTERNAL TABLE sdkOutputTable(dateType date, course_details struct<course_struct_course_time:
date>) STORED BY
+      s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
          |'carbondata' LOCATION
          |'$writerPath' """.stripMargin)
     checkAnswer(sql("select * from sdkOutputTable"), Seq(Row(java.sql.Date.valueOf("1970-04-12"),
Row(java.sql.Date.valueOf("1970-01-11")))))
@@ -2306,7 +2308,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll
{
     writer.write(record)
     writer.close()
     sql(
-      s"""CREATE EXTERNAL TABLE sdkOutputTable(dateType timestamp, course_details struct<course_struct_course_time:
timestamp>) STORED BY
+      s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
          |'carbondata' LOCATION
          |'$writerPath' """.stripMargin)
     checkAnswer(sql("select * from sdkOutputTable"), Seq(Row(Timestamp.valueOf("1970-01-02
16:00:00"), Row(Timestamp.valueOf("1970-01-02 16:00:00")))))
@@ -2353,7 +2355,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll
{
     writer.write(record)
     writer.close()
     sql(
-      s"""CREATE EXTERNAL TABLE sdkOutputTable(dateType timestamp, course_details struct<course_struct_course_time:
timestamp>) STORED BY
+      s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
          |'carbondata' LOCATION
          |'$writerPath' """.stripMargin)
     checkAnswer(sql("select * from sdkOutputTable"), Seq(Row(Timestamp.valueOf("1970-01-02
16:00:00"), Row(Timestamp.valueOf("1970-01-02 16:00:00")))))

http://git-wip-us.apache.org/repos/asf/carbondata/blob/af2c469b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTableWithAvroDataType.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTableWithAvroDataType.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTableWithAvroDataType.scala
index b50407c..7616ea3 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTableWithAvroDataType.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTableWithAvroDataType.scala
@@ -180,7 +180,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest
with Bef
     writer.write(record)
     writer.close()
     sql(
-      s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
+      s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
          |'carbondata' LOCATION
          |'$writerPath' """.stripMargin)
     checkAnswer(sql("select * from sdkOutputTable"),
@@ -241,7 +241,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest
with Bef
     writer.write(record)
     writer.close()
     sql(
-      s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
+      s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
          |'carbondata' LOCATION
          |'$writerPath' """.stripMargin)
     checkAnswer(sql("select * from sdkOutputTable"), Seq(Row(Row(true, null, null))))
@@ -271,7 +271,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest
with Bef
     writer.write(record)
     writer.close()
     sql(
-      s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
+      s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
          |'carbondata' LOCATION
          |'$writerPath' """.stripMargin)
     checkAnswer(sql("select * from sdkOutputTable"), Seq(Row(Row("abc", null, null))))
@@ -301,7 +301,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest
with Bef
     writer.write(record)
     writer.close()
     sql(
-      s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
+      s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
          |'carbondata' LOCATION
          |'$writerPath' """.stripMargin)
     checkAnswer(sql("select * from sdkOutputTable"), Seq(Row(Row(null, 10, null))))
@@ -331,7 +331,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest
with Bef
     writer.write(record)
     writer.close()
     sql(
-      s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
+      s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
          |'carbondata' LOCATION
          |'$writerPath' """.stripMargin)
     checkAnswer(sql("select * from sdkOutputTable"), Seq(Row(Row(null))))
@@ -398,7 +398,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest
with Bef
     writer.write(record)
     writer.close()
     sql(
-      s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
+      s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
          |'carbondata' LOCATION
          |'$writerPath' """.stripMargin)
     checkAnswer(sql("select * from sdkOutputTable"), Seq(Row(Row("VAL_0"))))
@@ -434,7 +434,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest
with Bef
     writer.write(record)
     writer.close()
     sql(
-      s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
+      s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
          |'carbondata' LOCATION
          |'$writerPath' """.stripMargin)
     sql("select * from sdkOutputTable").show(false)
@@ -472,7 +472,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest
with Bef
     writer.write(record)
     writer.close()
     sql(
-      s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
+      s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
          |'carbondata' LOCATION
          |'$writerPath' """.stripMargin)
     checkAnswer(sql("select * from sdkOutputTable"),
@@ -505,7 +505,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest
with Bef
     writer.write(record)
     writer.close()
     sql(
-      s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
+      s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
          |'carbondata' LOCATION
          |'$writerPath' """.stripMargin)
     sql("describe formatted sdkOutputTable").show(false)
@@ -564,7 +564,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest
with Bef
     writer.write(record)
     writer.close()
     sql(
-      s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
+      s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
          |'carbondata' LOCATION
          |'$writerPath' """.stripMargin)
     checkAnswer(sql("select * from sdkOutputTable"),
@@ -659,7 +659,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest
with Bef
     writer.write(record)
     writer.close()
     sql(
-      s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
+      s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
          |'carbondata' LOCATION
          |'$writerPath' """.stripMargin)
     sql("describe formatted sdkOutputTable").show(false)
@@ -700,7 +700,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest
with Bef
     writer.write(record)
     writer.close()
     sql(
-      s"""CREATE EXTERNAL TABLE sdkOutputTable(id decimal(4,3)) STORED BY
+      s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
          |'carbondata' LOCATION
          |'$writerPath' """.stripMargin)
     checkExistence(sql("select * from sdkOutputTable"), true, "1010.00")
@@ -738,7 +738,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest
with Bef
     writer.write(record)
     writer.close()
     sql(
-      s"""CREATE EXTERNAL TABLE sdkOutputTable(id struct<b:decimal(4,3)>) STORED BY
+      s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
          |'carbondata' LOCATION
          |'$writerPath' """.stripMargin)
     checkExistence(sql("select * from sdkOutputTable"), true, "32.00")
@@ -784,7 +784,7 @@ class TestNonTransactionalCarbonTableWithAvroDataType extends QueryTest
with Bef
     writer.write(record)
     writer.close()
     sql(
-      s"""CREATE EXTERNAL TABLE sdkOutputTable(id struct<b:decimal(4,3)>) STORED BY
+      s"""CREATE EXTERNAL TABLE sdkOutputTable STORED BY
          |'carbondata' LOCATION
          |'$writerPath' """.stripMargin)
     checkExistence(sql("select * from sdkOutputTable"), true, "32.0")

http://git-wip-us.apache.org/repos/asf/carbondata/blob/af2c469b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
index 693a8c4..16cee96 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
@@ -324,9 +324,7 @@ object CarbonSource {
       tableDesc.copy(storage = updatedFormat)
     } else {
       val tableInfo = CarbonUtil.convertGsonToTableInfo(properties.asJava)
-      val isExternal = properties.getOrElse("isExternal", "false")
-      val isTransactionalTable = properties.getOrElse("isTransactional", "true")
-        .contains("true")
+      val isTransactionalTable = properties.getOrElse("isTransactional", "true").contains("true")
       tableInfo.setTransactionalTable(isTransactionalTable)
       if (isTransactionalTable && !metaStore.isReadFromHiveMetaStore) {
         // save to disk
@@ -350,10 +348,7 @@ object CarbonSource {
       query: Option[LogicalPlan]): Map[String, String] = {
     val model = createTableInfoFromParams(properties, dataSchema, identifier, query, sparkSession)
     val tableInfo: TableInfo = TableNewProcessor(model)
-    val isExternal = properties.getOrElse("isExternal", "false")
-    val isTransactionalTable = properties.getOrElse("isTransactional", "true")
-      .contains("true")
-    val tablePath = properties.getOrElse("path", "")
+    val isTransactionalTable = properties.getOrElse("isTransactional", "true").contains("true")
     tableInfo.setTablePath(identifier.getTablePath)
     tableInfo.setTransactionalTable(isTransactionalTable)
     tableInfo.setDatabaseName(identifier.getDatabaseName)

http://git-wip-us.apache.org/repos/asf/carbondata/blob/af2c469b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParserUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParserUtil.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParserUtil.scala
index 9752535..1594c18 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParserUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParserUtil.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql.parser
 
+import scala.collection.JavaConverters._
 import scala.collection.mutable
 
 import org.antlr.v4.runtime.tree.TerminalNode
@@ -127,6 +128,16 @@ object CarbonSparkSqlParserUtil {
     var isTransactionalTable: Boolean = true
 
     val tableInfo = if (external) {
+      if (fields.nonEmpty) {
+        // user provided schema for this external table, this is not allow currently
+        // see CARBONDATA-2866
+        operationNotAllowed(
+          "Schema may not be specified for external table", columns)
+      }
+      if (partitionByStructFields.nonEmpty) {
+        operationNotAllowed(
+          "Partition is not supported for external table", partitionColumns)
+      }
       // read table info from schema file in the provided table path
       // external table also must convert table name to lower case
       val identifier = AbsoluteTableIdentifier.from(
@@ -142,15 +153,14 @@ object CarbonSparkSqlParserUtil {
             isTransactionalTable = false
             SchemaReader.inferSchema(identifier, false)
           }
-        }
-        else {
+        } else {
           SchemaReader.getTableInfo(identifier)
         }
-      }
-      catch {
+      } catch {
         case e: Throwable =>
           operationNotAllowed(s"Invalid table path provided: ${ tablePath.get } ", tableHeader)
       }
+
       // set "_external" property, so that DROP TABLE will not delete the data
       if (provider.equalsIgnoreCase("'carbonfile'")) {
         table.getFactTable.getTableProperties.put("_filelevelformat", "true")


Mime
View raw message