carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbono...@apache.org
Subject [47/56] [abbrv] incubator-carbondata git commit: [issue-581] fix the bug of load parse failure when tblname is in carbon's datatype in DDL (#704)
Date Thu, 23 Jun 2016 14:16:35 GMT
[issue-581] fix the bug of load parse failure when tblname is in carbon's datatype in DDL (#704)

* fix the bug of load parse failure when tblname is in carbon's datatype keywords in DDL

Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/5e2e66f1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/5e2e66f1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/5e2e66f1

Branch: refs/heads/master
Commit: 5e2e66f1f79ec981c733b0c3ce7798230ab8ca81
Parents: 5c7b043
Author: Zhangshunyu <zhangshunyu@huawei.com>
Authored: Mon Jun 20 22:32:52 2016 +0800
Committer: Venkata Ramana G <g.ramana.v@gmail.com>
Committed: Mon Jun 20 20:02:52 2016 +0530

----------------------------------------------------------------------
 .../org/apache/spark/sql/CarbonSqlParser.scala  | 33 +++----
 .../dataload/TestLoadTblNameIsKeyword.scala     | 96 ++++++++++++++++++++
 2 files changed, 110 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5e2e66f1/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
index 56a8a8e..7474f4f 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonSqlParser.scala
@@ -23,6 +23,7 @@ import java.util.regex.{Matcher, Pattern}
 import scala.collection.JavaConverters._
 import scala.collection.mutable.LinkedHashSet
 import scala.language.implicitConversions
+import scala.util.matching.Regex
 
 import org.apache.hadoop.hive.ql.lib.Node
 import org.apache.hadoop.hive.ql.parse._
@@ -36,13 +37,10 @@ import org.apache.spark.sql.execution.datasources.DescribeCommand
 import org.apache.spark.sql.hive.HiveQlWrapper
 
 import org.carbondata.core.carbon.metadata.datatype.DataType
-import org.carbondata.core.constants.CarbonCommonConstants
 import org.carbondata.core.util.DataTypeUtil
 import org.carbondata.spark.exception.MalformedCarbonCommandException
 import org.carbondata.spark.util.CommonUtil
 
-
-
 /**
  * Parser for All Carbon DDL, DML cases in Unified context
  */
@@ -73,7 +71,6 @@ class CarbonSqlParser()
   protected val DETAIL = Keyword("DETAIL")
   protected val DIMENSIONS = Keyword("DIMENSIONS")
   protected val DIMFOLDERPATH = Keyword("DIMFOLDERPATH")
-  protected val DOUBLE = Keyword("DOUBLE")
   protected val DROP = Keyword("DROP")
   protected val ESCAPECHAR = Keyword("ESCAPECHAR")
   protected val EXCLUDE = Keyword("EXCLUDE")
@@ -88,25 +85,17 @@ class CarbonSqlParser()
   protected val IN = Keyword("IN")
   protected val INCLUDE = Keyword("INCLUDE")
   protected val INPATH = Keyword("INPATH")
-  protected val INT = Keyword("INT")
-  protected val INTEGER = Keyword("INTEGER")
   protected val INTO = Keyword("INTO")
   protected val LEVELS = Keyword("LEVELS")
   protected val LIKE = Keyword("LIKE")
   protected val LOAD = Keyword("LOAD")
   protected val LOADS = Keyword("LOADS")
   protected val LOCAL = Keyword("LOCAL")
-  protected val LONG = Keyword("LONG")
   protected val MAPPED = Keyword("MAPPED")
   protected val MEASURES = Keyword("MEASURES")
   protected val MULTILINE = Keyword("MULTILINE")
   protected val COMPLEX_DELIMITER_LEVEL_1 = Keyword("COMPLEX_DELIMITER_LEVEL_1")
   protected val COMPLEX_DELIMITER_LEVEL_2 = Keyword("COMPLEX_DELIMITER_LEVEL_2")
-  protected val NUMERIC = Keyword("NUMERIC")
-  protected val ARRAY = Keyword("ARRAY")
-  protected val STRUCT = Keyword("STRUCT")
-  protected val BIGINT = Keyword("BIGINT")
-  protected val DECIMAL = Keyword("DECIMAL")
   protected val OPTIONS = Keyword("OPTIONS")
   protected val OUTPATH = Keyword("OUTPATH")
   protected val OVERWRITE = Keyword("OVERWRITE")
@@ -117,11 +106,9 @@ class CarbonSqlParser()
   protected val RELATION = Keyword("RELATION")
   protected val SCHEMAS = Keyword("SCHEMAS")
   protected val SHOW = Keyword("SHOW")
-  protected val STRING = Keyword("STRING")
   protected val TABLES = Keyword("TABLES")
   protected val TABLE = Keyword("TABLE")
   protected val TERMINATED = Keyword("TERMINATED")
-  protected val TIMESTAMP = Keyword("TIMESTAMP")
   protected val TYPE = Keyword("TYPE")
   protected val USE = Keyword("USE")
   protected val WHERE = Keyword("WHERE")
@@ -169,6 +156,11 @@ class CarbonSqlParser()
 
   }
 
+  import lexical.Identifier
+  implicit def regexToParser(regex: Regex): Parser[String] = acceptMatch(
+    s"identifier matching regex ${regex}",
+    { case Identifier(str) if regex.unapplySeq(str).isDefined => str }
+  )
   override def parse(input: String): LogicalPlan = synchronized {
     // Initialize the Keywords.
     initLexical
@@ -1131,7 +1123,9 @@ class CarbonSqlParser()
   protected lazy val dimCol: Parser[Field] = anyFieldDef
 
   protected lazy val primitiveTypes =
-    STRING | INTEGER | TIMESTAMP | NUMERIC | BIGINT | DECIMAL | INT | DOUBLE
+    "(?i)string".r ^^^ "string" | "(?i)integer".r ^^^ "integer" | "(?i)timestamp".r ^^^
+    "timestamp" | "(?i)numeric".r ^^^ "numeric" | "(?i)bigint".r ^^^ "bigint" |
+    "(?i)decimal".r ^^^ "decimal" | "(?i)int".r ^^^ "int" | "(?i)double".r ^^^ "double"
   protected lazy val nestedType: Parser[Field] = structFieldType | arrayFieldType |
     primitiveFieldType
 
@@ -1148,7 +1142,7 @@ class CarbonSqlParser()
     }
 
   protected lazy val arrayFieldType: Parser[Field] =
-    (ARRAY ~> "<" ~> nestedType <~ ">") ^^ {
+    (("(?i)array".r ^^^ "array") ~> "<" ~> nestedType <~ ">") ^^ {
       case e1 =>
         Field("unknown", Some("array"), Some("unknown"),
           Some(List(Field("val", e1.dataType, Some("val"),
@@ -1156,14 +1150,15 @@ class CarbonSqlParser()
     }
 
   protected lazy val structFieldType: Parser[Field] =
-    (STRUCT ~> "<" ~> repsep(anyFieldDef, ",") <~ ">") ^^ {
+    (("(?i)struct".r ^^^ "struct") ~> "<" ~> repsep(anyFieldDef, ",") <~ ">")
^^ {
       case e1 =>
         Field("unknown", Some("struct"), Some("unknown"), Some(e1))
     }
 
   protected lazy val measureCol: Parser[Field] =
-    (ident | stringLit) ~ (INTEGER | NUMERIC | BIGINT | DECIMAL).? ~ (AS ~> (ident | stringLit)).?
~
-      (IN ~> (ident | stringLit)).? ^^ {
+    (ident | stringLit) ~ ("(?i)integer".r ^^^ "integer" | "(?i)numeric".r ^^^ "numeric"
|
+      "(?i)bigint".r ^^^ "bigint" | "(?i)decimal".r ^^^ "decimal").? ~
+      (AS ~> (ident | stringLit)).? ~ (IN ~> (ident | stringLit)).? ^^ {
       case e1 ~ e2 ~ e3 ~ e4 => Field(e1, e2, e3, Some(null))
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5e2e66f1/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadTblNameIsKeyword.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadTblNameIsKeyword.scala
b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadTblNameIsKeyword.scala
new file mode 100644
index 0000000..6e52fb8
--- /dev/null
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadTblNameIsKeyword.scala
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.carbondata.integration.spark.testsuite.dataload
+
+import java.io.File
+
+import org.apache.spark.sql.common.util.CarbonHiveContext._
+import org.apache.spark.sql.common.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+import org.apache.spark.sql.Row
+
+/**
+  * Test Class for data loading into table whose name is key word
+  *
+  */
+class TestLoadTblNameIsKeyword extends QueryTest with BeforeAndAfterAll {
+  def currentPath: String = new File(this.getClass.getResource("/").getPath + "/../../")
+    .getCanonicalPath
+  val testData = new File(currentPath + "/../../examples/src/main/resources/dimSample.csv")
+    .getCanonicalPath
+  override def beforeAll {
+    sql("drop table if exists STRING")
+    sql("drop table if exists DoUbLe")
+    sql("drop table if exists timestamp")
+    sql("""
+          CREATE TABLE IF NOT EXISTS STRING
+          (id Int, name String, city String)
+          STORED BY 'org.apache.carbondata.format'
+        """)
+    sql("""
+          CREATE TABLE IF NOT EXISTS DoUbLe
+          (id Int, name String, city String)
+          STORED BY 'org.apache.carbondata.format'
+        """)
+    sql("""
+          CREATE TABLE IF NOT EXISTS timestamp
+          (id Int, name String, city String)
+          STORED BY 'org.apache.carbondata.format'
+        """)
+  }
+
+  test("test load data whose name is a keyword of data type") {
+    sql(s"""
+          LOAD DATA LOCAL INPATH '$testData' into table STRING
+        """)
+    checkAnswer(
+      sql("""
+            SELECT count(*) from STRING
+          """),
+      Seq(Row(20)))
+  }
+
+  test("test case in-sensitiveness") {
+    sql(s"""
+          LOAD DATA LOCAL INPATH '$testData' into table DoUbLe
+        """)
+    checkAnswer(
+      sql("""
+            SELECT count(*) from DoUbLe
+          """),
+      Seq(Row(20)))
+  }
+
+  test("test other ddl whose table name a keyword of data type") {
+    sql("describe timestamp")
+    sql(s"""
+          LOAD DATA LOCAL INPATH '$testData' into table timestamp
+        """)
+    sql("show segments for table timestamp")
+    sql("delete segments from table timestamp where starttime before '2099-10-01 18:00:00'")
+    sql("clean files for table timestamp")
+  }
+
+  override def afterAll {
+    sql("drop table STRING")
+    sql("drop table DoUbLe")
+    sql("drop table timestamp")
+  }
+}


Mime
View raw message