carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kunalkap...@apache.org
Subject carbondata git commit: [CARBONDATA-2792][schema restructure] Create external table fails post schema restructure.
Date Thu, 02 Aug 2018 11:09:03 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master 625a2efa7 -> b483a5746


[CARBONDATA-2792][schema restructure] Create external table fails post schema restructure.

Problem
Once the table schema is restructured "(do column drop and column add)".
The api org.apache.carbondata.spark.util.CarbonSparkUtil.getRawSchema(carbonRelation: CarbonRelation)
: String
to get visible columns raw schema string in ascending order of their ordinal value throws
ArrayIndexOutOfBoundException while creating external table.
The api prepares the array of raw column schema of visible columns in the ascending order
of schema ordinal.
It uses the schemaOrdinal as index to prepare the array, so the colum having the schemaOrdinal
value more
than the visible column count will cause the ArrayIndexOutOfBoundException.

Solution
Filter the visible and valid columns
sort the columns based on the schema ordinal.
Prepare the raw column schema based on the sorted columns index.

This closes #2571


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/b483a574
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/b483a574
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/b483a574

Branch: refs/heads/master
Commit: b483a57464a860fe9dbd2d074a3e08fd59141edc
Parents: 625a2ef
Author: mohammadshahidkhan <mohdshahidkhan1987@gmail.com>
Authored: Fri Jul 27 12:51:49 2018 +0530
Committer: kunal642 <kunalkapoor642@gmail.com>
Committed: Thu Aug 2 16:38:02 2018 +0530

----------------------------------------------------------------------
 .../createTable/TestCreateExternalTable.scala         | 14 ++++++++++++++
 .../carbondata/spark/util/CarbonSparkUtil.scala       | 14 +++++++++-----
 2 files changed, 23 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/b483a574/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala
index a9b8d57..6fb24c7 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala
@@ -32,6 +32,8 @@ class TestCreateExternalTable extends QueryTest with BeforeAndAfterAll {
 
   override def beforeAll(): Unit = {
     sql("DROP TABLE IF EXISTS origin")
+    sql("drop table IF EXISTS rsext")
+    sql("drop table IF EXISTS rstest1")
     // create carbon table and insert data
     sql("CREATE TABLE origin(key INT, value STRING) STORED BY 'carbondata'")
     sql("INSERT INTO origin select 100,'spark'")
@@ -41,6 +43,8 @@ class TestCreateExternalTable extends QueryTest with BeforeAndAfterAll {
 
   override def afterAll(): Unit = {
     sql("DROP TABLE IF EXISTS origin")
+    sql("drop table IF EXISTS rsext")
+    sql("drop table IF EXISTS rstest1")
   }
 
   test("create external table with existing files") {
@@ -111,5 +115,15 @@ class TestCreateExternalTable extends QueryTest with BeforeAndAfterAll
{
     }
     assert(exception.getMessage().contains("Create external table as select"))
   }
+  test("create external table with post schema resturcture") {
+    sql("create table rstest1 (c1 string,c2 int) STORED BY 'org.apache.carbondata.format'")
+    sql("Alter table rstest1 drop columns(c2)")
+    sql(
+      "Alter table rstest1 add columns(c4 string) TBLPROPERTIES('DICTIONARY_EXCLUDE'='c4',
" +
+      "'DEFAULT.VALUE.c4'='def')")
+    sql(s"""CREATE EXTERNAL TABLE rsext STORED BY 'carbondata' LOCATION '$storeLocation/rstest1'""")
+    sql("insert into rsext select 'shahid', 1")
+    checkAnswer(sql("select * from rstest1"),  sql("select * from rsext"))
+  }
 
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/b483a574/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/CarbonSparkUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/CarbonSparkUtil.scala
b/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/CarbonSparkUtil.scala
index b9e2442..a0c0545 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/CarbonSparkUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/CarbonSparkUtil.scala
@@ -18,13 +18,14 @@
 package org.apache.carbondata.spark.util
 
 import scala.collection.JavaConverters._
+import scala.collection.mutable
 
 import org.apache.spark.sql.hive.{CarbonMetaData, CarbonRelation, DictionaryMap}
 
 import org.apache.carbondata.core.constants.CarbonCommonConstants
 import org.apache.carbondata.core.metadata.encoder.Encoding
 import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo}
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn
+import org.apache.carbondata.core.metadata.schema.table.column.{CarbonColumn, ColumnSchema}
 import org.apache.carbondata.core.util.CarbonUtil
 
 case class TransformHolder(rdd: Any, mataData: CarbonMetaData)
@@ -87,18 +88,21 @@ object CarbonSparkUtil {
     val fields = new Array[String](
       carbonRelation.dimensionsAttr.size + carbonRelation.measureAttr.size)
     val carbonTable = carbonRelation.carbonTable
+    val columnSchemas: mutable.Buffer[ColumnSchema] = carbonTable.getTableInfo.getFactTable.
+      getListOfColumns.asScala
+      .filter(cSchema => !cSchema.isInvisible && cSchema.getSchemaOrdinal != -1).
+      sortWith(_.getSchemaOrdinal < _.getSchemaOrdinal)
+    val columnList = columnSchemas.toList.asJava
     carbonRelation.dimensionsAttr.foreach(attr => {
-      val carbonDimension = carbonTable.getDimensionByName(carbonRelation.tableName, attr.name)
       val carbonColumn = carbonTable.getColumnByName(carbonRelation.tableName, attr.name)
       val columnComment = getColumnComment(carbonColumn)
-      fields(carbonDimension.getSchemaOrdinal) =
+      fields(columnList.indexOf(carbonColumn.getColumnSchema)) =
         '`' + attr.name + '`' + ' ' + attr.dataType.catalogString + columnComment
     })
     carbonRelation.measureAttr.foreach(msrAtrr => {
-      val carbonMeasure = carbonTable.getMeasureByName(carbonRelation.tableName, msrAtrr.name)
       val carbonColumn = carbonTable.getColumnByName(carbonRelation.tableName, msrAtrr.name)
       val columnComment = getColumnComment(carbonColumn)
-      fields(carbonMeasure.getSchemaOrdinal) =
+      fields(columnList.indexOf(carbonColumn.getColumnSchema)) =
         '`' + msrAtrr.name + '`' + ' ' + msrAtrr.dataType.catalogString + columnComment
     })
     fields.mkString(",")


Mime
View raw message