carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [carbondata] 05/41: [CARBONDATA-2447] Block update operation on range/list/hash partition table
Date Tue, 02 Apr 2019 02:41:25 GMT
This is an automated email from the ASF dual-hosted git repository.

ravipesala pushed a commit to branch branch-1.5
in repository https://gitbox.apache.org/repos/asf/carbondata.git

commit 1c4c6dc511e461d27eed7699cf87706609fce6ff
Author: qiuchenjian <807169000@qq.com>
AuthorDate: Tue Jan 22 10:17:00 2019 +0800

    [CARBONDATA-2447] Block update operation on range/list/hash partition table
    
    [problem]
    when update the data on range partition table, it will lost data or update failed , see
the jira or new test case
    
    [Cause]
    Range partition table take taskNo in filename as partitionId, when update the taskNo is
inscreasing ,the taskNo didn't changed with partitionId
    
    [Solution]
    (1) When query the range partition table, don't match the partitionid ---this method losses
the meaning of partition
    (2) Range partition table use directory or seperate part as partitionid ---this is not
necessary and suggest to use standard partition
    (3) Range partition table doesn't support update opretion ---this PR use this method
    
    This closes #3091
---
 .../partition/TestUpdateForPartitionTable.scala    | 71 ++++++++++++++++++++++
 .../mutation/CarbonProjectForUpdateCommand.scala   |  9 ++-
 2 files changed, 79 insertions(+), 1 deletion(-)

diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestUpdateForPartitionTable.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestUpdateForPartitionTable.scala
new file mode 100644
index 0000000..14dab1e
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestUpdateForPartitionTable.scala
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.partition
+
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest. BeforeAndAfterAll
+
+class TestUpdateForPartitionTable extends QueryTest with BeforeAndAfterAll {
+
+  override def beforeAll: Unit = {
+    dropTable
+
+    sql("create table test_range_partition_table (id int) partitioned by (name string) "
+
+      "stored by 'carbondata' TBLPROPERTIES('PARTITION_TYPE' = 'RANGE','RANGE_INFO' = 'a,e,f')")
+    sql("create table test_hive_partition_table (id int) partitioned by (name string) " +
+      "stored by 'carbondata'")
+    sql("create table test_hash_partition_table (id int) partitioned by (name string) " +
+      "stored by 'carbondata' TBLPROPERTIES('PARTITION_TYPE' = 'HASH','NUM_PARTITIONS' =
'2')")
+    sql("create table test_list_partition_table (id int) partitioned by (name string) " +
+      "stored by 'carbondata' TBLPROPERTIES('PARTITION_TYPE' = 'LIST','LIST_INFO' = 'a,e,f')")
+  }
+
+  def dropTable = {
+    sql("drop table if exists test_hash_partition_table")
+    sql("drop table if exists test_list_partition_table")
+    sql("drop table if exists test_range_partition_table")
+    sql("drop table if exists test_hive_partition_table")
+  }
+
+
+  test ("test update for unsupported partition table") {
+    val updateTables = Array(
+      "test_range_partition_table",
+      "test_list_partition_table",
+      "test_hash_partition_table")
+
+    updateTables.foreach(table => {
+      sql("insert into " + table + " select 1,'b' ")
+      val ex = intercept[UnsupportedOperationException] {
+        sql("update " + table + " set (name) = ('c') where id = 1").collect()
+      }
+      assertResult("Unsupported update operation for range/hash/list partition table")(ex.getMessage)
+    })
+
+  }
+
+  test ("test update for hive(standard) partition table") {
+
+    sql("insert into test_hive_partition_table select 1,'b' ")
+    sql("update test_hive_partition_table set (name) = ('c') where id = 1").collect()
+    assertResult(1)(sql("select * from test_hive_partition_table where name = 'c'").collect().length)
+  }
+
+  override def afterAll() : Unit = {
+    dropTable
+  }
+}
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForUpdateCommand.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForUpdateCommand.scala
index 0f23081..e4abae1 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForUpdateCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/mutation/CarbonProjectForUpdateCommand.scala
@@ -32,7 +32,7 @@ import org.apache.carbondata.core.datamap.Segment
 import org.apache.carbondata.core.exception.ConcurrentOperationException
 import org.apache.carbondata.core.features.TableOperation
 import org.apache.carbondata.core.locks.{CarbonLockFactory, CarbonLockUtil, LockUsage}
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
+import org.apache.carbondata.core.metadata.schema.partition.PartitionType
 import org.apache.carbondata.core.mutate.CarbonUpdateUtil
 import org.apache.carbondata.core.statusmanager.SegmentStatusManager
 import org.apache.carbondata.core.util.CarbonProperties
@@ -60,6 +60,13 @@ private[sql] case class CarbonProjectForUpdateCommand(
       return Seq.empty
     }
     val carbonTable = CarbonEnv.getCarbonTable(databaseNameOp, tableName)(sparkSession)
+    if (carbonTable.getPartitionInfo != null &&
+      (carbonTable.getPartitionInfo.getPartitionType == PartitionType.RANGE ||
+        carbonTable.getPartitionInfo.getPartitionType == PartitionType.HASH ||
+        carbonTable.getPartitionInfo.getPartitionType == PartitionType.LIST)) {
+      throw new UnsupportedOperationException("Unsupported update operation for range/" +
+        "hash/list partition table")
+    }
     setAuditTable(carbonTable)
     setAuditInfo(Map("plan" -> plan.simpleString))
     columns.foreach { col =>


Mime
View raw message