carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From manishgupt...@apache.org
Subject carbondata git commit: [CARBONDATA-2436] Block pruning problem post the carbon schema restructure.
Date Mon, 07 May 2018 05:59:03 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master 729286919 -> a7926ea13


[CARBONDATA-2436] Block pruning problem post the carbon schema restructure.

Currently DataMap is pruning with segmentproperties from the 0th block of BlockletDataMap
is not correct. As post restructure if the table is updated then all the block
will not have symmetric schema within the same segments.

Solution : It must be ensured the block could be pruned with the same schema.

This closes #2263


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/a7926ea1
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/a7926ea1
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/a7926ea1

Branch: refs/heads/master
Commit: a7926ea133170bd027756969410e043eb9a4b7a3
Parents: 7292869
Author: mohammadshahidkhan <mohdshahidkhan1987@gmail.com>
Authored: Fri May 4 16:36:57 2018 +0530
Committer: manishgupta88 <tomanishgupta18@gmail.com>
Committed: Mon May 7 11:32:18 2018 +0530

----------------------------------------------------------------------
 .../blockletindex/BlockletDataMap.java          |  6 ++++-
 .../AlterTableValidationTestCase.scala          | 23 ++++++++++++++++++--
 2 files changed, 26 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/a7926ea1/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
index 906291a..f72dc06 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
@@ -685,7 +685,11 @@ public class BlockletDataMap extends CoarseGrainDataMap implements Cacheable
{
       }
     }
     // Prune with filters if the partitions are existed in this datamap
-    return prune(filterExp, segmentProperties);
+    // changed segmentProperties to this.segmentProperties to make sure the pruning with
its own
+    // segmentProperties.
+    // Its a temporary fix. The Interface DataMap.prune(FilterResolverIntf filterExp,
+    // SegmentProperties segmentProperties, List<PartitionSpec> partitions) should
be corrected
+    return prune(filterExp, this.segmentProperties);
   }
 
   private boolean isCorrectUUID(String[] fileDetails, PartitionSpec spec) {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/a7926ea1/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
index a86bf17..648ed11 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala
@@ -43,7 +43,7 @@ class AlterTableValidationTestCase extends Spark2QueryTest with BeforeAndAfterAl
     sql("drop table if exists testalterwithboolean")
     sql("drop table if exists testalterwithbooleanwithoutdefaultvalue")
     sql("drop table if exists test")
-
+    sql("drop table if exists retructure_iud")
 
     // clean data folder
     CarbonProperties.getInstance()
@@ -690,7 +690,25 @@ test("test alter command for boolean data type with correct default measure
valu
     testFilterWithDefaultValue(true)
     testFilterWithDefaultValue(false)
   }
-
+  test("Filter query on Restructure and updated table") {
+    sql(
+      """
+         CREATE TABLE retructure_iud(id int, name string, city string, age int)
+         STORED BY 'org.apache.carbondata.format'
+      """)
+    val testData = s"$resourcesPath/sample.csv"
+    sql(s"LOAD DATA LOCAL INPATH '$testData' into table retructure_iud")
+    sql("ALTER TABLE retructure_iud ADD COLUMNS (newField STRING) " +
+        "TBLPROPERTIES ('DEFAULT.VALUE.newField'='def', 'DICTIONARY_INCLUDE'='newField')").show()
+    sql("ALTER TABLE retructure_iud ADD COLUMNS (newField1 STRING) " +
+        "TBLPROPERTIES ('DEFAULT.VALUE.newField1'='def', 'DICTIONARY_EXCLUDE'='newField1')").show()
+    // update operation
+    sql("""update retructure_iud d  set (d.id) = (d.id + 1) where d.id > 2""").show()
+    checkAnswer(
+      sql("select count(*) from retructure_iud where id = 2 and newfield1='def'"),
+      Seq(Row(1))
+    )
+  }
   override def afterAll {
     sql("DROP TABLE IF EXISTS restructure")
     sql("drop table if exists table1")
@@ -707,5 +725,6 @@ test("test alter command for boolean data type with correct default measure
valu
     sql("drop table if exists testalterwithboolean")
     sql("drop table if exists testalterwithbooleanwithoutdefaultvalue")
     sql("drop table if exists test")
+    sql("drop table if exists retructure_iud")
   }
 }


Mime
View raw message