carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [1/2] incubator-carbondata git commit: Fixed compaction with multiple blocklet issue
Date Fri, 13 Jan 2017 08:45:48 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 6a2afc179 -> e547fd00a


Fixed compaction with multiple blocklet issue


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/822f23b8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/822f23b8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/822f23b8

Branch: refs/heads/master
Commit: 822f23b8046f692b80f29832a1c58c97acf3b43f
Parents: 6a2afc1
Author: kumarvishal <kumarvishal.1802@gmail.com>
Authored: Tue Jan 10 21:37:15 2017 +0800
Committer: jackylk <jacky.likun@huawei.com>
Committed: Fri Jan 13 16:31:46 2017 +0800

----------------------------------------------------------------------
 .../datacompaction/DataCompactionBlockletBoundryTest.scala      | 2 +-
 .../scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala | 5 ++++-
 2 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/822f23b8/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBlockletBoundryTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBlockletBoundryTest.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBlockletBoundryTest.scala
index 8b8b3c5..149ab6c 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBlockletBoundryTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBlockletBoundryTest.scala
@@ -36,7 +36,7 @@ class DataCompactionBlockletBoundryTest extends QueryTest with BeforeAndAfterAll
       .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "mm/dd/yyyy")
     CarbonProperties.getInstance()
       .addProperty(CarbonCommonConstants.BLOCKLET_SIZE,
-        "55")
+        "120")
     sql(
       "CREATE TABLE IF NOT EXISTS blocklettest (country String, ID String, date Timestamp,
name " +
         "String, " +

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/822f23b8/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
index 09c35ae..4923753 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
@@ -318,7 +318,10 @@ class CarbonMergerRDD[K, V](
 
     // prepare the details required to extract the segment properties using last segment.
     if (null != carbonInputSplits && carbonInputSplits.nonEmpty) {
-      val carbonInputSplit = carbonInputSplits.last
+      // taking head as scala sequence is use and while adding it will add at first
+      // so as we need to update the update the key of older segments with latest keygenerator
+      // we need to take the top of the split
+      val carbonInputSplit = carbonInputSplits.head
       var dataFileFooter: DataFileFooter = null
 
       try {


Mime
View raw message