carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [1/2] incubator-carbondata git commit: Made partition distribution as configurable
Date Tue, 21 Feb 2017 10:18:22 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master b400b82ce -> b4db25164


Made partition distribution as configurable

Fixed comment


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/e60a7b66
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/e60a7b66
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/e60a7b66

Branch: refs/heads/master
Commit: e60a7b669384c27467d7b5c86abc307c42a5a9f3
Parents: b400b82
Author: ravipesala <ravi.pesala@gmail.com>
Authored: Wed Feb 15 15:26:39 2017 +0530
Committer: jackylk <jacky.likun@huawei.com>
Committed: Tue Feb 21 18:17:21 2017 +0800

----------------------------------------------------------------------
 .../org/apache/carbondata/spark/rdd/CarbonScanRDD.scala | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e60a7b66/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
index 3415a81..09612cb 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
@@ -116,7 +116,8 @@ class CarbonScanRDD(
           i += 1
           result.add(partition)
         }
-      } else {
+      } else if (sparkContext.getConf.contains("spark.carbon.custom.distribution") &&
+                 sparkContext.getConf.getBoolean("spark.carbon.custom.distribution", false))
{
         // create a list of block based on split
         val blockList = splits.asScala.map(_.asInstanceOf[Distributable])
 
@@ -141,6 +142,15 @@ class CarbonScanRDD(
           }
         }
         noOfNodes = nodeBlockMapping.size
+      } else {
+        splits.asScala.zipWithIndex.foreach { splitWithIndex =>
+          val multiBlockSplit =
+            new CarbonMultiBlockSplit(identifier,
+              Seq(splitWithIndex._1.asInstanceOf[CarbonInputSplit]).asJava,
+              splitWithIndex._1.getLocations)
+          val partition = new CarbonSparkPartition(id, splitWithIndex._2, multiBlockSplit)
+          result.add(partition)
+        }
       }
 
       noOfBlocks = splits.size


Mime
View raw message