carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [04/50] [abbrv] carbondata git commit: [CARBONDATA-1931]DataLoad failed for Aggregate table when measure is …
Date Sun, 07 Jan 2018 03:05:12 GMT
[CARBONDATA-1931]DataLoad failed for Aggregate table when measure is …

Root Cause , during data loading ,dataframe columns are sorted based on selection order but
in RowConverter fileds values are compared based on ordinal column. Solution = while making
CarbonLoadModel set header sorted by ordinal columns instead of selection order.

This closes 1712


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/525920c2
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/525920c2
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/525920c2

Branch: refs/heads/carbonstore
Commit: 525920c2561efab6390c21db358b0ea6dddf5854
Parents: 7dcc2e7
Author: BJangir <babulaljangir111@gmail.com>
Authored: Fri Dec 22 14:10:48 2017 +0530
Committer: kumarvishal <kumarvishal.1802@gmail.com>
Committed: Tue Dec 26 21:46:02 2017 +0530

----------------------------------------------------------------------
 .../testsuite/preaggregate/TestPreAggregateLoad.scala  | 13 +++++++++++++
 .../command/preaaggregate/PreAggregateUtil.scala       |  3 ++-
 2 files changed, 15 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/525920c2/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateLoad.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateLoad.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateLoad.scala
index 6a5f221..ff1c330 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateLoad.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateLoad.scala
@@ -21,6 +21,9 @@ import org.apache.spark.sql.Row
 import org.apache.spark.sql.test.util.QueryTest
 import org.scalatest.{BeforeAndAfterAll, Ignore}
 
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
 @Ignore
 class TestPreAggregateLoad extends QueryTest with BeforeAndAfterAll {
 
@@ -203,4 +206,14 @@ class TestPreAggregateLoad extends QueryTest with BeforeAndAfterAll {
     checkAnswer(sql("select * from maintable_preagg_sum"), Row(1, 29))
   }
 
+  test("test load in aggregate table with Measure col") {
+    val originalBadRecordsAction = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION)
+    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION,
"FAIL")
+    sql("drop table if exists y ")
+    sql("create table y(year int,month int,name string,salary int) stored by 'carbondata'")
+    sql("insert into y select 10,11,'babu',12")
+    sql("create datamap y1_sum1 on table y using 'preaggregate' as select year,name,sum(salary)
from y group by year,name")
+    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION,
originalBadRecordsAction)
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/525920c2/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
index 8614d66..1f5bd41 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/preaaggregate/PreAggregateUtil.scala
@@ -508,7 +508,8 @@ object PreAggregateUtil {
     val headers = dataMapSchemas.find(_.getChildSchema.getTableName.equalsIgnoreCase(
       dataMapIdentifier.table)) match {
       case Some(dataMapSchema) =>
-        dataMapSchema.getChildSchema.getListOfColumns.asScala.map(_.getColumnName).mkString(",")
+        dataMapSchema.getChildSchema.getListOfColumns.asScala.sortBy(_.getSchemaOrdinal).map(
+          _.getColumnName).mkString(",")
       case None =>
         throw new RuntimeException(
           s"${ dataMapIdentifier.table} datamap not found in DataMapSchema list: ${


Mime
View raw message