carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From manishgupt...@apache.org
Subject [2/2] carbondata git commit: [CARBONDATA-2684] [PR-2442] Distinct count fails on complex columns
Date Fri, 06 Jul 2018 05:02:19 GMT
[CARBONDATA-2684] [PR-2442] Distinct count fails on complex columns

This PR fixes Code Generator Error thrown when Select filter contains more than one count
of distinct of ComplexColumn with group by Clause

This closes #2449


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/19a99e15
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/19a99e15
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/19a99e15

Branch: refs/heads/master
Commit: 19a99e15e5c9489ded3013cbc6bb02cab38d0720
Parents: cbecadf
Author: ravipesala <ravi.pesala@gmail.com>
Authored: Wed Jul 4 19:57:54 2018 +0530
Committer: manishgupta88 <tomanishgupta18@gmail.com>
Committed: Fri Jul 6 10:35:06 2018 +0530

----------------------------------------------------------------------
 .../complexType/TestComplexDataType.scala       |  2 +-
 .../sql/optimizer/CarbonLateDecodeRule.scala    | 31 +++++++++++++++-----
 2 files changed, 24 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/19a99e15/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
index ba0dc66..ab574c9 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
@@ -645,7 +645,7 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll {
       "sno:array<int>,sal:array<double>,state:array<string>,date1:array<timestamp>>)
stored by " +
       "'carbondata'")
     sql("insert into test values('cus_01','1$2017/01/01$1:2$2.0:3.0$ab:ac$2018/01/01')")
-    sql("select *from test").show(false)
+//    sql("select *from test").show(false)
     sql(
       "select struct_of_array.state[0],count(distinct struct_of_array.id) as count_int,count"
+
       "(distinct struct_of_array.state[0]) as count_string from test group by struct_of_array"
+

http://git-wip-us.apache.org/repos/asf/carbondata/blob/19a99e15/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonLateDecodeRule.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonLateDecodeRule.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonLateDecodeRule.scala
index 68e7f07..9738ab0 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonLateDecodeRule.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonLateDecodeRule.scala
@@ -30,7 +30,7 @@ import org.apache.spark.sql.catalyst.rules.Rule
 import org.apache.spark.sql.execution.command.RunnableCommand
 import org.apache.spark.sql.execution.datasources.LogicalRelation
 import org.apache.spark.sql.profiler.{Optimizer, Profiler}
-import org.apache.spark.sql.types.{IntegerType, StringType}
+import org.apache.spark.sql.types._
 
 import org.apache.carbondata.common.logging.LogServiceFactory
 import org.apache.carbondata.core.constants.CarbonCommonConstants
@@ -90,12 +90,7 @@ class CarbonLateDecodeRule extends Rule[LogicalPlan] with PredicateHelper
{
         LOGGER.info("skip CarbonOptimizer for scalar/predicate sub query")
         return false
       }
-      if(relations.exists(_.dictionaryMap.dictionaryMap.exists(_._2))) {
-        true
-      } else {
-        false
-      }
-
+      true
     } else {
       LOGGER.info("skip CarbonOptimizer")
       false
@@ -678,7 +673,9 @@ class CarbonLateDecodeRule extends Rule[LogicalPlan] with PredicateHelper
{
         val updatedProj = ex.projections.map { projs =>
           projs.zipWithIndex.map { case(p, index) =>
             p.transform {
-              case l: Literal if l.dataType != ex.output(index).dataType =>
+              case l: Literal
+                if l.dataType != ex.output(index).dataType &&
+                   !isComplexColumn(ex.output(index), ex.child.output) =>
                 Literal(l.value, ex.output(index).dataType)
             }
           }
@@ -734,6 +731,24 @@ class CarbonLateDecodeRule extends Rule[LogicalPlan] with PredicateHelper
{
     }
   }
 
+  /**
+   * Check whether given column is derived from complex column.
+   */
+  def isComplexColumn(attribute: Attribute, output: Seq[Attribute]): Boolean = {
+    val attrName = attribute.name.replace("`", "")
+    output.exists { a =>
+      a.dataType match {
+        case s: StructType =>
+          s.fields.map(sf => a.name + "." + sf.name).exists(n => {
+            attrName.contains(n)
+          })
+        case ar : ArrayType =>
+          attrName.contains(a.name + "[")
+        case _ => false
+      }
+    }
+  }
+
   private def updateProjection(plan: LogicalPlan): LogicalPlan = {
     val transFormedPlan = plan transform {
       case p@Project(projectList: Seq[NamedExpression], cd: CarbonDictionaryCatalystDecoder)
=>


Mime
View raw message