carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chenliang...@apache.org
Subject [01/50] [abbrv] incubator-carbondata git commit: [CARBONDATA-27] - Fixed filter expression to_date( productdate ) = '2012-12-12' (#784)
Date Wed, 20 Jul 2016 10:13:29 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 383b7db1b -> 1730082ff


[CARBONDATA-27] - Fixed filter expression to_date( productdate ) = '2012-12-12' (#784)

Also fixed cases if incompatible types filter match

Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/17c8128e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/17c8128e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/17c8128e

Branch: refs/heads/master
Commit: 17c8128e6399bbda2e14405c7eeeb02043a01a00
Parents: 8aaa5d1
Author: Mohammad Shahid Khan <mohdshahidkhan1987@gmail.com>
Authored: Fri Jul 15 00:16:54 2016 +0530
Committer: Venkata Ramana G <g.ramana.v1@gmail.com>
Committed: Fri Jul 15 00:16:54 2016 +0530

----------------------------------------------------------------------
 .../org/apache/spark/sql/CarbonOperators.scala  | 105 +++++++++++++------
 1 file changed, 74 insertions(+), 31 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/17c8128e/integration/spark/src/main/scala/org/apache/spark/sql/CarbonOperators.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonOperators.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonOperators.scala
index 3a5f975..d5c3397 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonOperators.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonOperators.scala
@@ -29,19 +29,17 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.execution.LeafNode
 import org.apache.spark.sql.hive.CarbonMetastoreCatalog
+import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 
 import org.carbondata.common.logging.LogServiceFactory
-import org.carbondata.core.carbon.{AbsoluteTableIdentifier}
 import org.carbondata.core.constants.CarbonCommonConstants
 import org.carbondata.core.util.CarbonProperties
 import org.carbondata.hadoop.CarbonInputFormat
 import org.carbondata.query.aggregator.impl.CountAggregator
-import org.carbondata.query.carbon.model.{CarbonQueryPlan, QueryDimension, QueryMeasure,
QueryModel, SortOrderType}
+import org.carbondata.query.carbon.model._
 import org.carbondata.query.carbon.result.RowResult
-import org.carbondata.query.expression.{ColumnExpression => CarbonColumnExpression}
-import org.carbondata.query.expression.{Expression => CarbonExpression}
-import org.carbondata.query.expression.{LiteralExpression => CarbonLiteralExpression}
+import org.carbondata.query.expression.{ColumnExpression => CarbonColumnExpression, Expression
=> CarbonExpression, LiteralExpression => CarbonLiteralExpression}
 import org.carbondata.query.expression.arithmetic.{AddExpression, DivideExpression, MultiplyExpression,
SubstractExpression}
 import org.carbondata.query.expression.conditional._
 import org.carbondata.query.expression.logical.{AndExpression, OrExpression}
@@ -398,63 +396,108 @@ case class CarbonTableScan(
     }
   }
 
+
+  def isCarbonSupportedDataTypes(expr: Expression): Boolean = {
+    expr.dataType match {
+      case StringType => true
+      case IntegerType => true
+      case LongType => true
+      case DoubleType => true
+      case FloatType => true
+      case BooleanType => true
+      case TimestampType => true
+      case ArrayType(_, _) => true
+      case StructType(_) => true
+      case DecimalType() => true
+      case _ => false
+    }
+  }
+
   def transformExpression(expr: Expression): CarbonExpression = {
     expr match {
-      case Or(left, right) => new
+      case Or(left, right)
+        if (isCarbonSupportedDataTypes(left) && isCarbonSupportedDataTypes(right))
=> new
           OrExpression(transformExpression(left), transformExpression(right))
-      case And(left, right) => new
+      case And(left, right)
+        if (isCarbonSupportedDataTypes(left) && isCarbonSupportedDataTypes(right))
=> new
           AndExpression(transformExpression(left), transformExpression(right))
-      case EqualTo(left, right) => new
+      case EqualTo(left, right)
+        if (isCarbonSupportedDataTypes(left) && isCarbonSupportedDataTypes(right))
=> new
           EqualToExpression(transformExpression(left), transformExpression(right))
-      case Not(EqualTo(left, right)) => new
+      case Not(EqualTo(left, right))
+        if (isCarbonSupportedDataTypes(left) && isCarbonSupportedDataTypes(right))
=> new
           NotEqualsExpression(transformExpression(left), transformExpression(right))
-      case IsNotNull(child) => new
+      case IsNotNull(child)
+        if (isCarbonSupportedDataTypes(child)) => new
           NotEqualsExpression(transformExpression(child), transformExpression(Literal(null)))
-      case Not(In(left, right)) => new NotInExpression(transformExpression(left),
-        new ListExpression(right.map(transformExpression).asJava))
-      case In(left, right) => new InExpression(transformExpression(left),
-        new ListExpression(right.map(transformExpression).asJava))
-      case Add(left, right) => new
+      case Not(In(left, right))
+        if (isCarbonSupportedDataTypes(left)) => new
+          NotInExpression(transformExpression(left),
+            new ListExpression(right.map(transformExpression).asJava)
+          )
+      case In(left, right)
+        if (isCarbonSupportedDataTypes(left)) => new
+          InExpression(transformExpression(left),
+            new ListExpression(right.map(transformExpression).asJava)
+          )
+      case Add(left, right)
+        if (isCarbonSupportedDataTypes(left) && isCarbonSupportedDataTypes(right))
=> new
           AddExpression(transformExpression(left), transformExpression(right))
-      case Subtract(left, right) => new
+      case Subtract(left, right)
+        if (isCarbonSupportedDataTypes(left) && isCarbonSupportedDataTypes(right))
=> new
           SubstractExpression(transformExpression(left), transformExpression(right))
-      case Multiply(left, right) => new
+      case Multiply(left, right)
+        if (isCarbonSupportedDataTypes(left) && isCarbonSupportedDataTypes(right))
=> new
           MultiplyExpression(transformExpression(left), transformExpression(right))
-      case Divide(left, right) => new
+      case Divide(left, right)
+        if (isCarbonSupportedDataTypes(left) && isCarbonSupportedDataTypes(right))
=> new
           DivideExpression(transformExpression(left), transformExpression(right))
-      case GreaterThan(left, right) => new
+      case GreaterThan(left, right)
+        if (isCarbonSupportedDataTypes(left) && isCarbonSupportedDataTypes(right))
=> new
           GreaterThanExpression(transformExpression(left), transformExpression(right))
-      case LessThan(left, right) => new
+      case LessThan(left, right)
+        if (isCarbonSupportedDataTypes(left) && isCarbonSupportedDataTypes(right))
=> new
           LessThanExpression(transformExpression(left), transformExpression(right))
-      case GreaterThanOrEqual(left, right) => new
+      case GreaterThanOrEqual(left, right)
+        if (isCarbonSupportedDataTypes(left) && isCarbonSupportedDataTypes(right))
=> new
           GreaterThanEqualToExpression(transformExpression(left), transformExpression(right))
-      case LessThanOrEqual(left, right) => new
+      case LessThanOrEqual(left, right)
+        if (isCarbonSupportedDataTypes(left) && isCarbonSupportedDataTypes(right))
=> new
           LessThanEqualToExpression(transformExpression(left), transformExpression(right))
       // convert StartWith('abc') or like(col 'abc%') to col >= 'abc' and col < 'abd'
-      case StartsWith(left, right @ Literal(pattern, dataType)) if pattern.toString.nonEmpty
=>
+      case StartsWith(left, right@Literal(pattern, dataType))
+        if (pattern.toString.size > 0 &&
+          isCarbonSupportedDataTypes(left) && isCarbonSupportedDataTypes(right))
=>
         val l = new GreaterThanEqualToExpression(
-          transformExpression(left), transformExpression(right))
+          transformExpression(left), transformExpression(right)
+        )
         val value = pattern.toString
         val maxValueLimit = value.substring(0, value.length - 1) +
           (value.charAt(value.length - 1).toInt + 1).toChar
         val r = new LessThanExpression(
           transformExpression(left),
-            new CarbonLiteralExpression(maxValueLimit,
-              CarbonScalaUtil.convertSparkToCarbonDataType(dataType)))
+          new CarbonLiteralExpression(maxValueLimit,
+            CarbonScalaUtil.convertSparkToCarbonDataType(dataType)
+          )
+        )
         new AndExpression(l, r)
       case AttributeReference(name, dataType, _, _) => new CarbonColumnExpression(name.toString,
-        CarbonScalaUtil.convertSparkToCarbonDataType(dataType))
+        CarbonScalaUtil.convertSparkToCarbonDataType(dataType)
+      )
       case Literal(name, dataType) => new
           CarbonLiteralExpression(name, CarbonScalaUtil.convertSparkToCarbonDataType(dataType))
-      case Cast(left, right) if !left.isInstanceOf[Literal] => transformExpression(left)
+      // case Cast(left, right) if !left.isInstanceOf[Literal] => transformExpression(left)
       case aggExpr: AggregateExpression =>
-          throw new UnsupportedOperationException(s"Cannot evaluate expression: $aggExpr")
+        throw new UnsupportedOperationException(s"Cannot evaluate expression: $aggExpr")
       case _ =>
         new SparkUnknownExpression(expr.transform {
           case AttributeReference(name, dataType, _, _) =>
             CarbonBoundReference(new CarbonColumnExpression(name.toString,
-              CarbonScalaUtil.convertSparkToCarbonDataType(dataType)), dataType, expr.nullable)
-        })
+              CarbonScalaUtil.convertSparkToCarbonDataType(dataType)
+            ), dataType, expr.nullable
+            )
+        }
+        )
     }
   }
 


Mime
View raw message