carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [13/47] incubator-carbondata git commit: [CARBONDATA-99] For complex type filter queries if query contains filter expression rather than BinaryExpression the system was not able to get the dimensions which are involved (#854)
Date Mon, 01 Aug 2016 10:05:11 GMT
[CARBONDATA-99] For complex type filter queries if query contains filter expression rather
than BinaryExpression the system was not able to get the dimensions which are involved (#854)

in the particular filter expression for executing complex type filter column expressions.

Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/7159dce7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/7159dce7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/7159dce7

Branch: refs/heads/master
Commit: 7159dce7b17646c67c9eed36f4d22d2577940055
Parents: fb8d807
Author: sujith71955 <sujithchacko.2010@gmail.com>
Authored: Sun Jul 24 21:44:46 2016 +0530
Committer: Kumar Vishal <kumarvishal.1802@gmail.com>
Committed: Sun Jul 24 21:44:46 2016 +0530

----------------------------------------------------------------------
 .../query/carbon/executor/util/QueryUtil.java   | 20 ++++++++++++++++++++
 .../filter/resolver/AndFilterResolverImpl.java  |  5 +++--
 .../resolver/LogicalFilterResolverImpl.java     | 11 +++++++----
 .../filters/FilterExpressionProcessor.java      | 12 +++++-------
 integration/spark/src/test/resources/array1.csv |  2 ++
 .../complexType/TestComplexTypeQuery.scala      | 19 ++++++++++++++++++-
 6 files changed, 55 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7159dce7/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java b/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java
index 2433cbc..b1de455 100644
--- a/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java
+++ b/core/src/main/java/org/carbondata/query/carbon/executor/util/QueryUtil.java
@@ -1108,6 +1108,8 @@ public class QueryUtil {
     if (filterExpression instanceof BinaryLogicalExpression) {
       BinaryLogicalExpression logicalExpression = (BinaryLogicalExpression) filterExpression;
       dimensionResolvedInfos.addAll(logicalExpression.getColumnList());
+    } else {
+      addColumnDimensions(filterExpression, filterDimensions);
     }
     for (ColumnExpression info : dimensionResolvedInfos) {
       if (info.isDimension() && info.getDimension().getNumberOfChild() > 0) {
@@ -1118,4 +1120,22 @@ public class QueryUtil {
 
   }
 
+  /**
+   * This method will check if a given expression contains a column expression
+   * recursively and add the dimension instance to the set which holds the dimension
+   * instances of the complex filter expressions.
+   *
+   * @param filterDimensions
+   * @return
+   */
+  private static void addColumnDimensions(Expression expression,
+      Set<CarbonDimension> filterDimensions) {
+    if (null != expression && expression instanceof ColumnExpression) {
+      filterDimensions.add(((ColumnExpression) expression).getDimension());
+      return;
+    }
+    for (Expression child : expression.getChildren()) {
+      addColumnDimensions(child, filterDimensions);
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7159dce7/core/src/main/java/org/carbondata/query/filter/resolver/AndFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/filter/resolver/AndFilterResolverImpl.java
b/core/src/main/java/org/carbondata/query/filter/resolver/AndFilterResolverImpl.java
index 442b17e..62ebf99 100644
--- a/core/src/main/java/org/carbondata/query/filter/resolver/AndFilterResolverImpl.java
+++ b/core/src/main/java/org/carbondata/query/filter/resolver/AndFilterResolverImpl.java
@@ -24,6 +24,7 @@ import org.carbondata.core.carbon.AbsoluteTableIdentifier;
 import org.carbondata.core.carbon.datastore.block.SegmentProperties;
 import org.carbondata.query.carbon.executor.exception.QueryExecutionException;
 import org.carbondata.query.carbonfilterinterface.ExpressionType;
+import org.carbondata.query.expression.BinaryExpression;
 
 public class AndFilterResolverImpl extends LogicalFilterResolverImpl {
 
@@ -33,8 +34,8 @@ public class AndFilterResolverImpl extends LogicalFilterResolverImpl {
   private static final long serialVersionUID = -761688076874662001L;
 
   public AndFilterResolverImpl(FilterResolverIntf leftEvalutor, FilterResolverIntf rightEvalutor,
-      ExpressionType filterExpressionType) {
-    super(leftEvalutor, rightEvalutor, filterExpressionType);
+      ExpressionType filterExpressionType,BinaryExpression expression) {
+    super(leftEvalutor, rightEvalutor, expression);
   }
 
   @Override public void getStartKey(SegmentProperties segmentProperties, long[] startKeys,

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7159dce7/core/src/main/java/org/carbondata/query/filter/resolver/LogicalFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/filter/resolver/LogicalFilterResolverImpl.java
b/core/src/main/java/org/carbondata/query/filter/resolver/LogicalFilterResolverImpl.java
index 46b03e4..b0ec737 100644
--- a/core/src/main/java/org/carbondata/query/filter/resolver/LogicalFilterResolverImpl.java
+++ b/core/src/main/java/org/carbondata/query/filter/resolver/LogicalFilterResolverImpl.java
@@ -25,6 +25,7 @@ import org.carbondata.core.carbon.datastore.block.SegmentProperties;
 import org.carbondata.query.carbon.executor.exception.QueryExecutionException;
 import org.carbondata.query.carbonfilterinterface.ExpressionType;
 import org.carbondata.query.carbonfilterinterface.FilterExecuterType;
+import org.carbondata.query.expression.BinaryExpression;
 import org.carbondata.query.expression.Expression;
 import org.carbondata.query.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
 
@@ -40,11 +41,14 @@ public class LogicalFilterResolverImpl implements FilterResolverIntf {
 
   protected ExpressionType filterExpressionType;
 
+  private BinaryExpression filterExpression;
+
   public LogicalFilterResolverImpl(FilterResolverIntf leftEvalutor,
-      FilterResolverIntf rightEvalutor, ExpressionType filterExpressionType) {
+      FilterResolverIntf rightEvalutor,BinaryExpression currentExpression) {
     this.leftEvalutor = leftEvalutor;
     this.rightEvalutor = rightEvalutor;
-    this.filterExpressionType = filterExpressionType;
+    this.filterExpressionType = currentExpression.getFilterExpressionType();
+    this.filterExpression=currentExpression;
   }
 
   /**
@@ -104,7 +108,6 @@ public class LogicalFilterResolverImpl implements FilterResolverIntf {
   }
 
   @Override public Expression getFilterExpression() {
-    // TODO Auto-generated method stub
-    return null;
+    return filterExpression;
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7159dce7/core/src/main/java/org/carbondata/query/filters/FilterExpressionProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/filters/FilterExpressionProcessor.java
b/core/src/main/java/org/carbondata/query/filters/FilterExpressionProcessor.java
index 2d69010..9482efe 100644
--- a/core/src/main/java/org/carbondata/query/filters/FilterExpressionProcessor.java
+++ b/core/src/main/java/org/carbondata/query/filters/FilterExpressionProcessor.java
@@ -41,7 +41,6 @@ import org.carbondata.query.expression.Expression;
 import org.carbondata.query.expression.conditional.BinaryConditionalExpression;
 import org.carbondata.query.expression.conditional.ConditionalExpression;
 import org.carbondata.query.expression.exception.FilterUnsupportedException;
-import org.carbondata.query.expression.logical.BinaryLogicalExpression;
 import org.carbondata.query.filter.executer.FilterExecuter;
 import org.carbondata.query.filter.resolver.ConditionalFilterResolverImpl;
 import org.carbondata.query.filter.resolver.FilterResolverIntf;
@@ -213,7 +212,6 @@ public class FilterExpressionProcessor implements FilterProcessor {
       AbsoluteTableIdentifier tableIdentifier, Expression intermediateExpression) {
     ExpressionType filterExpressionType = expressionTree.getFilterExpressionType();
     BinaryExpression currentExpression = null;
-    BinaryLogicalExpression logicalExpression = null;
     switch (filterExpressionType) {
       case OR:
         currentExpression = (BinaryExpression) expressionTree;
@@ -221,14 +219,14 @@ public class FilterExpressionProcessor implements FilterProcessor {
             createFilterResolverTree(currentExpression.getLeft(), tableIdentifier,
                 currentExpression),
             createFilterResolverTree(currentExpression.getRight(), tableIdentifier,
-                currentExpression), filterExpressionType);
+                currentExpression),currentExpression);
       case AND:
-        logicalExpression = (BinaryLogicalExpression) expressionTree;
+        currentExpression = (BinaryExpression) expressionTree;
         return new LogicalFilterResolverImpl(
-            createFilterResolverTree(logicalExpression.getLeft(), tableIdentifier,
+            createFilterResolverTree(currentExpression.getLeft(), tableIdentifier,
                 currentExpression),
-            createFilterResolverTree(logicalExpression.getRight(), tableIdentifier,
-                currentExpression), filterExpressionType);
+            createFilterResolverTree(currentExpression.getRight(), tableIdentifier,
+                currentExpression), currentExpression);
       case EQUALS:
       case IN:
         return getFilterResolverBasedOnExpressionType(ExpressionType.EQUALS, false, expressionTree,

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7159dce7/integration/spark/src/test/resources/array1.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/array1.csv b/integration/spark/src/test/resources/array1.csv
new file mode 100644
index 0000000..c7a0f89
--- /dev/null
+++ b/integration/spark/src/test/resources/array1.csv
@@ -0,0 +1,2 @@
+1,hello$bye,12345$5678,123456$3456,12.0$13.0,2017-07-09 12:00:00$2016-07-09 13:00:00,123456789$987654321
+2,welcome$hi,9876$1234,13456$356,15.0$18.0,2019-07-09 12:00:00$2015-07-09 13:00:00,8888888888$99999999999
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7159dce7/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
b/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
index ab48133..3f6c3d8 100644
--- a/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
+++ b/integration/spark/src/test/scala/org/carbondata/integration/spark/testsuite/complexType/TestComplexTypeQuery.scala
@@ -37,6 +37,7 @@ class TestComplexTypeQuery extends QueryTest with BeforeAndAfterAll {
   override def beforeAll: Unit = {
      sql("drop table if exists complexcarbontable").show
      sql("drop table if exists complexhivetable").show
+     sql("drop table if exists complex_filter").show
      sql("drop table if exists structusingstructCarbon").show
      sql("drop table if exists structusingstructHive").show
      sql("drop table if exists structusingarraycarbon").show
@@ -45,6 +46,8 @@ class TestComplexTypeQuery extends QueryTest with BeforeAndAfterAll {
      sql("LOAD DATA local inpath './src/test/resources/complextypesample.csv' INTO table
complexcarbontable  OPTIONS('DELIMITER'=',', 'QUOTECHAR'='\"', 'FILEHEADER'='deviceInformationId,channelsId,ROMSize,ROMName,purchasedate,mobile,MAC,locationinfo,proddate,gamePointId,contractNumber',
'COMPLEX_DELIMITER_LEVEL_1'='$', 'COMPLEX_DELIMITER_LEVEL_2'=':')");
      sql("create table complexhivetable(deviceInformationId int, channelsId string, ROMSize
string, ROMName String, purchasedate string, mobile struct<imei:string, imsi:string>,
MAC array<string>, locationinfo array<struct<ActiveAreaId:int, ActiveCountry:string,
ActiveProvince:string, Activecity:string, ActiveDistrict:string, ActiveStreet:string>>,
proddate struct<productionDate:string,activeDeactivedate:array<string>>, gamePointId
double,contractNumber double)row format delimited fields terminated by ',' collection items
terminated by '$' map keys terminated by ':'")
      sql("LOAD DATA local inpath './src/test/resources/complextypesample.csv' INTO table
complexhivetable");
+     sql("create table complex_filter(test1 int, test2 array<String>,test3 array<bigint>,test4
array<int>,test5 array<decimal>,test6 array<timestamp>,test7 array<double>)
STORED BY 'org.apache.carbondata.format'")
+     sql("LOAD DATA INPATH './src/test/resources/array1.csv'  INTO TABLE complex_filter options
('DELIMITER'=',', 'QUOTECHAR'='\"', 'COMPLEX_DELIMITER_LEVEL_1'='$', 'FILEHEADER'= 'test1,test2,test3,test4,test5,test6,test7')").show()
      
      sql("create table structusingarraycarbon (MAC struct<MAC1:array<string>,ActiveCountry:array<string>>)
STORED BY 'org.apache.carbondata.format'");
      sql("LOAD DATA local INPATH './src/test/resources/struct_all.csv' INTO table structusingarraycarbon
options ('DELIMITER'=',', 'QUOTECHAR'='\"', 'FILEHEADER'='MAC','COMPLEX_DELIMITER_LEVEL_1'='$','COMPLEX_DELIMITER_LEVEL_2'='&')")
@@ -57,7 +60,21 @@ class TestComplexTypeQuery extends QueryTest with BeforeAndAfterAll {
      sql("LOAD DATA local INPATH './src/test/resources/structusingstruct.csv' INTO table
structusingstructhive")
      
   }
-
+  
+   
+ 
+   test("complex filter set1") {
+    checkAnswer(
+      sql("select test3[1] from complex_filter where test4[1] not like'%1%' order by test1"),
+      Seq(Row(5678), Row(1234))
+    )
+  }
+   test("complex filter set2") {
+    checkAnswer(
+      sql("select test2[0] from complex_filter  where  test3[0] like '%1234%'"),
+      Seq(Row("hello"))
+    )
+  }
   test("select * from structusingarraycarbon") {
      checkAnswer(sql("select * from structusingarraycarbon"),
      sql("select * from structusingarrayhive"))


Mime
View raw message