carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [11/50] [abbrv] carbondata git commit: [CARBONDATA-2655][BloomDataMap] BloomFilter datamap support in operator
Date Wed, 18 Jul 2018 02:19:44 GMT
[CARBONDATA-2655][BloomDataMap] BloomFilter datamap support in operator

Now queries with in expression on bloom index column can leverage the
BloomFilter datamap.

This closes #2445


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/202d099d
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/202d099d
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/202d099d

Branch: refs/heads/carbonstore
Commit: 202d099d631571a96acdf781749561ad3f0da36a
Parents: f911403
Author: xuchuanyin <xuchuanyin@hust.edu.cn>
Authored: Sat Jul 7 22:19:53 2018 +0800
Committer: Jacky Li <jacky.likun@qq.com>
Committed: Thu Jul 12 16:38:51 2018 +0800

----------------------------------------------------------------------
 .../datamap/bloom/BloomCoarseGrainDataMap.java  | 48 ++++++++++++++++++--
 .../bloom/BloomCoarseGrainDataMapFactory.java   |  1 +
 .../bloom/BloomCoarseGrainDataMapSuite.scala    | 36 +++++++++++++++
 3 files changed, 82 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/202d099d/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java
----------------------------------------------------------------------
diff --git a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java
b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java
index 01bd804..96f3495 100644
--- a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java
+++ b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java
@@ -55,6 +55,8 @@ import org.apache.carbondata.core.scan.expression.ColumnExpression;
 import org.apache.carbondata.core.scan.expression.Expression;
 import org.apache.carbondata.core.scan.expression.LiteralExpression;
 import org.apache.carbondata.core.scan.expression.conditional.EqualToExpression;
+import org.apache.carbondata.core.scan.expression.conditional.InExpression;
+import org.apache.carbondata.core.scan.expression.conditional.ListExpression;
 import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
 import org.apache.carbondata.core.util.CarbonProperties;
 import org.apache.carbondata.core.util.CarbonUtil;
@@ -178,6 +180,7 @@ public class BloomCoarseGrainDataMap extends CoarseGrainDataMap {
   private List<BloomQueryModel> createQueryModel(Expression expression)
       throws DictionaryGenerationException, UnsupportedEncodingException {
     List<BloomQueryModel> queryModels = new ArrayList<BloomQueryModel>();
+    // bloomdatamap only support equalTo and In operators now
     if (expression instanceof EqualToExpression) {
       Expression left = ((EqualToExpression) expression).getLeft();
       Expression right = ((EqualToExpression) expression).getRight();
@@ -186,7 +189,7 @@ public class BloomCoarseGrainDataMap extends CoarseGrainDataMap {
         column = ((ColumnExpression) left).getColumnName();
         if (this.name2Col.containsKey(column)) {
           BloomQueryModel bloomQueryModel =
-              buildQueryModelFromExpression((ColumnExpression) left, (LiteralExpression)
right);
+              buildQueryModelForEqual((ColumnExpression) left, (LiteralExpression) right);
           queryModels.add(bloomQueryModel);
         }
         return queryModels;
@@ -194,10 +197,35 @@ public class BloomCoarseGrainDataMap extends CoarseGrainDataMap {
         column = ((ColumnExpression) right).getColumnName();
         if (this.name2Col.containsKey(column)) {
           BloomQueryModel bloomQueryModel =
-              buildQueryModelFromExpression((ColumnExpression) right, (LiteralExpression)
left);
+              buildQueryModelForEqual((ColumnExpression) right, (LiteralExpression) left);
           queryModels.add(bloomQueryModel);
         }
         return queryModels;
+      } else {
+        LOGGER.warn("BloomFilter can only support the 'equal' filter like 'Col = PlainValue'");
+      }
+    } else if (expression instanceof InExpression) {
+      Expression left = ((InExpression) expression).getLeft();
+      Expression right = ((InExpression) expression).getRight();
+      String column;
+      if (left instanceof ColumnExpression && right instanceof ListExpression) {
+        column = ((ColumnExpression) left).getColumnName();
+        if (this.name2Col.containsKey(column)) {
+          List<BloomQueryModel> models =
+              buildQueryModelForIn((ColumnExpression) left, (ListExpression) right);
+          queryModels.addAll(models);
+        }
+        return queryModels;
+      } else if (left instanceof ListExpression && right instanceof ColumnExpression)
{
+        column = ((ColumnExpression) right).getColumnName();
+        if (this.name2Col.containsKey(column)) {
+          List<BloomQueryModel> models =
+              buildQueryModelForIn((ColumnExpression) right, (ListExpression) left);
+          queryModels.addAll(models);
+        }
+        return queryModels;
+      } else {
+        LOGGER.warn("BloomFilter can only support the 'in' filter like 'Col in (PlainValues)'");
       }
     }
 
@@ -207,7 +235,7 @@ public class BloomCoarseGrainDataMap extends CoarseGrainDataMap {
     return queryModels;
   }
 
-  private BloomQueryModel buildQueryModelFromExpression(ColumnExpression ce,
+  private BloomQueryModel buildQueryModelForEqual(ColumnExpression ce,
       LiteralExpression le) throws DictionaryGenerationException, UnsupportedEncodingException
{
     String columnName = ce.getColumnName();
     DataType dataType = ce.getDataType();
@@ -234,6 +262,20 @@ public class BloomCoarseGrainDataMap extends CoarseGrainDataMap {
     return buildQueryModelInternal(this.name2Col.get(columnName), literalValue, dataType);
   }
 
+  /**
+   * for `in` expressions, we use `equal` to handle it.
+   * Note that `in` operator needs at least one match not exactly match. since while doing
pruning,
+   * we collect all the blocklets that will match the querymodel, this will not be a problem.
+   */
+  private List<BloomQueryModel> buildQueryModelForIn(ColumnExpression ce, ListExpression
le)
+      throws DictionaryGenerationException, UnsupportedEncodingException {
+    List<BloomQueryModel> queryModels = new ArrayList<>();
+    for (Expression child : le.getChildren()) {
+      queryModels.add(buildQueryModelForEqual(ce, (LiteralExpression) child));
+    }
+    return queryModels;
+  }
+
   private BloomQueryModel buildQueryModelInternal(CarbonColumn carbonColumn,
       Object filterLiteralValue, DataType filterValueDataType) throws
       DictionaryGenerationException, UnsupportedEncodingException {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/202d099d/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
----------------------------------------------------------------------
diff --git a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
index bf56043..68cf45c 100644
--- a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
+++ b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
@@ -112,6 +112,7 @@ public class BloomCoarseGrainDataMapFactory extends DataMapFactory<CoarseGrainDa
     List<ExpressionType> optimizedOperations = new ArrayList<ExpressionType>();
     // todo: support more optimize operations
     optimizedOperations.add(ExpressionType.EQUALS);
+    optimizedOperations.add(ExpressionType.IN);
     this.dataMapMeta = new DataMapMeta(this.dataMapName, indexedColumns, optimizedOperations);
     LOGGER.info(String.format("DataMap %s works for %s with bloom size %d",
         this.dataMapName, this.dataMapMeta, this.bloomFilterSize));

http://git-wip-us.apache.org/repos/asf/carbondata/blob/202d099d/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
index 14f8966..4dc1837 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
@@ -57,6 +57,9 @@ class BloomCoarseGrainDataMapSuite extends QueryTest with BeforeAndAfterAll
with
   }
 
   private def checkQuery(dataMapName: String, shouldHit: Boolean = true) = {
+    /**
+     * queries that use equal operator
+     */
     checkAnswer(
       checkSqlHitDataMap(s"select * from $bloomDMSampleTable where id = 1", dataMapName,
shouldHit),
       sql(s"select * from $normalTable where id = 1"))
@@ -85,6 +88,39 @@ class BloomCoarseGrainDataMapSuite extends QueryTest with BeforeAndAfterAll
with
     checkAnswer(
       checkSqlHitDataMap(s"select * from $bloomDMSampleTable where city = 'city_999' and
name='n1'", dataMapName, shouldHit),
       sql(s"select * from $normalTable where city = 'city_999' and name='n1'"))
+
+    /**
+     * queries that use in operator
+     */
+    checkAnswer(
+      checkSqlHitDataMap(s"select * from $bloomDMSampleTable where id in (1)", dataMapName,
shouldHit),
+      sql(s"select * from $normalTable where id in (1)"))
+    checkAnswer(
+      checkSqlHitDataMap(s"select * from $bloomDMSampleTable where id in (999)", dataMapName,
shouldHit),
+      sql(s"select * from $normalTable where id in (999)"))
+    checkAnswer(
+      checkSqlHitDataMap(s"select * from $bloomDMSampleTable where city in( 'city_1')", dataMapName,
shouldHit),
+      sql(s"select * from $normalTable where city in( 'city_1')"))
+    checkAnswer(
+      checkSqlHitDataMap(s"select * from $bloomDMSampleTable where city in ('city_999')",
dataMapName, shouldHit),
+      sql(s"select * from $normalTable where city in ('city_999')"))
+    // query with two index_columns
+    checkAnswer(
+      checkSqlHitDataMap(s"select * from $bloomDMSampleTable where id in (1) and city in
('city_1')", dataMapName, shouldHit),
+      sql(s"select * from $normalTable where id in (1) and city in ('city_1')"))
+    checkAnswer(
+      checkSqlHitDataMap(s"select * from $bloomDMSampleTable where id in (999) and city in
('city_999')", dataMapName, shouldHit),
+      sql(s"select * from $normalTable where id in (999) and city in ('city_999')"))
+    checkAnswer(
+      checkSqlHitDataMap(s"select * from $bloomDMSampleTable where city in ('city_1') and
id in (0)", dataMapName, shouldHit),
+      sql(s"select * from $normalTable where city in ('city_1') and id in (0)"))
+    checkAnswer(
+      checkSqlHitDataMap(s"select * from $bloomDMSampleTable where city in ('city_999') and
name in ('n999')", dataMapName, shouldHit),
+      sql(s"select * from $normalTable where city in ('city_999') and name in ('n999')"))
+    checkAnswer(
+      checkSqlHitDataMap(s"select * from $bloomDMSampleTable where city in ('city_999') and
name in ('n1')", dataMapName, shouldHit),
+      sql(s"select * from $normalTable where city in ('city_999') and name in ('n1')"))
+
     checkAnswer(
       sql(s"select min(id), max(id), min(name), max(name), min(city), max(city)" +
           s" from $bloomDMSampleTable"),


Mime
View raw message