carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject carbondata git commit: [CARBONDATA-2654][Datamap] Optimize output for explaining querying with datamap
Date Thu, 05 Jul 2018 06:19:31 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master aeb2ec4cd -> c0de9f160


[CARBONDATA-2654][Datamap] Optimize output for explaining querying with datamap

Currently if we have multiple datamaps and the query hits all the
datamaps, carbondata explain command will only show the first datamap
and all the datamaps are not shown. In this commit, we show all the
datamaps that are hitted in this query.

This closes #2411


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c0de9f16
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c0de9f16
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c0de9f16

Branch: refs/heads/master
Commit: c0de9f16010e0cb83db1dbd89e94e8bd3681dda7
Parents: aeb2ec4
Author: xuchuanyin <xuchuanyin@hust.edu.cn>
Authored: Thu Jun 28 20:01:03 2018 +0800
Committer: Jacky Li <jacky.likun@qq.com>
Committed: Thu Jul 5 14:19:19 2018 +0800

----------------------------------------------------------------------
 .../datamap/dev/expr/AndDataMapExprWrapper.java | 10 ++-
 .../datamap/dev/expr/DataMapExprWrapper.java    |  9 +-
 .../dev/expr/DataMapExprWrapperImpl.java        | 11 ++-
 .../dev/expr/DataMapWrapperSimpleInfo.java      | 90 ++++++++++++++++++++
 .../datamap/dev/expr/OrDataMapExprWrapper.java  | 11 ++-
 .../core/profiler/ExplainCollector.java         | 17 ++--
 .../core/profiler/TablePruningInfo.java         | 31 ++++---
 .../hadoop/api/CarbonInputFormat.java           | 20 +++--
 .../testsuite/datamap/CGDataMapTestCase.scala   |  5 +-
 .../testsuite/datamap/FGDataMapTestCase.scala   | 12 +--
 .../bloom/BloomCoarseGrainDataMapSuite.scala    | 79 ++++++++++++++++-
 11 files changed, 242 insertions(+), 53 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/c0de9f16/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/AndDataMapExprWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/AndDataMapExprWrapper.java
b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/AndDataMapExprWrapper.java
index ec674de..c5099db 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/AndDataMapExprWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/AndDataMapExprWrapper.java
@@ -25,7 +25,6 @@ import org.apache.carbondata.core.datamap.DataMapLevel;
 import org.apache.carbondata.core.datamap.Segment;
 import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
 import org.apache.carbondata.core.indexstore.PartitionSpec;
-import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
 import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
 
 /**
@@ -116,8 +115,13 @@ public class AndDataMapExprWrapper implements DataMapExprWrapper {
     return left.getDataMapLevel();
   }
 
-  @Override public DataMapSchema getDataMapSchema() {
-    return left.getDataMapSchema();
+  @Override
+  public DataMapExprWrapper getLeftDataMapWrapper() {
+    return left;
   }
 
+  @Override
+  public DataMapExprWrapper getRightDataMapWrapprt() {
+    return right;
+  }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/c0de9f16/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapper.java
b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapper.java
index 901cfc7..ec26107 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapper.java
@@ -25,7 +25,6 @@ import org.apache.carbondata.core.datamap.DataMapLevel;
 import org.apache.carbondata.core.datamap.Segment;
 import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
 import org.apache.carbondata.core.indexstore.PartitionSpec;
-import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
 import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
 
 /**
@@ -90,8 +89,12 @@ public interface DataMapExprWrapper extends Serializable {
   DataMapLevel getDataMapLevel();
 
   /**
-   * Get the datamap schema
+   * get the left datamap wrapper
    */
-  DataMapSchema getDataMapSchema();
+  DataMapExprWrapper getLeftDataMapWrapper();
 
+  /**
+   * get the right datamap wrapper
+   */
+  DataMapExprWrapper getRightDataMapWrapprt();
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/c0de9f16/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java
b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java
index 6537976..4643b47 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapExprWrapperImpl.java
@@ -97,8 +97,17 @@ public class DataMapExprWrapperImpl implements DataMapExprWrapper {
     return dataMap.getDataMapFactory().getDataMapLevel();
   }
 
-  @Override public DataMapSchema getDataMapSchema() {
+  public DataMapSchema getDataMapSchema() {
     return dataMap.getDataMapSchema();
   }
 
+  @Override
+  public DataMapExprWrapper getLeftDataMapWrapper() {
+    return null;
+  }
+
+  @Override
+  public DataMapExprWrapper getRightDataMapWrapprt() {
+    return null;
+  }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/c0de9f16/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapWrapperSimpleInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapWrapperSimpleInfo.java
b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapWrapperSimpleInfo.java
new file mode 100644
index 0000000..331e77e
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/DataMapWrapperSimpleInfo.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datamap.dev.expr;
+
+import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
+
+/**
+ * schema for datamap wrapper.
+ * Currently a DataMapWrapper contains more than one datamap, this class is used to describe
its
+ * schema. For example a AndDataMapExprWrapper contains BloomFilter in its left and Lucene
in
+ * its right, then its schema would be AND(BloomFilter, Lucene)
+ */
+public class DataMapWrapperSimpleInfo {
+  enum WrapperType {
+    PRIMITIVE,
+    AND,
+    OR
+  }
+
+  private WrapperType wrapperType;
+  private DataMapWrapperSimpleInfo left;
+  private DataMapWrapperSimpleInfo right;
+  private DataMapSchema schema;
+
+  private DataMapWrapperSimpleInfo(WrapperType wrapperType, DataMapWrapperSimpleInfo left,
+      DataMapWrapperSimpleInfo right) {
+    this.wrapperType = wrapperType;
+    this.left = left;
+    this.right = right;
+  }
+
+  private DataMapWrapperSimpleInfo(DataMapSchema schema) {
+    this.wrapperType = WrapperType.PRIMITIVE;
+    this.schema = schema;
+  }
+
+  public static DataMapWrapperSimpleInfo fromDataMapWrapper(DataMapExprWrapper dataMapExprWrapper)
{
+    if (dataMapExprWrapper instanceof DataMapExprWrapperImpl) {
+      return new DataMapWrapperSimpleInfo(
+          ((DataMapExprWrapperImpl) dataMapExprWrapper).getDataMapSchema());
+    } else if (dataMapExprWrapper instanceof AndDataMapExprWrapper) {
+      return new DataMapWrapperSimpleInfo(WrapperType.AND,
+          fromDataMapWrapper(dataMapExprWrapper.getLeftDataMapWrapper()),
+          fromDataMapWrapper(dataMapExprWrapper.getRightDataMapWrapprt()));
+    } else {
+      return new DataMapWrapperSimpleInfo(WrapperType.OR,
+          fromDataMapWrapper(dataMapExprWrapper.getLeftDataMapWrapper()),
+          fromDataMapWrapper(dataMapExprWrapper.getRightDataMapWrapprt()));
+    }
+  }
+
+  public String getDataMapWrapperName() {
+    if (WrapperType.PRIMITIVE == wrapperType) {
+      return schema.getDataMapName();
+    } else {
+      return String.format("%s(%s, %s)",
+          wrapperType, left.getDataMapWrapperName(), right.getDataMapWrapperName());
+    }
+  }
+
+  public String getDataMapWrapperProvider() {
+    if (WrapperType.PRIMITIVE == wrapperType) {
+      return schema.getProviderName();
+    } else {
+      return String.format("%s(%s, %s)",
+          wrapperType, left.getDataMapWrapperProvider(), right.getDataMapWrapperProvider());
+    }
+  }
+
+  @Override
+  public String toString() {
+    return "DatamapWrapperSchema: Name->" + getDataMapWrapperName()
+        + ", Provider->" + getDataMapWrapperProvider();
+  }
+}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/c0de9f16/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/OrDataMapExprWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/OrDataMapExprWrapper.java
b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/OrDataMapExprWrapper.java
index bb98535..7812969 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/OrDataMapExprWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/expr/OrDataMapExprWrapper.java
@@ -27,7 +27,6 @@ import org.apache.carbondata.core.datamap.DataMapLevel;
 import org.apache.carbondata.core.datamap.Segment;
 import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
 import org.apache.carbondata.core.indexstore.PartitionSpec;
-import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
 import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
 
 /**
@@ -109,7 +108,13 @@ public class OrDataMapExprWrapper implements DataMapExprWrapper {
     return left.getDataMapLevel();
   }
 
-  @Override public DataMapSchema getDataMapSchema() {
-    return left.getDataMapSchema();
+  @Override
+  public DataMapExprWrapper getLeftDataMapWrapper() {
+    return left;
+  }
+
+  @Override
+  public DataMapExprWrapper getRightDataMapWrapprt() {
+    return right;
   }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/c0de9f16/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java
b/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java
index 5d69bb9..02e45f6 100644
--- a/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/profiler/ExplainCollector.java
@@ -26,7 +26,7 @@ import java.util.Objects;
 import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.carbondata.common.annotations.InterfaceAudience;
-import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
+import org.apache.carbondata.core.datamap.dev.expr.DataMapWrapperSimpleInfo;
 
 /**
  * An information collector used for EXPLAIN command, to print out
@@ -94,24 +94,27 @@ public class ExplainCollector {
     }
   }
 
-  public static void recordDefaultDataMapPruning(DataMapSchema dataMapSchema, int numBlocklets)
{
+  public static void recordDefaultDataMapPruning(DataMapWrapperSimpleInfo dataMapWrapperSimpleInfo,
+      int numBlocklets) {
     if (enabled()) {
       TablePruningInfo scan = getCurrentTablePruningInfo();
-      scan.setNumBlockletsAfterDefaultPruning(dataMapSchema, numBlocklets);
+      scan.setNumBlockletsAfterDefaultPruning(dataMapWrapperSimpleInfo, numBlocklets);
     }
   }
 
-  public static void recordCGDataMapPruning(DataMapSchema dataMapSchema, int numBlocklets)
{
+  public static void recordCGDataMapPruning(DataMapWrapperSimpleInfo dataMapWrapperSimpleInfo,
+      int numBlocklets) {
     if (enabled()) {
       TablePruningInfo scan = getCurrentTablePruningInfo();
-      scan.setNumBlockletsAfterCGPruning(dataMapSchema, numBlocklets);
+      scan.setNumBlockletsAfterCGPruning(dataMapWrapperSimpleInfo, numBlocklets);
     }
   }
 
-  public static void recordFGDataMapPruning(DataMapSchema dataMapSchema, int numBlocklets)
{
+  public static void recordFGDataMapPruning(DataMapWrapperSimpleInfo dataMapWrapperSimpleInfo,
+      int numBlocklets) {
     if (enabled()) {
       TablePruningInfo scan = getCurrentTablePruningInfo();
-      scan.setNumBlockletsAfterFGPruning(dataMapSchema, numBlocklets);
+      scan.setNumBlockletsAfterFGPruning(dataMapWrapperSimpleInfo, numBlocklets);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/c0de9f16/core/src/main/java/org/apache/carbondata/core/profiler/TablePruningInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/profiler/TablePruningInfo.java
b/core/src/main/java/org/apache/carbondata/core/profiler/TablePruningInfo.java
index cec8827..651af9d 100644
--- a/core/src/main/java/org/apache/carbondata/core/profiler/TablePruningInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/profiler/TablePruningInfo.java
@@ -18,7 +18,7 @@
 package org.apache.carbondata.core.profiler;
 
 import org.apache.carbondata.common.annotations.InterfaceAudience;
-import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;
+import org.apache.carbondata.core.datamap.dev.expr.DataMapWrapperSimpleInfo;
 
 /**
  * Used for EXPLAIN command
@@ -29,13 +29,13 @@ public class TablePruningInfo {
   private int totalBlocklets;
   private String filterStatement;
 
-  private DataMapSchema defaultDataMap;
+  private DataMapWrapperSimpleInfo defaultDataMap;
   private int numBlockletsAfterDefaultPruning;
 
-  private DataMapSchema cgDataMap;
+  private DataMapWrapperSimpleInfo cgDataMap;
   private int numBlockletsAfterCGPruning;
 
-  private DataMapSchema fgDataMap;
+  private DataMapWrapperSimpleInfo fgDataMap;
   private int numBlockletsAfterFGPruning;
 
   void addTotalBlocklets(int numBlocklets) {
@@ -46,18 +46,21 @@ public class TablePruningInfo {
     this.filterStatement = filterStatement;
   }
 
-  void setNumBlockletsAfterDefaultPruning(DataMapSchema dataMapSchema, int numBlocklets)
{
-    this.defaultDataMap = dataMapSchema;
+  void setNumBlockletsAfterDefaultPruning(DataMapWrapperSimpleInfo dataMapWrapperSimpleInfo,
+      int numBlocklets) {
+    this.defaultDataMap = dataMapWrapperSimpleInfo;
     this.numBlockletsAfterDefaultPruning = numBlocklets;
   }
 
-  void setNumBlockletsAfterCGPruning(DataMapSchema dataMapSchema, int numBlocklets) {
-    this.cgDataMap = dataMapSchema;
+  void setNumBlockletsAfterCGPruning(DataMapWrapperSimpleInfo dataMapWrapperSimpleInfo,
+      int numBlocklets) {
+    this.cgDataMap = dataMapWrapperSimpleInfo;
     this.numBlockletsAfterCGPruning = numBlocklets;
   }
 
-  void setNumBlockletsAfterFGPruning(DataMapSchema dataMapSchema, int numBlocklets) {
-    this.fgDataMap = dataMapSchema;
+  void setNumBlockletsAfterFGPruning(DataMapWrapperSimpleInfo dataMapWrapperSimpleInfo,
+      int numBlocklets) {
+    this.fgDataMap = dataMapWrapperSimpleInfo;
     this.numBlockletsAfterFGPruning = numBlocklets;
   }
 
@@ -77,8 +80,8 @@ public class TablePruningInfo {
       int skipBlocklets = numBlockletsAfterDefaultPruning - numBlockletsAfterCGPruning;
       builder
           .append(" - pruned by CG DataMap").append("\n")
-          .append("    - name: ").append(cgDataMap.getDataMapName()).append("\n")
-          .append("    - provider: ").append(cgDataMap.getProviderName()).append("\n")
+          .append("    - name: ").append(cgDataMap.getDataMapWrapperName()).append("\n")
+          .append("    - provider: ").append(cgDataMap.getDataMapWrapperProvider()).append("\n")
           .append("    - skipped blocklets: ").append(skipBlocklets).append("\n");
     }
     if (fgDataMap != null) {
@@ -90,8 +93,8 @@ public class TablePruningInfo {
       }
       builder
           .append(" - pruned by FG DataMap").append("\n")
-          .append("    - name: ").append(fgDataMap.getDataMapName()).append("\n")
-          .append("    - provider: ").append(fgDataMap.getProviderName()).append("\n")
+          .append("    - name: ").append(fgDataMap.getDataMapWrapperName()).append("\n")
+          .append("    - provider: ").append(fgDataMap.getDataMapWrapperProvider()).append("\n")
           .append("    - skipped blocklets: ").append(skipBlocklets).append("\n");
     }
     return builder.toString();

http://git-wip-us.apache.org/repos/asf/carbondata/blob/c0de9f16/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java
b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java
index 4b30ddf..356dd5a 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonInputFormat.java
@@ -33,6 +33,7 @@ import org.apache.carbondata.core.datamap.DataMapJob;
 import org.apache.carbondata.core.datamap.DataMapUtil;
 import org.apache.carbondata.core.datamap.Segment;
 import org.apache.carbondata.core.datamap.dev.expr.DataMapExprWrapper;
+import org.apache.carbondata.core.datamap.dev.expr.DataMapWrapperSimpleInfo;
 import org.apache.carbondata.core.exception.InvalidConfigurationException;
 import org.apache.carbondata.core.indexstore.ExtendedBlocklet;
 import org.apache.carbondata.core.indexstore.PartitionSpec;
@@ -433,12 +434,13 @@ m filterExpression
     DataMapJob dataMapJob = DataMapUtil.getDataMapJob(job.getConfiguration());
     List<PartitionSpec> partitionsToPrune = getPartitionsToPrune(job.getConfiguration());
     // First prune using default datamap on driver side.
-    DataMapExprWrapper dataMapExprWrapper = DataMapChooser.getDefaultDataMap(
-        getOrCreateCarbonTable(job.getConfiguration()), resolver);
-    List<ExtendedBlocklet> prunedBlocklets = dataMapExprWrapper.prune(segmentIds,
-        partitionsToPrune);
-    ExplainCollector.recordDefaultDataMapPruning(dataMapExprWrapper.getDataMapSchema(),
-        prunedBlocklets.size());
+    DataMapExprWrapper dataMapExprWrapper = DataMapChooser
+        .getDefaultDataMap(getOrCreateCarbonTable(job.getConfiguration()), resolver);
+    List<ExtendedBlocklet> prunedBlocklets =
+        dataMapExprWrapper.prune(segmentIds, partitionsToPrune);
+
+    ExplainCollector.recordDefaultDataMapPruning(
+        DataMapWrapperSimpleInfo.fromDataMapWrapper(dataMapExprWrapper), prunedBlocklets.size());
     if (prunedBlocklets.size() == 0) {
       return prunedBlocklets;
     }
@@ -463,7 +465,8 @@ m filterExpression
       prunedBlocklets = (List) CollectionUtils.intersection(
           cgPrunedBlocklets, prunedBlocklets);
       ExplainCollector.recordCGDataMapPruning(
-          cgDataMapExprWrapper.getDataMapSchema(), prunedBlocklets.size());
+          DataMapWrapperSimpleInfo.fromDataMapWrapper(cgDataMapExprWrapper),
+          prunedBlocklets.size());
     }
 
     if (prunedBlocklets.size() == 0) {
@@ -481,7 +484,8 @@ m filterExpression
         // 'prunedBlocklets', so the intersection should keep the elements in 'fgPrunedBlocklets'
         prunedBlocklets = (List) CollectionUtils.intersection(fgPrunedBlocklets,
             prunedBlocklets);
-        ExplainCollector.recordFGDataMapPruning(fgDataMapExprWrapper.getDataMapSchema(),
+        ExplainCollector.recordFGDataMapPruning(
+            DataMapWrapperSimpleInfo.fromDataMapWrapper(fgDataMapExprWrapper),
             prunedBlocklets.size());
       }
     }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/c0de9f16/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
index e141a09..bc8b28b 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
@@ -425,10 +425,7 @@ class CGDataMapTestCase extends QueryTest with BeforeAndAfterAll {
     val df1 = sql(s"EXPLAIN EXTENDED SELECT * FROM $tableName WHERE name='n502670' AND city='c2670'").collect()
     assert(df1(0).getString(0).contains("CG DataMap"))
     assert(df1(0).getString(0).contains(dataMapName1))
-    val e11 = intercept[Exception] {
-      assert(df1(0).getString(0).contains(dataMapName2))
-    }
-    assert(e11.getMessage.contains("did not contain \"" + dataMapName2))
+    assert(df1(0).getString(0).contains(dataMapName2))
 
     // make datamap1 invisible
     sql(s"SET ${CarbonCommonConstants.CARBON_DATAMAP_VISIBLE}default.$tableName.$dataMapName1
= false")

http://git-wip-us.apache.org/repos/asf/carbondata/blob/c0de9f16/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
index 20a76f8..6f8310c 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
@@ -516,10 +516,7 @@ class FGDataMapTestCase extends QueryTest with BeforeAndAfterAll {
     val df1 = sql(s"EXPLAIN EXTENDED SELECT * FROM $tableName WHERE name='n502670' AND city='c2670'").collect()
     assert(df1(0).getString(0).contains("FG DataMap"))
     assert(df1(0).getString(0).contains(dataMapName1))
-    val e11 = intercept[Exception] {
-      assert(df1(0).getString(0).contains(dataMapName2))
-    }
-    assert(e11.getMessage.contains("did not contain \"" + dataMapName2))
+    assert(df1(0).getString(0).contains(dataMapName2))
 
     // make datamap1 invisible
     sql(s"SET ${CarbonCommonConstants.CARBON_DATAMAP_VISIBLE}default.$tableName.$dataMapName1
= false")
@@ -548,15 +545,12 @@ class FGDataMapTestCase extends QueryTest with BeforeAndAfterAll {
 
     // make datamap1,datamap2 visible
     sql(s"SET ${CarbonCommonConstants.CARBON_DATAMAP_VISIBLE}default.$tableName.$dataMapName1
= true")
-    sql(s"SET ${CarbonCommonConstants.CARBON_DATAMAP_VISIBLE}default.$tableName.$dataMapName1
= true")
+    sql(s"SET ${CarbonCommonConstants.CARBON_DATAMAP_VISIBLE}default.$tableName.$dataMapName2
= true")
     checkAnswer(sql(s"SELECT * FROM $tableName WHERE name='n502670' AND city='c2670'"),
       sql("SELECT * FROM normal_test WHERE name='n502670' AND city='c2670'"))
     val df4 = sql(s"EXPLAIN EXTENDED SELECT * FROM $tableName WHERE name='n502670' AND city='c2670'").collect()
     assert(df4(0).getString(0).contains(dataMapName1))
-    val e41 = intercept[Exception] {
-      assert(df3(0).getString(0).contains(dataMapName2))
-    }
-    assert(e41.getMessage.contains("did not contain \"" + dataMapName2))
+    assert(df4(0).getString(0).contains(dataMapName2))
   }
 
   override protected def afterAll(): Unit = {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/c0de9f16/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
index a29f0f2..836c8e7 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
@@ -38,7 +38,7 @@ class BloomCoarseGrainDataMapSuite extends QueryTest with BeforeAndAfterAll
with
 
   override protected def beforeAll(): Unit = {
     new File(CarbonProperties.getInstance().getSystemFolderLocation).delete()
-    createFile(bigFile, line = 500000)
+    createFile(bigFile, line = 50000)
     createFile(smallFile)
     sql(s"DROP TABLE IF EXISTS $normalTable")
     sql(s"DROP TABLE IF EXISTS $bloomDMSampleTable")
@@ -300,6 +300,83 @@ class BloomCoarseGrainDataMapSuite extends QueryTest with BeforeAndAfterAll
with
     sql(s"DROP TABLE IF EXISTS $bloomDMSampleTable")
   }
 
+  test("test bloom datamap: multiple datamaps with each on one column vs one datamap on multiple
columns") {
+    val iterations = 1
+    // 500000 lines will result to 3 blocklets and bloomfilter datamap will prune 2 blocklets.
+    val datamap11 = "datamap11"
+    val datamap12 = "datamap12"
+    val datamap13 = "datamap13"
+    val datamap2 = "datamap2"
+
+    sql(s"DROP TABLE IF EXISTS $bloomDMSampleTable")
+    // create a table and 3 bloom datamaps on it, each datamap contains one index column
+    sql(
+      s"""
+         | CREATE TABLE $bloomDMSampleTable(id INT, name STRING, city STRING, age INT,
+         | s1 STRING, s2 STRING, s3 STRING, s4 STRING, s5 STRING, s6 STRING, s7 STRING, s8
STRING)
+         | STORED BY 'carbondata' TBLPROPERTIES('table_blocksize'='128', 'SORT_COLUMNS'='s1')
+         |  """.stripMargin)
+    sql(
+      s"""
+         | CREATE DATAMAP $datamap11 ON TABLE $bloomDMSampleTable
+         | USING 'bloomfilter'
+         | DMProperties('INDEX_COLUMNS'='id', 'BLOOM_SIZE'='64000', 'BLOOM_FPP'='0.00001')
+      """.stripMargin)
+    sql(
+      s"""
+         | CREATE DATAMAP $datamap12 ON TABLE $bloomDMSampleTable
+         | USING 'bloomfilter'
+         | DMProperties('INDEX_COLUMNS'='name', 'BLOOM_SIZE'='64000', 'BLOOM_FPP'='0.00001')
+      """.stripMargin)
+    sql(
+      s"""
+         | CREATE DATAMAP $datamap13 ON TABLE $bloomDMSampleTable
+         | USING 'bloomfilter'
+         | DMProperties('INDEX_COLUMNS'='city', 'BLOOM_SIZE'='64000', 'BLOOM_FPP'='0.00001')
+      """.stripMargin)
+
+    // create a table and 1 bloom datamap on it, this datamap contains 3 index columns
+    sql(
+      s"""
+         | CREATE TABLE $normalTable(id INT, name STRING, city STRING, age INT,
+         | s1 STRING, s2 STRING, s3 STRING, s4 STRING, s5 STRING, s6 STRING, s7 STRING, s8
STRING)
+         | STORED BY 'carbondata' TBLPROPERTIES('table_blocksize'='128', 'SORT_COLUMNS'='s1')
+         |  """.stripMargin)
+    sql(
+      s"""
+         | CREATE DATAMAP $datamap2 ON TABLE $normalTable
+         | USING 'bloomfilter'
+         | DMProperties('INDEX_COLUMNS'='id, name, city', 'BLOOM_SIZE'='64000', 'BLOOM_FPP'='0.00001')
+      """.stripMargin)
+
+    (0 until iterations).foreach { p =>
+      sql(
+        s"""
+           | LOAD DATA LOCAL INPATH '${bigFile}' INTO TABLE $bloomDMSampleTable
+           | OPTIONS('header'='false')
+         """.stripMargin)
+      sql(
+        s"""
+           | LOAD DATA LOCAL INPATH '${bigFile}' INTO TABLE $normalTable
+           | OPTIONS('header'='false')
+         """.stripMargin)
+    }
+
+    var res = sql(s"explain select * from $bloomDMSampleTable where id = 1 and city = 'city_1'
and name='n1'")
+    checkExistence(res, true, datamap11, datamap12, datamap13)
+    res = sql(s"explain select * from $normalTable where id = 1 and city = 'city_1' and name='n1'")
+    checkExistence(res, true, datamap2)
+    // in the following cases, default blocklet datamap will prune all the blocklets
+    // and bloomfilter datamap will not take effects
+    res = sql(s"explain select * from $bloomDMSampleTable where id < 0")
+    checkExistence(res, false, datamap11, datamap12, datamap13)
+    res = sql(s"explain select * from $normalTable where id < 0")
+    checkExistence(res, false, datamap2)
+
+    // we do not care about the datamap name here, only to validate the query results are
them same
+    checkQuery("fakeDm", shouldHit = false)
+  }
+
   override protected def afterAll(): Unit = {
     deleteFile(bigFile)
     deleteFile(smallFile)


Mime
View raw message