carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kunalkap...@apache.org
Subject carbondata git commit: [CARBONDATA-2799][BloomDataMap] Fix bugs in querying with bloom datamap on preagg with dictionary column
Date Thu, 02 Aug 2018 11:26:43 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master b65bf9bc7 -> bd6abbbff


[CARBONDATA-2799][BloomDataMap] Fix bugs in querying with bloom datamap on preagg with dictionary
column

For preaggregate table, if the groupby column is dictionary column in
parent table, the preaggregate table will inherit the dictionary
encoding as well as the dictionary file from the parent table.

So for dictionary columns, during query with bloom, we need to
convert the plain filter value to dictionarty encoded value based on
parent table's dictionary file.

This closes #2580


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/bd6abbbf
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/bd6abbbf
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/bd6abbbf

Branch: refs/heads/master
Commit: bd6abbbffd36b5ca0aaad9d937d401982d1d60eb
Parents: b65bf9b
Author: xuchuanyin <xuchuanyin@hust.edu.cn>
Authored: Mon Jul 30 17:50:51 2018 +0800
Committer: kunal642 <kunalkapoor642@gmail.com>
Committed: Thu Aug 2 16:55:59 2018 +0530

----------------------------------------------------------------------
 .../datamap/bloom/BloomCoarseGrainDataMap.java  | 21 ++++-
 .../BloomCoarseGrainDataMapFunctionSuite.scala  | 97 ++++++++++++++++++++
 2 files changed, 117 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/bd6abbbf/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java
----------------------------------------------------------------------
diff --git a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java
b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java
index be531d6..71b1c55 100644
--- a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java
+++ b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMap.java
@@ -47,10 +47,12 @@ import org.apache.carbondata.core.devapi.DictionaryGenerationException;
 import org.apache.carbondata.core.indexstore.Blocklet;
 import org.apache.carbondata.core.indexstore.PartitionSpec;
 import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonMetadata;
 import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.metadata.datatype.DataTypes;
 import org.apache.carbondata.core.metadata.encoder.Encoding;
 import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.RelationIdentifier;
 import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
 import org.apache.carbondata.core.scan.expression.ColumnExpression;
 import org.apache.carbondata.core.scan.expression.Expression;
@@ -108,6 +110,7 @@ public class BloomCoarseGrainDataMap extends CoarseGrainDataMap {
     for (CarbonColumn col : indexedColumn) {
       this.name2Col.put(col.getColName(), col);
     }
+    String parentTablePath = getAncestorTablePath(carbonTable);
 
     try {
       this.name2Converters = new HashMap<>(indexedColumn.size());
@@ -129,7 +132,7 @@ public class BloomCoarseGrainDataMap extends CoarseGrainDataMap {
         dataField.setTimestampFormat(tsFormat);
         FieldConverter fieldConverter = FieldEncoderFactory.getInstance()
             .createFieldEncoder(dataField, absoluteTableIdentifier, i, nullFormat, null,
false,
-                localCaches[i], false, carbonTable.getTablePath());
+                localCaches[i], false, parentTablePath);
         this.name2Converters.put(indexedColumn.get(i).getColName(), fieldConverter);
       }
     } catch (IOException e) {
@@ -140,6 +143,22 @@ public class BloomCoarseGrainDataMap extends CoarseGrainDataMap {
     this.badRecordLogHolder.setLogged(false);
   }
 
+  /**
+   * recursively find the ancestor's table path. This is used for dictionary scenario
+   * where preagg will use the dictionary of the parent table.
+   */
+  private String getAncestorTablePath(CarbonTable currentTable) {
+    if (!currentTable.isChildDataMap()) {
+      return currentTable.getTablePath();
+    }
+
+    RelationIdentifier parentIdentifier =
+        currentTable.getTableInfo().getParentRelationIdentifiers().get(0);
+    CarbonTable parentTable = CarbonMetadata.getInstance().getCarbonTable(
+        parentIdentifier.getDatabaseName(), parentIdentifier.getTableName());
+    return getAncestorTablePath(parentTable);
+  }
+
   @Override
   public List<Blocklet> prune(FilterResolverIntf filterExp, SegmentProperties segmentProperties,
       List<PartitionSpec> partitions) throws IOException {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/bd6abbbf/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFunctionSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFunctionSuite.scala
b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFunctionSuite.scala
index 496a506..fd1345c 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFunctionSuite.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFunctionSuite.scala
@@ -832,6 +832,103 @@ class BloomCoarseGrainDataMapFunctionSuite  extends QueryTest with BeforeAndAfte
       CarbonCommonConstants.BLOCKLET_SIZE_DEFAULT_VAL)
   }
 
+  /**
+   * create bloom and preagg on base table, then create bloom on preagg table,
+   * index column and group by column is dictionary column.
+   * note that the test steps are copied from issue.
+   * In the CI env, sometime it will become timeout, so we ignore the newly added tests
+   */
+  ignore("test bloom datamap: CARBONDATA-2799 bloom datamap on preaggregate") {
+    sql(
+      s"""
+         | CREATE TABLE $normalTable (id int, name string, salary float,dob date)
+         | STORED BY 'carbondata'
+         | TBLPROPERTIES('dictionary_include'='id')
+       """.stripMargin)
+    sql(
+      s"""
+         | CREATE TABLE $bloomDMSampleTable (id int, name string, salary float,dob date)
+         | STORED BY 'carbondata'
+         | TBLPROPERTIES('dictionary_include'='id')
+       """.stripMargin)
+    (1 to 2).foreach { _ =>
+      sql(
+        s"""
+           | INSERT INTO $bloomDMSampleTable VALUES
+           | ('1', 'name1', '11.1', '2018-07-01'),
+           | ('2', 'name2', '21.1', '2018-07-02'),
+           | ('3', 'name3', '31.1', '2018-07-03'),
+           | ('4', 'name4', '41.1', '2018-07-04')
+       """.stripMargin)
+      sql(
+        s"""
+           | INSERT INTO $normalTable VALUES
+           | ('1', 'name1', '11.1', '2018-07-01'),
+           | ('2', 'name2', '21.1', '2018-07-02'),
+           | ('3', 'name3', '31.1', '2018-07-03'),
+           | ('4', 'name4', '41.1', '2018-07-04')
+       """.stripMargin)
+    }
+    sql(
+      s"""
+         | CREATE DATAMAP $dataMapName ON TABLE $bloomDMSampleTable
+         | USING 'bloomfilter'
+         | DMPROPERTIES('INDEX_COLUMNS'='id', 'BLOOM_SIZE'='320000', 'BLOOM_FPP'='0.01',
'BLOOM_COMPRESS'='TRUE')
+       """.stripMargin)
+    sql(
+      s"""
+         | INSERT INTO $bloomDMSampleTable VALUES
+         | ('1', 'name1', '11.1', '2018-07-01'),
+         | ('2', 'name2', '21.1', '2018-07-02'),
+         | ('3', 'name3', '31.1', '2018-07-03'),
+         | ('4', 'name4', '41.1', '2018-07-04')
+       """.stripMargin)
+    sql(
+      s"""
+         | INSERT INTO $normalTable VALUES
+         | ('1', 'name1', '11.1', '2018-07-01'),
+         | ('2', 'name2', '21.1', '2018-07-02'),
+         | ('3', 'name3', '31.1', '2018-07-03'),
+         | ('4', 'name4', '41.1', '2018-07-04')
+       """.stripMargin)
+    val preAggOnBase = "preagg_on_base"
+    sql(
+      s"""
+         | CREATE DATAMAP $preAggOnBase ON TABLE $bloomDMSampleTable
+         | USING 'preaggregate' AS
+         | select id, count(id) from $bloomDMSampleTable group by id
+       """.stripMargin)
+    checkAnswer(sql(s"SELECT id, count(id) from $bloomDMSampleTable where id = 3 group by
id"),
+      sql(s"SELECT id, count(id) from $normalTable where id = 3 group by id"))
+
+    val bloomOnPreAgg = "bloom_on_pre_agg"
+    sql(
+      s"""
+         | CREATE DATAMAP $bloomOnPreAgg ON TABLE ${bloomDMSampleTable}_${preAggOnBase}
+         | USING 'bloomfilter'
+         | DMPROPERTIES('INDEX_COLUMNS'='${bloomDMSampleTable}_id')
+       """.stripMargin)
+    checkAnswer(sql(s"SELECT id, count(id) from $bloomDMSampleTable where id = 3 group by
id"),
+      sql(s"SELECT id, count(id) from $normalTable where id = 3 group by id"))
+
+    sql(s"DROP DATAMAP $bloomOnPreAgg on table ${bloomDMSampleTable}_${preAggOnBase}")
+    checkAnswer(sql(s"SELECT id, count(id) from $bloomDMSampleTable where id = 3 group by
id"),
+      sql(s"SELECT id, count(id) from $normalTable where id = 3 group by id"))
+
+    sql(
+      s"""
+         | CREATE DATAMAP $bloomOnPreAgg ON TABLE ${bloomDMSampleTable}_${preAggOnBase}
+         | USING 'bloomfilter'
+         | DMPROPERTIES('INDEX_COLUMNS'='${bloomDMSampleTable}_id')
+       """.stripMargin)
+    checkAnswer(sql(s"SELECT id, count(id) from $bloomDMSampleTable where id = 3 group by
id"),
+      sql(s"SELECT id, count(id) from $normalTable where id = 3 group by id"))
+
+    sql(s"DROP DATAMAP $bloomOnPreAgg on table ${bloomDMSampleTable}_${preAggOnBase}")
+    checkAnswer(sql(s"SELECT id, count(id) from $bloomDMSampleTable where id = 3 group by
id"),
+      sql(s"SELECT id, count(id) from $normalTable where id = 3 group by id"))
+  }
+
   override def afterAll(): Unit = {
     deleteFile(bigFile)
     sql(s"DROP TABLE IF EXISTS $normalTable")


Mime
View raw message