carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ajan...@apache.org
Subject [carbondata] branch master updated: [CARBONDATA-3351] Block bloom datamap creation on binary datatype column
Date Wed, 11 Dec 2019 01:36:46 GMT
This is an automated email from the ASF dual-hosted git repository.

ajantha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git


The following commit(s) were added to refs/heads/master by this push:
     new 27a689a  [CARBONDATA-3351] Block bloom datamap creation on binary datatype column
27a689a is described below

commit 27a689a12a0cfce7fb151ec3348e48b4793656e5
Author: akkio-97 <akshay.nuthala@gmail.com>
AuthorDate: Thu Dec 5 20:22:39 2019 +0530

    [CARBONDATA-3351] Block bloom datamap creation on binary datatype column
    
    Why this PR?
    As per Binary Datatype design Bloom data-map creation on Binary datatype
    column is not supported.
    Handled the same.
    
    This closes #3500
---
 .../testsuite/datamap/TestDataMapCommand.scala     | 61 +---------------------
 .../datamap/CarbonCreateDataMapCommand.scala       |  7 +++
 .../bloom/BloomCoarseGrainDataMapSuite.scala       | 19 +++++++
 3 files changed, 27 insertions(+), 60 deletions(-)

diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
index d9461b2..0da9428 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
@@ -266,66 +266,7 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
     sql(s"drop table if exists $tableName")
   }
 
-    test("test support bloomFilter on binary data type") {
-        val tableName = "datamapshowtest"
-        val datamapName = "bloomdatamap"
-        val datamapName2 = "bloomdatamap2"
-        sql(s"drop table if exists $tableName")
-        // for index datamap
-        sql(s"create table $tableName (a string, b string, c string, d binary, e binary)
stored by 'carbondata'")
-
-        sql(s"insert into $tableName  values('a1','b1','c1','d1','e1')")
-        sql(s"insert into $tableName  values('a1','b2','c2','d1','e2')")
-        sql(s"insert into $tableName  values('a3','b3','c1','d2','e2')")
-        sql(
-            s"""
-               | create datamap $datamapName on table $tableName using 'bloomfilter'
-               | DMPROPERTIES ('index_columns'='d', 'bloom_size'='32000', 'bloom_fpp'='0.001')
-       """.stripMargin)
-        sql(
-            s"""
-               | create datamap $datamapName2 on table $tableName using 'bloomfilter'
-               | DMPROPERTIES ('index_columns'='e')
-       """.stripMargin)
-        var bloom1 = sql(s"select * from $tableName where d=cast('d1' as binary)")
-        assert(2 == bloom1.collect().length)
-        bloom1.collect().foreach { each =>
-            assert(5 == each.length)
-            assert("a1".equals(each.get(0)))
-            assert("d1".equals(new String(each.getAs[Array[Byte]](3))))
-            if ("b1".equals(each.get(1))) {
-                assert("c1".equals(each.get(2)))
-                assert("e1".equals(new String(each.getAs[Array[Byte]](4))))
-            } else if ("b2".equals(each.get(1))) {
-                assert("c2".equals(each.get(2)))
-                assert("e2".equals(new String(each.getAs[Array[Byte]](4))))
-            } else {
-                assert(false)
-            }
-        }
-
-        bloom1 = sql(s"select * from $tableName where d=cast('d1' as binary) and e=cast('e1'
as binary)")
-        assert(1 == bloom1.collect().length)
-        bloom1.collect().foreach { each =>
-            assert(5 == each.length)
-            assert("a1".equals(each.get(0)))
-            assert("d1".equals(new String(each.getAs[Array[Byte]](3))))
-            if ("b1".equals(each.get(1))) {
-                assert("c1".equals(each.get(2)))
-                assert("e1".equals(new String(each.getAs[Array[Byte]](4))))
-            } else {
-                assert(false)
-            }
-        }
-
-        val result = sql(s"show datamap on table $tableName").cache()
-
-        checkAnswer(sql(s"show datamap on table $tableName"),
-            Seq(Row(datamapName, "bloomfilter", s"default.$tableName", "'bloom_fpp'='0.001',
'bloom_size'='32000', 'index_columns'='d'", "ENABLED", "NA"),
-                Row(datamapName2, "bloomfilter", s"default.$tableName", "'index_columns'='e'",
"ENABLED", "NA")))
-        result.unpersist()
-        sql(s"drop table if exists $tableName")
-    }
+    
 
     test("test don't support timeseries on binary data type") {
         val tableName = "datamapshowtest"
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
index bf4106b..d4524d0 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
@@ -27,6 +27,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory
 import org.apache.carbondata.core.datamap.{DataMapProvider, DataMapStoreManager}
 import org.apache.carbondata.core.datamap.status.DataMapStatusManager
 import org.apache.carbondata.core.metadata.ColumnarFormatVersion
+import org.apache.carbondata.core.metadata.datatype.DataTypes
 import org.apache.carbondata.core.metadata.schema.datamap.{DataMapClassProvider, DataMapProperty}
 import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, DataMapSchema}
 import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
@@ -145,6 +146,12 @@ case class CarbonCreateDataMapCommand(
               "column '%s' already has %s index datamap created",
               column.getColName, thisDmProviderName))
           } else if (isBloomFilter) {
+            if (column.getDataType == DataTypes.BINARY) {
+              throw new MalformedDataMapCommandException(
+                s"BloomFilter datamap does not support Binary datatype column: ${
+                  column.getColName
+                }")
+            }
             // if datamap provider is bloomfilter,the index column datatype cannot be complex
type
             if (column.isComplex) {
               throw new MalformedDataMapCommandException(
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
index 06e81bd..bedcc4a 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
@@ -590,6 +590,25 @@ class BloomCoarseGrainDataMapSuite extends QueryTest with BeforeAndAfterAll
with
       "BloomFilter datamap does not support complex datatype column"))
   }
 
+  test("test create bloomfilter datamap which index column datatype is Binary ") {
+    sql("drop table if exists binaryTable")
+    sql(
+      "CREATE TABLE binaryTable (CUST_ID binary,CUST_NAME String,ACTIVE_EMUI_VERSION string,
DOB " +
+      "timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1
" +
+      "decimal(30,10), DECIMAL_COLUMN2 decimal(36,36),Double_COLUMN1 double, Double_COLUMN2
" +
+      "double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format'")
+    val exception: MalformedDataMapCommandException = intercept[MalformedDataMapCommandException]
{
+      sql(
+        s"""
+           | CREATE DATAMAP binaryBloom ON TABLE binaryTable
+           | USING 'bloomfilter'
+           | DMProperties('INDEX_COLUMNS'='cust_id', 'BLOOM_SIZE'='640000')
+           | """.stripMargin)
+    }
+    assert(exception.getMessage.equalsIgnoreCase(
+      "BloomFilter datamap does not support binary datatype column: cust_id"  ))
+  }
+
   test("test create bloom datamap on newly added column") {
     val datamap1 = "datamap1"
     val datamap2 = "datamap2"


Mime
View raw message