carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [31/50] [abbrv] carbondata git commit: [CARBONDATA-2698][CARBONDATA-2700][CARBONDATA-2732][BloomDataMap] block some operations of bloomfilter datamap
Date Wed, 18 Jul 2018 02:20:04 GMT
[CARBONDATA-2698][CARBONDATA-2700][CARBONDATA-2732][BloomDataMap] block some operations of
bloomfilter datamap

1.Block create bloomfilter datamap index on column which its datatype is complex type;
2.Block change datatype for bloomfilter index datamap;
3.Block dropping index columns for bloomfilter index datamap

This closes #2505


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/1c4358e8
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/1c4358e8
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/1c4358e8

Branch: refs/heads/carbonstore
Commit: 1c4358e89f5cba1132e9512107d3a0cb22087b7b
Parents: 8e78957
Author: Sssan520 <liangap2008@aliyun.com>
Authored: Mon Jul 16 10:59:43 2018 +0800
Committer: xuchuanyin <xuchuanyin@hust.edu.cn>
Committed: Tue Jul 17 16:34:14 2018 +0800

----------------------------------------------------------------------
 .../core/datamap/dev/DataMapFactory.java        | 13 +++
 .../core/metadata/schema/table/CarbonTable.java | 13 ++-
 .../bloom/BloomCoarseGrainDataMapFactory.java   | 37 +++++++-
 .../datamap/CarbonCreateDataMapCommand.scala    | 10 ++
 .../CarbonAlterTableDataTypeChangeCommand.scala |  3 +-
 .../CarbonAlterTableDropColumnCommand.scala     |  3 +-
 .../bloom/BloomCoarseGrainDataMapSuite.scala    | 99 ++++++++++++++++++++
 7 files changed, 171 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/1c4358e8/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
index 0889f8b..ab0f8ea 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
@@ -144,4 +144,17 @@ public abstract class DataMapFactory<T extends DataMap> {
     }
   }
 
+  /**
+   * whether to block operation on corresponding table or column.
+   * For example, bloomfilter datamap will block changing datatype for bloomindex column.
+   * By default it will not block any operation.
+   *
+   * @param operation table operation
+   * @param targets objects which the operation impact on
+   * @return true the operation will be blocked;false the operation will not be blocked
+   */
+  public boolean isOperationBlocked(TableOperation operation, Object... targets) {
+    return false;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/1c4358e8/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index 71256d4..995f943 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -1054,11 +1054,12 @@ public class CarbonTable implements Serializable {
   /**
    * methods returns true if operation is allowed for the corresponding datamap or not
    * if this operation makes datamap stale it is not allowed
-   * @param carbonTable
-   * @param operation
-   * @return
+   * @param carbonTable carbontable to be operated
+   * @param operation which operation on the table,such as drop column,change datatype.
+   * @param targets objects which the operation impact on,such as column
+   * @return true allow;false not allow
    */
-  public boolean canAllow(CarbonTable carbonTable, TableOperation operation) {
+  public boolean canAllow(CarbonTable carbonTable, TableOperation operation, Object... targets)
{
     try {
       List<TableDataMap> datamaps = DataMapStoreManager.getInstance().getAllDataMap(carbonTable);
       if (!datamaps.isEmpty()) {
@@ -1069,6 +1070,10 @@ public class CarbonTable implements Serializable {
           if (factoryClass.willBecomeStale(operation)) {
             return false;
           }
+          // check whether the operation is blocked for datamap
+          if (factoryClass.isOperationBlocked(operation, targets)) {
+            return false;
+          }
         }
       }
     } catch (Exception e) {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/1c4358e8/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
----------------------------------------------------------------------
diff --git a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
index 0fe0175..d9646d5 100644
--- a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
+++ b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
@@ -377,7 +377,8 @@ public class BloomCoarseGrainDataMapFactory extends DataMapFactory<CoarseGrainDa
     }
   }
 
-  @Override public boolean willBecomeStale(TableOperation operation) {
+  @Override
+  public boolean willBecomeStale(TableOperation operation) {
     switch (operation) {
       case ALTER_RENAME:
         return false;
@@ -401,6 +402,40 @@ public class BloomCoarseGrainDataMapFactory extends DataMapFactory<CoarseGrainDa
   }
 
   @Override
+  public boolean isOperationBlocked(TableOperation operation, Object... targets) {
+    switch (operation) {
+      case ALTER_DROP: {
+        // alter table drop columns
+        // will be blocked if the columns in bloomfilter datamap
+        List<String> columnsToDrop = (List<String>) targets[0];
+        List<String> indexedColumnNames = dataMapMeta.getIndexedColumnNames();
+        for (String indexedcolumn : indexedColumnNames) {
+          for (String column : columnsToDrop) {
+            if (column.equalsIgnoreCase(indexedcolumn)) {
+              return true;
+            }
+          }
+        }
+        return false;
+      }
+      case ALTER_CHANGE_DATATYPE: {
+        // alter table change one column datatype
+        // will be blocked if the column in bloomfilter datamap
+        String columnToChangeDatatype = (String) targets[0];
+        List<String> indexedColumnNames = dataMapMeta.getIndexedColumnNames();
+        for (String indexedcolumn : indexedColumnNames) {
+          if (indexedcolumn.equalsIgnoreCase(columnToChangeDatatype)) {
+            return true;
+          }
+        }
+        return false;
+      }
+      default:
+        return false;
+    }
+  }
+
+  @Override
   public DataMapMeta getMeta() {
     return this.dataMapMeta;
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/1c4358e8/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
index a1a86a4..33dba28 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
@@ -95,6 +95,8 @@ case class CarbonCreateDataMapCommand(
     // If it is index datamap, check whether the column has datamap created already
     dataMapProvider match {
       case provider: IndexDataMapProvider =>
+        val isBloomFilter = DataMapClassProvider.BLOOMFILTER.getShortName
+          .equalsIgnoreCase(dmProviderName)
         val datamaps = DataMapStoreManager.getInstance.getAllDataMap(mainTable).asScala
         val thisDmProviderName =
           dataMapProvider.asInstanceOf[IndexDataMapProvider].getDataMapSchema.getProviderName
@@ -109,6 +111,14 @@ case class CarbonCreateDataMapCommand(
             throw new MalformedDataMapCommandException(String.format(
               "column '%s' already has %s index datamap created",
               column.getColName, thisDmProviderName))
+          } else if (isBloomFilter) {
+            // if datamap provider is bloomfilter,the index column datatype cannot be complex
type
+            if (column.isComplex) {
+              throw new MalformedDataMapCommandException(
+                s"BloomFilter datamap does not support complex datatype column: ${
+                  column.getColName
+                }")
+            }
           }
         }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/1c4358e8/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala
index fddbb7b..9ce79e9 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDataTypeChangeCommand.scala
@@ -55,7 +55,8 @@ private[sql] case class CarbonAlterTableDataTypeChangeCommand(
         .validateTableAndAcquireLock(dbName, tableName, locksToBeAcquired)(sparkSession)
       val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
       carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
-      if (!carbonTable.canAllow(carbonTable, TableOperation.ALTER_CHANGE_DATATYPE)) {
+      if (!carbonTable.canAllow(carbonTable, TableOperation.ALTER_CHANGE_DATATYPE,
+        alterTableDataTypeChangeModel.columnName)) {
         throw new MalformedCarbonCommandException(
           "alter table change datatype is not supported for index datamap")
       }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/1c4358e8/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala
index 7022e98..23dbf9e 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/schema/CarbonAlterTableDropColumnCommand.scala
@@ -58,7 +58,8 @@ private[sql] case class CarbonAlterTableDropColumnCommand(
         .validateTableAndAcquireLock(dbName, tableName, locksToBeAcquired)(sparkSession)
       val metastore = CarbonEnv.getInstance(sparkSession).carbonMetastore
       carbonTable = CarbonEnv.getCarbonTable(Some(dbName), tableName)(sparkSession)
-      if (!carbonTable.canAllow(carbonTable, TableOperation.ALTER_DROP)) {
+      if (!carbonTable.canAllow(carbonTable, TableOperation.ALTER_DROP,
+          alterTableDropColumnModel.columns.asJava)) {
         throw new MalformedCarbonCommandException(
           "alter table drop column is not supported for index datamap")
       }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/1c4358e8/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
index 4dc1837..df5ee18 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
@@ -26,6 +26,8 @@ import org.apache.spark.sql.{CarbonSession, DataFrame}
 import org.apache.spark.sql.test.util.QueryTest
 import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
 
+import org.apache.carbondata.common.exceptions.sql.{MalformedCarbonCommandException, MalformedDataMapCommandException}
+import org.apache.carbondata.core.constants.CarbonCommonConstants
 import org.apache.carbondata.core.datamap.status.DataMapStatusManager
 import org.apache.carbondata.core.util.CarbonProperties
 
@@ -413,6 +415,103 @@ class BloomCoarseGrainDataMapSuite extends QueryTest with BeforeAndAfterAll
with
     checkQuery("fakeDm", shouldHit = false)
   }
 
+  test("test block change datatype for bloomfilter index datamap") {
+    sql(
+      s"""
+         | CREATE TABLE $normalTable(id INT, name STRING, city STRING, age INT,
+         | s1 STRING, s2 STRING, s3 STRING, s4 STRING, s5 STRING, s6 STRING, s7 STRING, s8
STRING)
+         | STORED BY 'carbondata' TBLPROPERTIES('table_blocksize'='128')
+         | """.stripMargin)
+
+    sql(
+      s"""
+         | CREATE DATAMAP $dataMapName ON TABLE $normalTable
+         | USING 'bloomfilter' WITH DEFERRED REBUILD
+         | DMProperties( 'INDEX_COLUMNS'='city,id', 'BLOOM_SIZE'='640000')
+      """.stripMargin)
+    val exception: MalformedCarbonCommandException = intercept[MalformedCarbonCommandException]
{
+      sql(s"ALTER TABLE $normalTable CHANGE id id bigint")
+    }
+    assert(exception.getMessage.contains(
+      "alter table change datatype is not supported for index datamap"))
+  }
+
+  test("test drop index columns for bloomfilter datamap") {
+    sql(
+      s"""
+         | CREATE TABLE $normalTable(id INT, name STRING, city STRING, age INT,
+         | s1 STRING, s2 STRING, s3 STRING, s4 STRING, s5 STRING, s6 STRING, s7 STRING, s8
STRING)
+         | STORED BY 'carbondata' TBLPROPERTIES('table_blocksize'='128')
+         |  """.stripMargin)
+    sql(
+      s"""
+         | CREATE DATAMAP $dataMapName ON TABLE $normalTable
+         | USING 'bloomfilter'
+         | WITH DEFERRED REBUILD
+         | DMProperties('INDEX_COLUMNS'='city,id', 'BLOOM_SIZE'='640000')
+      """.stripMargin)
+    val exception: MalformedCarbonCommandException = intercept[MalformedCarbonCommandException]
{
+      sql(s"alter table $normalTable drop columns(name, id)")
+    }
+    assert(exception.getMessage.contains(
+      "alter table drop column is not supported for index datamap"))
+  }
+
+  test("test bloom datamap: bloom index column is local dictionary") {
+    sql(
+      s"""
+         | CREATE TABLE $normalTable(c1 string, c2 int, c3 string)
+         | STORED BY 'carbondata'
+         | """.stripMargin)
+    // c1 is local dictionary and will create bloom index on it
+    sql(
+      s"""
+         | CREATE TABLE $bloomDMSampleTable(c1 string, c2 int, c3 string)
+         | STORED BY 'carbondata'
+         | TBLPROPERTIES('local_dictionary_include'='c1', 'local_dictionary_threshold'='1000')
+         | """.stripMargin)
+    sql(
+      s"""
+         | CREATE DATAMAP $dataMapName on table $bloomDMSampleTable
+         | using 'bloomfilter'
+         | DMPROPERTIES('index_columns'='c1, c2')
+         | """.stripMargin)
+    sql(
+      s"""
+         | INSERT INTO $bloomDMSampleTable
+         | values ('c1v11', 11, 'c3v11'), ('c1v12', 12, 'c3v12')
+         | """.stripMargin)
+    sql(
+      s"""
+         | INSERT INTO $normalTable values ('c1v11', 11, 'c3v11'),
+         | ('c1v12', 12, 'c3v12')
+         | """.stripMargin)
+    checkAnswer(sql(s"select * from $bloomDMSampleTable"),
+      sql(s"select * from $normalTable"))
+    checkAnswer(sql(s"select * from $bloomDMSampleTable where c1 = 'c1v12'"),
+      sql(s"select * from $normalTable where c1 = 'c1v12'"))
+  }
+
+  test("test create bloomfilter datamap which index column datatype is complex ") {
+    sql(
+      s"""
+         | CREATE TABLE $normalTable(id INT, name STRING, city Array<INT>, age INT,
+         | s1 STRING, s2 STRING, s3 STRING, s4 STRING, s5 STRING, s6 STRING, s7 STRING, s8
STRING)
+         | STORED BY 'carbondata'
+         | """.stripMargin)
+    val exception: MalformedDataMapCommandException = intercept[MalformedDataMapCommandException]
{
+      sql(
+        s"""
+           | CREATE DATAMAP $dataMapName ON TABLE $normalTable
+           | USING 'bloomfilter'
+           | WITH DEFERRED REBUILD
+           | DMProperties('INDEX_COLUMNS'='city,id', 'BLOOM_SIZE'='640000')
+           | """.stripMargin)
+    }
+    assert(exception.getMessage.contains(
+      "BloomFilter datamap does not support complex datatype column"))
+  }
+
   override protected def afterAll(): Unit = {
     deleteFile(bigFile)
     deleteFile(smallFile)


Mime
View raw message