carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject carbondata git commit: [CARBONDATA-2706][BloomDataMap] clear bloom index files after segment is deleted
Date Tue, 10 Jul 2018 11:27:09 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master 0112ed096 -> d8562e5bd


[CARBONDATA-2706][BloomDataMap] clear bloom index files after segment is deleted

clear bloom index files after corresponding segment is deleted and
cleaned

This closes #2461


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d8562e5b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d8562e5b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d8562e5b

Branch: refs/heads/master
Commit: d8562e5bd2bb7ed2ecb6a56b905e6b219b539caf
Parents: 0112ed0
Author: xuchuanyin <xuchuanyin@hust.edu.cn>
Authored: Mon Jul 9 11:36:13 2018 +0800
Committer: Jacky Li <jacky.likun@qq.com>
Committed: Tue Jul 10 19:26:52 2018 +0800

----------------------------------------------------------------------
 .../carbondata/core/datamap/TableDataMap.java   |  8 +++++
 .../core/datamap/dev/DataMapFactory.java        |  5 ++++
 .../blockletindex/BlockletDataMapFactory.java   |  5 ++++
 .../carbondata/core/util/DeleteLoadFolders.java | 28 +++++++++++++++++-
 .../bloom/BloomCoarseGrainDataMapFactory.java   | 27 +++++++++++------
 .../examples/MinMaxIndexDataMapFactory.java     |  5 ++++
 .../lucene/LuceneDataMapFactoryBase.java        | 30 +++++++++++--------
 .../testsuite/datamap/CGDataMapTestCase.scala   |  7 +++++
 .../testsuite/datamap/DataMapWriterSuite.scala  |  7 +++++
 .../testsuite/datamap/FGDataMapTestCase.scala   |  7 +++++
 .../testsuite/datamap/TestDataMapStatus.scala   |  7 +++++
 .../TestInsertAndOtherCommandConcurrent.scala   |  7 +++++
 .../management/CarbonCleanFilesCommand.scala    |  5 ++++
 .../BloomCoarseGrainDataMapFunctionSuite.scala  | 31 ++++++++++++++++++++
 14 files changed, 157 insertions(+), 22 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
index cb7ec03..89a4c86 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
@@ -208,6 +208,14 @@ public final class TableDataMap extends OperationEventListener {
   }
 
   /**
+   * delete only the datamaps of the segments
+   */
+  public void deleteDatamapData(List<Segment> segments) {
+    for (Segment segment: segments) {
+      dataMapFactory.deleteDatamapData(segment);
+    }
+  }
+  /**
    * delete datamap data if any
    */
   public void deleteDatamapData() {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
index ff4ef72..b115462 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
@@ -111,6 +111,11 @@ public abstract class DataMapFactory<T extends DataMap> {
   public abstract DataMapLevel getDataMapLevel();
 
   /**
+   * delete datamap data in the specified segment
+   */
+  public abstract void deleteDatamapData(Segment segment);
+
+  /**
    * delete datamap data if any
    */
   public abstract void deleteDatamapData();

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
index 51e6e21..175a2a4 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
@@ -330,6 +330,11 @@ public class BlockletDataMapFactory extends CoarseGrainDataMapFactory
     return null;
   }
 
+  @Override
+  public void deleteDatamapData(Segment segment) {
+
+  }
+
   @Override public void deleteDatamapData() {
 
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/core/src/main/java/org/apache/carbondata/core/util/DeleteLoadFolders.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DeleteLoadFolders.java b/core/src/main/java/org/apache/carbondata/core/util/DeleteLoadFolders.java
index 25ba553..0433ba4 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DeleteLoadFolders.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DeleteLoadFolders.java
@@ -18,10 +18,14 @@
 package org.apache.carbondata.core.util;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datamap.DataMapStoreManager;
+import org.apache.carbondata.core.datamap.Segment;
+import org.apache.carbondata.core.datamap.TableDataMap;
 import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
 import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
 import org.apache.carbondata.core.datastore.impl.FileFactory;
@@ -31,6 +35,7 @@ import org.apache.carbondata.core.locks.ICarbonLock;
 import org.apache.carbondata.core.locks.LockUsage;
 import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
 import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
 import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
 import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
 import org.apache.carbondata.core.statusmanager.SegmentStatus;
@@ -85,7 +90,22 @@ public final class DeleteLoadFolders {
       LoadMetadataDetails[] loadDetails,
       boolean isForceDelete,
       List<PartitionSpec> specs) {
-    for (LoadMetadataDetails oneLoad : loadDetails) {
+    CarbonTable carbonTable = DataMapStoreManager.getInstance().getCarbonTable(
+        absoluteTableIdentifier);
+    List<TableDataMap> indexDataMaps = new ArrayList<>();
+    try {
+      for (TableDataMap dataMap : DataMapStoreManager.getInstance().getAllDataMap(carbonTable))
{
+        if (dataMap.getDataMapSchema().isIndexDataMap()) {
+          indexDataMaps.add(dataMap);
+        }
+      }
+    } catch (IOException e) {
+      LOGGER.warn(String.format(
+          "Failed to get datamaps for %s.%s, therefore the datamap files could not be cleaned.",
+          absoluteTableIdentifier.getDatabaseName(), absoluteTableIdentifier.getTableName()));
+    }
+
+    for (final LoadMetadataDetails oneLoad : loadDetails) {
       if (checkIfLoadCanBeDeletedPhysically(oneLoad, isForceDelete)) {
         try {
           if (oneLoad.getSegmentFile() != null) {
@@ -135,6 +155,12 @@ public final class DeleteLoadFolders {
             }
 
           }
+          List<Segment> segments = new ArrayList<>(1);
+          for (TableDataMap dataMap : indexDataMaps) {
+            segments.clear();
+            segments.add(new Segment(oneLoad.getLoadName()));
+            dataMap.deleteDatamapData(segments);
+          }
         } catch (IOException e) {
           LOGGER.warn("Unable to delete the file as per delete command " + oneLoad.getLoadName());
         }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
----------------------------------------------------------------------
diff --git a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
index 57c645f..aa6c5e1 100644
--- a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
+++ b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
@@ -345,22 +345,31 @@ public class BloomCoarseGrainDataMapFactory extends DataMapFactory<CoarseGrainDa
   }
 
   @Override
+  public void deleteDatamapData(Segment segment) {
+    try {
+      String segmentId = segment.getSegmentNo();
+      String datamapPath = CarbonTablePath
+          .getDataMapStorePath(getCarbonTable().getTablePath(), segmentId, dataMapName);
+      if (FileFactory.isFileExist(datamapPath)) {
+        CarbonFile file = FileFactory.getCarbonFile(datamapPath,
+            FileFactory.getFileType(datamapPath));
+        CarbonUtil.deleteFoldersAndFilesSilent(file);
+      }
+    } catch (IOException | InterruptedException ex) {
+      LOGGER.error("Failed to delete datamap for segment_" + segment.getSegmentNo());
+    }
+  }
+
+  @Override
   public void deleteDatamapData() {
     SegmentStatusManager ssm =
         new SegmentStatusManager(getCarbonTable().getAbsoluteTableIdentifier());
     try {
       List<Segment> validSegments = ssm.getValidAndInvalidSegments().getValidSegments();
       for (Segment segment : validSegments) {
-        String segmentId = segment.getSegmentNo();
-        String datamapPath = CarbonTablePath
-            .getDataMapStorePath(getCarbonTable().getTablePath(), segmentId, dataMapName);
-        if (FileFactory.isFileExist(datamapPath)) {
-          CarbonFile file = FileFactory.getCarbonFile(datamapPath,
-              FileFactory.getFileType(datamapPath));
-          CarbonUtil.deleteFoldersAndFilesSilent(file);
-        }
+        deleteDatamapData(segment);
       }
-    } catch (IOException | InterruptedException ex) {
+    } catch (IOException e) {
       LOGGER.error("drop datamap failed, failed to delete datamap directory");
     }
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMapFactory.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMapFactory.java
b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMapFactory.java
index bc65a93..506b91a 100644
--- a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMapFactory.java
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMapFactory.java
@@ -156,6 +156,11 @@ public class MinMaxIndexDataMapFactory extends CoarseGrainDataMapFactory
{
     return this.dataMapMeta;
   }
 
+  @Override
+  public void deleteDatamapData(Segment segment) {
+
+  }
+
   @Override public void deleteDatamapData() {
 
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
----------------------------------------------------------------------
diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
index 5393746..8186841 100644
--- a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
+++ b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
@@ -181,19 +181,9 @@ abstract class LuceneDataMapFactoryBase<T extends DataMap> extends
DataMapFactor
     try {
       List<Segment> validSegments = ssm.getValidAndInvalidSegments().getValidSegments();
       for (Segment segment : validSegments) {
-        String segmentId = segment.getSegmentNo();
-        String datamapPath = CarbonTablePath
-            .getDataMapStorePath(tableIdentifier.getTablePath(), segmentId, dataMapName);
-        if (FileFactory.isFileExist(datamapPath)) {
-          CarbonFile file =
-              FileFactory.getCarbonFile(datamapPath, FileFactory.getFileType(datamapPath));
-          CarbonUtil.deleteFoldersAndFilesSilent(file);
-        }
+        deleteDatamapData(segment);
       }
-    } catch (IOException ex) {
-      throw new MalformedDataMapCommandException(
-          "drop datamap failed, failed to delete datamap directory");
-    } catch (InterruptedException ex) {
+    } catch (IOException | RuntimeException ex) {
       throw new MalformedDataMapCommandException(
           "drop datamap failed, failed to delete datamap directory");
     }
@@ -270,6 +260,22 @@ abstract class LuceneDataMapFactoryBase<T extends DataMap> extends
DataMapFactor
 
   }
 
+  @Override public void deleteDatamapData(Segment segment) {
+    try {
+      String segmentId = segment.getSegmentNo();
+      String datamapPath = CarbonTablePath
+          .getDataMapStorePath(tableIdentifier.getTablePath(), segmentId, dataMapName);
+      if (FileFactory.isFileExist(datamapPath)) {
+        CarbonFile file = FileFactory.getCarbonFile(datamapPath,
+            FileFactory.getFileType(datamapPath));
+        CarbonUtil.deleteFoldersAndFilesSilent(file);
+      }
+    } catch (IOException | InterruptedException ex) {
+      throw new RuntimeException(
+          "drop datamap failed, failed to delete datamap directory");
+    }
+  }
+
   @Override public void deleteDatamapData() {
     try {
       deleteDatamap();

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
index bc8b28b..9a7cb99 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
@@ -136,6 +136,13 @@ class CGDataMapFactory(
   }
 
   /**
+   * delete datamap of the segment
+   */
+  override def deleteDatamapData(segment: Segment): Unit = {
+    ???
+  }
+
+  /**
    * delete datamap data if any
    */
   override def deleteDatamapData(): Unit = {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
index d4d78c9..434c363 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
@@ -71,6 +71,13 @@ class C2DataMapFactory(
   }
 
   /**
+   * delete datamap of the segment
+   */
+  override def deleteDatamapData(segment: Segment): Unit = {
+    ???
+  }
+
+  /**
    * delete datamap data if any
    */
   override def deleteDatamapData(): Unit = {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
index 02ada2e..f4c12f7 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
@@ -129,6 +129,13 @@ class FGDataMapFactory(carbonTable: CarbonTable,
   }
 
   /**
+   * delete datamap of the segment
+   */
+  override def deleteDatamapData(segment: Segment): Unit = {
+    ???
+  }
+
+  /**
    * delete datamap data if any
    */
   override def deleteDatamapData(): Unit = {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapStatus.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapStatus.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapStatus.scala
index c615078..0638a07 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapStatus.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapStatus.scala
@@ -267,6 +267,13 @@ class TestDataMapFactory(
   }
 
   /**
+   * delete datamap of the segment
+   */
+  override def deleteDatamapData(segment: Segment): Unit = {
+    ???
+  }
+
+  /**
    * delete datamap data if any
    */
   override def deleteDatamapData(): Unit = {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala
index 2441b53..dbbcd4d 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala
@@ -340,6 +340,13 @@ class WaitingDataMapFactory(
   }
 
   /**
+   * delete datamap of the segment
+   */
+  override def deleteDatamapData(segment: Segment): Unit = {
+
+  }
+
+  /**
    * delete datamap data if any
    */
   override def deleteDatamapData(): Unit = {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala
index 1c4d068..9d0f149 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala
@@ -29,6 +29,7 @@ import org.apache.carbondata.api.CarbonStore
 import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
 import org.apache.carbondata.common.logging.LogServiceFactory
 import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datamap.DataMapStoreManager
 import org.apache.carbondata.core.exception.ConcurrentOperationException
 import org.apache.carbondata.core.indexstore.PartitionSpec
 import org.apache.carbondata.core.metadata.schema.table.CarbonTable
@@ -57,6 +58,10 @@ case class CarbonCleanFilesCommand(
   override def processMetadata(sparkSession: SparkSession): Seq[Row] = {
     carbonTable = CarbonEnv.getCarbonTable(databaseNameOp, tableName.get)(sparkSession)
 
+    val dms = carbonTable.getTableInfo.getDataMapSchemaList.asScala.map(_.getDataMapName)
+    val indexDms = DataMapStoreManager.getInstance.getAllDataMap(carbonTable).asScala
+      .filter(_.getDataMapSchema.isIndexDataMap)
+
     if (carbonTable.hasAggregationDataMap) {
       cleanFileCommands = carbonTable.getTableInfo.getDataMapSchemaList.asScala.map {
         dataMapSchema =>

http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFunctionSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFunctionSuite.scala
b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFunctionSuite.scala
index d0e2cbd..92943b2 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFunctionSuite.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFunctionSuite.scala
@@ -2,12 +2,16 @@ package org.apache.carbondata.datamap.bloom
 
 import java.io.File
 
+import org.apache.commons.io.FileUtils
+import org.apache.spark.sql.CarbonEnv
+import org.apache.spark.sql.test.Spark2TestQueryExecutor
 import org.apache.spark.sql.test.util.QueryTest
 import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
 
 import org.apache.carbondata.core.constants.CarbonCommonConstants
 import org.apache.carbondata.core.datamap.status.DataMapStatusManager
 import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.path.CarbonTablePath
 import org.apache.carbondata.datamap.bloom.BloomCoarseGrainDataMapTestUtil.deleteFile
 import org.apache.carbondata.datamap.bloom.BloomCoarseGrainDataMapTestUtil.createFile
 import org.apache.carbondata.datamap.bloom.BloomCoarseGrainDataMapTestUtil.checkBasicQuery
@@ -759,6 +763,33 @@ class BloomCoarseGrainDataMapFunctionSuite  extends QueryTest with BeforeAndAfte
       sql(s"SELECT * FROM $normalTable WHERE isNull(c2)"))
   }
 
+  test("test bloom datamap: deleting & clearning segment will clear datamap files") {
+    sql(s"CREATE TABLE $bloomDMSampleTable(c1 string, c2 int, c3 string) STORED BY 'carbondata'")
+    sql(
+      s"""
+         | CREATE DATAMAP $dataMapName on table $bloomDMSampleTable
+         | using 'bloomfilter'
+         | DMPROPERTIES('index_columns'='c1, c2')
+       """.stripMargin)
+    sql(s"INSERT INTO $bloomDMSampleTable SELECT 'c1v1', 1, 'c3v1'")
+    sql(s"INSERT INTO $bloomDMSampleTable SELECT 'c1v2', 2, 'c3v2'")
+
+    // two segments both has datamap files
+    val carbonTable = CarbonEnv.getCarbonTable(Option("default"), bloomDMSampleTable)(Spark2TestQueryExecutor.spark)
+    import scala.collection.JavaConverters._
+    (0 to 1).foreach { segId =>
+      val datamapPath = CarbonTablePath.getDataMapStorePath(carbonTable.getTablePath, segId.toString,
dataMapName)
+      assert(FileUtils.listFiles(FileUtils.getFile(datamapPath), Array("bloomindex"), true).asScala.nonEmpty)
+    }
+    // delete and clean the first segment, the corresponding datamap files should be cleaned
too
+    sql(s"DELETE FROM TABLE $bloomDMSampleTable WHERE SEGMENT.ID IN (0)")
+    sql(s"CLEAN FILES FOR TABLE $bloomDMSampleTable")
+    var datamapPath = CarbonTablePath.getDataMapStorePath(carbonTable.getTablePath, "0",
dataMapName)
+    assert(!FileUtils.getFile(datamapPath).exists(), "index file of this segment has been
deleted, should not exist")
+    datamapPath = CarbonTablePath.getDataMapStorePath(carbonTable.getTablePath, "1", dataMapName)
+    assert(FileUtils.listFiles(FileUtils.getFile(datamapPath), Array("bloomindex"), true).asScala.nonEmpty)
+  }
+
   override def afterAll(): Unit = {
     deleteFile(bigFile)
     sql(s"DROP TABLE IF EXISTS $normalTable")


Mime
View raw message