carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [12/50] [abbrv] carbondata git commit: [CARBONDATA-1947]fix select * issue after compaction, delete and clean files operation
Date Sun, 07 Jan 2018 03:05:20 GMT
[CARBONDATA-1947]fix select * issue after compaction, delete and clean files operation

When updating the metadata details for merged segment, get the load starttime from carbonloadModel
facttime stamp

This closes #1735


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/83e92715
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/83e92715
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/83e92715

Branch: refs/heads/carbonstore
Commit: 83e92715a6819d2ff5c3e5cd50fe073be5ee01a6
Parents: 1937a21
Author: akashrn5 <akashnilugal@gmail.com>
Authored: Thu Dec 28 13:43:15 2017 +0530
Committer: Jacky Li <jacky.likun@qq.com>
Committed: Fri Dec 29 14:41:30 2017 +0800

----------------------------------------------------------------------
 .../testsuite/iud/DeleteCarbonTableTestCase.scala    | 15 +++++++++++++++
 .../carbondata/spark/rdd/CarbonTableCompactor.scala  |  3 +--
 .../processing/merger/CarbonDataMergerUtil.java      |  4 ++--
 3 files changed, 18 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/83e92715/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
index deaad20..6521657 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala
@@ -165,6 +165,21 @@ class DeleteCarbonTableTestCase extends QueryTest with BeforeAndAfterAll
{
     sql("drop table if exists preaggmain_preagg1")
   }
 
+  test("test select query after compaction, delete and clean files") {
+    sql("drop table if exists select_after_clean")
+    sql("create table select_after_clean(id int, name string) stored by 'carbondata'")
+    sql("insert into select_after_clean select 1,'abc'")
+    sql("insert into select_after_clean select 2,'def'")
+    sql("insert into select_after_clean select 3,'uhj'")
+    sql("insert into select_after_clean select 4,'frg'")
+    sql("alter table select_after_clean compact 'minor'")
+    sql("clean files for table select_after_clean")
+    sql("delete from select_after_clean where name='def'")
+    sql("clean files for table select_after_clean")
+    checkAnswer(sql("""select * from select_after_clean"""),
+      Seq(Row(1, "abc"), Row(3, "uhj"), Row(4, "frg")))
+  }
+
 
   override def afterAll {
     sql("use default")

http://git-wip-us.apache.org/repos/asf/carbondata/blob/83e92715/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
index 9768625..0dc856d 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala
@@ -135,7 +135,6 @@ class CarbonTableCompactor(carbonLoadModel: CarbonLoadModel,
     val factTableName = carbonLoadModel.getTableName
     val validSegments: Array[String] = CarbonDataMergerUtil
       .getValidSegments(loadsToMerge).split(',')
-    val mergeLoadStartTime = CarbonUpdateUtil.readCurrentTime()
     val partitionMapper = if (carbonTable.isHivePartitionTable) {
       var partitionMap: util.Map[String, util.List[String]] = null
       validSegments.foreach { segmentId =>
@@ -245,7 +244,7 @@ class CarbonTableCompactor(carbonLoadModel: CarbonLoadModel,
              carbonLoadModel)) ||
         CarbonDataMergerUtil
           .updateLoadMetadataWithMergeStatus(loadsToMerge, carbonTable.getMetaDataFilepath,
-            mergedLoadNumber, carbonLoadModel, mergeLoadStartTime, compactionType)
+            mergedLoadNumber, carbonLoadModel, compactionType)
 
       if (!statusFileUpdation) {
         LOGGER.audit(s"Compaction request failed for table ${ carbonLoadModel.getDatabaseName
}." +

http://git-wip-us.apache.org/repos/asf/carbondata/blob/83e92715/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
index e245927..26648d0 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
@@ -280,7 +280,7 @@ public final class CarbonDataMergerUtil {
    */
   public static boolean updateLoadMetadataWithMergeStatus(List<LoadMetadataDetails>
loadsToMerge,
       String metaDataFilepath, String mergedLoadNumber, CarbonLoadModel carbonLoadModel,
-      long mergeLoadStartTime, CompactionType compactionType) throws IOException {
+      CompactionType compactionType) throws IOException {
     boolean tableStatusUpdationStatus = false;
     AbsoluteTableIdentifier absoluteTableIdentifier =
         carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier();
@@ -327,7 +327,7 @@ public final class CarbonDataMergerUtil {
         loadMetadataDetails.setLoadName(mergedLoadNumber);
         CarbonLoaderUtil
             .addDataIndexSizeIntoMetaEntry(loadMetadataDetails, mergedLoadNumber, carbonTable);
-        loadMetadataDetails.setLoadStartTime(mergeLoadStartTime);
+        loadMetadataDetails.setLoadStartTime(carbonLoadModel.getFactTimeStamp());
         loadMetadataDetails.setPartitionCount("0");
         // if this is a major compaction then set the segment as major compaction.
         if (CompactionType.MAJOR == compactionType) {


Mime
View raw message