carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject carbondata git commit: [CARBONDATA-2658][DataLoad] Fix bugs in spilling in-memory pages
Date Mon, 09 Jul 2018 07:18:26 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master c2c5b18ba -> c3bc1ba10


[CARBONDATA-2658][DataLoad] Fix bugs in spilling in-memory pages

the parameter carbon.load.sortMemory.spill.percentage configured the value range 0-100,according
to configuration merge and spill in-memory pages to disk

This closes #2414


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c3bc1ba1
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c3bc1ba1
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c3bc1ba1

Branch: refs/heads/master
Commit: c3bc1ba10ef4c468f199fdea6e38d8b197908fcd
Parents: c2c5b18
Author: ndwangsen <luffy.wang@huawei.com>
Authored: Fri Jun 22 12:02:36 2018 +0800
Committer: Jacky Li <jacky.likun@qq.com>
Committed: Mon Jul 9 15:18:12 2018 +0800

----------------------------------------------------------------------
 .../processing/loading/sort/unsafe/UnsafeSortDataRows.java  | 1 -
 .../sort/unsafe/merger/UnsafeIntermediateMerger.java        | 9 +++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3bc1ba1/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java
b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java
index fb0bcc3..76532f0 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java
@@ -149,7 +149,6 @@ public class UnsafeSortDataRows {
     if (isMemoryAvailable) {
       UnsafeSortMemoryManager.INSTANCE.allocateDummyMemory(baseBlock.size());
     } else {
-      LOGGER.info("trigger in-memory merge and spill for table " + parameters.getTableName());
       // merge and spill in-memory pages to disk if memory is not enough
       unsafeInMemoryIntermediateFileMerger.tryTriggerInmemoryMerging(true);
     }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3bc1ba1/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java
b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java
index bd2f5d9..e6e24ec 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java
@@ -84,6 +84,14 @@ public class UnsafeIntermediateMerger {
     Integer spillPercentage = CarbonProperties.getInstance().getSortMemorySpillPercentage();
     this.spillSizeInSortMemory =
         UnsafeSortMemoryManager.INSTANCE.getUsableMemory() * spillPercentage / 100;
+    // get memory chunk size
+    long inMemoryChunkSizeInMB = CarbonProperties.getInstance().getSortMemoryChunkSizeInMB();
+    if (spillSizeInSortMemory < inMemoryChunkSizeInMB * 1024 * 1024) {
+      LOGGER.warn("the configure spill size is " + spillSizeInSortMemory +
+          " less than the page size " + inMemoryChunkSizeInMB * 1024 * 1024 +
+          ",so no merge and spill in-memory pages to disk");
+    }
+
   }
 
   public void addDataChunkToMerge(UnsafeCarbonRowPage rowPage) {
@@ -147,6 +155,7 @@ public class UnsafeIntermediateMerger {
         UnsafeCarbonRowPage page = iter.next();
         if (!spillDisk || sizeAdded + page.getDataBlock().size() < this.spillSizeInSortMemory)
{
           pages2Merge.add(page);
+          sizeAdded += page.getDataBlock().size();
           totalRows2Merge += page.getBuffer().getActualSize();
           iter.remove();
         } else {


Mime
View raw message