carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kunalkap...@apache.org
Subject carbondata git commit: [CARBONDATA-2265] [DFX]-Load]: Load job fails if 1 folder contains 1000 files
Date Wed, 21 Mar 2018 10:16:54 GMT
Repository: carbondata
Updated Branches:
  refs/heads/branch-1.3 aff822ccf -> 87de69729


[CARBONDATA-2265] [DFX]-Load]: Load job fails if 1 folder contains 1000 files

Problem : We are keeping the rawTablePages also in memory, but we are no where using it
Solution : Removed storing rawTablePages

This closes #2082


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/87de6972
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/87de6972
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/87de6972

Branch: refs/heads/branch-1.3
Commit: 87de69729e916e1ca5ad1be00a091ccbe65a990d
Parents: aff822c
Author: dhatchayani <dhatcha.official@gmail.com>
Authored: Tue Mar 20 18:11:58 2018 +0530
Committer: kunal642 <kunalkapoor642@gmail.com>
Committed: Wed Mar 21 15:45:04 2018 +0530

----------------------------------------------------------------------
 .../processing/store/writer/v3/BlockletDataHolder.java        | 7 -------
 .../store/writer/v3/CarbonFactDataWriterImplV3.java           | 5 +++--
 2 files changed, 3 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/87de6972/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/BlockletDataHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/BlockletDataHolder.java
b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/BlockletDataHolder.java
index 68aee95..36fda3c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/BlockletDataHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/BlockletDataHolder.java
@@ -24,24 +24,20 @@ import org.apache.carbondata.processing.store.TablePage;
 
 public class BlockletDataHolder {
   private List<EncodedTablePage> encodedTablePage;
-  private List<TablePage> rawTablePages;
   private long currentSize;
 
   public BlockletDataHolder() {
     this.encodedTablePage = new ArrayList<>();
-    this.rawTablePages = new ArrayList<>();
   }
 
   public void clear() {
     encodedTablePage.clear();
-    rawTablePages.clear();
     currentSize = 0;
   }
 
   public void addPage(TablePage rawTablePage) {
     EncodedTablePage encodedTablePage = rawTablePage.getEncodedTablePage();
     this.encodedTablePage.add(encodedTablePage);
-    this.rawTablePages.add(rawTablePage);
     currentSize += encodedTablePage.getEncodedSize();
   }
 
@@ -66,7 +62,4 @@ public class BlockletDataHolder {
     return encodedTablePage;
   }
 
-  public List<TablePage> getRawTablePages() {
-    return rawTablePages;
-  }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/87de6972/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
index ddf444d..fc1bd7d 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java
@@ -187,9 +187,10 @@ public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter
{
     } catch (IOException e) {
       LOGGER.error(e, "Problem while writing file");
       throw new CarbonDataWriterException("Problem while writing file", e);
+    } finally {
+      // clear the data holder
+      blockletDataHolder.clear();
     }
-    // clear the data holder
-    blockletDataHolder.clear();
 
   }
 


Mime
View raw message