carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [carbondata] branch master updated: [CARBONDATA-3764] Reduce Reusable buffer creation when few projections selected out of many columns
Date Sun, 05 Apr 2020 10:33:55 GMT
This is an automated email from the ASF dual-hosted git repository.

jackylk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git


The following commit(s) were added to refs/heads/master by this push:
     new 7ed7cf1  [CARBONDATA-3764] Reduce Reusable buffer creation when few projections selected
out of many columns
7ed7cf1 is described below

commit 7ed7cf1f69b9b852c8f3a5bf41f6d2a9b5f175e7
Author: ajantha-bhat <ajanthabhat@gmail.com>
AuthorDate: Fri Apr 3 12:31:34 2020 +0530

    [CARBONDATA-3764] Reduce Reusable buffer creation when few projections selected out of
many columns
    
    Why is this PR needed?
    
    If few projections selected out of many columns, reusable buffer is based on columns count
instead of projections count. Hence many unused objects were created.
    
    What changes were proposed in this PR?
    
    Create reusable buffers only as per the projection count.
    Fix spelling error from Resusable to Reusable
    
    Does this PR introduce any user interface change?
    No
    
    Is any new testcase added?
    No. Internal change
    
    This closes #3693
---
 .../scan/executor/impl/AbstractQueryExecutor.java  | 25 ++++++++++------------
 .../scan/executor/infos/BlockExecutionInfo.java    | 20 ++++++++---------
 .../core/scan/result/BlockletScannedResult.java    |  4 ++--
 3 files changed, 23 insertions(+), 26 deletions(-)

diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
index 05d3876..dcb2c0f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
@@ -382,16 +382,16 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E>
{
               dataRefNode.getBlockInfos().get(0).getDeletedDeltaFilePath(),
               dataRefNode.getBlockInfos().get(0).getSegment());
       if (null == dimensionReusableDataBuffers || null == measureReusableDataBuffers) {
-        dimensionReusableDataBuffers = blockExecutionInfoForBlock.getDimensionResusableDataBuffer();
-        measureReusableDataBuffers = blockExecutionInfoForBlock.getMeasureResusableDataBuffer();
+        dimensionReusableDataBuffers = blockExecutionInfoForBlock.getDimensionReusableDataBuffer();
+        measureReusableDataBuffers = blockExecutionInfoForBlock.getMeasureReusableDataBuffer();
       } else {
         if (dimensionReusableDataBuffers.length == blockExecutionInfoForBlock
-            .getDimensionResusableDataBuffer().length) {
-          blockExecutionInfoForBlock.setDimensionResusableDataBuffer(dimensionReusableDataBuffers);
+            .getDimensionReusableDataBuffer().length) {
+          blockExecutionInfoForBlock.setDimensionReusableDataBuffer(dimensionReusableDataBuffers);
         }
         if (measureReusableDataBuffers.length == blockExecutionInfoForBlock
-            .getMeasureResusableDataBuffer().length) {
-          blockExecutionInfoForBlock.setMeasureResusableDataBuffer(measureReusableDataBuffers);
+            .getMeasureReusableDataBuffer().length) {
+          blockExecutionInfoForBlock.setMeasureReusableDataBuffer(measureReusableDataBuffers);
         }
       }
       blockExecutionInfoList.add(blockExecutionInfoForBlock);
@@ -510,13 +510,11 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E>
{
     int[] dimensionChunkIndexes = QueryUtil.getDimensionChunkIndexes(projectDimensions,
         segmentProperties.getDimensionOrdinalToChunkMapping(),
         currentBlockFilterDimensions, allProjectionListDimensionIdexes);
-    int reusableBufferSize = Math.max(segmentProperties.getDimensionOrdinalToChunkMapping().size(),
-        projectDimensions.size());
-    ReusableDataBuffer[] dimensionBuffer = new ReusableDataBuffer[reusableBufferSize];
+    ReusableDataBuffer[] dimensionBuffer = new ReusableDataBuffer[projectDimensions.size()];
     for (int i = 0; i < dimensionBuffer.length; i++) {
       dimensionBuffer[i] = new ReusableDataBuffer();
     }
-    blockExecutionInfo.setDimensionResusableDataBuffer(dimensionBuffer);
+    blockExecutionInfo.setDimensionReusableDataBuffer(dimensionBuffer);
     int numberOfColumnToBeReadInOneIO = Integer.parseInt(CarbonProperties.getInstance()
         .getProperty(CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO,
             CarbonV3DataFormatConstants.NUMBER_OF_COLUMN_TO_READ_IN_IO_DEFAULTVALUE));
@@ -541,13 +539,12 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E>
{
         projectionMeasures, expressionMeasures,
         segmentProperties.getMeasuresOrdinalToChunkMapping(), filterMeasures,
         allProjectionListMeasureIndexes);
-    reusableBufferSize = Math.max(segmentProperties.getMeasuresOrdinalToChunkMapping().size(),
-        allProjectionListMeasureIndexes.size());
-    ReusableDataBuffer[] measureBuffer = new ReusableDataBuffer[reusableBufferSize];
+    ReusableDataBuffer[] measureBuffer =
+        new ReusableDataBuffer[allProjectionListMeasureIndexes.size()];
     for (int i = 0; i < measureBuffer.length; i++) {
       measureBuffer[i] = new ReusableDataBuffer();
     }
-    blockExecutionInfo.setMeasureResusableDataBuffer(measureBuffer);
+    blockExecutionInfo.setMeasureReusableDataBuffer(measureBuffer);
     if (measureChunkIndexes.length > 0) {
 
       numberOfElementToConsider = measureChunkIndexes[measureChunkIndexes.length - 1]
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
index eb19852..e3b4cd9 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
@@ -203,9 +203,9 @@ public class BlockExecutionInfo {
    */
   private boolean isDirectVectorFill;
 
-  private ReusableDataBuffer[] dimensionResusableDataBuffer;
+  private ReusableDataBuffer[] dimensionReusableDataBuffer;
 
-  private ReusableDataBuffer[] measureResusableDataBuffer;
+  private ReusableDataBuffer[] measureReusableDataBuffer;
 
   /**
    * It is used to read only the deleted data of a particular version. It will be used to
get the
@@ -588,20 +588,20 @@ public class BlockExecutionInfo {
     isDirectVectorFill = directVectorFill;
   }
 
-  public ReusableDataBuffer[] getDimensionResusableDataBuffer() {
-    return dimensionResusableDataBuffer;
+  public ReusableDataBuffer[] getDimensionReusableDataBuffer() {
+    return dimensionReusableDataBuffer;
   }
 
-  public void setDimensionResusableDataBuffer(ReusableDataBuffer[] dimensionResusableDataBuffer)
{
-    this.dimensionResusableDataBuffer = dimensionResusableDataBuffer;
+  public void setDimensionReusableDataBuffer(ReusableDataBuffer[] dimensionReusableDataBuffer)
{
+    this.dimensionReusableDataBuffer = dimensionReusableDataBuffer;
   }
 
-  public ReusableDataBuffer[] getMeasureResusableDataBuffer() {
-    return measureResusableDataBuffer;
+  public ReusableDataBuffer[] getMeasureReusableDataBuffer() {
+    return measureReusableDataBuffer;
   }
 
-  public void setMeasureResusableDataBuffer(ReusableDataBuffer[] measureResusableDataBuffer)
{
-    this.measureResusableDataBuffer = measureResusableDataBuffer;
+  public void setMeasureReusableDataBuffer(ReusableDataBuffer[] measureReusableDataBuffer)
{
+    this.measureReusableDataBuffer = measureReusableDataBuffer;
   }
 
   public boolean isReadOnlyDelta() {
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
b/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
index 6172e09..5f333ae 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/BlockletScannedResult.java
@@ -157,8 +157,8 @@ public abstract class BlockletScannedResult {
 
   public BlockletScannedResult(BlockExecutionInfo blockExecutionInfo,
       QueryStatisticsModel queryStatisticsModel) {
-    this.dimensionReusableBuffer = blockExecutionInfo.getDimensionResusableDataBuffer();
-    this.measureReusableBuffer = blockExecutionInfo.getMeasureResusableDataBuffer();
+    this.dimensionReusableBuffer = blockExecutionInfo.getDimensionReusableDataBuffer();
+    this.measureReusableBuffer = blockExecutionInfo.getMeasureReusableDataBuffer();
     this.fixedLengthKeySize = blockExecutionInfo.getFixedLengthKeySize();
     this.noDictionaryColumnChunkIndexes = blockExecutionInfo.getNoDictionaryColumnChunkIndexes();
     this.dictionaryColumnChunkIndexes = blockExecutionInfo.getDictionaryColumnChunkIndex();


Mime
View raw message