carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [carbondata] 05/22: [CARBONDATA-3353 ]Fixed MinMax Based Pruning for Measure column in case of Legacy store
Date Thu, 16 May 2019 19:05:38 GMT
This is an automated email from the ASF dual-hosted git repository.

ravipesala pushed a commit to branch branch-1.5
in repository https://gitbox.apache.org/repos/asf/carbondata.git

commit 743b843676e8736beaf55286df3369772224bb90
Author: Indhumathi27 <indhumathim27@gmail.com>
AuthorDate: Fri Apr 12 12:25:38 2019 +0530

    [CARBONDATA-3353 ]Fixed MinMax Based Pruning for Measure column in case of Legacy store
    
    Why this PR needed?
    
    Problem:
    For table created and loaded with legacy store having a measure column, while building
the page min max,
    min is written as max and viceversa, so blocklet level minmax is wrong. With current version,
when we query with filter on measure column, measure filter pruning is skipping some blocks
and giving wrong results.
    
    Solution:
    Skip MinMax based pruning in case of legacy store for measure column.
    
    This closes #3176
---
 .../indexstore/blockletindex/BlockDataMap.java     | 15 +------
 .../scan/executor/impl/AbstractQueryExecutor.java  | 15 +++----
 .../carbondata/core/scan/filter/FilterUtil.java    |  8 ++++
 .../filter/executer/IncludeFilterExecuterImpl.java | 11 +++--
 .../RowLevelRangeGrtThanFiterExecuterImpl.java     | 10 +++--
 ...LevelRangeGrtrThanEquaToFilterExecuterImpl.java | 10 +++--
 ...wLevelRangeLessThanEqualFilterExecuterImpl.java | 10 +++--
 .../RowLevelRangeLessThanFilterExecuterImpl.java   | 10 +++--
 .../apache/carbondata/core/util/CarbonUtil.java    | 47 ----------------------
 .../carbondata/core/util/CarbonUtilTest.java       | 46 ---------------------
 10 files changed, 51 insertions(+), 131 deletions(-)

diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockDataMap.java
b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockDataMap.java
index 5b2132c..1fc5831 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockDataMap.java
@@ -67,7 +67,6 @@ import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
 import org.apache.carbondata.core.scan.model.QueryModel;
 import org.apache.carbondata.core.util.BlockletDataMapUtil;
 import org.apache.carbondata.core.util.ByteUtil;
-import org.apache.carbondata.core.util.CarbonUtil;
 import org.apache.carbondata.core.util.DataFileFooterConverter;
 import org.apache.carbondata.core.util.path.CarbonTablePath;
 
@@ -219,7 +218,7 @@ public class BlockDataMap extends CoarseGrainDataMap
     DataMapRowImpl summaryRow = null;
     CarbonRowSchema[] schema = getFileFooterEntrySchema();
     boolean[] minMaxFlag = new boolean[segmentProperties.getColumnsValueSize().length];
-    Arrays.fill(minMaxFlag, true);
+    FilterUtil.setMinMaxFlagForLegacyStore(minMaxFlag, segmentProperties);
     long totalRowCount = 0;
     for (DataFileFooter fileFooter : indexInfo) {
       TableBlockInfo blockInfo = fileFooter.getBlockInfo().getTableBlockInfo();
@@ -232,19 +231,9 @@ public class BlockDataMap extends CoarseGrainDataMap
       if (null != blockMetaInfo) {
         BlockletIndex blockletIndex = fileFooter.getBlockletIndex();
         BlockletMinMaxIndex minMaxIndex = blockletIndex.getMinMaxIndex();
-        byte[][] minValues =
-            BlockletDataMapUtil.updateMinValues(segmentProperties, minMaxIndex.getMinValues());
-        byte[][] maxValues =
-            BlockletDataMapUtil.updateMaxValues(segmentProperties, minMaxIndex.getMaxValues());
-        // update min max values in case of old store for measures as measure min/max in
-        // old stores in written opposite
-        byte[][] updatedMinValues =
-            CarbonUtil.updateMinMaxValues(fileFooter, maxValues, minValues, true);
-        byte[][] updatedMaxValues =
-            CarbonUtil.updateMinMaxValues(fileFooter, maxValues, minValues, false);
         summaryRow = loadToUnsafeBlock(schema, taskSummarySchema, fileFooter, segmentProperties,
             getMinMaxCacheColumns(), blockInfo.getFilePath(), summaryRow,
-            blockMetaInfo, updatedMinValues, updatedMaxValues, minMaxFlag);
+            blockMetaInfo, minMaxIndex.getMinValues(), minMaxIndex.getMaxValues(), minMaxFlag);
         totalRowCount += fileFooter.getNumberOfRows();
       }
     }
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
index f81a3dc..b15bdb5 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
@@ -238,6 +238,12 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E>
{
           LOGGER.warn("Skipping Direct Vector Filling as it is not Supported "
               + "for Legacy store prior to V3 store");
           queryModel.setDirectVectorFill(false);
+          // Skip minmax based pruning for measure column in case of legacy store
+          boolean[] minMaxFlag = new boolean[segmentProperties.getColumnsValueSize().length];
+          FilterUtil.setMinMaxFlagForLegacyStore(minMaxFlag, segmentProperties);
+          for (BlockletInfo blockletInfo : fileFooter.getBlockletList()) {
+            blockletInfo.getBlockletIndex().getMinMaxIndex().setIsMinMaxSet(minMaxFlag);
+          }
         }
         readAndFillBlockletInfo(tableBlockInfos, blockInfo,
             blockletDetailInfo, fileFooter, segmentProperties);
@@ -386,15 +392,6 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E>
{
     byte[][] maxValues = blockletInfo.getBlockletIndex().getMinMaxIndex().getMaxValues();
     byte[][] minValues = blockletInfo.getBlockletIndex().getMinMaxIndex().getMinValues();
     if (blockletDetailInfo.isLegacyStore()) {
-      minValues = BlockletDataMapUtil.updateMinValues(segmentProperties,
-          blockletInfo.getBlockletIndex().getMinMaxIndex().getMinValues());
-      maxValues = BlockletDataMapUtil.updateMaxValues(segmentProperties,
-          blockletInfo.getBlockletIndex().getMinMaxIndex().getMaxValues());
-      // update min and max values in case of old store for measures as min and max is written
-      // opposite for measures in old store ( store <= 1.1 version)
-      byte[][] tempMaxValues = maxValues;
-      maxValues = CarbonUtil.updateMinMaxValues(fileFooter, maxValues, minValues, false);
-      minValues = CarbonUtil.updateMinMaxValues(fileFooter, tempMaxValues, minValues, true);
       info.setDataBlockFromOldStore(true);
     }
     blockletInfo.getBlockletIndex().getMinMaxIndex().setMaxValues(maxValues);
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index 6cc13e2..9d8fe8d 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -2326,4 +2326,12 @@ public final class FilterUtil {
     return defaultValue;
   }
 
+  public static void setMinMaxFlagForLegacyStore(boolean[] minMaxFlag,
+      SegmentProperties segmentProperties) {
+    int index = segmentProperties.getEachDimColumnValueSize().length + segmentProperties
+        .getEachComplexDimColumnValueSize().length;
+    Arrays.fill(minMaxFlag, 0, index, true);
+    Arrays.fill(minMaxFlag, index, minMaxFlag.length, false);
+  }
+
 }
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
index 4668242..33a337b 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
@@ -524,10 +524,13 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
           isMinMaxSet[chunkIndex]);
       }
     } else if (isMeasurePresentInCurrentBlock) {
-      chunkIndex = msrColumnEvaluatorInfo.getColumnIndexInMinMaxByteArray();
-      isScanRequired = isScanRequired(blkMaxVal[chunkIndex], blkMinVal[chunkIndex],
-          msrColumnExecutorInfo.getFilterKeys(),
-          msrColumnEvaluatorInfo.getType());
+      if (isMinMaxSet[chunkIndex]) {
+        chunkIndex = msrColumnEvaluatorInfo.getColumnIndexInMinMaxByteArray();
+        isScanRequired = isScanRequired(blkMaxVal[chunkIndex], blkMinVal[chunkIndex],
+            msrColumnExecutorInfo.getFilterKeys(), msrColumnEvaluatorInfo.getType());
+      } else {
+        isScanRequired = true;
+      }
     }
 
     if (isScanRequired) {
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
index d2c4b05..6b37e60 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
@@ -122,9 +122,13 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
     byte[] maxValue = null;
     if (isMeasurePresentInCurrentBlock[0] || isDimensionPresentInCurrentBlock[0]) {
       if (isMeasurePresentInCurrentBlock[0]) {
-        maxValue = blockMaxValue[measureChunkIndex[0]];
-        isScanRequired =
-            isScanRequired(maxValue, msrFilterRangeValues, msrColEvalutorInfoList.get(0).getType());
+        if (isMinMaxSet[measureChunkIndex[0]]) {
+          maxValue = blockMaxValue[measureChunkIndex[0]];
+          isScanRequired = isScanRequired(maxValue, msrFilterRangeValues,
+              msrColEvalutorInfoList.get(0).getType());
+        } else {
+          isScanRequired = true;
+        }
       } else {
         maxValue = blockMaxValue[dimensionChunkIndex[0]];
         DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType();
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
index cf31033..24c2e3c 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
@@ -120,9 +120,13 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
     byte[] maxValue = null;
     if (isMeasurePresentInCurrentBlock[0] || isDimensionPresentInCurrentBlock[0]) {
       if (isMeasurePresentInCurrentBlock[0]) {
-        maxValue = blockMaxValue[measureChunkIndex[0]];
-        isScanRequired =
-            isScanRequired(maxValue, msrFilterRangeValues, msrColEvalutorInfoList.get(0).getType());
+        if (isMinMaxSet[measureChunkIndex[0]]) {
+          maxValue = blockMaxValue[measureChunkIndex[0]];
+          isScanRequired = isScanRequired(maxValue, msrFilterRangeValues,
+              msrColEvalutorInfoList.get(0).getType());
+        } else {
+          isScanRequired = true;
+        }
       } else {
         maxValue = blockMaxValue[dimensionChunkIndex[0]];
         DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType();
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
index 8ea6e0d..0dbdf79 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
@@ -120,9 +120,13 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
     boolean isScanRequired = false;
     if (isMeasurePresentInCurrentBlock[0] || isDimensionPresentInCurrentBlock[0]) {
       if (isMeasurePresentInCurrentBlock[0]) {
-        minValue = blockMinValue[measureChunkIndex[0]];
-        isScanRequired =
-            isScanRequired(minValue, msrFilterRangeValues, msrColEvalutorInfoList.get(0).getType());
+        if (isMinMaxSet[measureChunkIndex[0]]) {
+          minValue = blockMinValue[measureChunkIndex[0]];
+          isScanRequired = isScanRequired(minValue, msrFilterRangeValues,
+              msrColEvalutorInfoList.get(0).getType());
+        } else {
+          isScanRequired = true;
+        }
       } else {
         minValue = blockMinValue[dimensionChunkIndex[0]];
         DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType();
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java
index df1afc4..acd918a 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFilterExecuterImpl.java
@@ -120,9 +120,13 @@ public class RowLevelRangeLessThanFilterExecuterImpl extends RowLevelFilterExecu
     boolean isScanRequired = false;
     if (isMeasurePresentInCurrentBlock[0] || isDimensionPresentInCurrentBlock[0]) {
       if (isMeasurePresentInCurrentBlock[0]) {
-        minValue = blockMinValue[measureChunkIndex[0]];
-        isScanRequired =
-            isScanRequired(minValue, msrFilterRangeValues, msrColEvalutorInfoList.get(0).getType());
+        if (isMinMaxSet[measureChunkIndex[0]]) {
+          minValue = blockMinValue[measureChunkIndex[0]];
+          isScanRequired = isScanRequired(minValue, msrFilterRangeValues,
+              msrColEvalutorInfoList.get(0).getType());
+        } else {
+          isScanRequired = true;
+        }
       } else {
         minValue = blockMinValue[dimensionChunkIndex[0]];
         DataType dataType = dimColEvaluatorInfoList.get(0).getDimension().getDataType();
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index d9f69e3..a4af9cc 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -88,8 +88,6 @@ import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
 import org.apache.carbondata.core.statusmanager.SegmentStatus;
 import org.apache.carbondata.core.statusmanager.SegmentStatusManager;
 import org.apache.carbondata.core.statusmanager.SegmentUpdateStatusManager;
-import org.apache.carbondata.core.util.comparator.Comparator;
-import org.apache.carbondata.core.util.comparator.SerializableComparator;
 import org.apache.carbondata.core.util.path.CarbonTablePath;
 import org.apache.carbondata.format.BlockletHeader;
 import org.apache.carbondata.format.DataChunk2;
@@ -2826,51 +2824,6 @@ public final class CarbonUtil {
   }
 
   /**
-   * This method will be used to update the min and max values and this will be used in case
of
-   * old store where min and max values for measures are written opposite
-   * (i.e max values in place of min and min in place of max values)
-   *
-   * @param dataFileFooter
-   * @param maxValues
-   * @param minValues
-   * @param isMinValueComparison
-   * @return
-   */
-  public static byte[][] updateMinMaxValues(DataFileFooter dataFileFooter, byte[][] maxValues,
-      byte[][] minValues, boolean isMinValueComparison) {
-    byte[][] updatedMinMaxValues = new byte[maxValues.length][];
-    if (isMinValueComparison) {
-      System.arraycopy(minValues, 0, updatedMinMaxValues, 0, minValues.length);
-    } else {
-      System.arraycopy(maxValues, 0, updatedMinMaxValues, 0, maxValues.length);
-    }
-    for (int i = 0; i < maxValues.length; i++) {
-      // update min and max values only for measures
-      if (!dataFileFooter.getColumnInTable().get(i).isDimensionColumn()) {
-        DataType dataType = dataFileFooter.getColumnInTable().get(i).getDataType();
-        SerializableComparator comparator = Comparator.getComparator(dataType);
-        int compare;
-        if (isMinValueComparison) {
-          compare = comparator
-              .compare(DataTypeUtil.getMeasureObjectFromDataType(maxValues[i], dataType),
-                  DataTypeUtil.getMeasureObjectFromDataType(minValues[i], dataType));
-          if (compare < 0) {
-            updatedMinMaxValues[i] = maxValues[i];
-          }
-        } else {
-          compare = comparator
-              .compare(DataTypeUtil.getMeasureObjectFromDataType(minValues[i], dataType),
-                  DataTypeUtil.getMeasureObjectFromDataType(maxValues[i], dataType));
-          if (compare > 0) {
-            updatedMinMaxValues[i] = minValues[i];
-          }
-        }
-      }
-    }
-    return updatedMinMaxValues;
-  }
-
-  /**
    * Generate the blockid as per the block path
    *
    * @param identifier
diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
index a82a8aa..a8d30b1 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
@@ -1011,52 +1011,6 @@ public class CarbonUtilTest {
     Assert.assertTrue(schemaString.length() > schema.length());
   }
 
-  @Test
-  public void testUpdateMinMaxValues() {
-    // create dimension and measure column schema
-    ColumnSchema dimensionColumnSchema = createColumnSchema(DataTypes.STRING, true);
-    ColumnSchema measureColumnSchema = createColumnSchema(DataTypes.DOUBLE, false);
-    List<ColumnSchema> columnSchemas = new ArrayList<>(2);
-    columnSchemas.add(dimensionColumnSchema);
-    columnSchemas.add(measureColumnSchema);
-    // create data file footer object
-    DataFileFooter fileFooter = new DataFileFooter();
-    fileFooter.setColumnInTable(columnSchemas);
-    // initialise the expected values
-    int expectedMaxValue = 5;
-    int expectedMinValue = 2;
-    double expectedMeasureMaxValue = 28.74;
-    double expectedMeasureMinValue = -21.46;
-    // initialise the minValues
-    byte[][] minValues = new byte[2][];
-    minValues[0] = new byte[] { 2 };
-    ByteBuffer buffer = ByteBuffer.allocate(8);
-    minValues[1] = (byte[]) buffer.putDouble(28.74).flip().array();
-    buffer = ByteBuffer.allocate(8);
-    // initialise the maxValues
-    byte[][] maxValues = new byte[2][];
-    maxValues[0] = new byte[] { 5 };
-    maxValues[1] = (byte[]) buffer.putDouble(-21.46).flip().array();
-    byte[][] updateMaxValues =
-        CarbonUtil.updateMinMaxValues(fileFooter, maxValues, minValues, false);
-    byte[][] updateMinValues =
-        CarbonUtil.updateMinMaxValues(fileFooter, maxValues, minValues, true);
-    // compare max values
-    assert (expectedMaxValue == ByteBuffer.wrap(updateMaxValues[0]).get());
-    assert (expectedMeasureMaxValue == ByteBuffer.wrap(updateMaxValues[1]).getDouble());
-
-    // compare min values
-    assert (expectedMinValue == ByteBuffer.wrap(updateMinValues[0]).get());
-    assert (expectedMeasureMinValue == ByteBuffer.wrap(updateMinValues[1]).getDouble());
-  }
-
-  private ColumnSchema createColumnSchema(DataType dataType, boolean isDimensionColumn) {
-    ColumnSchema columnSchema = new ColumnSchema();
-    columnSchema.setDataType(dataType);
-    columnSchema.setDimensionColumn(isDimensionColumn);
-    return columnSchema;
-  }
-
   private String generateString(int length) {
     StringBuilder builder = new StringBuilder();
     for (int i = 0; i < length; i++) {


Mime
View raw message