carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From manishgupt...@apache.org
Subject carbondata git commit: [CARBONDATA-1979] ][IMPLICIT COLUMN] Modified implicit column filtering logic to directly validate the blocklet ID
Date Thu, 04 Jan 2018 10:01:11 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master e2f19979b -> aee5213b8


[CARBONDATA-1979] ][IMPLICIT COLUMN] Modified implicit column filtering logic to directly
validate the blocklet ID

As BlockletDataMap directly identified the blocklets after pruning, implict column filter
should validate directly for blocklet id instead of block Id

This closes #1760


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/aee5213b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/aee5213b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/aee5213b

Branch: refs/heads/master
Commit: aee5213b8a6f137e14d1b902c523f836224a1953
Parents: e2f1997
Author: rahulforallp <rahul.kumar@knoldus.in>
Authored: Wed Jan 3 15:21:37 2018 +0530
Committer: manishgupta88 <tomanishgupta18@gmail.com>
Committed: Thu Jan 4 15:34:21 2018 +0530

----------------------------------------------------------------------
 .../blockletindex/BlockletDataMap.java          |  8 ++-
 .../core/scan/filter/ColumnFilterInfo.java      | 20 ------
 .../ImplicitIncludeFilterExecutorImpl.java      |  7 +--
 .../blockletindex/TestBlockletDataMap.java      | 66 ++++++++++++++++++++
 4 files changed, 72 insertions(+), 29 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/aee5213b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
index 1e54019..43953ce 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
@@ -517,7 +517,7 @@ public class BlockletDataMap implements DataMap, Cacheable {
             CarbonCommonConstants.DEFAULT_CHARSET_CLASS);
         boolean isValid =
             addBlockBasedOnMinMaxValue(filterExecuter, getMinMaxValue(unsafeRow, MAX_VALUES_INDEX),
-                getMinMaxValue(unsafeRow, MIN_VALUES_INDEX), filePath);
+                getMinMaxValue(unsafeRow, MIN_VALUES_INDEX), filePath, startIndex);
         if (isValid) {
           blocklets.add(createBlocklet(unsafeRow, startIndex));
         }
@@ -557,13 +557,15 @@ public class BlockletDataMap implements DataMap, Cacheable {
    * @param maxValue
    * @param minValue
    * @param filePath
+   * @param blockletId
    * @return
    */
   private boolean addBlockBasedOnMinMaxValue(FilterExecuter filterExecuter, byte[][] maxValue,
-      byte[][] minValue, String filePath) {
+      byte[][] minValue, String filePath, int blockletId) {
     BitSet bitSet = null;
     if (filterExecuter instanceof ImplicitColumnFilterExecutor) {
-      String uniqueBlockPath = filePath.substring(filePath.lastIndexOf("/Part") + 1);
+      String uniqueBlockPath = filePath.substring(filePath.lastIndexOf("/Part") + 1)
+          + CarbonCommonConstants.FILE_SEPARATOR + blockletId;
       bitSet = ((ImplicitColumnFilterExecutor) filterExecuter)
           .isFilterValuesPresentInBlockOrBlocklet(maxValue, minValue, uniqueBlockPath);
     } else {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/aee5213b/core/src/main/java/org/apache/carbondata/core/scan/filter/ColumnFilterInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/ColumnFilterInfo.java
b/core/src/main/java/org/apache/carbondata/core/scan/filter/ColumnFilterInfo.java
index aeb3e6d..6a7fced 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/ColumnFilterInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/ColumnFilterInfo.java
@@ -18,11 +18,7 @@
 package org.apache.carbondata.core.scan.filter;
 
 import java.io.Serializable;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
 
 public class ColumnFilterInfo implements Serializable {
 
@@ -37,7 +33,6 @@ public class ColumnFilterInfo implements Serializable {
    */
   private List<String> implicitColumnFilterList;
   private List<Integer> excludeFilterList;
-  private transient Set<String> implicitDriverColumnFilterList;
   /**
    * maintain the no dictionary filter values list.
    */
@@ -91,7 +86,6 @@ public class ColumnFilterInfo implements Serializable {
 
   public void setImplicitColumnFilterList(List<String> implicitColumnFilterList) {
     this.implicitColumnFilterList = implicitColumnFilterList;
-    populateBlockIdListForDriverBlockPruning();
   }
 
   public List<Object> getMeasuresFilterValuesList() {
@@ -101,18 +95,4 @@ public class ColumnFilterInfo implements Serializable {
   public void setMeasuresFilterValuesList(List<Object> measuresFilterValuesList) {
     this.measuresFilterValuesList = measuresFilterValuesList;
   }
-
-  public Set<String> getImplicitDriverColumnFilterList() {
-    return implicitDriverColumnFilterList;
-  }
-
-  private void populateBlockIdListForDriverBlockPruning() {
-    implicitDriverColumnFilterList = new HashSet<>(implicitColumnFilterList.size());
-    String blockId = null;
-    for (String blockletId : implicitColumnFilterList) {
-      blockId =
-          blockletId.substring(0, blockletId.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR));
-      implicitDriverColumnFilterList.add(blockId);
-    }
-  }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/aee5213b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java
b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java
index 9c4c7ba..5289e45 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java
@@ -70,12 +70,7 @@ public class ImplicitIncludeFilterExecutorImpl
     BitSet bitSet = new BitSet(1);
     boolean isScanRequired = false;
     String shortBlockId = CarbonTablePath.getShortBlockId(uniqueBlockPath);
-    if (uniqueBlockPath.endsWith(".carbondata")) {
-      if (dimColumnEvaluatorInfo.getFilterValues().getImplicitDriverColumnFilterList()
-          .contains(shortBlockId)) {
-        isScanRequired = true;
-      }
-    } else if (dimColumnEvaluatorInfo.getFilterValues().getImplicitColumnFilterList()
+    if (dimColumnEvaluatorInfo.getFilterValues().getImplicitColumnFilterList()
         .contains(shortBlockId)) {
       isScanRequired = true;
     }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/aee5213b/core/src/test/java/org/apache/carbondata/core/indexstore/blockletindex/TestBlockletDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/indexstore/blockletindex/TestBlockletDataMap.java
b/core/src/test/java/org/apache/carbondata/core/indexstore/blockletindex/TestBlockletDataMap.java
new file mode 100644
index 0000000..88ac3ed
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/indexstore/blockletindex/TestBlockletDataMap.java
@@ -0,0 +1,66 @@
+package org.apache.carbondata.core.indexstore.blockletindex;
+
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.List;
+import java.util.UUID;
+
+import org.apache.carbondata.core.cache.dictionary.AbstractDictionaryCacheTest;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonImplicitDimension;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
+import org.apache.carbondata.core.scan.filter.executer.ImplicitIncludeFilterExecutorImpl;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import org.apache.carbondata.core.util.ByteUtil;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestBlockletDataMap extends AbstractDictionaryCacheTest {
+
+  ImplicitIncludeFilterExecutorImpl implicitIncludeFilterExecutor;
+  @Before public void setUp() throws Exception {
+    CarbonImplicitDimension carbonImplicitDimension =
+        new CarbonImplicitDimension(0, CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_POSITIONID);
+    DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = new DimColumnResolvedFilterInfo();
+    dimColumnEvaluatorInfo.setColumnIndex(0);
+    dimColumnEvaluatorInfo.setRowIndex(0);
+    dimColumnEvaluatorInfo.setDimension(carbonImplicitDimension);
+    dimColumnEvaluatorInfo.setDimensionExistsInCurrentSilce(false);
+    implicitIncludeFilterExecutor =
+        new ImplicitIncludeFilterExecutorImpl(dimColumnEvaluatorInfo);
+  }
+
+  @Test public void testaddBlockBasedOnMinMaxValue() throws Exception {
+
+    new MockUp<ImplicitIncludeFilterExecutorImpl>() {
+      @Mock BitSet isFilterValuesPresentInBlockOrBlocklet(byte[][] maxValue, byte[][] minValue,
+          String uniqueBlockPath) {
+        BitSet bitSet = new BitSet(1);
+        bitSet.set(8);
+        return bitSet;
+      }
+    };
+
+    BlockletDataMap blockletDataMap = new BlockletDataMap();
+    Method method = BlockletDataMap.class
+        .getDeclaredMethod("addBlockBasedOnMinMaxValue", FilterExecuter.class, byte[][].class,
+            byte[][].class, String.class, int.class);
+    method.setAccessible(true);
+
+    byte[][] minValue = { ByteUtil.toBytes("sfds") };
+    byte[][] maxValue = { ByteUtil.toBytes("resa") };
+    Object result = method
+        .invoke(blockletDataMap, implicitIncludeFilterExecutor, minValue, maxValue,
+            "/opt/store/default/carbon_table/Fact/Part0/Segment_0/part-0-0_batchno0-0-1514989110586.carbondata",
+            0);
+    assert ((boolean) result);
+  }
+}


Mime
View raw message