carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject carbondata git commit: [CARBONDATA-1538] Added code to support BITSET PIPE LINE in filter
Date Thu, 12 Oct 2017 08:05:01 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master ac6c1d2b8 -> 75e0bd419


[CARBONDATA-1538] Added code to support BITSET PIPE LINE in filter

This closes #1403


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/75e0bd41
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/75e0bd41
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/75e0bd41

Branch: refs/heads/master
Commit: 75e0bd4199e4c8123bd4dde6dbae208ab5f8311b
Parents: ac6c1d2
Author: kumarvishal <kumarvishal.1802@gmail.com>
Authored: Sun Oct 8 16:48:59 2017 +0530
Committer: ravipesala <ravi.pesala@gmail.com>
Committed: Thu Oct 12 13:34:21 2017 +0530

----------------------------------------------------------------------
 .../core/constants/CarbonCommonConstants.java   |   9 ++
 .../chunk/DimensionColumnDataChunk.java         |   7 +
 .../chunk/impl/AbstractDimensionDataChunk.java  |   7 +
 .../impl/ColumnGroupDimensionDataChunk.java     |   8 +
 .../chunk/store/ColumnPageWrapper.java          |   4 +
 .../chunk/store/DimensionDataChunkStore.java    |   7 +
 .../SafeAbsractDimensionDataChunkStore.java     |  10 ++
 .../UnsafeAbstractDimensionDataChunkStore.java  |  12 ++
 .../filter/executer/AndFilterExecuterImpl.java  |   8 +-
 .../executer/ExcludeFilterExecuterImpl.java     | 152 +++++++++++++++++--
 .../scan/filter/executer/FilterExecuter.java    |   2 +-
 .../IncludeColGroupFilterExecuterImpl.java      |   4 +-
 .../executer/IncludeFilterExecuterImpl.java     | 151 ++++++++++++++++--
 .../filter/executer/OrFilterExecuterImpl.java   |   9 +-
 .../executer/RangeValueFilterExecuterImpl.java  |   2 +-
 .../RestructureExcludeFilterExecutorImpl.java   |   4 +-
 .../RestructureIncludeFilterExecutorImpl.java   |   4 +-
 .../executer/RowLevelFilterExecuterImpl.java    |  50 ++++--
 .../RowLevelRangeGrtThanFiterExecuterImpl.java  |   3 +-
 ...elRangeGrtrThanEquaToFilterExecuterImpl.java |   3 +-
 ...velRangeLessThanEqualFilterExecuterImpl.java |   3 +-
 .../RowLevelRangeLessThanFiterExecuterImpl.java |   3 +-
 .../filter/executer/TrueFilterExecutor.java     |   2 +-
 .../executer/ValueBasedFilterExecuterImpl.java  |   3 +-
 .../core/scan/processor/BlocksChunkHolder.java  |  11 ++
 .../core/scan/scanner/impl/FilterScanner.java   |  11 +-
 .../apache/carbondata/core/util/CarbonUtil.java |  46 ++++++
 27 files changed, 482 insertions(+), 53 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index c1a3ec7..51b92cc 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1367,6 +1367,15 @@ public final class CarbonCommonConstants {
 
   public static final String CARBON_USE_BLOCKLET_DISTRIBUTION_DEFAULT = "true";
 
+  /**
+   * this will be used to pass bitset value in filter to another filter for
+   * faster execution of filter query
+   */
+  @CarbonProperty
+  public static final String BITSET_PIPE_LINE = "carbon.use.bitset.pipe.line";
+
+  public static final String BITSET_PIPE_LINE_DEFAULT = "true";
+
   private CarbonCommonConstants() {
   }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnDataChunk.java
index db54885..3791314 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnDataChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnDataChunk.java
@@ -78,6 +78,13 @@ public interface DimensionColumnDataChunk {
   int getInvertedIndex(int index);
 
   /**
+   *
+   * @param invertedIndex
+   * @return index reverse index
+   */
+  int getInvertedReverseIndex(int invertedIndex);
+
+  /**
    * @return whether column is dictionary column or not
    */
   boolean isNoDicitionaryColumn();

http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/AbstractDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/AbstractDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/AbstractDimensionDataChunk.java
index ac2b2cb..eac062f 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/AbstractDimensionDataChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/AbstractDimensionDataChunk.java
@@ -54,6 +54,13 @@ public abstract class AbstractDimensionDataChunk implements DimensionColumnDataC
   }
 
   /**
+   * @param invertedIndex
+   * @return inverted index reverse
+   */
+  @Override public int getInvertedReverseIndex(int invertedIndex) {
+    return dataChunkStore.getInvertedReverseIndex(invertedIndex);
+  }
+  /**
    * @return length of each column
    */
   @Override public int getColumnValueSize() {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java
index 8234f67..b76ae53 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java
@@ -103,6 +103,14 @@ public class ColumnGroupDimensionDataChunk extends AbstractDimensionDataChunk {
   }
 
   /**
+   * @param invertedIndex
+   * @return inverted index reverse
+   */
+  @Override public int getInvertedReverseIndex(int invertedIndex) {
+    throw new UnsupportedOperationException("Operation not supported in case of cloumn group");
+  }
+
+  /**
    * @return whether columns where explictly sorted or not
    */
   @Override public boolean isExplicitSorted() {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java
index 21e130b..fbdb499 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/ColumnPageWrapper.java
@@ -66,6 +66,10 @@ public class ColumnPageWrapper implements DimensionColumnDataChunk {
     throw new UnsupportedOperationException("internal error");
   }
 
+  @Override public int getInvertedReverseIndex(int invertedIndex) {
+    throw new UnsupportedOperationException("internal error");
+  }
+
   @Override
   public boolean isNoDicitionaryColumn() {
     return true;

http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionDataChunkStore.java
index 5301945..5072c75 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionDataChunkStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionDataChunkStore.java
@@ -68,6 +68,13 @@ public interface DimensionDataChunkStore {
   int getInvertedIndex(int rowId);
 
   /**
+   * Below method will be used to get the reverse Inverted Index
+   * @param invertedIndex
+   * @return reverse Inverted Index
+   */
+  int getInvertedReverseIndex(int invertedIndex);
+
+  /**
    * Below method will be used to get the surrogate key of the
    * based on the row id passed
    *

http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeAbsractDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeAbsractDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeAbsractDimensionDataChunkStore.java
index cb4c1a7..f7189e6 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeAbsractDimensionDataChunkStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/safe/SafeAbsractDimensionDataChunkStore.java
@@ -86,6 +86,16 @@ public abstract class SafeAbsractDimensionDataChunkStore implements DimensionDat
   }
 
   /**
+   * Below method will be used to get the inverted index reverse
+   *
+   * @param rowId row id
+   * @return inverted index based on row id passed
+   */
+  @Override public int getInvertedReverseIndex(int rowId) {
+    return invertedIndexReverse[rowId];
+  }
+
+  /**
    * Below method will be used to get the surrogate key of the based on the row
    * id passed
    *

http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java
index 22c2e16..940ca1a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java
@@ -142,6 +142,18 @@ public abstract class UnsafeAbstractDimensionDataChunkStore implements Dimension
   }
 
   /**
+   * Below method will be used to get the reverse inverted index
+   *
+   * @param rowId row id
+   * @return inverted index based on row id passed
+   */
+  @Override public int getInvertedReverseIndex(int rowId) {
+    return CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(),
+        dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + ((long)rowId
+            * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+  }
+
+  /**
    * Below method will be used to get the surrogate key of the based on the row
    * id passed
    *

http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/AndFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/AndFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/AndFilterExecuterImpl.java
index 971f9b4..f79e788 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/AndFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/AndFilterExecuterImpl.java
@@ -33,17 +33,19 @@ public class AndFilterExecuterImpl implements FilterExecuter {
     this.rightExecuter = rightExecuter;
   }
 
-  @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
+  @Override
+  public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
       throws FilterUnsupportedException, IOException {
-    BitSetGroup leftFilters = leftExecuter.applyFilter(blockChunkHolder);
+    BitSetGroup leftFilters = leftExecuter.applyFilter(blockChunkHolder, useBitsetPipeLine);
     if (leftFilters.isEmpty()) {
       return leftFilters;
     }
-    BitSetGroup rightFilter = rightExecuter.applyFilter(blockChunkHolder);
+    BitSetGroup rightFilter = rightExecuter.applyFilter(blockChunkHolder, useBitsetPipeLine);
     if (rightFilter.isEmpty()) {
       return rightFilter;
     }
     leftFilters.and(rightFilter);
+    blockChunkHolder.setBitSetGroup(leftFilters);
     return leftFilters;
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
index 4471ced..841ecdc 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
@@ -74,7 +74,9 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
 
   }
 
-  @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder) throws IOException {
+  @Override
+  public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
+      throws IOException {
     if (isDimensionPresentInCurrentBlock) {
       int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
           .get(dimColEvaluatorInfo.getColumnIndex());
@@ -89,7 +91,8 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
       BitSetGroup bitSetGroup = new BitSetGroup(dimensionRawColumnChunk.getPagesCount());
       for (int i = 0; i < dimensionColumnDataChunks.length; i++) {
         BitSet bitSet = getFilteredIndexes(dimensionColumnDataChunks[i],
-            dimensionRawColumnChunk.getRowCount()[i]);
+            dimensionRawColumnChunk.getRowCount()[i], useBitsetPipeLine,
+            blockChunkHolder.getBitSetGroup(), i);
         bitSetGroup.setBitSet(bitSet, i);
       }
 
@@ -109,8 +112,9 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
       DataType msrType = getMeasureDataType(msrColumnEvaluatorInfo);
       for (int i = 0; i < ColumnPages.length; i++) {
         BitSet bitSet =
-            getFilteredIndexes(ColumnPages[i], measureRawColumnChunk.getRowCount()[i],
-                msrType);
+            getFilteredIndexesForMeasure(measureRawColumnChunk.convertToColumnPage(i),
+                measureRawColumnChunk.getRowCount()[i], useBitsetPipeLine,
+                blockChunkHolder.getBitSetGroup(), i, msrType);;
         bitSetGroup.setBitSet(bitSet, i);
       }
       return bitSetGroup;
@@ -167,14 +171,144 @@ public class ExcludeFilterExecuterImpl implements FilterExecuter {
     return bitSet;
   }
 
-  protected BitSet getFilteredIndexes(DimensionColumnDataChunk dimColumnDataChunk,
-      int numerOfRows) {
-    if (dimColumnDataChunk.isExplicitSorted()) {
-      return setFilterdIndexToBitSetWithColumnIndex(dimColumnDataChunk, numerOfRows);
+  /**
+   * Below method will be used to apply filter on measure column
+   * @param measureColumnPage
+   * @param numberOfRows
+   * @param useBitsetPipeLine
+   * @param prvBitSetGroup
+   * @param pageNumber
+   * @param msrDataType
+   * @return filtered indexes bitset
+   */
+  private BitSet getFilteredIndexesForMeasure(ColumnPage measureColumnPage, int numberOfRows,
+      boolean useBitsetPipeLine, BitSetGroup prvBitSetGroup, int pageNumber, DataType msrDataType) {
+    // check whether previous indexes can be optimal to apply filter on measure column
+    if (CarbonUtil.usePreviousFilterBitsetGroup(useBitsetPipeLine, prvBitSetGroup, pageNumber,
+        msrColumnExecutorInfo.getFilterKeys().length)) {
+      return getFilteredIndexesForMsrUsingPrvBitSet(measureColumnPage, prvBitSetGroup, pageNumber,
+          numberOfRows, msrDataType);
+    } else {
+      return getFilteredIndexes(measureColumnPage, numberOfRows, msrDataType);
+    }
+  }
+  /**
+   * Below method will be used to apply filter on measure column based on previous filtered indexes
+   * @param measureColumnPage
+   * @param prvBitSetGroup
+   * @param pageNumber
+   * @param numberOfRows
+   * @param msrDataType
+   * @return filtred indexes bitset
+   */
+  private BitSet getFilteredIndexesForMsrUsingPrvBitSet(ColumnPage measureColumnPage,
+      BitSetGroup prvBitSetGroup, int pageNumber, int numberOfRows, DataType msrDataType) {
+    BitSet bitSet = new BitSet(numberOfRows);
+    bitSet.flip(0, numberOfRows);
+    Object[] filterValues = msrColumnExecutorInfo.getFilterKeys();
+    BitSet nullBitSet = measureColumnPage.getNullBits();
+    BitSet prvPageBitSet = prvBitSetGroup.getBitSet(pageNumber);
+    SerializableComparator comparator = Comparator.getComparatorByDataTypeForMeasure(msrDataType);
+    for (int i = 0; i < filterValues.length; i++) {
+      if (filterValues[i] == null) {
+        for (int j = nullBitSet.nextSetBit(0); j >= 0; j = nullBitSet.nextSetBit(j + 1)) {
+          bitSet.flip(j);
+        }
+        continue;
+      }
+      for (int index = prvPageBitSet.nextSetBit(0);
+           index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) {
+        if (!nullBitSet.get(index)) {
+          // Check if filterValue[i] matches with measure Values.
+          Object msrValue = DataTypeUtil
+              .getMeasureObjectBasedOnDataType(measureColumnPage, index,
+                  msrDataType, msrColumnEvaluatorInfo.getMeasure());
+
+          if (comparator.compare(msrValue, filterValues[i]) == 0) {
+            // This is a match.
+            bitSet.flip(index);
+          }
+        }
+      }
+    }
+    return bitSet;
+  }
+
+  /**
+   * Below method will be used to apply filter on dimension column
+   * @param dimensionColumnDataChunk
+   * @param numberOfRows
+   * @param useBitsetPipeLine
+   * @param prvBitSetGroup
+   * @param pageNumber
+   * @return filtered indexes bitset
+   */
+  private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+      int numberOfRows, boolean useBitsetPipeLine, BitSetGroup prvBitSetGroup, int pageNumber) {
+    // check whether applying filtered based on previous bitset will be optimal
+    if (CarbonUtil.usePreviousFilterBitsetGroup(useBitsetPipeLine, prvBitSetGroup, pageNumber,
+        dimColumnExecuterInfo.getFilterKeys().length)) {
+      return getFilteredIndexesUisngPrvBitset(dimensionColumnDataChunk, prvBitSetGroup, pageNumber,
+          numberOfRows);
+    } else {
+      return getFilteredIndexes(dimensionColumnDataChunk, numberOfRows);
+    }
+  }
+
+  private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+      int numberOfRows) {
+    if (dimensionColumnDataChunk.isExplicitSorted()) {
+      return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnDataChunk, numberOfRows);
     }
-    return setFilterdIndexToBitSet(dimColumnDataChunk, numerOfRows);
+    return setFilterdIndexToBitSet(dimensionColumnDataChunk, numberOfRows);
   }
 
+  /**
+   * Below method will be used to apply filter based on previous filtered bitset
+   * @param dimensionColumnDataChunk
+   * @param prvBitSetGroup
+   * @param pageNumber
+   * @param numberOfRows
+   * @return filtered indexes bitset
+   */
+  private BitSet getFilteredIndexesUisngPrvBitset(DimensionColumnDataChunk dimensionColumnDataChunk,
+      BitSetGroup prvBitSetGroup, int pageNumber, int numberOfRows) {
+    BitSet prvPageBitSet = prvBitSetGroup.getBitSet(pageNumber);
+    BitSet bitSet = new BitSet(numberOfRows);
+    byte[][] filterKeys = dimColumnExecuterInfo.getFilterKeys();
+    int compareResult = 0;
+    // if dimension data was natural sorted then get the index from previous bitset
+    // and use the same in next column data, otherwise use the inverted index reverse
+    if (!dimensionColumnDataChunk.isExplicitSorted()) {
+      for (int index = prvPageBitSet.nextSetBit(0);
+           index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) {
+        compareResult = CarbonUtil
+            .isFilterPresent(filterKeys, dimensionColumnDataChunk, 0, filterKeys.length - 1, index);
+        if (compareResult != 0) {
+          bitSet.set(index);
+        } else {
+          if (bitSet.get(index)) {
+            bitSet.flip(index);
+          }
+        }
+      }
+    } else {
+      for (int index = prvPageBitSet.nextSetBit(0);
+           index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) {
+        compareResult = CarbonUtil
+            .isFilterPresent(filterKeys, dimensionColumnDataChunk, 0, filterKeys.length - 1,
+                dimensionColumnDataChunk.getInvertedReverseIndex(index));
+        if (compareResult != 0) {
+          bitSet.set(index);
+        } else {
+          if (bitSet.get(index)) {
+            bitSet.flip(index);
+          }
+        }
+      }
+    }
+    return bitSet;
+  }
   private BitSet setFilterdIndexToBitSetWithColumnIndex(
       DimensionColumnDataChunk dimensionColumnDataChunk, int numerOfRows) {
     BitSet bitSet = new BitSet(numerOfRows);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
index 7182dd5..93640fa 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
@@ -31,7 +31,7 @@ public interface FilterExecuter {
    * @return
    * @throws FilterUnsupportedException
    */
-  BitSetGroup applyFilter(BlocksChunkHolder blocksChunkHolder)
+  BitSetGroup applyFilter(BlocksChunkHolder blocksChunkHolder, boolean useBitsetPipeLine)
       throws FilterUnsupportedException, IOException;
 
   /**

http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
index 45831e3..1cbc0bc 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
@@ -84,7 +84,9 @@ public class IncludeColGroupFilterExecuterImpl extends IncludeFilterExecuterImpl
     return bitSet;
   }
 
-  @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder) throws IOException {
+  @Override
+  public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
+      throws IOException {
     int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
         .get(dimColumnEvaluatorInfo.getColumnIndex());
     if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
index 2195c0a..7d6fd4f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
@@ -81,7 +81,9 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
 
   }
 
-  @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder) throws IOException {
+  @Override
+  public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
+      throws IOException {
     if (isDimensionPresentInCurrentBlock == true) {
       int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
           .get(dimColumnEvaluatorInfo.getColumnIndex());
@@ -97,12 +99,14 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
           if (isScanRequired(dimensionRawColumnChunk.getMaxValues()[i],
               dimensionRawColumnChunk.getMinValues()[i], dimColumnExecuterInfo.getFilterKeys())) {
             BitSet bitSet = getFilteredIndexes(dimensionRawColumnChunk.convertToDimColDataChunk(i),
-                dimensionRawColumnChunk.getRowCount()[i]);
+                dimensionRawColumnChunk.getRowCount()[i], useBitsetPipeLine,
+                blockChunkHolder.getBitSetGroup(), i);
             bitSetGroup.setBitSet(bitSet, i);
           }
         } else {
           BitSet bitSet = getFilteredIndexes(dimensionRawColumnChunk.convertToDimColDataChunk(i),
-              dimensionRawColumnChunk.getRowCount()[i]);
+              dimensionRawColumnChunk.getRowCount()[i], useBitsetPipeLine,
+              blockChunkHolder.getBitSetGroup(), i);
           bitSetGroup.setBitSet(bitSet, i);
         }
       }
@@ -124,14 +128,16 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
               measureRawColumnChunk.getMinValues()[i], msrColumnExecutorInfo.getFilterKeys(),
               msrColumnEvaluatorInfo.getType())) {
             BitSet bitSet =
-                getFilteredIndexesForMeasures(measureRawColumnChunk.convertToColumnPage(i),
-                    measureRawColumnChunk.getRowCount()[i], msrType);
+                getFilteredIndexesForMeasure(measureRawColumnChunk.convertToColumnPage(i),
+                    measureRawColumnChunk.getRowCount()[i], useBitsetPipeLine,
+                    blockChunkHolder.getBitSetGroup(), i, msrType);
             bitSetGroup.setBitSet(bitSet, i);
           }
         } else {
           BitSet bitSet =
-              getFilteredIndexesForMeasures(measureRawColumnChunk.convertToColumnPage(i),
-                  measureRawColumnChunk.getRowCount()[i], msrType);
+              getFilteredIndexesForMeasure(measureRawColumnChunk.convertToColumnPage(i),
+                  measureRawColumnChunk.getRowCount()[i], useBitsetPipeLine,
+                  blockChunkHolder.getBitSetGroup(), i, msrType);
           bitSetGroup.setBitSet(bitSet, i);
         }
       }
@@ -189,14 +195,137 @@ public class IncludeFilterExecuterImpl implements FilterExecuter {
     return bitSet;
   }
 
-  protected BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
-      int numerOfRows) {
+  /**
+   * Below method will be used to apply filter on measure column
+   * @param measureColumnPage
+   * @param numberOfRows
+   * @param useBitsetPipeLine
+   * @param prvBitSetGroup
+   * @param pageNumber
+   * @param msrDataType
+   * @return filtered indexes bitset
+   */
+  private BitSet getFilteredIndexesForMeasure(ColumnPage measureColumnPage, int numberOfRows,
+      boolean useBitsetPipeLine, BitSetGroup prvBitSetGroup, int pageNumber, DataType msrDataType) {
+    // check whether previous indexes can be optimal to apply filter on measure column
+    if (CarbonUtil.usePreviousFilterBitsetGroup(useBitsetPipeLine, prvBitSetGroup, pageNumber,
+        msrColumnExecutorInfo.getFilterKeys().length)) {
+      return getFilteredIndexesForMsrUsingPrvBitSet(measureColumnPage, prvBitSetGroup, pageNumber,
+          numberOfRows, msrDataType);
+    } else {
+      return getFilteredIndexesForMeasures(measureColumnPage, numberOfRows, msrDataType);
+    }
+  }
+
+  /**
+   * Below method will be used to apply filter on measure column based on previous filtered indexes
+   * @param measureColumnPage
+   * @param prvBitSetGroup
+   * @param pageNumber
+   * @param numberOfRows
+   * @param msrDataType
+   * @return filtred indexes bitset
+   */
+  private BitSet getFilteredIndexesForMsrUsingPrvBitSet(ColumnPage measureColumnPage,
+      BitSetGroup prvBitSetGroup, int pageNumber, int numberOfRows, DataType msrDataType) {
+    BitSet bitSet = new BitSet(numberOfRows);
+    Object[] filterValues = msrColumnExecutorInfo.getFilterKeys();
+    BitSet nullBitSet = measureColumnPage.getNullBits();
+    BitSet prvPageBitSet = prvBitSetGroup.getBitSet(pageNumber);
+    SerializableComparator comparator = Comparator.getComparatorByDataTypeForMeasure(msrDataType);
+    for (int i = 0; i < filterValues.length; i++) {
+      if (filterValues[i] == null) {
+        for (int j = nullBitSet.nextSetBit(0); j >= 0; j = nullBitSet.nextSetBit(j + 1)) {
+          bitSet.set(j);
+        }
+        continue;
+      }
+      for (int index = prvPageBitSet.nextSetBit(0);
+           index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) {
+        if (!nullBitSet.get(index)) {
+          // Check if filterValue[i] matches with measure Values.
+          Object msrValue = DataTypeUtil
+              .getMeasureObjectBasedOnDataType(measureColumnPage, index,
+                  msrDataType, msrColumnEvaluatorInfo.getMeasure());
+
+          if (comparator.compare(msrValue, filterValues[i]) == 0) {
+            // This is a match.
+            bitSet.set(index);
+          }
+        }
+      }
+    }
+    return bitSet;
+  }
+
+  /**
+   * Below method will be used to apply filter on dimension column
+   * @param dimensionColumnDataChunk
+   * @param numberOfRows
+   * @param useBitsetPipeLine
+   * @param prvBitSetGroup
+   * @param pageNumber
+   * @return filtered indexes bitset
+   */
+  private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+      int numberOfRows, boolean useBitsetPipeLine, BitSetGroup prvBitSetGroup, int pageNumber) {
+    // check whether previous indexes can be optimal to apply filter on dimension column
+    if (CarbonUtil.usePreviousFilterBitsetGroup(useBitsetPipeLine, prvBitSetGroup, pageNumber,
+        dimColumnExecuterInfo.getFilterKeys().length)) {
+      return getFilteredIndexesUisngPrvBitset(dimensionColumnDataChunk, prvBitSetGroup, pageNumber,
+          numberOfRows);
+    } else {
+      return getFilteredIndexes(dimensionColumnDataChunk, numberOfRows);
+    }
+  }
+
+  private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+      int numberOfRows) {
     if (dimensionColumnDataChunk.isExplicitSorted()) {
-      return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnDataChunk, numerOfRows);
+      return setFilterdIndexToBitSetWithColumnIndex(dimensionColumnDataChunk, numberOfRows);
     }
-    return setFilterdIndexToBitSet(dimensionColumnDataChunk, numerOfRows);
+    return setFilterdIndexToBitSet(dimensionColumnDataChunk, numberOfRows);
   }
 
+  /**
+   * Below method will be used to apply filter on dimension
+   * column based on previous filtered indexes
+   * @param dimensionColumnDataChunk
+   * @param prvBitSetGroup
+   * @param pageNumber
+   * @param numberOfRows
+   * @return filtered bitset
+   */
+  private BitSet getFilteredIndexesUisngPrvBitset(DimensionColumnDataChunk dimensionColumnDataChunk,
+      BitSetGroup prvBitSetGroup, int pageNumber, int numberOfRows) {
+    BitSet prvPageBitSet = prvBitSetGroup.getBitSet(pageNumber);
+    BitSet bitSet = new BitSet(numberOfRows);
+    byte[][] filterKeys = dimColumnExecuterInfo.getFilterKeys();
+    int compareResult = 0;
+    // if dimension data was natural sorted then get the index from previous bitset
+    // and use the same in next column data, otherwise use the inverted index reverse
+    if (!dimensionColumnDataChunk.isExplicitSorted()) {
+      for (int index = prvPageBitSet.nextSetBit(0);
+           index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) {
+        compareResult = CarbonUtil
+            .isFilterPresent(filterKeys, dimensionColumnDataChunk, 0, filterKeys.length - 1, index);
+        if (compareResult == 0) {
+          bitSet.set(index);
+        }
+      }
+    } else {
+      for (int index = prvPageBitSet.nextSetBit(0);
+           index >= 0; index = prvPageBitSet.nextSetBit(index + 1)) {
+        compareResult = CarbonUtil
+            .isFilterPresent(filterKeys, dimensionColumnDataChunk, 0, filterKeys.length - 1,
+                dimensionColumnDataChunk.getInvertedReverseIndex(index));
+        if (compareResult == 0) {
+          bitSet.set(index);
+        }
+      }
+    }
+    return bitSet;
+  }
   private BitSet setFilterdIndexToBitSetWithColumnIndex(
       DimensionColumnDataChunk dimensionColumnDataChunk, int numerOfRows) {
     BitSet bitSet = new BitSet(numerOfRows);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java
index 119bda7..7eed8ee 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java
@@ -33,12 +33,13 @@ public class OrFilterExecuterImpl implements FilterExecuter {
     this.rightExecuter = rightExecuter;
   }
 
-  @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
+  @Override
+  public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
       throws FilterUnsupportedException, IOException {
-    BitSetGroup leftFilters = leftExecuter.applyFilter(blockChunkHolder);
-    BitSetGroup rightFilters = rightExecuter.applyFilter(blockChunkHolder);
+    BitSetGroup leftFilters = leftExecuter.applyFilter(blockChunkHolder, false);
+    BitSetGroup rightFilters = rightExecuter.applyFilter(blockChunkHolder, false);
     leftFilters.or(rightFilters);
-
+    blockChunkHolder.setBitSetGroup(leftFilters);
     return leftFilters;
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
index c7a0ae7..acce53e 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RangeValueFilterExecuterImpl.java
@@ -140,7 +140,7 @@ public class RangeValueFilterExecuterImpl extends ValueBasedFilterExecuterImpl {
    * @throws FilterUnsupportedException
    * @throws IOException
    */
-  public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
+  public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
       throws FilterUnsupportedException, IOException {
     return applyNoAndDirectFilter(blockChunkHolder);
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java
index 1509949..2c0d39f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java
@@ -45,7 +45,9 @@ public class RestructureExcludeFilterExecutorImpl extends RestructureEvaluatorIm
     }
   }
 
-  @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder) throws IOException {
+  @Override
+  public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
+      throws IOException {
     int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
     return FilterUtil
         .createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(),

http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java
index 0e9a5f0..5ec6971 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java
@@ -44,7 +44,9 @@ public class RestructureIncludeFilterExecutorImpl extends RestructureEvaluatorIm
     }
   }
 
-  @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder) throws IOException {
+  @Override
+  public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
+      throws IOException {
     int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
     return FilterUtil
         .createBitSetGroupWithDefaultValue(blockChunkHolder.getDataBlock().numberOfPages(),

http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index 8f3eb93..08ce565 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -186,7 +186,8 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
     }
   }
 
-  @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
+  @Override
+  public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
       throws FilterUnsupportedException, IOException {
     readBlocks(blockChunkHolder);
     // CHECKSTYLE:ON
@@ -222,20 +223,41 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
     for (int i = 0; i < pageNumbers; i++) {
       BitSet set = new BitSet(numberOfRows[i]);
       RowIntf row = new RowImpl();
-      for (int index = 0; index < numberOfRows[i]; index++) {
-        createRow(blockChunkHolder, row ,i, index);
-        Boolean rslt = false;
-        try {
-          rslt = exp.evaluate(row).getBoolean();
-        }
-        // Any invalid member while evaluation shall be ignored, system will log the
-        // error only once since all rows the evaluation happens so inorder to avoid
-        // too much log inforation only once the log will be printed.
-        catch (FilterIllegalMemberException e) {
-          FilterUtil.logError(e, false);
+      BitSet prvBitset = null;
+      // if bitset pipe line is enabled then use rowid from previous bitset
+      // otherwise use older flow
+      if (!useBitsetPipeLine || null == blockChunkHolder.getBitSetGroup() || null == bitSetGroup
+          .getBitSet(i) || blockChunkHolder.getBitSetGroup().getBitSet(i).isEmpty()) {
+        for (int index = 0; index < numberOfRows[i]; index++) {
+          createRow(blockChunkHolder, row, i, index);
+          Boolean rslt = false;
+          try {
+            rslt = exp.evaluate(row).getBoolean();
+          }
+          // Any invalid member while evaluation shall be ignored, system will log the
+          // error only once since all rows the evaluation happens so inorder to avoid
+          // too much log inforation only once the log will be printed.
+          catch (FilterIllegalMemberException e) {
+            FilterUtil.logError(e, false);
+          }
+          if (null != rslt && rslt) {
+            set.set(index);
+          }
         }
-        if (null != rslt && rslt) {
-          set.set(index);
+      } else {
+        prvBitset = blockChunkHolder.getBitSetGroup().getBitSet(i);
+        for (int index = prvBitset.nextSetBit(0);
+             index >= 0; index = prvBitset.nextSetBit(index + 1)) {
+          createRow(blockChunkHolder, row, i, index);
+          Boolean rslt = false;
+          try {
+            rslt = exp.evaluate(row).getBoolean();
+          } catch (FilterIllegalMemberException e) {
+            FilterUtil.logError(e, false);
+          }
+          if (null != rslt && rslt) {
+            set.set(index);
+          }
         }
       }
       bitSetGroup.setBitSet(set, i);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
index d34ab2c..306f3fa 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
@@ -166,7 +166,8 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
     return false;
   }
 
-  @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
+  @Override
+  public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
       throws FilterUnsupportedException, IOException {
     // select all rows if dimension does not exists in the current block
     if (!isDimensionPresentInCurrentBlock[0] && !isMeasurePresentInCurrentBlock[0]) {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
index 3f8eddd..db55e42 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
@@ -164,7 +164,8 @@ public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilte
     return false;
   }
 
-  @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
+  @Override
+  public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
       throws FilterUnsupportedException, IOException {
     // select all rows if dimension does not exists in the current block
     if (!isDimensionPresentInCurrentBlock[0] && !isMeasurePresentInCurrentBlock[0]) {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
index f8886f9..67fe848 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
@@ -165,7 +165,8 @@ public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilter
     return false;
   }
 
-  @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
+  @Override
+  public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
       throws FilterUnsupportedException, IOException {
     // select all rows if dimension does not exists in the current block
     if (!isDimensionPresentInCurrentBlock[0] && !isMeasurePresentInCurrentBlock[0]) {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
index 580f963..8db37c4 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
@@ -167,7 +167,8 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
     return false;
   }
 
-  @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
+  @Override
+  public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
       throws FilterUnsupportedException, IOException {
     // select all rows if dimension does not exists in the current block
     if (!isDimensionPresentInCurrentBlock[0] && !isMeasurePresentInCurrentBlock[0]) {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/TrueFilterExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/TrueFilterExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/TrueFilterExecutor.java
index ac8c423..91cebc5 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/TrueFilterExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/TrueFilterExecutor.java
@@ -32,7 +32,7 @@ public class TrueFilterExecutor implements FilterExecuter {
    * @return
    * @throws FilterUnsupportedException
    */
-  public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
+  public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
       throws FilterUnsupportedException, IOException {
     int numberOfPages = blockChunkHolder.getDataBlock().numberOfPages();
     BitSetGroup group = new BitSetGroup(numberOfPages);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java
index 3e33d04..6dc1375 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ValueBasedFilterExecuterImpl.java
@@ -25,7 +25,8 @@ import org.apache.carbondata.core.util.BitSetGroup;
 
 public class ValueBasedFilterExecuterImpl implements FilterExecuter {
 
-  @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
+  @Override
+  public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder, boolean useBitsetPipeLine)
       throws FilterUnsupportedException, IOException {
 
     return new BitSetGroup(0);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java
index 5227115..60090d0 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java
@@ -20,6 +20,7 @@ import org.apache.carbondata.core.datastore.DataRefNode;
 import org.apache.carbondata.core.datastore.FileHolder;
 import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
 import org.apache.carbondata.core.datastore.chunk.impl.MeasureRawColumnChunk;
+import org.apache.carbondata.core.util.BitSetGroup;
 
 /**
  * Block chunk holder which will hold the dimension and
@@ -47,6 +48,8 @@ public class BlocksChunkHolder {
    */
   private DataRefNode dataBlock;
 
+  private BitSetGroup bitSetGroup;
+
   public BlocksChunkHolder(int numberOfDimensionBlock, int numberOfMeasureBlock) {
     dimensionRawDataChunk = new DimensionRawColumnChunk[numberOfDimensionBlock];
     measureRawDataChunk = new MeasureRawColumnChunk[numberOfMeasureBlock];
@@ -127,4 +130,12 @@ public class BlocksChunkHolder {
       this.dimensionRawDataChunk[i] = null;
     }
   }
+
+  public BitSetGroup getBitSetGroup() {
+    return bitSetGroup;
+  }
+
+  public void setBitSetGroup(BitSetGroup bitSetGroup) {
+    this.bitSetGroup = bitSetGroup;
+  }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
index 919b837..abfc5f4 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
@@ -64,6 +64,8 @@ public class FilterScanner extends AbstractBlockletScanner {
 
   private QueryStatisticsModel queryStatisticsModel;
 
+  private boolean useBitSetPipeLine;
+
   public FilterScanner(BlockExecutionInfo blockExecutionInfo,
       QueryStatisticsModel queryStatisticsModel) {
     super(blockExecutionInfo);
@@ -77,6 +79,13 @@ public class FilterScanner extends AbstractBlockletScanner {
     // get the filter tree
     this.filterExecuter = blockExecutionInfo.getFilterExecuterTree();
     this.queryStatisticsModel = queryStatisticsModel;
+
+    String useBitSetPipeLine = CarbonProperties.getInstance()
+        .getProperty(CarbonCommonConstants.BITSET_PIPE_LINE,
+            CarbonCommonConstants.BITSET_PIPE_LINE_DEFAULT);
+    if (null != useBitSetPipeLine) {
+      this.useBitSetPipeLine = Boolean.parseBoolean(useBitSetPipeLine);
+    }
   }
 
   /**
@@ -145,7 +154,7 @@ public class FilterScanner extends AbstractBlockletScanner {
     totalBlockletStatistic.addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM,
         totalBlockletStatistic.getCount() + 1);
     // apply filter on actual data
-    BitSetGroup bitSetGroup = this.filterExecuter.applyFilter(blocksChunkHolder);
+    BitSetGroup bitSetGroup = this.filterExecuter.applyFilter(blocksChunkHolder, useBitSetPipeLine);
     // if indexes is empty then return with empty result
     if (bitSetGroup.isEmpty()) {
       CarbonUtil.freeMemory(blocksChunkHolder.getDimensionRawDataChunk(),

http://git-wip-us.apache.org/repos/asf/carbondata/blob/75e0bd41/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index d0171b9..b5b3e63 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -1994,5 +1994,51 @@ public final class CarbonUtil {
     }
   }
 
+  /**
+   * Below method will be used to check whether bitset applied on previous filter
+   * can be used to apply on next column filter
+   * @param usePrvBitSetGroup
+   * @param prvBitsetGroup
+   * @param pageNumber
+   * @param numberOfFilterValues
+   * @return
+   */
+  public static boolean usePreviousFilterBitsetGroup(boolean usePrvBitSetGroup,
+      BitSetGroup prvBitsetGroup, int pageNumber, int numberOfFilterValues) {
+    if (!usePrvBitSetGroup || null == prvBitsetGroup || null == prvBitsetGroup.getBitSet(pageNumber)
+        || prvBitsetGroup.getBitSet(pageNumber).isEmpty()) {
+      return false;
+    }
+    int numberOfRowSelected = prvBitsetGroup.getBitSet(pageNumber).cardinality();
+    return numberOfFilterValues > numberOfRowSelected;
+  }
+
+  /**
+   * Below method will be used to check filter value is present in the data chunk or not
+   * @param filterValues
+   * @param dimensionColumnDataChunk
+   * @param low
+   * @param high
+   * @param chunkRowIndex
+   * @return
+   */
+  public static int isFilterPresent(byte[][] filterValues,
+      DimensionColumnDataChunk dimensionColumnDataChunk, int low, int high, int chunkRowIndex) {
+    int compareResult = 0;
+    int mid = 0;
+    while (low <= high) {
+      mid = (low + high) >>> 1;
+      compareResult = dimensionColumnDataChunk.compareTo(chunkRowIndex, filterValues[mid]);
+      if (compareResult < 0) {
+        high = mid - 1;
+      } else if (compareResult > 0) {
+        low = mid + 1;
+      } else {
+        return compareResult;
+      }
+    }
+    return -1;
+  }
+
 }
 


Mime
View raw message