carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gvram...@apache.org
Subject [05/22] incubator-carbondata git commit: IUD implicit tupleid
Date Fri, 06 Jan 2017 13:57:05 GMT
IUD implicit tupleid


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/ff84a2ed
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/ff84a2ed
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/ff84a2ed

Branch: refs/heads/master
Commit: ff84a2edfe439c93d4571fc582d811091258a638
Parents: 4d688f2
Author: nareshpr <prnaresh.naresh@gmail.com>
Authored: Fri Dec 30 18:44:39 2016 +0530
Committer: Venkata Ramana G <ramana.gollamudi@huawei.com>
Committed: Fri Jan 6 19:16:29 2017 +0530

----------------------------------------------------------------------
 .../core/carbon/metadata/encoder/Encoding.java  |   3 +-
 .../metadata/schema/table/CarbonTable.java      |  55 +++++-
 .../table/column/CarbonImplicitDimension.java   | 176 +++++++++++++++++++
 .../core/carbon/path/CarbonTablePath.java       |  71 ++++++++
 .../core/constants/CarbonCommonConstants.java   |  11 ++
 .../carbondata/core/update/TupleIdEnum.java     |  41 +++++
 .../apache/carbondata/core/util/CarbonUtil.java |   9 +
 .../impl/DictionaryBasedResultCollector.java    |  21 ++-
 .../collector/impl/RawBasedResultCollector.java |   4 +
 .../executor/impl/AbstractQueryExecutor.java    |  16 +-
 .../scan/executor/util/QueryUtil.java           |  24 ++-
 .../scan/executor/util/RestructureUtil.java     |  13 +-
 .../scan/filter/DimColumnFilterInfo.java        |  19 ++
 .../executer/ImplicitColumnFilterExecutor.java  |  43 +++++
 .../ImplicitIncludeFilterExecutorImpl.java      |  96 ++++++++++
 .../scan/result/AbstractScannedResult.java      |  37 ++++
 .../scan/scanner/AbstractBlockletScanner.java   |   5 +
 .../scan/scanner/impl/FilterScanner.java        |  23 ++-
 .../scan/wrappers/ByteArrayWrapper.java         |  19 ++
 .../org/apache/spark/sql/CarbonContext.scala    |   1 +
 .../scala/org/apache/spark/sql/CarbonScan.scala |   1 +
 .../spark/sql/hive/CarbonAnalysisRules.scala    |   2 +-
 .../spark/sql/hive/CarbonStrategies.scala       |  24 ++-
 .../spark/sql/optimizer/CarbonOptimizer.scala   |  33 +++-
 24 files changed, 705 insertions(+), 42 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff84a2ed/core/src/main/java/org/apache/carbondata/core/carbon/metadata/encoder/Encoding.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/encoder/Encoding.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/encoder/Encoding.java
index 5a46502..a22f98c 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/encoder/Encoding.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/encoder/Encoding.java
@@ -27,5 +27,6 @@ public enum Encoding {
   RLE,
   INVERTED_INDEX,
   BIT_PACKED,
-  DIRECT_DICTIONARY
+  DIRECT_DICTIONARY,
+  IMPLICIT;
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff84a2ed/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTable.java
index eacacc1..d02aebf 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/CarbonTable.java
@@ -20,13 +20,8 @@
 package org.apache.carbondata.core.carbon.metadata.schema.table;
 
 import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
@@ -36,6 +31,7 @@ import org.apache.carbondata.core.carbon.metadata.encoder.Encoding;
 import org.apache.carbondata.core.carbon.metadata.schema.BucketingInfo;
 import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonColumn;
 import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonImplicitDimension;
 import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonMeasure;
 import org.apache.carbondata.core.carbon.metadata.schema.table.column.ColumnSchema;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
@@ -75,6 +71,11 @@ public class CarbonTable implements Serializable {
   /**
    * table measures list.
    */
+  private Map<String, List<CarbonDimension>> tableImplicitDimensionsMap;
+
+  /**
+   * table measures list.
+   */
   private Map<String, List<CarbonMeasure>> tableMeasuresMap;
 
   /**
@@ -109,6 +110,7 @@ public class CarbonTable implements Serializable {
 
   public CarbonTable() {
     this.tableDimensionsMap = new HashMap<String, List<CarbonDimension>>();
+    this.tableImplicitDimensionsMap = new HashMap<String, List<CarbonDimension>>();
     this.tableMeasuresMap = new HashMap<String, List<CarbonMeasure>>();
     this.tableBucketMap = new HashMap<>();
     this.aggregateTablesName = new ArrayList<String>();
@@ -202,10 +204,12 @@ public class CarbonTable implements Serializable {
   private void fillDimensionsAndMeasuresForTables(TableSchema tableSchema) {
     List<CarbonDimension> dimensions = new ArrayList<CarbonDimension>();
     List<CarbonDimension> primitiveDimensions = new ArrayList<CarbonDimension>();
+    List<CarbonDimension> implicitDimensions = new ArrayList<CarbonDimension>();
     List<CarbonMeasure> measures = new ArrayList<CarbonMeasure>();
     this.tableDimensionsMap.put(tableSchema.getTableName(), dimensions);
     this.tablePrimitiveDimensionsMap.put(this.tableUniqueName, primitiveDimensions);
     this.tableMeasuresMap.put(tableSchema.getTableName(), measures);
+    this.tableImplicitDimensionsMap.put(tableSchema.getTableName(), implicitDimensions);
     int dimensionOrdinal = 0;
     int measureOrdinal = 0;
     int keyOrdinal = 0;
@@ -257,6 +261,20 @@ public class CarbonTable implements Serializable {
                  columnSchema.getSchemaOrdinal()));
       }
     }
+    addImplicitDimension(dimensionOrdinal, implicitDimensions);
+  }
+
+  /**
+   * This method will add implict dimension into carbontable
+   *
+   * @param dimensionOrdinal
+   * @param dimensions
+   */
+  private void addImplicitDimension(int dimensionOrdinal, List<CarbonDimension> dimensions) {
+    dimensions.add(new CarbonImplicitDimension(dimensionOrdinal,
+        CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_POSITIONID));
+    dimensions.add(new CarbonImplicitDimension(dimensionOrdinal+1,
+        CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID));
   }
 
   /**
@@ -404,6 +422,16 @@ public class CarbonTable implements Serializable {
   }
 
   /**
+   * to get the all dimension of a table
+   *
+   * @param tableName
+   * @return all dimension of a table
+   */
+  public List<CarbonDimension> getImplicitDimensionByTableName(String tableName) {
+    return tableImplicitDimensionsMap.get(tableName);
+  }
+
+  /**
    * This will give user created order column
    *
    * @return
@@ -437,13 +465,22 @@ public class CarbonTable implements Serializable {
    * @return
    */
   public CarbonDimension getDimensionByName(String tableName, String columnName) {
+    CarbonDimension carbonDimension = null;
     List<CarbonDimension> dimList = tableDimensionsMap.get(tableName);
     for (CarbonDimension dim : dimList) {
       if (dim.getColName().equalsIgnoreCase(columnName)) {
-        return dim;
+        carbonDimension = dim;
+        break;
       }
     }
-    return null;
+    List<CarbonDimension> implicitDimList = tableImplicitDimensionsMap.get(tableName);
+    for (CarbonDimension dim : implicitDimList) {
+      if (dim.getColName().equalsIgnoreCase(columnName)) {
+        carbonDimension = dim;
+        break;
+      }
+    }
+    return carbonDimension;
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff84a2ed/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonImplicitDimension.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonImplicitDimension.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonImplicitDimension.java
new file mode 100644
index 0000000..2292b10
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonImplicitDimension.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.carbon.metadata.schema.table.column;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
+import org.apache.carbondata.core.carbon.metadata.encoder.Encoding;
+
+public class CarbonImplicitDimension extends CarbonDimension {
+  /**
+   * serialization version
+   */
+  private static final long serialVersionUID = 3648269871656322681L;
+
+  /**
+   * List of encoding that are chained to encode the data for this column
+   */
+  private List<Encoding> encodingList;
+
+  /**
+   *
+   * @param ordinal
+   */
+  private String implicitDimensionName;
+
+  public CarbonImplicitDimension(int ordinal, String implicitDimensionName) {
+    super(null, ordinal, -1, -1, -1);
+    encodingList = new ArrayList<Encoding>();
+    encodingList.add(Encoding.IMPLICIT);
+    this.implicitDimensionName = implicitDimensionName;
+  }
+
+  public boolean hasEncoding(Encoding encoding) {
+    return encodingList.contains(encoding);
+  }
+
+  /**
+   * @return column unique id
+   */
+  public String getColumnId() {
+    return UUID.randomUUID().toString();
+  }
+
+  public Map<String, String> getColumnProperties() {
+    return null;
+  }
+
+  /**
+   * @return if DataType is ARRAY or STRUCT, this method return true, else
+   * false.
+   */
+  public Boolean isComplex() {
+    return false;
+  }
+
+  /**
+   * @return row group id if it is row based
+   */
+  @Override public int columnGroupId() {
+    return -1;
+  }
+
+  /**
+   * @return the list of encoder used in dimension
+   */
+  @Override public List<Encoding> getEncoder() {
+    return encodingList;
+  }
+
+  /**
+   * @return return the number of child present in case of complex type
+   */
+  @Override public int numberOfChild() {
+    return 0;
+  }
+
+  /**
+   * @return the colName
+   */
+  @Override public String getColName() {
+    return implicitDimensionName;
+  }
+
+  /**
+   * @return if column is dimension return true, else false.
+   */
+  @Override public Boolean isDimesion() {
+    return true;
+  }
+
+  /**
+   * @return number of children for complex type
+   */
+  public int getNumberOfChild() {
+    return 0;
+  }
+
+  /**
+   * @return the dataType
+   */
+  @Override public DataType getDataType() {
+    return DataType.STRING;
+  }
+
+  /**
+   * @return columnar or row based
+   */
+  public boolean isColumnar() {
+    return true;
+  }
+
+  /**
+   * To specify the visibily of the column by default its false
+   */
+    public boolean isInvisible() {
+    return true;
+  }
+
+  /**
+   * to generate the hash code for this class
+   */
+  @Override public int hashCode() {
+    final int prime = 31;
+    int result = 1;
+    result = prime * result + ((implicitDimensionName == null) ?
+        super.hashCode() :
+        super.hashCode() + implicitDimensionName.hashCode());
+    return result;
+  }
+
+  /**
+   * to check whether to dimension are equal or not
+   */
+  @Override public boolean equals(Object obj) {
+    if (this == obj) {
+      return true;
+    }
+    if (obj == null) {
+      return false;
+    }
+    if (!(obj instanceof CarbonImplicitDimension)) {
+      return false;
+    }
+    CarbonImplicitDimension other = (CarbonImplicitDimension) obj;
+    if (columnSchema == null) {
+      if (other.columnSchema != null) {
+        return false;
+      }
+    }
+    if (!getColName().equals(other.getColName())) {
+      return false;
+    }
+    return true;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff84a2ed/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java
index 8cf6edb..e2d49c9 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java
@@ -441,3 +441,74 @@ public class CarbonTablePath extends Path {
     return carbonDataFileName;
   }
 }
+  /**
+   * This method will remove strings in path and return short block id
+   *
+   * @param blockId
+   * @return shortBlockId
+   */
+  public static String getShortBlockId(String blockId) {
+    return blockId.replace(PARTITION_PREFIX, "")
+            .replace(SEGMENT_PREFIX, "")
+            .replace(DATA_PART_PREFIX, "")
+            .replace(CARBON_DATA_EXT, "");
+  }
+
+  /**
+   * This method will append strings in path and return block id
+   *
+   * @param shortBlockId
+   * @return blockId
+   */
+  public static String getBlockId(String shortBlockId) {
+    String[] splitRecords = shortBlockId.split(CarbonCommonConstants.FILE_SEPARATOR);
+    StringBuffer sb = new StringBuffer();
+    for (int i = 0; i < splitRecords.length; i++) {
+      if (i == 0) {
+        sb.append(PARTITION_PREFIX);
+        sb.append(splitRecords[i]);
+      } else if (i == 1) {
+        sb.append(CarbonCommonConstants.FILE_SEPARATOR);
+        sb.append(SEGMENT_PREFIX);
+        sb.append(splitRecords[i]);
+      } else if (i == 2) {
+        sb.append(CarbonCommonConstants.FILE_SEPARATOR);
+        sb.append(DATA_PART_PREFIX);
+        sb.append(splitRecords[i]);
+      } else if (i == 3) {
+        sb.append(CarbonCommonConstants.FILE_SEPARATOR);
+        sb.append(splitRecords[i]);
+        sb.append(CARBON_DATA_EXT);
+      } else {
+        sb.append(CarbonCommonConstants.FILE_SEPARATOR);
+        sb.append(splitRecords[i]);
+      }
+    }
+    return sb.toString();
+  }
+
+
+  /**
+   * adds data part prefix to given value
+   * @return partition prefix
+   */
+  public static String addDataPartPrefix(String value) {
+    return DATA_PART_PREFIX + value;
+  }
+
+  /**
+   * adds part prefix to given value
+   * @return partition prefix
+   */
+  public static String addPartPrefix(String value) {
+    return PARTITION_PREFIX + value;
+  }
+
+  /**
+   * adds part prefix to given value
+   * @return partition prefix
+   */
+  public static String addSegmentPrefix(String value) {
+    return SEGMENT_PREFIX + value;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff84a2ed/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 32339ff..1088e5c 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -556,10 +556,21 @@ public final class CarbonCommonConstants {
    */
   public static final String SCHEMAS_MODIFIED_TIME_FILE = "modifiedTime.mdt";
   public static final String DEFAULT_INVISIBLE_DUMMY_MEASURE = "default_dummy_measure";
+  public static final String CARBON_IMPLICIT_COLUMN_POSITIONID = "positionId";
+  public static final String CARBON_IMPLICIT_COLUMN_TUPLEID = "tupleId";
   /**
    * max driver lru cache size upto which lru cache will be loaded in memory
    */
   public static final String CARBON_MAX_DRIVER_LRU_CACHE_SIZE = "carbon.max.driver.lru.cache.size";
+  public static final String POSITION_REFERENCE = "positionReference";
+  /**
+   * implicit column which will be added to each carbon table
+   */
+  public static final String POSITION_ID = "positionId";
+  /**
+   * max driver lru cache size upto which lru cache will be loaded in memory
+   */
+  public static final String CARBON_MAX_LEVEL_CACHE_SIZE = "carbon.max.level.cache.size";
   /**
    * max executor lru cache size upto which lru cache will be loaded in memory
    */

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff84a2ed/core/src/main/java/org/apache/carbondata/core/update/TupleIdEnum.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/update/TupleIdEnum.java b/core/src/main/java/org/apache/carbondata/core/update/TupleIdEnum.java
new file mode 100644
index 0000000..d4d9eb2
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/update/TupleIdEnum.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.huawei.carbondata.core.update;
+
+/**
+ * Enum class for tupleID.
+ */
+public enum TupleIdEnum {
+  PART_ID(0),
+  SEGMENT_ID(1),
+  BLOCK_ID(2),
+  BLOCKLET_ID(3),
+  OFFSET(4);
+
+  private int index;
+
+  TupleIdEnum(int index) {
+    this.index = index;
+  }
+
+  public int getTupleIdIndex(){
+    return this.index;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff84a2ed/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index dc28513..c61aaef 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -831,6 +831,15 @@ public final class CarbonUtil {
     return dictionaryEncodingArray;
   }
 
+  public static boolean[] getImplicitColumnArray(QueryDimension[] queryDimensions) {
+    boolean[] implicitColumnArray = new boolean[queryDimensions.length];
+    for (int i = 0; i < queryDimensions.length; i++) {
+      implicitColumnArray[i] = queryDimensions[i]
+          .getDimension().hasEncoding(Encoding.IMPLICIT);
+    }
+    return implicitColumnArray;
+  }
+
   public static boolean[] getComplexDataTypeArray(QueryDimension[] queryDimensions) {
     boolean[] dictionaryEncodingArray = new boolean[queryDimensions.length];
     for (int i = 0; i < queryDimensions.length; i++) {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff84a2ed/core/src/main/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedResultCollector.java b/core/src/main/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedResultCollector.java
index 52815b7..f9be188 100644
--- a/core/src/main/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedResultCollector.java
@@ -25,6 +25,8 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.carbondata.core.carbon.metadata.encoder.Encoding;
+import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
 import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
 import org.apache.carbondata.core.util.CarbonUtil;
@@ -81,6 +83,7 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
     boolean[] dictionaryEncodingArray = CarbonUtil.getDictionaryEncodingArray(queryDimensions);
     boolean[] directDictionaryEncodingArray =
         CarbonUtil.getDirectDictionaryEncodingArray(queryDimensions);
+    boolean[] implictColumnArray = CarbonUtil.getImplicitColumnArray(queryDimensions);
     boolean[] complexDataTypeArray = CarbonUtil.getComplexDataTypeArray(queryDimensions);
     int dimSize = queryDimensions.length;
     boolean isDimensionsExist = dimSize > 0;
@@ -110,9 +113,21 @@ public class DictionaryBasedResultCollector extends AbstractScannedResultCollect
         complexTypeColumnIndex = 0;
         for (int i = 0; i < dimSize; i++) {
           if (!dictionaryEncodingArray[i]) {
-            row[order[i]] = DataTypeUtil
-                .getDataBasedOnDataType(noDictionaryKeys[noDictionaryColumnIndex++],
-                    queryDimensions[i].getDimension().getDataType());
+            if (implictColumnArray[i]) {
+              if (CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID
+                  .equals(queryDimensions[i].getDimension().getColName())) {
+                row[order[i]] = DataTypeUtil.getDataBasedOnDataType(
+                    scannedResult.getBlockletId() + CarbonCommonConstants.FILE_SEPARATOR
+                        + scannedResult.getCurrenrRowId(), DataType.STRING);
+              } else {
+                row[order[i]] = DataTypeUtil
+                    .getDataBasedOnDataType(scannedResult.getBlockletId(), DataType.STRING);
+              }
+            } else {
+              row[order[i]] = DataTypeUtil
+                  .getDataBasedOnDataType(noDictionaryKeys[noDictionaryColumnIndex++],
+                      queryDimensions[i].getDimension().getDataType());
+            }
           } else if (directDictionaryEncodingArray[i]) {
             DirectDictionaryGenerator directDictionaryGenerator =
                 DirectDictionaryKeyGeneratorFactory

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff84a2ed/core/src/main/java/org/apache/carbondata/scan/collector/impl/RawBasedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/collector/impl/RawBasedResultCollector.java b/core/src/main/java/org/apache/carbondata/scan/collector/impl/RawBasedResultCollector.java
index aa885d3..4489c26 100644
--- a/core/src/main/java/org/apache/carbondata/scan/collector/impl/RawBasedResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/scan/collector/impl/RawBasedResultCollector.java
@@ -18,9 +18,11 @@
  */
 package org.apache.carbondata.scan.collector.impl;
 
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
 import org.apache.carbondata.scan.model.QueryMeasure;
 import org.apache.carbondata.scan.result.AbstractScannedResult;
@@ -51,6 +53,8 @@ public class RawBasedResultCollector extends AbstractScannedResultCollector {
       wrapper.setDictionaryKey(scannedResult.getDictionaryKeyArray());
       wrapper.setNoDictionaryKeys(scannedResult.getNoDictionaryKeyArray());
       wrapper.setComplexTypesKeys(scannedResult.getComplexTypeKeyArray());
+      wrapper.setImplicitColumnByteArray(scannedResult.getBlockletId()
+          .getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)));
       row[0] = wrapper;
       fillMeasureData(row, 1, scannedResult);
       listBasedResult.add(row);

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff84a2ed/core/src/main/java/org/apache/carbondata/scan/executor/impl/AbstractQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/apache/carbondata/scan/executor/impl/AbstractQueryExecutor.java
index deb9179..7e8d8eb 100644
--- a/core/src/main/java/org/apache/carbondata/scan/executor/impl/AbstractQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/scan/executor/impl/AbstractQueryExecutor.java
@@ -44,6 +44,7 @@ import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
 import org.apache.carbondata.core.carbon.metadata.encoder.Encoding;
 import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension;
 import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonMeasure;
+import org.apache.carbondata.core.carbon.path.CarbonStorePath;
 import org.apache.carbondata.core.carbon.querystatistics.QueryStatistic;
 import org.apache.carbondata.core.carbon.querystatistics.QueryStatisticsConstants;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
@@ -218,8 +219,8 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
       blockExecutionInfoList.add(
           getBlockExecutionInfoForBlock(queryModel, queryProperties.dataBlocks.get(i),
               queryModel.getTableBlockInfos().get(i).getBlockletInfos().getStartBlockletNumber(),
-              queryModel.getTableBlockInfos().get(i).getBlockletInfos()
-                  .getNumberOfBlockletToScan()));
+              queryModel.getTableBlockInfos().get(i).getBlockletInfos().getNumberOfBlockletToScan(),
+              queryModel.getTableBlockInfos().get(i).getFilePath()));
     }
     if (null != queryModel.getStatisticsRecorder()) {
       QueryStatistic queryStatistic = new QueryStatistic();
@@ -240,7 +241,7 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
    * @throws QueryExecutionException any failure during block info creation
    */
   protected BlockExecutionInfo getBlockExecutionInfoForBlock(QueryModel queryModel,
-      AbstractIndex blockIndex, int startBlockletIndex, int numberOfBlockletToScan)
+      AbstractIndex blockIndex, int startBlockletIndex, int numberOfBlockletToScan, String filePath)
       throws QueryExecutionException {
     BlockExecutionInfo blockExecutionInfo = new BlockExecutionInfo();
     SegmentProperties segmentProperties = blockIndex.getSegmentProperties();
@@ -257,6 +258,11 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
         QueryUtil.getMaskedByteRange(updatedQueryDimension, blockKeyGenerator);
     int[] maksedByte =
         QueryUtil.getMaskedByte(blockKeyGenerator.getKeySizeInBytes(), maskByteRangesForBlock);
+    int tableFactPathLength = CarbonStorePath
+        .getCarbonTablePath(queryModel.getAbsoluteTableIdentifier().getStorePath(),
+            queryModel.getAbsoluteTableIdentifier().getCarbonTableIdentifier()).getFactDir()
+        .length() + 1;
+    blockExecutionInfo.setBlockId(filePath.substring(tableFactPathLength));
     blockExecutionInfo.setStartBlockletIndex(startBlockletIndex);
     blockExecutionInfo.setNumberOfBlockletToScan(numberOfBlockletToScan);
     blockExecutionInfo.setQueryDimensions(
@@ -317,9 +323,11 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
         new ArrayList<CarbonMeasure>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
     // setting all the dimension chunk indexes to be read from file
     int numberOfElementToConsider = 0;
+    // list of dimensions to be projected
+    List<Integer> allProjectionListDimensionIdexes = new ArrayList<>();
     int[] dimensionsBlockIndexes = QueryUtil.getDimensionsBlockIndexes(updatedQueryDimension,
         segmentProperties.getDimensionOrdinalToBlockMapping(), expressionDimensions,
-        queryProperties.complexFilterDimension);
+        queryProperties.complexFilterDimension, allProjectionListDimensionIdexes);
     if (dimensionsBlockIndexes.length > 0) {
       numberOfElementToConsider = dimensionsBlockIndexes[dimensionsBlockIndexes.length - 1]
           == segmentProperties.getBlockTodimensionOrdinalMapping().size() - 1 ?

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff84a2ed/core/src/main/java/org/apache/carbondata/scan/executor/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/util/QueryUtil.java b/core/src/main/java/org/apache/carbondata/scan/executor/util/QueryUtil.java
index 3fd81f8..88088ee 100644
--- a/core/src/main/java/org/apache/carbondata/scan/executor/util/QueryUtil.java
+++ b/core/src/main/java/org/apache/carbondata/scan/executor/util/QueryUtil.java
@@ -210,12 +210,18 @@ public class QueryUtil {
    */
   public static int[] getDimensionsBlockIndexes(List<QueryDimension> queryDimensions,
       Map<Integer, Integer> dimensionOrdinalToBlockMapping,
-      List<CarbonDimension> customAggregationDimension, Set<CarbonDimension> filterDimensions) {
+      List<CarbonDimension> customAggregationDimension, Set<CarbonDimension> filterDimensions,
+      List<Integer> allProjectionListDimensionIndexes) {
     // using set as in row group columns will point to same block
     Set<Integer> dimensionBlockIndex = new HashSet<Integer>();
     Set<Integer> filterDimensionOrdinal = getFilterDimensionOrdinal(filterDimensions);
     int blockIndex = 0;
     for (int i = 0; i < queryDimensions.size(); i++) {
+      if (queryDimensions.get(i).getDimension().hasEncoding(Encoding.IMPLICIT)) {
+        continue;
+      }
+      allProjectionListDimensionIndexes.add(queryDimensions.get(i).getDimension().getOrdinal());
+
       if (!filterDimensionOrdinal.contains(queryDimensions.get(i).getDimension().getOrdinal())) {
         blockIndex =
             dimensionOrdinalToBlockMapping.get(queryDimensions.get(i).getDimension().getOrdinal());
@@ -277,7 +283,8 @@ public class QueryUtil {
       // TODO need to remove the data type check for parent column in complex type no need to
       // write encoding dictionary
       if (CarbonUtil.hasEncoding(encodingList, Encoding.DICTIONARY) && !CarbonUtil
-          .hasEncoding(encodingList, Encoding.DIRECT_DICTIONARY)) {
+          .hasEncoding(encodingList, Encoding.DIRECT_DICTIONARY) && !CarbonUtil
+          .hasEncoding(encodingList, Encoding.IMPLICIT)) {
 
         if (queryDimensions.get(i).getDimension().numberOfChild() == 0) {
           dictionaryDimensionFromQuery.add(queryDimensions.get(i).getDimension().getColumnId());
@@ -534,7 +541,10 @@ public class QueryUtil {
       // column group id
       // then we need to add ordinal of that column as it belongs to same
       // column group
-      if (!dimensions.get(index).getDimension().isColumnar()
+      if (dimensions.get(index).getDimension().hasEncoding(Encoding.IMPLICIT)) {
+        index++;
+        continue;
+      } else if (!dimensions.get(index).getDimension().isColumnar()
           && dimensions.get(index).getDimension().columnGroupId() == prvColumnGroupId
           && null != currentColumnGroup) {
         currentColumnGroup.add(dimensions.get(index).getDimension().getOrdinal());
@@ -599,7 +609,9 @@ public class QueryUtil {
           && queryDimension.getDimension().numberOfChild() == 0) {
         dictionaryDimensionBlockIndex
             .add(columnOrdinalToBlockIndexMapping.get(queryDimension.getDimension().getOrdinal()));
-      } else if (queryDimension.getDimension().numberOfChild() == 0) {
+      } else if (
+          !CarbonUtil.hasEncoding(queryDimension.getDimension().getEncoder(), Encoding.IMPLICIT)
+              && queryDimension.getDimension().numberOfChild() == 0) {
         noDictionaryDimensionBlockIndex
             .add(columnOrdinalToBlockIndexMapping.get(queryDimension.getDimension().getOrdinal()));
       }
@@ -738,6 +750,10 @@ public class QueryUtil {
     }
     if (null != filterDimensions) {
       for (CarbonDimension filterDimension : filterDimensions) {
+        // do not fill nay details for implicit dimension type
+        if (filterDimension.hasEncoding(Encoding.IMPLICIT)) {
+          continue;
+        }
         fillParentDetails(dimensionToBlockIndexMap, filterDimension, complexTypeMap,
             eachComplexColumnValueSize, columnIdToDictionaryMap);
       }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff84a2ed/core/src/main/java/org/apache/carbondata/scan/executor/util/RestructureUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/util/RestructureUtil.java b/core/src/main/java/org/apache/carbondata/scan/executor/util/RestructureUtil.java
index a20fd53..37f8909 100644
--- a/core/src/main/java/org/apache/carbondata/scan/executor/util/RestructureUtil.java
+++ b/core/src/main/java/org/apache/carbondata/scan/executor/util/RestructureUtil.java
@@ -21,6 +21,7 @@ package org.apache.carbondata.scan.executor.util;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.carbondata.core.carbon.metadata.encoder.Encoding;
 import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension;
 import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonMeasure;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
@@ -48,10 +49,14 @@ public class RestructureUtil {
     List<QueryDimension> presentDimension =
         new ArrayList<QueryDimension>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
     // selecting only those dimension which is present in the query
-    for (QueryDimension queryDimimension : queryDimensions) {
-      for (CarbonDimension tableDimension : tableBlockDimensions) {
-        if (tableDimension.equals(queryDimimension.getDimension())) {
-          presentDimension.add(queryDimimension);
+    for (QueryDimension queryDimension : queryDimensions) {
+      if (queryDimension.getDimension().hasEncoding(Encoding.IMPLICIT)) {
+        presentDimension.add(queryDimension);
+      } else {
+        for (CarbonDimension tableDimension : tableBlockDimensions) {
+          if (tableDimension.equals(queryDimension.getDimension())) {
+            presentDimension.add(queryDimension);
+          }
         }
       }
     }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff84a2ed/core/src/main/java/org/apache/carbondata/scan/filter/DimColumnFilterInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/DimColumnFilterInfo.java b/core/src/main/java/org/apache/carbondata/scan/filter/DimColumnFilterInfo.java
index 0d51286..f97fffc 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/DimColumnFilterInfo.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/DimColumnFilterInfo.java
@@ -31,6 +31,11 @@ public class DimColumnFilterInfo implements Serializable {
   private List<Integer> filterList;
 
   /**
+   * Implicit column filter values to be used for block and blocklet pruning
+   */
+  private List<String> implicitColumnFilterList;
+  private List<Integer> excludeFilterList;
+  /**
    * maintain the no dictionary filter values list.
    */
   private List<byte[]> noDictionaryFilterValuesList;
@@ -58,4 +63,18 @@ public class DimColumnFilterInfo implements Serializable {
   public void setFilterListForNoDictionaryCols(List<byte[]> noDictionaryFilterValuesList) {
     this.noDictionaryFilterValuesList = noDictionaryFilterValuesList;
   }
+
+  public List<Integer> getExcludeFilterList() {
+    return excludeFilterList;
+  }
+  public void setExcludeFilterList(List<Integer> excludeFilterList) {
+    this.excludeFilterList = excludeFilterList;
+  }
+  public List<String> getImplicitColumnFilterList() {
+    return implicitColumnFilterList;
+  }
+
+  public void setImplicitColumnFilterList(List<String> implicitColumnFilterList) {
+    this.implicitColumnFilterList = implicitColumnFilterList;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff84a2ed/core/src/main/java/org/apache/carbondata/scan/filter/executer/ImplicitColumnFilterExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/ImplicitColumnFilterExecutor.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/ImplicitColumnFilterExecutor.java
new file mode 100644
index 0000000..de0eb5f
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/executer/ImplicitColumnFilterExecutor.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.scan.filter.executer;
+
+import java.util.BitSet;
+
+import org.apache.carbondata.core.carbon.datastore.DataRefNode;
+
+/**
+ * Implementation of this interface will involve block
+ * and blocklet pruning based on block/blocklet id where
+ * the filter values are present.
+ */
+public interface ImplicitColumnFilterExecutor extends FilterExecuter {
+
+  /**
+   * This method will validate the block or blocklet id with the implicit
+   * column filter value list and decide whether the required block or
+   * blocklet has to be scanned for the data or not
+   *
+   * @param dataRefNode
+   * @param uniqueBlockPath
+   * @return
+   */
+  BitSet isFilterValuesPresentInBlockOrBlocklet(DataRefNode dataRefNode, String uniqueBlockPath);
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff84a2ed/core/src/main/java/org/apache/carbondata/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java
new file mode 100644
index 0000000..8eec63e
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.scan.filter.executer;
+
+import java.util.BitSet;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.carbondata.core.carbon.datastore.DataRefNode;
+import org.apache.carbondata.core.carbon.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.carbon.path.CarbonTablePath;
+import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import org.apache.carbondata.scan.processor.BlocksChunkHolder;
+
+/**
+ * This class will implement the blocklet and block pruning logic based
+ * on the implicit column filter values
+ */
+public class ImplicitIncludeFilterExecutorImpl implements ImplicitColumnFilterExecutor {
+
+  private final DimColumnResolvedFilterInfo dimColumnEvaluatorInfo;
+  private final SegmentProperties segmentProperties;
+  /**
+   * List with implicit column filter values
+   */
+  private Set<String> implicitFilterValueList;
+
+  public ImplicitIncludeFilterExecutorImpl(DimColumnResolvedFilterInfo dimColumnEvaluatorInfo,
+      SegmentProperties segmentProperties) {
+    this.dimColumnEvaluatorInfo = dimColumnEvaluatorInfo;
+    this.segmentProperties = segmentProperties;
+    this.implicitFilterValueList =
+        new HashSet(dimColumnEvaluatorInfo.getFilterValues().getImplicitColumnFilterList());
+  }
+
+
+  public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
+      throws FilterUnsupportedException {
+    return setBitSetForCompleteDimensionData(blockChunkHolder.getDataBlock().nodeSize());
+  }
+
+  public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+    return null;
+  }
+   public BitSet isFilterValuesPresentInBlockOrBlocklet(DataRefNode dataRefNode,
+      String uniqueBlockPath) {
+    BitSet bitSet = new BitSet(1);
+    boolean isScanRequired = false;
+    if (uniqueBlockPath.endsWith(".carbondata")) {
+      String shortBlockId = CarbonTablePath.getShortBlockId(uniqueBlockPath);
+      for (String blockletId : implicitFilterValueList) {
+        if (blockletId.startsWith(shortBlockId)) {
+          isScanRequired = true;
+          break;
+        }
+      }
+    } else if (implicitFilterValueList.contains(uniqueBlockPath)) {
+      isScanRequired = true;
+    }
+    if (isScanRequired) {
+      bitSet.set(0);
+    }
+    return bitSet;
+  }
+
+  /**
+   * For implicit column filtering, complete data need to be selected. As it is a special case
+   * no data need to be discarded, implicit filtering is only for slecting block and blocklets
+   *
+   * @param numberOfRows
+   * @return
+   */
+  private BitSet setBitSetForCompleteDimensionData(int numberOfRows) {
+    BitSet bitSet = new BitSet();
+    bitSet.set(0, numberOfRows, true);
+    return bitSet;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff84a2ed/core/src/main/java/org/apache/carbondata/scan/result/AbstractScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/result/AbstractScannedResult.java b/core/src/main/java/org/apache/carbondata/scan/result/AbstractScannedResult.java
index cd0ce9c..bb2d132 100644
--- a/core/src/main/java/org/apache/carbondata/scan/result/AbstractScannedResult.java
+++ b/core/src/main/java/org/apache/carbondata/scan/result/AbstractScannedResult.java
@@ -29,6 +29,7 @@ import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
 import org.apache.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk;
 import org.apache.carbondata.core.carbon.datastore.chunk.MeasureColumnDataChunk;
+import org.apache.carbondata.core.carbon.path.CarbonTablePath;
 import org.apache.carbondata.core.util.CarbonUtil;
 import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
 import org.apache.carbondata.scan.executor.infos.KeyStructureInfo;
@@ -99,6 +100,13 @@ public abstract class AbstractScannedResult {
   private int totalDimensionsSize;
 
   /**
+   * blockedId which will be blockId + blocklet number in the block
+   */
+  private String blockletId;
+
+  private long rowId;
+
+  /**
    * parent block indexes
    */
   private int[] complexParentBlockIndexes;
@@ -301,6 +309,35 @@ public abstract class AbstractScannedResult {
   }
 
   /**
+   * @return blockletId
+   */
+  public String getBlockletId() {
+    return blockletId;
+  }
+
+  /**
+   * @param blockletId
+   */
+  public void setBlockletId(String blockletId) {
+    this.blockletId = CarbonTablePath.getShortBlockId(blockletId);
+  }
+
+  /**
+   * @return blockletId
+   */
+  public long getRowId() {
+    return rowId;
+  }
+
+  /**
+   * @param blockletId
+   */
+  public void setRowId(long rowId) {
+    this.rowId = rowId;
+  }
+
+
+  /**
    * Below method will be used to get the complex type keys array based
    * on row id for all the complex type dimension selected in query
    *

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff84a2ed/core/src/main/java/org/apache/carbondata/scan/scanner/AbstractBlockletScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/scanner/AbstractBlockletScanner.java b/core/src/main/java/org/apache/carbondata/scan/scanner/AbstractBlockletScanner.java
index 84c0cb4..325fac8 100644
--- a/core/src/main/java/org/apache/carbondata/scan/scanner/AbstractBlockletScanner.java
+++ b/core/src/main/java/org/apache/carbondata/scan/scanner/AbstractBlockletScanner.java
@@ -23,6 +23,8 @@ import java.io.IOException;
 import org.apache.carbondata.core.carbon.querystatistics.QueryStatistic;
 import org.apache.carbondata.core.carbon.querystatistics.QueryStatisticsConstants;
 import org.apache.carbondata.core.carbon.querystatistics.QueryStatisticsModel;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
 import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
 import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
 import org.apache.carbondata.scan.processor.BlocksChunkHolder;
@@ -69,6 +71,9 @@ public abstract class AbstractBlockletScanner implements BlockletScanner {
                     validScannedBlockletStatistic.getCount() + 1);
     queryStatisticsModel.getRecorder().recordStatistics(validScannedBlockletStatistic);
     scannedResult.reset();
+    scannedResult.setBlockletId(
+        blockExecutionInfo.getBlockId() + CarbonCommonConstants.FILE_SEPARATOR
+            + blocksChunkHolder.getDataBlock().nodeNumber());
     scannedResult.setNumberOfRows(blocksChunkHolder.getDataBlock().nodeSize());
     scannedResult.setDimensionChunks(blocksChunkHolder.getDataBlock()
         .getDimensionChunks(blocksChunkHolder.getFileReader(),

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff84a2ed/core/src/main/java/org/apache/carbondata/scan/scanner/impl/FilterScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/scanner/impl/FilterScanner.java b/core/src/main/java/org/apache/carbondata/scan/scanner/impl/FilterScanner.java
index 5ff61d4..f50d498 100644
--- a/core/src/main/java/org/apache/carbondata/scan/scanner/impl/FilterScanner.java
+++ b/core/src/main/java/org/apache/carbondata/scan/scanner/impl/FilterScanner.java
@@ -33,6 +33,7 @@ import org.apache.carbondata.core.util.CarbonProperties;
 import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
 import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
 import org.apache.carbondata.scan.filter.executer.FilterExecuter;
+import org.apache.carbondata.scan.filter.executer.ImplicitColumnFilterExecutor;
 import org.apache.carbondata.scan.processor.BlocksChunkHolder;
 import org.apache.carbondata.scan.result.AbstractScannedResult;
 import org.apache.carbondata.scan.result.impl.FilterQueryScannedResult;
@@ -109,16 +110,22 @@ public class FilterScanner extends AbstractBlockletScanner {
   private void fillScannedResult(BlocksChunkHolder blocksChunkHolder)
       throws FilterUnsupportedException, IOException {
     scannedResult.reset();
-    QueryStatistic totalBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap()
-        .get(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM);
-    totalBlockletStatistic.addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM,
-        totalBlockletStatistic.getCount() + 1);
-    queryStatisticsModel.getRecorder().recordStatistics(totalBlockletStatistic);
+    scannedResult.setBlockletId(
+        blockExecutionInfo.getBlockId() + CarbonCommonConstants.FILE_SEPARATOR + blocksChunkHolder
+            .getDataBlock().nodeNumber());
     // apply min max
     if (isMinMaxEnabled) {
-      BitSet bitSet = this.filterExecuter
-          .isScanRequired(blocksChunkHolder.getDataBlock().getColumnsMaxValue(),
-              blocksChunkHolder.getDataBlock().getColumnsMinValue());
+      BitSet bitSet = null;
+      // check for implicit include filter instance
+      if (filterExecuter instanceof ImplicitColumnFilterExecutor) {
+        bitSet = ((ImplicitColumnFilterExecutor) filterExecuter)
+            .isFilterValuesPresentInBlockOrBlocklet(blocksChunkHolder.getDataBlock(),
+                scannedResult.getBlockletId());
+      } else {
+        bitSet = this.filterExecuter
+            .isScanRequired(blocksChunkHolder.getDataBlock().getColumnsMaxValue(),
+                blocksChunkHolder.getDataBlock().getColumnsMinValue());
+      }
       if (bitSet.isEmpty()) {
         scannedResult.setNumberOfRows(0);
         scannedResult.setIndexes(new int[0]);

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff84a2ed/core/src/main/java/org/apache/carbondata/scan/wrappers/ByteArrayWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/wrappers/ByteArrayWrapper.java b/core/src/main/java/org/apache/carbondata/scan/wrappers/ByteArrayWrapper.java
index 1b3a4c6..1805636 100644
--- a/core/src/main/java/org/apache/carbondata/scan/wrappers/ByteArrayWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/scan/wrappers/ByteArrayWrapper.java
@@ -42,6 +42,11 @@ public class ByteArrayWrapper implements Comparable<ByteArrayWrapper> {
    */
   protected byte[][] noDictionaryKeys;
 
+  /**
+   * contains value of implicit columns in byte array format
+   */
+  protected byte[] implicitColumnByteArray;
+
   public ByteArrayWrapper() {
   }
 
@@ -199,4 +204,18 @@ public class ByteArrayWrapper implements Comparable<ByteArrayWrapper> {
   public void setComplexTypesKeys(byte[][] complexTypesKeys) {
     this.complexTypesKeys = complexTypesKeys;
   }
+
+  /**
+   * @return
+   */
+  public byte[] getImplicitColumnByteArray() {
+    return implicitColumnByteArray;
+  }
+
+  /**
+   * @param implicitColumnByteArray
+   */
+  public void setImplicitColumnByteArray(byte[] implicitColumnByteArray) {
+    this.implicitColumnByteArray = implicitColumnByteArray;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff84a2ed/integration/spark/src/main/scala/org/apache/spark/sql/CarbonContext.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonContext.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonContext.scala
index 51b1943..2c9b7b2 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonContext.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonContext.scala
@@ -56,6 +56,7 @@ class CarbonContext(
 
   CarbonContext.addInstance(sc, this)
   CodeGenerateFactory.init(sc.version)
+  udf.register("getTupleId", () => "")
   CarbonEnv.init(this)
 
   var lastSchemaUpdatedTime = System.currentTimeMillis()

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff84a2ed/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala
index 405d9d1..40a85a1 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala
@@ -81,6 +81,7 @@ case class CarbonScan(
     }
 
     val columns = carbonTable.getCreateOrderColumn(carbonTable.getFactTableName)
+    columns.addAll(carbonTable.getImplicitDimensionByTableName(carbonTable.getFactTableName))
     val colAttr = new Array[Attribute](columns.size())
     columnProjection.foreach { attr =>
     val column =

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff84a2ed/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonAnalysisRules.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonAnalysisRules.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonAnalysisRules.scala
index 15ac2f5..c224de8 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonAnalysisRules.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonAnalysisRules.scala
@@ -78,7 +78,7 @@ object CarbonIUDAnalysisRule extends Rule[LogicalPlan] {
       val tupleId = UnresolvedAlias(Alias(UnresolvedFunction("getTupleId",
         Seq.empty, isDistinct = false), "tupleId")())
       val projList = Seq(
-        UnresolvedAlias(UnresolvedStar(table.alias))/*, tupleId*/)
+        UnresolvedAlias(UnresolvedStar(table.alias)), tupleId)
       // include tuple id and rest of the required columns in subqury
       Subquery(table.alias.getOrElse(""), Project(projList, relation))
     }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff84a2ed/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonStrategies.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonStrategies.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonStrategies.scala
index 91c96af..bd55f73 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonStrategies.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonStrategies.scala
@@ -36,8 +36,10 @@ import org.apache.spark.sql.hive.execution.{DropTable, HiveNativeCommand}
 import org.apache.spark.sql.hive.execution.command._
 import org.apache.spark.sql.optimizer.CarbonDecoderRelation
 import org.apache.spark.sql.types.IntegerType
+import org.apache.spark.sql.types.StringType
 
 import org.apache.carbondata.common.logging.LogServiceFactory
+import org.apache.carbondata.core.constants.CarbonCommonConstants
 import org.apache.carbondata.spark.CarbonAliasDecoderRelation
 import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
 
@@ -93,12 +95,22 @@ class CarbonStrategies(sqlContext: SQLContext) extends QueryPlanner[SparkPlan] {
         relation.carbonRelation.metaData.carbonTable.getFactTableName.toLowerCase
       // Check out any expressions are there in project list. if they are present then we need to
       // decode them as well.
-
-      val projectSet = AttributeSet(projectList.flatMap(_.references))
+      val newProjectList = projectList.map { element =>
+        element match {
+          case a@Alias(s: ScalaUDF, name)
+            if (name.equalsIgnoreCase(CarbonCommonConstants.POSITION_ID) ||
+              name.equalsIgnoreCase(
+                CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID)) =>
+            AttributeReference(name, StringType, true)().withExprId(a.exprId)
+          case other => other
+        }
+      }
+      val projectSet = AttributeSet(newProjectList.flatMap(_.references))
       val filterSet = AttributeSet(predicates.flatMap(_.references))
-
-      val scan = CarbonScan(projectSet.toSeq, relation.carbonRelation, predicates)(sqlContext)
-      projectList.map {
+      val scan = CarbonScan(projectSet.toSeq,
+        relation.carbonRelation,
+        predicates)(sqlContext)
+      newProjectList.map {
         case attr: AttributeReference =>
         case Alias(attr: AttributeReference, _) =>
         case others =>
@@ -135,7 +147,7 @@ class CarbonStrategies(sqlContext: SQLContext) extends QueryPlanner[SparkPlan] {
         // just do a scan with no extra project.
         scanWithDecoder
       } else {
-        Project(projectList, scanWithDecoder)
+        Project(newProjectList, scanWithDecoder)
       }
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ff84a2ed/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala
index 34763ae..752b2b9 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonOptimizer.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.optimizer
 
 import java.util
 
+import org.apache.carbondata.core.constants.CarbonCommonConstants
 import org.apache.spark.sql.execution.command.ProjectForUpdateCommand
 
 import scala.collection.JavaConverters._
@@ -77,10 +78,11 @@ class ResolveCarbonFunctions(relations: Seq[CarbonDecoderRelation])
   def apply(logicalPlan: LogicalPlan): LogicalPlan = {
     if (relations.nonEmpty && !isOptimized(logicalPlan)) {
       val plan = processPlan(logicalPlan)
+      val udfTransformedPlan = pushDownUDFToJoinLeftRelation(plan)
       LOGGER.info("Starting to optimize plan")
       val recorder = CarbonTimeStatisticsFactory.createExecutorRecorder("")
       val queryStatistic = new QueryStatistic()
-      val result = transformCarbonPlan(plan, relations)
+      val result = transformCarbonPlan(udfTransformedPlan, relations)
       queryStatistic.addStatistics("Time taken for Carbon Optimizer to optimize: ",
         System.currentTimeMillis)
       recorder.recordStatistics(queryStatistic)
@@ -110,7 +112,34 @@ class ResolveCarbonFunctions(relations: Seq[CarbonDecoderRelation])
         ProjectForUpdateCommand(newPlan, table.tableIdentifier)
     }
   }
-
+  private def pushDownUDFToJoinLeftRelation(plan: LogicalPlan): LogicalPlan = {
+    val output = plan match {
+      case proj@Project(cols, Join(
+      left, right, jointype: org.apache.spark.sql.catalyst.plans.JoinType, condition)) =>
+        var projectionToBeAdded: Seq[org.apache.spark.sql.catalyst.expressions.Alias] = Seq.empty
+        val newCols = cols.map { col =>
+          col match {
+            case a@Alias(s: ScalaUDF, name)
+              if (name.equalsIgnoreCase(CarbonCommonConstants.POSITION_ID) ||
+                name.equalsIgnoreCase(
+                  CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID)) =>
+              projectionToBeAdded :+= a
+              AttributeReference(name, StringType, true)().withExprId(a.exprId)
+            case other => other
+          }
+        }
+        val newLeft = left match {
+          case Project(columns, logicalPlan) =>
+            Project(columns ++ projectionToBeAdded, logicalPlan)
+          case filter: Filter =>
+            Project(filter.output ++ projectionToBeAdded, filter)
+          case other => other
+        }
+        Project(newCols, Join(newLeft, right, jointype, condition))
+      case other => other
+    }
+    output
+  }
   def isOptimized(plan: LogicalPlan): Boolean = {
     plan find {
       case cd: CarbonDictionaryCatalystDecoder => true


Mime
View raw message