drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vita...@apache.org
Subject [4/5] drill git commit: DRILL-6199: Add support for filter push down and partition pruning with several nested star sub-queries
Date Tue, 20 Mar 2018 20:58:09 GMT
DRILL-6199: Add support for filter push down and partition pruning with several nested star
sub-queries

closes #1152


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/ddb73a50
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/ddb73a50
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/ddb73a50

Branch: refs/heads/master
Commit: ddb73a506c6cbb6492d6ff91ca174dbff1b8a045
Parents: cdc21ce
Author: Arina Ielchiieva <arina.yelchiyeva@gmail.com>
Authored: Sun Mar 4 22:12:06 2018 +0200
Committer: Vitalii Diravka <vitalii.diravka@gmail.com>
Committed: Tue Mar 20 22:40:00 2018 +0200

----------------------------------------------------------------------
 .../apache/drill/exec/planner/PlannerPhase.java |  33 +-
 .../DrillFilterItemStarReWriterRule.java        | 301 +++++++++++++------
 .../TestPushDownAndPruningWithItemStar.java     | 114 ++++++-
 3 files changed, 343 insertions(+), 105 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/ddb73a50/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java
index f46a7ee..f71c281 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java
@@ -29,6 +29,7 @@ import org.apache.calcite.tools.RuleSets;
 import org.apache.drill.exec.ops.OptimizerRulesContext;
 import org.apache.drill.exec.planner.logical.DrillAggregateRule;
 import org.apache.drill.exec.planner.logical.DrillFilterAggregateTransposeRule;
+import org.apache.drill.exec.planner.logical.DrillFilterItemStarReWriterRule;
 import org.apache.drill.exec.planner.logical.DrillFilterJoinRules;
 import org.apache.drill.exec.planner.logical.DrillFilterRule;
 import org.apache.drill.exec.planner.logical.DrillJoinRel;
@@ -41,7 +42,6 @@ import org.apache.drill.exec.planner.logical.DrillPushLimitToScanRule;
 import org.apache.drill.exec.planner.logical.DrillPushProjectIntoScanRule;
 import org.apache.drill.exec.planner.logical.DrillPushProjectPastFilterRule;
 import org.apache.drill.exec.planner.logical.DrillPushProjectPastJoinRule;
-import org.apache.drill.exec.planner.logical.DrillFilterItemStarReWriterRule;
 import org.apache.drill.exec.planner.logical.DrillReduceAggregatesRule;
 import org.apache.drill.exec.planner.logical.DrillReduceExpressionsRule;
 import org.apache.drill.exec.planner.logical.DrillRelFactories;
@@ -80,6 +80,7 @@ import org.apache.drill.exec.store.parquet.ParquetPushDownFilter;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
+import java.util.Set;
 
 /**
  * Returns RuleSet for concrete planner phase.
@@ -227,7 +228,7 @@ public enum PlannerPhase {
 
     // This list is used to store rules that can be turned on an off
     // by user facing planning options
-    final Builder<RelOptRule> userConfigurableRules = ImmutableSet.<RelOptRule>builder();
+    final Builder<RelOptRule> userConfigurableRules = ImmutableSet.builder();
 
     if (ps.isConstantFoldingEnabled()) {
       // TODO - DRILL-2218
@@ -337,8 +338,8 @@ public enum PlannerPhase {
    */
   static RuleSet getPruneScanRules(OptimizerRulesContext optimizerRulesContext) {
     final ImmutableSet<RelOptRule> pruneRules = ImmutableSet.<RelOptRule>builder()
+        .addAll(getItemStarRules())
         .add(
-            DrillFilterItemStarReWriterRule.INSTANCE,
             PruneScanRule.getDirFilterOnProject(optimizerRulesContext),
             PruneScanRule.getDirFilterOnScan(optimizerRulesContext),
             ParquetPruneScanRule.getFilterOnProjectParquet(optimizerRulesContext),
@@ -373,14 +374,15 @@ public enum PlannerPhase {
   }
 
   /**
-   *  Get an immutable list of directory-based partition pruing rules that will be used in
Calcite logical planning.
-   * @param optimizerRulesContext
-   * @return
+   *  Get an immutable list of directory-based partition pruning rules that will be used
in Calcite logical planning.
+   *
+   * @param optimizerRulesContext rules context
+   * @return directory-based partition pruning rules
    */
   static RuleSet getDirPruneScanRules(OptimizerRulesContext optimizerRulesContext) {
-    final ImmutableSet<RelOptRule> pruneRules = ImmutableSet.<RelOptRule>builder()
+    final Set<RelOptRule> pruneRules = ImmutableSet.<RelOptRule>builder()
+        .addAll(getItemStarRules())
         .add(
-            DrillFilterItemStarReWriterRule.INSTANCE,
             PruneScanRule.getDirFilterOnProject(optimizerRulesContext),
             PruneScanRule.getDirFilterOnScan(optimizerRulesContext)
         )
@@ -402,7 +404,7 @@ public enum PlannerPhase {
       ProjectPrule.INSTANCE
     ));
 
-  static final RuleSet getPhysicalRules(OptimizerRulesContext optimizerRulesContext) {
+  static RuleSet getPhysicalRules(OptimizerRulesContext optimizerRulesContext) {
     final List<RelOptRule> ruleList = new ArrayList<>();
     final PlannerSettings ps = optimizerRulesContext.getPlannerSettings();
 
@@ -472,4 +474,17 @@ public enum PlannerPhase {
     return RuleSets.ofList(relOptRuleSetBuilder.build());
   }
 
+  /**
+   * @return collection of rules to re-write item star operator for filter push down and
partition pruning
+   */
+  private static ImmutableSet<RelOptRule> getItemStarRules() {
+    return ImmutableSet.<RelOptRule>builder()
+       .add(
+             DrillFilterItemStarReWriterRule.PROJECT_ON_SCAN,
+             DrillFilterItemStarReWriterRule.FILTER_ON_SCAN,
+             DrillFilterItemStarReWriterRule.FILTER_PROJECT_SCAN
+       ).build();
+  }
+
+
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/ddb73a50/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterItemStarReWriterRule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterItemStarReWriterRule.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterItemStarReWriterRule.java
index 84ae674..27f8c49 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterItemStarReWriterRule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterItemStarReWriterRule.java
@@ -17,21 +17,10 @@
  */
 package org.apache.drill.exec.planner.logical;
 
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import org.apache.calcite.adapter.enumerable.EnumerableTableScan;
 import org.apache.calcite.plan.RelOptRule;
 import org.apache.calcite.plan.RelOptRuleCall;
 import org.apache.calcite.plan.RelOptRuleOperand;
-import org.apache.calcite.plan.RelOptTable;
-import org.apache.calcite.prepare.RelOptTableImpl;
-import org.apache.calcite.rel.RelNode;
-import org.apache.calcite.rel.core.CorrelationId;
-import org.apache.calcite.rel.core.Filter;
-import org.apache.calcite.rel.core.Project;
-import org.apache.calcite.rel.core.TableScan;
-import org.apache.calcite.rel.logical.LogicalFilter;
-import org.apache.calcite.rel.logical.LogicalProject;
+import org.apache.calcite.rel.rules.ProjectRemoveRule;
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.rel.type.RelDataTypeField;
@@ -39,10 +28,11 @@ import org.apache.calcite.rex.RexCall;
 import org.apache.calcite.rex.RexInputRef;
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.rex.RexVisitorImpl;
-import org.apache.calcite.schema.Table;
+import org.apache.drill.common.expression.PathSegment;
+import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.exec.planner.types.RelDataTypeDrillImpl;
 import org.apache.drill.exec.planner.types.RelDataTypeHolder;
-import org.apache.drill.exec.util.Utilities;
+import org.apache.drill.exec.store.parquet.ParquetGroupScan;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -54,83 +44,189 @@ import static org.apache.drill.exec.planner.logical.FieldsReWriterUtil.DesiredFi
 import static org.apache.drill.exec.planner.logical.FieldsReWriterUtil.FieldsReWriter;
 
 /**
- * Rule will transform filter -> project -> scan call with item star fields in filter
- * into project -> filter -> project -> scan where item star fields are pushed into
scan
- * and replaced with actual field references.
+ * Rule will transform item star fields in filter and replaced with actual field references.
  *
  * This will help partition pruning and push down rules to detect fields that can be pruned
or push downed.
  * Item star operator appears when sub-select or cte with star are used as source.
  */
-public class DrillFilterItemStarReWriterRule extends RelOptRule {
+public class DrillFilterItemStarReWriterRule {
 
-  public static final DrillFilterItemStarReWriterRule INSTANCE = new DrillFilterItemStarReWriterRule(
-      RelOptHelper.some(Filter.class, RelOptHelper.some(Project.class, RelOptHelper.any(
TableScan.class))),
-      "DrillFilterItemStarReWriterRule");
+  public static final ProjectOnScan PROJECT_ON_SCAN = new ProjectOnScan(
+          RelOptHelper.some(DrillProjectRel.class, RelOptHelper.any(DrillScanRel.class)),
+          "DrillFilterItemStarReWriterRule.ProjectOnScan");
 
-  private DrillFilterItemStarReWriterRule(RelOptRuleOperand operand, String id) {
-    super(operand, id);
-  }
+  public static final FilterOnScan FILTER_ON_SCAN = new FilterOnScan(
+      RelOptHelper.some(DrillFilterRel.class, RelOptHelper.any(DrillScanRel.class)),
+      "DrillFilterItemStarReWriterRule.FilterOnScan");
 
-  @Override
-  public void onMatch(RelOptRuleCall call) {
-    Filter filterRel = call.rel(0);
-    Project projectRel = call.rel(1);
-    TableScan scanRel = call.rel(2);
+  public static final FilterProjectScan FILTER_PROJECT_SCAN = new FilterProjectScan(
+      RelOptHelper.some(DrillFilterRel.class, RelOptHelper.some(DrillProjectRel.class, RelOptHelper.any(DrillScanRel.class))),
+      "DrillFilterItemStarReWriterRule.FilterProjectScan");
 
-    ItemStarFieldsVisitor itemStarFieldsVisitor = new ItemStarFieldsVisitor(filterRel.getRowType().getFieldNames());
-    filterRel.getCondition().accept(itemStarFieldsVisitor);
 
-    // there are no item fields, no need to proceed further
-    if (!itemStarFieldsVisitor.hasItemStarFields()) {
-      return;
+  private static class ProjectOnScan extends RelOptRule {
+
+    ProjectOnScan(RelOptRuleOperand operand, String id) {
+      super(operand, id);
     }
 
-    Map<String, DesiredField> itemStarFields = itemStarFieldsVisitor.getItemStarFields();
+    @Override
+    public boolean matches(RelOptRuleCall call) {
+      DrillScanRel scan = call.rel(1);
+      return scan.getGroupScan() instanceof ParquetGroupScan && super.matches(call);
+    }
+
+    @Override
+    public void onMatch(RelOptRuleCall call) {
+      DrillProjectRel projectRel = call.rel(0);
+      DrillScanRel scanRel = call.rel(1);
+
+      ItemStarFieldsVisitor itemStarFieldsVisitor = new ItemStarFieldsVisitor(scanRel.getRowType().getFieldNames());
+      List<RexNode> projects = projectRel.getProjects();
+      for (RexNode project : projects) {
+        project.accept(itemStarFieldsVisitor);
+      }
 
-    // create new scan
-    RelNode newScan = constructNewScan(scanRel, itemStarFields.keySet());
+      // if there are no item fields, no need to proceed further
+      if (itemStarFieldsVisitor.hasNoItemStarFields()) {
+        return;
+      }
 
-    // combine original and new projects
-    List<RexNode> newProjects = new ArrayList<>(projectRel.getProjects());
+      Map<String, DesiredField> itemStarFields = itemStarFieldsVisitor.getItemStarFields();
 
-    // prepare node mapper to replace item star calls with new input field references
-    Map<RexNode, Integer> fieldMapper = new HashMap<>();
+      DrillScanRel newScan = createNewScan(scanRel, itemStarFields);
 
-    // since scan might have already some fields, new field reference index should start
from the last used in scan
-    // NB: field reference index starts from 0 thus original field count can be taken as
starting index
-    int index = scanRel.getRowType().getFieldCount();
+      // re-write projects
+      Map<RexNode, Integer> fieldMapper = createFieldMapper(itemStarFields.values(),
scanRel.getRowType().getFieldCount());
+      FieldsReWriter fieldsReWriter = new FieldsReWriter(fieldMapper);
+      List<RexNode> newProjects = new ArrayList<>();
+      for (RexNode node : projectRel.getChildExps()) {
+        newProjects.add(node.accept(fieldsReWriter));
+      }
 
-    for (DesiredField desiredField : itemStarFields.values()) {
-      RexInputRef inputRef = new RexInputRef(index, desiredField.getType());
-      // add references to item star fields in new project
-      newProjects.add(inputRef);
-      for (RexNode node : desiredField.getNodes()) {
-        // if field is referenced in more then one call, add each call to field mapper
-        fieldMapper.put(node, index);
+      DrillProjectRel newProject = new DrillProjectRel(
+          projectRel.getCluster(),
+          projectRel.getTraitSet(),
+          newScan,
+          newProjects,
+          projectRel.getRowType());
+
+      if (ProjectRemoveRule.isTrivial(newProject)) {
+        call.transformTo(newScan);
+      } else {
+        call.transformTo(newProject);
       }
-      // increment index for the next node reference
-      index++;
     }
 
-    // create new project row type
-    RelDataType newProjectRowType = getNewRowType(
-        projectRel.getCluster().getTypeFactory(),
-        projectRel.getRowType().getFieldList(),
-        itemStarFields.keySet());
+  }
+
+  private static class FilterOnScan extends RelOptRule {
+
+    FilterOnScan(RelOptRuleOperand operand, String id) {
+      super(operand, id);
+    }
+
+    @Override
+    public boolean matches(RelOptRuleCall call) {
+      DrillScanRel scan = call.rel(1);
+      return scan.getGroupScan() instanceof ParquetGroupScan && super.matches(call);
+    }
+
+    @Override
+    public void onMatch(RelOptRuleCall call) {
+      DrillFilterRel filterRel = call.rel(0);
+      DrillScanRel scanRel = call.rel(1);
+      transformFilterCall(filterRel, null, scanRel, call);
+    }
+  }
+
+  private static class FilterProjectScan extends RelOptRule {
+
+    FilterProjectScan(RelOptRuleOperand operand, String id) {
+      super(operand, id);
+    }
+
+    @Override
+    public boolean matches(RelOptRuleCall call) {
+      DrillScanRel scan = call.rel(2);
+      return scan.getGroupScan() instanceof ParquetGroupScan && super.matches(call);
+    }
+
+    @Override
+    public void onMatch(RelOptRuleCall call) {
+      DrillFilterRel filterRel = call.rel(0);
+      DrillProjectRel projectRel = call.rel(1);
+      DrillScanRel scanRel = call.rel(2);
+      transformFilterCall(filterRel, projectRel, scanRel, call);
+    }
+  }
+
+
+  /**
+   * Removes item star call from filter expression and propagates changes into project (if
present) and scan.
+   *
+   * @param filterRel original filter expression
+   * @param projectRel original project expression
+   * @param scanRel original scan expression
+   * @param call original rule call
+   */
+  private static void transformFilterCall(DrillFilterRel filterRel, DrillProjectRel projectRel,
DrillScanRel scanRel, RelOptRuleCall call) {
+    List<String> fieldNames = projectRel == null ? scanRel.getRowType().getFieldNames()
: projectRel.getRowType().getFieldNames();
+    ItemStarFieldsVisitor itemStarFieldsVisitor = new ItemStarFieldsVisitor(fieldNames);
+    filterRel.getCondition().accept(itemStarFieldsVisitor);
+
+    // if there are no item fields, no need to proceed further
+    if (itemStarFieldsVisitor.hasNoItemStarFields()) {
+      return;
+    }
+
+    Map<String, DesiredField> itemStarFields = itemStarFieldsVisitor.getItemStarFields();
+
+    DrillScanRel newScan = createNewScan(scanRel, itemStarFields);
+
+    // create new project if was present in call
+    DrillProjectRel newProject = null;
+    if (projectRel != null) {
 
-    // create new project
-    RelNode newProject = new LogicalProject(projectRel.getCluster(), projectRel.getTraitSet(),
newScan, newProjects, newProjectRowType);
+      // add new projects to the already existing in original project
+      int projectIndex = scanRel.getRowType().getFieldCount();
+      List<RexNode> newProjects = new ArrayList<>(projectRel.getProjects());
+      for (DesiredField desiredField : itemStarFields.values()) {
+        newProjects.add(new RexInputRef(projectIndex, desiredField.getType()));
+        projectIndex++;
+      }
+
+      RelDataType newProjectRowType = createNewRowType(
+          projectRel.getCluster().getTypeFactory(),
+          projectRel.getRowType().getFieldList(),
+          itemStarFields.keySet());
+
+      newProject = new DrillProjectRel(
+          projectRel.getCluster(),
+          projectRel.getTraitSet(),
+          newScan,
+          newProjects,
+          newProjectRowType);
+    }
 
     // transform filter condition
+    Map<RexNode, Integer> fieldMapper = createFieldMapper(itemStarFields.values(),
scanRel.getRowType().getFieldCount());
     FieldsReWriter fieldsReWriter = new FieldsReWriter(fieldMapper);
     RexNode newCondition = filterRel.getCondition().accept(fieldsReWriter);
 
     // create new filter
-    RelNode newFilter = new LogicalFilter(filterRel.getCluster(), filterRel.getTraitSet(),
newProject, newCondition, ImmutableSet.<CorrelationId>of());
+    DrillFilterRel newFilter = DrillFilterRel.create(newProject != null ? newProject : newScan,
newCondition);
 
     // wrap with project to have the same row type as before
-    Project wrapper = projectRel.copy(projectRel.getTraitSet(), newFilter, projectRel.getProjects(),
projectRel.getRowType());
+    List<RexNode> newProjects = new ArrayList<>();
+    RelDataType rowType = filterRel.getRowType();
+    List<RelDataTypeField> fieldList = rowType.getFieldList();
+    for (RelDataTypeField field : fieldList) {
+      RexInputRef inputRef = new RexInputRef(field.getIndex(), field.getType());
+      newProjects.add(inputRef);
+    }
 
+    DrillProjectRel wrapper = new DrillProjectRel(filterRel.getCluster(), filterRel.getTraitSet(),
newFilter, newProjects, filterRel.getRowType());
     call.transformTo(wrapper);
   }
 
@@ -142,9 +238,9 @@ public class DrillFilterItemStarReWriterRule extends RelOptRule {
    * @param newFields new fields
    * @return new row type with original and new fields
    */
-  private RelDataType getNewRowType(RelDataTypeFactory typeFactory,
-                                    List<RelDataTypeField> originalFields,
-                                    Collection<String> newFields) {
+  private static RelDataType createNewRowType(RelDataTypeFactory typeFactory,
+                                              List<RelDataTypeField> originalFields,
+                                              Collection<String> newFields) {
     RelDataTypeHolder relDataTypeHolder = new RelDataTypeHolder();
 
     // add original fields
@@ -161,30 +257,55 @@ public class DrillFilterItemStarReWriterRule extends RelOptRule {
   }
 
   /**
-   * Constructs new scan based on the original scan.
-   * Preserves all original fields and add new fields.
+   * Creates new scan with fields from original scan and fields used in item star operator.
    *
-   * @param scanRel original scan
-   * @param newFields new fields
-   * @return new scan with original and new fields
+   * @param scanRel original scan expression
+   * @param itemStarFields item star fields
+   * @return new scan expression
+   */
+  private static DrillScanRel createNewScan(DrillScanRel scanRel, Map<String, DesiredField>
itemStarFields) {
+    RelDataType newScanRowType = createNewRowType(
+            scanRel.getCluster().getTypeFactory(),
+            scanRel.getRowType().getFieldList(),
+            itemStarFields.keySet());
+
+    List<SchemaPath> columns = new ArrayList<>(scanRel.getColumns());
+    for (DesiredField desiredField : itemStarFields.values()) {
+      String name = desiredField.getName();
+      PathSegment.NameSegment nameSegment = new PathSegment.NameSegment(name);
+      columns.add(new SchemaPath(nameSegment));
+    }
+
+    return new DrillScanRel(
+            scanRel.getCluster(),
+            scanRel.getTraitSet().plus(DrillRel.DRILL_LOGICAL),
+            scanRel.getTable(),
+            newScanRowType,
+            columns);
+  }
+
+  /**
+   * Creates node mapper to replace item star calls with new input field references.
+   * Starting index should be calculated from the last used input expression (i.e. scan expression).
+   * NB: field reference index starts from 0 thus original field count can be taken as starting
index
+   *
+   * @param desiredFields list of desired fields
+   * @param startingIndex starting index
+   * @return field mapper
    */
-  private RelNode constructNewScan(TableScan scanRel, Collection<String> newFields)
{
-    // create new scan row type
-    RelDataType newScanRowType = getNewRowType(
-        scanRel.getCluster().getTypeFactory(),
-        scanRel.getRowType().getFieldList(),
-        newFields);
-
-    // create new scan
-    RelOptTable table = scanRel.getTable();
-    Class elementType = EnumerableTableScan.deduceElementType(table.unwrap(Table.class));
-
-    DrillTable unwrap = Utilities.getDrillTable(table);
-    DrillTranslatableTable newTable = new DrillTranslatableTable(
-        new DynamicDrillTable(unwrap.getPlugin(), unwrap.getStorageEngineName(), unwrap.getUserName(),
unwrap.getSelection()));
-    RelOptTableImpl newOptTableImpl = RelOptTableImpl.create(table.getRelOptSchema(), newScanRowType,
newTable, ImmutableList.<String>of());
-
-    return new EnumerableTableScan(scanRel.getCluster(), scanRel.getTraitSet(), newOptTableImpl,
elementType);
+  private static Map<RexNode, Integer> createFieldMapper(Collection<DesiredField>
desiredFields, int startingIndex) {
+    Map<RexNode, Integer> fieldMapper = new HashMap<>();
+
+    int index = startingIndex;
+    for (DesiredField desiredField : desiredFields) {
+      for (RexNode node : desiredField.getNodes()) {
+        // if field is referenced in more then one call, add each call to field mapper
+        fieldMapper.put(node, index);
+      }
+      // increment index for the next node reference
+      index++;
+    }
+    return fieldMapper;
   }
 
   /**
@@ -192,7 +313,7 @@ public class DrillFilterItemStarReWriterRule extends RelOptRule {
    * For the fields with the same name, stores original calls in a list, does not duplicate
fields.
    * Holds state, should not be re-used.
    */
-  private class ItemStarFieldsVisitor extends RexVisitorImpl<RexNode> {
+  private static class ItemStarFieldsVisitor extends RexVisitorImpl<RexNode> {
 
     private final Map<String, DesiredField> itemStarFields = new HashMap<>();
     private final List<String> fieldNames;
@@ -202,8 +323,8 @@ public class DrillFilterItemStarReWriterRule extends RelOptRule {
       this.fieldNames = fieldNames;
     }
 
-    boolean hasItemStarFields() {
-      return !itemStarFields.isEmpty();
+    boolean hasNoItemStarFields() {
+      return itemStarFields.isEmpty();
     }
 
     Map<String, DesiredField> getItemStarFields() {

http://git-wip-us.apache.org/repos/asf/drill/blob/ddb73a50/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestPushDownAndPruningWithItemStar.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestPushDownAndPruningWithItemStar.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestPushDownAndPruningWithItemStar.java
index 24b9212..6ac08ee 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestPushDownAndPruningWithItemStar.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestPushDownAndPruningWithItemStar.java
@@ -44,7 +44,7 @@ public class TestPushDownAndPruningWithItemStar extends PlanTestBase {
   }
 
   @Test
-  public void testPushProjectIntoScan() throws Exception {
+  public void testPushProjectIntoScanWithGroupByClause() throws Exception {
     String query = String.format("select o_orderdate, count(*) from (select * from `%s`.`%s`)
group by o_orderdate", DFS_TMP_SCHEMA, TABLE_NAME);
 
     String[] expectedPlan = {"numFiles=3, numRowGroups=3, usedMetadataFile=false, columns=\\[`o_orderdate`\\]"};
@@ -79,7 +79,7 @@ public class TestPushDownAndPruningWithItemStar extends PlanTestBase {
   public void testPushProjectIntoScanWithExpressionInFilter() throws Exception {
     String query = String.format("select o_orderdate from (select * from `%s`.`%s`) where
o_custkey + o_orderkey < 5", DFS_TMP_SCHEMA, TABLE_NAME);
 
-    String[] expectedPlan = {"numFiles=3, numRowGroups=3, usedMetadataFile=false, columns=\\[`o_custkey`,
`o_orderkey`, `o_orderdate`\\]"};
+    String[] expectedPlan = {"numFiles=3, numRowGroups=3, usedMetadataFile=false, columns=\\[`o_orderdate`,
`o_custkey`, `o_orderkey`\\]"};
     String[] excludedPlan = {};
 
     PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan);
@@ -116,7 +116,7 @@ public class TestPushDownAndPruningWithItemStar extends PlanTestBase {
       "where t.user_info.cust_id > 28 and t.user_info.device = 'IOS5' and t.marketing_info.camp_id
> 5 and t.marketing_info.keywords[2] is not null";
 
     String[] expectedPlan = {"numFiles=1, numRowGroups=1, usedMetadataFile=false, " +
-      "columns=\\[`user_info`.`cust_id`, `user_info`.`device`, `marketing_info`.`camp_id`,
`marketing_info`.`keywords`\\[2\\], `trans_id`\\]"};
+      "columns=\\[`trans_id`, `user_info`.`cust_id`, `user_info`.`device`, `marketing_info`.`camp_id`,
`marketing_info`.`keywords`\\[2\\]\\]"};
     String[] excludedPlan = {};
 
     PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan);
@@ -131,10 +131,43 @@ public class TestPushDownAndPruningWithItemStar extends PlanTestBase
{
   }
 
   @Test
+  public void testProjectIntoScanWithNestedStarSubQuery() throws Exception {
+    String query = String.format("select *, o_orderdate from (select * from `%s`.`%s`)",
DFS_TMP_SCHEMA, TABLE_NAME);
+
+    String[] expectedPlan = {"numFiles=3, numRowGroups=3, usedMetadataFile=false, columns=\\[`\\*\\*`,
`o_orderdate`\\]"};
+    String[] excludedPlan = {};
+
+    PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan);
+
+    testBuilder()
+        .sqlQuery(query)
+        .unOrdered()
+        .sqlBaselineQuery("select *, o_orderdate from `%s`.`%s`", DFS_TMP_SCHEMA, TABLE_NAME)
+        .build();
+  }
+
+  @Test
+  public void testProjectIntoScanWithSeveralNestedStarSubQueries() throws Exception {
+    String subQuery = String.format("select * from `%s`.`%s`", DFS_TMP_SCHEMA, TABLE_NAME);
+    String query = String.format("select o_custkey + o_orderkey from (select * from (select
* from (%s)))", subQuery);
+
+    String[] expectedPlan = {"numFiles=3, numRowGroups=3, usedMetadataFile=false, columns=\\[`o_custkey`,
`o_orderkey`\\]"};
+    String[] excludedPlan = {};
+
+    PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan);
+
+    testBuilder()
+        .sqlQuery(query)
+        .unOrdered()
+        .sqlBaselineQuery("select o_custkey + o_orderkey from `%s`.`%s`", DFS_TMP_SCHEMA,
TABLE_NAME)
+        .build();
+  }
+
+  @Test
   public void testDirectoryPruning() throws Exception {
     String query = String.format("select * from (select * from `%s`.`%s`) where dir0 = 't1'",
DFS_TMP_SCHEMA, TABLE_NAME);
 
-    String[] expectedPlan = {"numFiles=1, numRowGroups=1, usedMetadataFile=false, columns=\\[`\\*\\*`\\]"};
+    String[] expectedPlan = {"numFiles=1, numRowGroups=1, usedMetadataFile=false, columns=\\[`\\*\\*`,
`dir0`\\]"};
     String[] excludedPlan = {};
 
     PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan);
@@ -147,10 +180,45 @@ public class TestPushDownAndPruningWithItemStar extends PlanTestBase
{
   }
 
   @Test
+  public void testDirectoryPruningWithNestedStarSubQuery() throws Exception {
+    String subQuery = String.format("select * from `%s`.`%s`", DFS_TMP_SCHEMA, TABLE_NAME);
+    String query = String.format("select * from (select * from (select * from (%s))) where
dir0 = 't1'", subQuery);
+
+    String[] expectedPlan = {"numFiles=1, numRowGroups=1, usedMetadataFile=false, columns=\\[`\\*\\*`,
`dir0`\\]"};
+    String[] excludedPlan = {};
+
+    PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan);
+
+    testBuilder()
+        .sqlQuery(query)
+        .unOrdered()
+        .sqlBaselineQuery("select * from `%s`.`%s` where dir0 = 't1'", DFS_TMP_SCHEMA, TABLE_NAME)
+        .build();
+  }
+
+    @Test
+    public void testDirectoryPruningWithNestedStarSubQueryAndAdditionalColumns() throws Exception
{
+      String subQuery = String.format("select * from `%s`.`%s`", DFS_TMP_SCHEMA, TABLE_NAME);
+      String query = String.format("select * from (select * from (select *, `o_orderdate`
from (%s))) where dir0 = 't1'", subQuery);
+
+      String[] expectedPlan = {"numFiles=1, numRowGroups=1, usedMetadataFile=false, columns=\\[`\\*\\*`,
`o_orderdate`, `dir0`\\]"};
+      String[] excludedPlan = {};
+
+      PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan);
+
+      testBuilder()
+         .sqlQuery(query)
+         .unOrdered()
+         .sqlBaselineQuery("select * from `%s`.`%s` where dir0 = 't1'", DFS_TMP_SCHEMA, TABLE_NAME)
+         .build();
+    }
+
+
+    @Test
   public void testFilterPushDownSingleCondition() throws Exception {
     String query = String.format("select * from (select * from `%s`.`%s`) where o_orderdate
= date '1992-01-01'", DFS_TMP_SCHEMA, TABLE_NAME);
 
-    String[] expectedPlan = {"numFiles=1, numRowGroups=1, usedMetadataFile=false, columns=\\[`\\*\\*`\\]"};
+    String[] expectedPlan = {"numFiles=1, numRowGroups=1, usedMetadataFile=false, columns=\\[`\\*\\*`,
`o_orderdate`\\]"};
     String[] excludedPlan = {};
 
     PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan);
@@ -167,7 +235,7 @@ public class TestPushDownAndPruningWithItemStar extends PlanTestBase {
     String query = String.format("select * from (select * from `%s`.`%s`) where o_orderdate
= date '1992-01-01' or o_orderdate = date '1992-01-09'",
         DFS_TMP_SCHEMA, TABLE_NAME);
 
-    String[] expectedPlan = {"numFiles=2, numRowGroups=2, usedMetadataFile=false, columns=\\[`\\*\\*`\\]"};
+    String[] expectedPlan = {"numFiles=2, numRowGroups=2, usedMetadataFile=false, columns=\\[`\\*\\*`,
`o_orderdate`\\]"};
     String[] excludedPlan = {};
 
     PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan);
@@ -180,4 +248,38 @@ public class TestPushDownAndPruningWithItemStar extends PlanTestBase
{
         .build();
   }
 
+  @Test
+  public void testFilterPushDownWithSeveralNestedStarSubQueries() throws Exception {
+    String subQuery = String.format("select * from `%s`.`%s`", DFS_TMP_SCHEMA, TABLE_NAME);
+    String query = String.format("select * from (select * from (select * from (%s))) where
o_orderdate = date '1992-01-01'", subQuery);
+
+    String[] expectedPlan = {"numFiles=1, numRowGroups=1, usedMetadataFile=false, columns=\\[`\\*\\*`,
`o_orderdate`\\]"};
+    String[] excludedPlan = {};
+
+    PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan);
+
+    testBuilder()
+        .sqlQuery(query)
+        .unOrdered()
+        .sqlBaselineQuery("select * from `%s`.`%s` where o_orderdate = date '1992-01-01'",
DFS_TMP_SCHEMA, TABLE_NAME)
+        .build();
+  }
+
+  @Test
+  public void testFilterPushDownWithSeveralNestedStarSubQueriesWithAdditionalColumns() throws
Exception {
+    String subQuery = String.format("select * from `%s`.`%s`", DFS_TMP_SCHEMA, TABLE_NAME);
+    String query = String.format("select * from (select * from (select *, o_custkey from
(%s))) where o_orderdate = date '1992-01-01'", subQuery);
+
+    String[] expectedPlan = {"numFiles=1, numRowGroups=1, usedMetadataFile=false, columns=\\[`\\*\\*`,
`o_custkey`, `o_orderdate`\\]"};
+    String[] excludedPlan = {};
+
+    PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan);
+
+    testBuilder()
+        .sqlQuery(query)
+        .unOrdered()
+        .sqlBaselineQuery("select *, o_custkey from `%s`.`%s` where o_orderdate = date '1992-01-01'",
DFS_TMP_SCHEMA, TABLE_NAME)
+        .build();
+  }
+
 }


Mime
View raw message