DRILL-6199: Add support for filter push down and partition pruning with several nested star sub-queries closes #1152 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/ddb73a50 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/ddb73a50 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/ddb73a50 Branch: refs/heads/master Commit: ddb73a506c6cbb6492d6ff91ca174dbff1b8a045 Parents: cdc21ce Author: Arina Ielchiieva Authored: Sun Mar 4 22:12:06 2018 +0200 Committer: Vitalii Diravka Committed: Tue Mar 20 22:40:00 2018 +0200 ---------------------------------------------------------------------- .../apache/drill/exec/planner/PlannerPhase.java | 33 +- .../DrillFilterItemStarReWriterRule.java | 301 +++++++++++++------ .../TestPushDownAndPruningWithItemStar.java | 114 ++++++- 3 files changed, 343 insertions(+), 105 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/ddb73a50/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java index f46a7ee..f71c281 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PlannerPhase.java @@ -29,6 +29,7 @@ import org.apache.calcite.tools.RuleSets; import org.apache.drill.exec.ops.OptimizerRulesContext; import org.apache.drill.exec.planner.logical.DrillAggregateRule; import org.apache.drill.exec.planner.logical.DrillFilterAggregateTransposeRule; +import org.apache.drill.exec.planner.logical.DrillFilterItemStarReWriterRule; import org.apache.drill.exec.planner.logical.DrillFilterJoinRules; import org.apache.drill.exec.planner.logical.DrillFilterRule; import org.apache.drill.exec.planner.logical.DrillJoinRel; @@ -41,7 +42,6 @@ import org.apache.drill.exec.planner.logical.DrillPushLimitToScanRule; import org.apache.drill.exec.planner.logical.DrillPushProjectIntoScanRule; import org.apache.drill.exec.planner.logical.DrillPushProjectPastFilterRule; import org.apache.drill.exec.planner.logical.DrillPushProjectPastJoinRule; -import org.apache.drill.exec.planner.logical.DrillFilterItemStarReWriterRule; import org.apache.drill.exec.planner.logical.DrillReduceAggregatesRule; import org.apache.drill.exec.planner.logical.DrillReduceExpressionsRule; import org.apache.drill.exec.planner.logical.DrillRelFactories; @@ -80,6 +80,7 @@ import org.apache.drill.exec.store.parquet.ParquetPushDownFilter; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Set; /** * Returns RuleSet for concrete planner phase. @@ -227,7 +228,7 @@ public enum PlannerPhase { // This list is used to store rules that can be turned on an off // by user facing planning options - final Builder userConfigurableRules = ImmutableSet.builder(); + final Builder userConfigurableRules = ImmutableSet.builder(); if (ps.isConstantFoldingEnabled()) { // TODO - DRILL-2218 @@ -337,8 +338,8 @@ public enum PlannerPhase { */ static RuleSet getPruneScanRules(OptimizerRulesContext optimizerRulesContext) { final ImmutableSet pruneRules = ImmutableSet.builder() + .addAll(getItemStarRules()) .add( - DrillFilterItemStarReWriterRule.INSTANCE, PruneScanRule.getDirFilterOnProject(optimizerRulesContext), PruneScanRule.getDirFilterOnScan(optimizerRulesContext), ParquetPruneScanRule.getFilterOnProjectParquet(optimizerRulesContext), @@ -373,14 +374,15 @@ public enum PlannerPhase { } /** - * Get an immutable list of directory-based partition pruing rules that will be used in Calcite logical planning. - * @param optimizerRulesContext - * @return + * Get an immutable list of directory-based partition pruning rules that will be used in Calcite logical planning. + * + * @param optimizerRulesContext rules context + * @return directory-based partition pruning rules */ static RuleSet getDirPruneScanRules(OptimizerRulesContext optimizerRulesContext) { - final ImmutableSet pruneRules = ImmutableSet.builder() + final Set pruneRules = ImmutableSet.builder() + .addAll(getItemStarRules()) .add( - DrillFilterItemStarReWriterRule.INSTANCE, PruneScanRule.getDirFilterOnProject(optimizerRulesContext), PruneScanRule.getDirFilterOnScan(optimizerRulesContext) ) @@ -402,7 +404,7 @@ public enum PlannerPhase { ProjectPrule.INSTANCE )); - static final RuleSet getPhysicalRules(OptimizerRulesContext optimizerRulesContext) { + static RuleSet getPhysicalRules(OptimizerRulesContext optimizerRulesContext) { final List ruleList = new ArrayList<>(); final PlannerSettings ps = optimizerRulesContext.getPlannerSettings(); @@ -472,4 +474,17 @@ public enum PlannerPhase { return RuleSets.ofList(relOptRuleSetBuilder.build()); } + /** + * @return collection of rules to re-write item star operator for filter push down and partition pruning + */ + private static ImmutableSet getItemStarRules() { + return ImmutableSet.builder() + .add( + DrillFilterItemStarReWriterRule.PROJECT_ON_SCAN, + DrillFilterItemStarReWriterRule.FILTER_ON_SCAN, + DrillFilterItemStarReWriterRule.FILTER_PROJECT_SCAN + ).build(); + } + + } http://git-wip-us.apache.org/repos/asf/drill/blob/ddb73a50/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterItemStarReWriterRule.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterItemStarReWriterRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterItemStarReWriterRule.java index 84ae674..27f8c49 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterItemStarReWriterRule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillFilterItemStarReWriterRule.java @@ -17,21 +17,10 @@ */ package org.apache.drill.exec.planner.logical; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import org.apache.calcite.adapter.enumerable.EnumerableTableScan; import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelOptRuleOperand; -import org.apache.calcite.plan.RelOptTable; -import org.apache.calcite.prepare.RelOptTableImpl; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.CorrelationId; -import org.apache.calcite.rel.core.Filter; -import org.apache.calcite.rel.core.Project; -import org.apache.calcite.rel.core.TableScan; -import org.apache.calcite.rel.logical.LogicalFilter; -import org.apache.calcite.rel.logical.LogicalProject; +import org.apache.calcite.rel.rules.ProjectRemoveRule; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeField; @@ -39,10 +28,11 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexVisitorImpl; -import org.apache.calcite.schema.Table; +import org.apache.drill.common.expression.PathSegment; +import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.exec.planner.types.RelDataTypeDrillImpl; import org.apache.drill.exec.planner.types.RelDataTypeHolder; -import org.apache.drill.exec.util.Utilities; +import org.apache.drill.exec.store.parquet.ParquetGroupScan; import java.util.ArrayList; import java.util.Collection; @@ -54,83 +44,189 @@ import static org.apache.drill.exec.planner.logical.FieldsReWriterUtil.DesiredFi import static org.apache.drill.exec.planner.logical.FieldsReWriterUtil.FieldsReWriter; /** - * Rule will transform filter -> project -> scan call with item star fields in filter - * into project -> filter -> project -> scan where item star fields are pushed into scan - * and replaced with actual field references. + * Rule will transform item star fields in filter and replaced with actual field references. * * This will help partition pruning and push down rules to detect fields that can be pruned or push downed. * Item star operator appears when sub-select or cte with star are used as source. */ -public class DrillFilterItemStarReWriterRule extends RelOptRule { +public class DrillFilterItemStarReWriterRule { - public static final DrillFilterItemStarReWriterRule INSTANCE = new DrillFilterItemStarReWriterRule( - RelOptHelper.some(Filter.class, RelOptHelper.some(Project.class, RelOptHelper.any( TableScan.class))), - "DrillFilterItemStarReWriterRule"); + public static final ProjectOnScan PROJECT_ON_SCAN = new ProjectOnScan( + RelOptHelper.some(DrillProjectRel.class, RelOptHelper.any(DrillScanRel.class)), + "DrillFilterItemStarReWriterRule.ProjectOnScan"); - private DrillFilterItemStarReWriterRule(RelOptRuleOperand operand, String id) { - super(operand, id); - } + public static final FilterOnScan FILTER_ON_SCAN = new FilterOnScan( + RelOptHelper.some(DrillFilterRel.class, RelOptHelper.any(DrillScanRel.class)), + "DrillFilterItemStarReWriterRule.FilterOnScan"); - @Override - public void onMatch(RelOptRuleCall call) { - Filter filterRel = call.rel(0); - Project projectRel = call.rel(1); - TableScan scanRel = call.rel(2); + public static final FilterProjectScan FILTER_PROJECT_SCAN = new FilterProjectScan( + RelOptHelper.some(DrillFilterRel.class, RelOptHelper.some(DrillProjectRel.class, RelOptHelper.any(DrillScanRel.class))), + "DrillFilterItemStarReWriterRule.FilterProjectScan"); - ItemStarFieldsVisitor itemStarFieldsVisitor = new ItemStarFieldsVisitor(filterRel.getRowType().getFieldNames()); - filterRel.getCondition().accept(itemStarFieldsVisitor); - // there are no item fields, no need to proceed further - if (!itemStarFieldsVisitor.hasItemStarFields()) { - return; + private static class ProjectOnScan extends RelOptRule { + + ProjectOnScan(RelOptRuleOperand operand, String id) { + super(operand, id); } - Map itemStarFields = itemStarFieldsVisitor.getItemStarFields(); + @Override + public boolean matches(RelOptRuleCall call) { + DrillScanRel scan = call.rel(1); + return scan.getGroupScan() instanceof ParquetGroupScan && super.matches(call); + } + + @Override + public void onMatch(RelOptRuleCall call) { + DrillProjectRel projectRel = call.rel(0); + DrillScanRel scanRel = call.rel(1); + + ItemStarFieldsVisitor itemStarFieldsVisitor = new ItemStarFieldsVisitor(scanRel.getRowType().getFieldNames()); + List projects = projectRel.getProjects(); + for (RexNode project : projects) { + project.accept(itemStarFieldsVisitor); + } - // create new scan - RelNode newScan = constructNewScan(scanRel, itemStarFields.keySet()); + // if there are no item fields, no need to proceed further + if (itemStarFieldsVisitor.hasNoItemStarFields()) { + return; + } - // combine original and new projects - List newProjects = new ArrayList<>(projectRel.getProjects()); + Map itemStarFields = itemStarFieldsVisitor.getItemStarFields(); - // prepare node mapper to replace item star calls with new input field references - Map fieldMapper = new HashMap<>(); + DrillScanRel newScan = createNewScan(scanRel, itemStarFields); - // since scan might have already some fields, new field reference index should start from the last used in scan - // NB: field reference index starts from 0 thus original field count can be taken as starting index - int index = scanRel.getRowType().getFieldCount(); + // re-write projects + Map fieldMapper = createFieldMapper(itemStarFields.values(), scanRel.getRowType().getFieldCount()); + FieldsReWriter fieldsReWriter = new FieldsReWriter(fieldMapper); + List newProjects = new ArrayList<>(); + for (RexNode node : projectRel.getChildExps()) { + newProjects.add(node.accept(fieldsReWriter)); + } - for (DesiredField desiredField : itemStarFields.values()) { - RexInputRef inputRef = new RexInputRef(index, desiredField.getType()); - // add references to item star fields in new project - newProjects.add(inputRef); - for (RexNode node : desiredField.getNodes()) { - // if field is referenced in more then one call, add each call to field mapper - fieldMapper.put(node, index); + DrillProjectRel newProject = new DrillProjectRel( + projectRel.getCluster(), + projectRel.getTraitSet(), + newScan, + newProjects, + projectRel.getRowType()); + + if (ProjectRemoveRule.isTrivial(newProject)) { + call.transformTo(newScan); + } else { + call.transformTo(newProject); } - // increment index for the next node reference - index++; } - // create new project row type - RelDataType newProjectRowType = getNewRowType( - projectRel.getCluster().getTypeFactory(), - projectRel.getRowType().getFieldList(), - itemStarFields.keySet()); + } + + private static class FilterOnScan extends RelOptRule { + + FilterOnScan(RelOptRuleOperand operand, String id) { + super(operand, id); + } + + @Override + public boolean matches(RelOptRuleCall call) { + DrillScanRel scan = call.rel(1); + return scan.getGroupScan() instanceof ParquetGroupScan && super.matches(call); + } + + @Override + public void onMatch(RelOptRuleCall call) { + DrillFilterRel filterRel = call.rel(0); + DrillScanRel scanRel = call.rel(1); + transformFilterCall(filterRel, null, scanRel, call); + } + } + + private static class FilterProjectScan extends RelOptRule { + + FilterProjectScan(RelOptRuleOperand operand, String id) { + super(operand, id); + } + + @Override + public boolean matches(RelOptRuleCall call) { + DrillScanRel scan = call.rel(2); + return scan.getGroupScan() instanceof ParquetGroupScan && super.matches(call); + } + + @Override + public void onMatch(RelOptRuleCall call) { + DrillFilterRel filterRel = call.rel(0); + DrillProjectRel projectRel = call.rel(1); + DrillScanRel scanRel = call.rel(2); + transformFilterCall(filterRel, projectRel, scanRel, call); + } + } + + + /** + * Removes item star call from filter expression and propagates changes into project (if present) and scan. + * + * @param filterRel original filter expression + * @param projectRel original project expression + * @param scanRel original scan expression + * @param call original rule call + */ + private static void transformFilterCall(DrillFilterRel filterRel, DrillProjectRel projectRel, DrillScanRel scanRel, RelOptRuleCall call) { + List fieldNames = projectRel == null ? scanRel.getRowType().getFieldNames() : projectRel.getRowType().getFieldNames(); + ItemStarFieldsVisitor itemStarFieldsVisitor = new ItemStarFieldsVisitor(fieldNames); + filterRel.getCondition().accept(itemStarFieldsVisitor); + + // if there are no item fields, no need to proceed further + if (itemStarFieldsVisitor.hasNoItemStarFields()) { + return; + } + + Map itemStarFields = itemStarFieldsVisitor.getItemStarFields(); + + DrillScanRel newScan = createNewScan(scanRel, itemStarFields); + + // create new project if was present in call + DrillProjectRel newProject = null; + if (projectRel != null) { - // create new project - RelNode newProject = new LogicalProject(projectRel.getCluster(), projectRel.getTraitSet(), newScan, newProjects, newProjectRowType); + // add new projects to the already existing in original project + int projectIndex = scanRel.getRowType().getFieldCount(); + List newProjects = new ArrayList<>(projectRel.getProjects()); + for (DesiredField desiredField : itemStarFields.values()) { + newProjects.add(new RexInputRef(projectIndex, desiredField.getType())); + projectIndex++; + } + + RelDataType newProjectRowType = createNewRowType( + projectRel.getCluster().getTypeFactory(), + projectRel.getRowType().getFieldList(), + itemStarFields.keySet()); + + newProject = new DrillProjectRel( + projectRel.getCluster(), + projectRel.getTraitSet(), + newScan, + newProjects, + newProjectRowType); + } // transform filter condition + Map fieldMapper = createFieldMapper(itemStarFields.values(), scanRel.getRowType().getFieldCount()); FieldsReWriter fieldsReWriter = new FieldsReWriter(fieldMapper); RexNode newCondition = filterRel.getCondition().accept(fieldsReWriter); // create new filter - RelNode newFilter = new LogicalFilter(filterRel.getCluster(), filterRel.getTraitSet(), newProject, newCondition, ImmutableSet.of()); + DrillFilterRel newFilter = DrillFilterRel.create(newProject != null ? newProject : newScan, newCondition); // wrap with project to have the same row type as before - Project wrapper = projectRel.copy(projectRel.getTraitSet(), newFilter, projectRel.getProjects(), projectRel.getRowType()); + List newProjects = new ArrayList<>(); + RelDataType rowType = filterRel.getRowType(); + List fieldList = rowType.getFieldList(); + for (RelDataTypeField field : fieldList) { + RexInputRef inputRef = new RexInputRef(field.getIndex(), field.getType()); + newProjects.add(inputRef); + } + DrillProjectRel wrapper = new DrillProjectRel(filterRel.getCluster(), filterRel.getTraitSet(), newFilter, newProjects, filterRel.getRowType()); call.transformTo(wrapper); } @@ -142,9 +238,9 @@ public class DrillFilterItemStarReWriterRule extends RelOptRule { * @param newFields new fields * @return new row type with original and new fields */ - private RelDataType getNewRowType(RelDataTypeFactory typeFactory, - List originalFields, - Collection newFields) { + private static RelDataType createNewRowType(RelDataTypeFactory typeFactory, + List originalFields, + Collection newFields) { RelDataTypeHolder relDataTypeHolder = new RelDataTypeHolder(); // add original fields @@ -161,30 +257,55 @@ public class DrillFilterItemStarReWriterRule extends RelOptRule { } /** - * Constructs new scan based on the original scan. - * Preserves all original fields and add new fields. + * Creates new scan with fields from original scan and fields used in item star operator. * - * @param scanRel original scan - * @param newFields new fields - * @return new scan with original and new fields + * @param scanRel original scan expression + * @param itemStarFields item star fields + * @return new scan expression + */ + private static DrillScanRel createNewScan(DrillScanRel scanRel, Map itemStarFields) { + RelDataType newScanRowType = createNewRowType( + scanRel.getCluster().getTypeFactory(), + scanRel.getRowType().getFieldList(), + itemStarFields.keySet()); + + List columns = new ArrayList<>(scanRel.getColumns()); + for (DesiredField desiredField : itemStarFields.values()) { + String name = desiredField.getName(); + PathSegment.NameSegment nameSegment = new PathSegment.NameSegment(name); + columns.add(new SchemaPath(nameSegment)); + } + + return new DrillScanRel( + scanRel.getCluster(), + scanRel.getTraitSet().plus(DrillRel.DRILL_LOGICAL), + scanRel.getTable(), + newScanRowType, + columns); + } + + /** + * Creates node mapper to replace item star calls with new input field references. + * Starting index should be calculated from the last used input expression (i.e. scan expression). + * NB: field reference index starts from 0 thus original field count can be taken as starting index + * + * @param desiredFields list of desired fields + * @param startingIndex starting index + * @return field mapper */ - private RelNode constructNewScan(TableScan scanRel, Collection newFields) { - // create new scan row type - RelDataType newScanRowType = getNewRowType( - scanRel.getCluster().getTypeFactory(), - scanRel.getRowType().getFieldList(), - newFields); - - // create new scan - RelOptTable table = scanRel.getTable(); - Class elementType = EnumerableTableScan.deduceElementType(table.unwrap(Table.class)); - - DrillTable unwrap = Utilities.getDrillTable(table); - DrillTranslatableTable newTable = new DrillTranslatableTable( - new DynamicDrillTable(unwrap.getPlugin(), unwrap.getStorageEngineName(), unwrap.getUserName(), unwrap.getSelection())); - RelOptTableImpl newOptTableImpl = RelOptTableImpl.create(table.getRelOptSchema(), newScanRowType, newTable, ImmutableList.of()); - - return new EnumerableTableScan(scanRel.getCluster(), scanRel.getTraitSet(), newOptTableImpl, elementType); + private static Map createFieldMapper(Collection desiredFields, int startingIndex) { + Map fieldMapper = new HashMap<>(); + + int index = startingIndex; + for (DesiredField desiredField : desiredFields) { + for (RexNode node : desiredField.getNodes()) { + // if field is referenced in more then one call, add each call to field mapper + fieldMapper.put(node, index); + } + // increment index for the next node reference + index++; + } + return fieldMapper; } /** @@ -192,7 +313,7 @@ public class DrillFilterItemStarReWriterRule extends RelOptRule { * For the fields with the same name, stores original calls in a list, does not duplicate fields. * Holds state, should not be re-used. */ - private class ItemStarFieldsVisitor extends RexVisitorImpl { + private static class ItemStarFieldsVisitor extends RexVisitorImpl { private final Map itemStarFields = new HashMap<>(); private final List fieldNames; @@ -202,8 +323,8 @@ public class DrillFilterItemStarReWriterRule extends RelOptRule { this.fieldNames = fieldNames; } - boolean hasItemStarFields() { - return !itemStarFields.isEmpty(); + boolean hasNoItemStarFields() { + return itemStarFields.isEmpty(); } Map getItemStarFields() { http://git-wip-us.apache.org/repos/asf/drill/blob/ddb73a50/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestPushDownAndPruningWithItemStar.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestPushDownAndPruningWithItemStar.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestPushDownAndPruningWithItemStar.java index 24b9212..6ac08ee 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestPushDownAndPruningWithItemStar.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestPushDownAndPruningWithItemStar.java @@ -44,7 +44,7 @@ public class TestPushDownAndPruningWithItemStar extends PlanTestBase { } @Test - public void testPushProjectIntoScan() throws Exception { + public void testPushProjectIntoScanWithGroupByClause() throws Exception { String query = String.format("select o_orderdate, count(*) from (select * from `%s`.`%s`) group by o_orderdate", DFS_TMP_SCHEMA, TABLE_NAME); String[] expectedPlan = {"numFiles=3, numRowGroups=3, usedMetadataFile=false, columns=\\[`o_orderdate`\\]"}; @@ -79,7 +79,7 @@ public class TestPushDownAndPruningWithItemStar extends PlanTestBase { public void testPushProjectIntoScanWithExpressionInFilter() throws Exception { String query = String.format("select o_orderdate from (select * from `%s`.`%s`) where o_custkey + o_orderkey < 5", DFS_TMP_SCHEMA, TABLE_NAME); - String[] expectedPlan = {"numFiles=3, numRowGroups=3, usedMetadataFile=false, columns=\\[`o_custkey`, `o_orderkey`, `o_orderdate`\\]"}; + String[] expectedPlan = {"numFiles=3, numRowGroups=3, usedMetadataFile=false, columns=\\[`o_orderdate`, `o_custkey`, `o_orderkey`\\]"}; String[] excludedPlan = {}; PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan); @@ -116,7 +116,7 @@ public class TestPushDownAndPruningWithItemStar extends PlanTestBase { "where t.user_info.cust_id > 28 and t.user_info.device = 'IOS5' and t.marketing_info.camp_id > 5 and t.marketing_info.keywords[2] is not null"; String[] expectedPlan = {"numFiles=1, numRowGroups=1, usedMetadataFile=false, " + - "columns=\\[`user_info`.`cust_id`, `user_info`.`device`, `marketing_info`.`camp_id`, `marketing_info`.`keywords`\\[2\\], `trans_id`\\]"}; + "columns=\\[`trans_id`, `user_info`.`cust_id`, `user_info`.`device`, `marketing_info`.`camp_id`, `marketing_info`.`keywords`\\[2\\]\\]"}; String[] excludedPlan = {}; PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan); @@ -131,10 +131,43 @@ public class TestPushDownAndPruningWithItemStar extends PlanTestBase { } @Test + public void testProjectIntoScanWithNestedStarSubQuery() throws Exception { + String query = String.format("select *, o_orderdate from (select * from `%s`.`%s`)", DFS_TMP_SCHEMA, TABLE_NAME); + + String[] expectedPlan = {"numFiles=3, numRowGroups=3, usedMetadataFile=false, columns=\\[`\\*\\*`, `o_orderdate`\\]"}; + String[] excludedPlan = {}; + + PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan); + + testBuilder() + .sqlQuery(query) + .unOrdered() + .sqlBaselineQuery("select *, o_orderdate from `%s`.`%s`", DFS_TMP_SCHEMA, TABLE_NAME) + .build(); + } + + @Test + public void testProjectIntoScanWithSeveralNestedStarSubQueries() throws Exception { + String subQuery = String.format("select * from `%s`.`%s`", DFS_TMP_SCHEMA, TABLE_NAME); + String query = String.format("select o_custkey + o_orderkey from (select * from (select * from (%s)))", subQuery); + + String[] expectedPlan = {"numFiles=3, numRowGroups=3, usedMetadataFile=false, columns=\\[`o_custkey`, `o_orderkey`\\]"}; + String[] excludedPlan = {}; + + PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan); + + testBuilder() + .sqlQuery(query) + .unOrdered() + .sqlBaselineQuery("select o_custkey + o_orderkey from `%s`.`%s`", DFS_TMP_SCHEMA, TABLE_NAME) + .build(); + } + + @Test public void testDirectoryPruning() throws Exception { String query = String.format("select * from (select * from `%s`.`%s`) where dir0 = 't1'", DFS_TMP_SCHEMA, TABLE_NAME); - String[] expectedPlan = {"numFiles=1, numRowGroups=1, usedMetadataFile=false, columns=\\[`\\*\\*`\\]"}; + String[] expectedPlan = {"numFiles=1, numRowGroups=1, usedMetadataFile=false, columns=\\[`\\*\\*`, `dir0`\\]"}; String[] excludedPlan = {}; PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan); @@ -147,10 +180,45 @@ public class TestPushDownAndPruningWithItemStar extends PlanTestBase { } @Test + public void testDirectoryPruningWithNestedStarSubQuery() throws Exception { + String subQuery = String.format("select * from `%s`.`%s`", DFS_TMP_SCHEMA, TABLE_NAME); + String query = String.format("select * from (select * from (select * from (%s))) where dir0 = 't1'", subQuery); + + String[] expectedPlan = {"numFiles=1, numRowGroups=1, usedMetadataFile=false, columns=\\[`\\*\\*`, `dir0`\\]"}; + String[] excludedPlan = {}; + + PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan); + + testBuilder() + .sqlQuery(query) + .unOrdered() + .sqlBaselineQuery("select * from `%s`.`%s` where dir0 = 't1'", DFS_TMP_SCHEMA, TABLE_NAME) + .build(); + } + + @Test + public void testDirectoryPruningWithNestedStarSubQueryAndAdditionalColumns() throws Exception { + String subQuery = String.format("select * from `%s`.`%s`", DFS_TMP_SCHEMA, TABLE_NAME); + String query = String.format("select * from (select * from (select *, `o_orderdate` from (%s))) where dir0 = 't1'", subQuery); + + String[] expectedPlan = {"numFiles=1, numRowGroups=1, usedMetadataFile=false, columns=\\[`\\*\\*`, `o_orderdate`, `dir0`\\]"}; + String[] excludedPlan = {}; + + PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan); + + testBuilder() + .sqlQuery(query) + .unOrdered() + .sqlBaselineQuery("select * from `%s`.`%s` where dir0 = 't1'", DFS_TMP_SCHEMA, TABLE_NAME) + .build(); + } + + + @Test public void testFilterPushDownSingleCondition() throws Exception { String query = String.format("select * from (select * from `%s`.`%s`) where o_orderdate = date '1992-01-01'", DFS_TMP_SCHEMA, TABLE_NAME); - String[] expectedPlan = {"numFiles=1, numRowGroups=1, usedMetadataFile=false, columns=\\[`\\*\\*`\\]"}; + String[] expectedPlan = {"numFiles=1, numRowGroups=1, usedMetadataFile=false, columns=\\[`\\*\\*`, `o_orderdate`\\]"}; String[] excludedPlan = {}; PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan); @@ -167,7 +235,7 @@ public class TestPushDownAndPruningWithItemStar extends PlanTestBase { String query = String.format("select * from (select * from `%s`.`%s`) where o_orderdate = date '1992-01-01' or o_orderdate = date '1992-01-09'", DFS_TMP_SCHEMA, TABLE_NAME); - String[] expectedPlan = {"numFiles=2, numRowGroups=2, usedMetadataFile=false, columns=\\[`\\*\\*`\\]"}; + String[] expectedPlan = {"numFiles=2, numRowGroups=2, usedMetadataFile=false, columns=\\[`\\*\\*`, `o_orderdate`\\]"}; String[] excludedPlan = {}; PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan); @@ -180,4 +248,38 @@ public class TestPushDownAndPruningWithItemStar extends PlanTestBase { .build(); } + @Test + public void testFilterPushDownWithSeveralNestedStarSubQueries() throws Exception { + String subQuery = String.format("select * from `%s`.`%s`", DFS_TMP_SCHEMA, TABLE_NAME); + String query = String.format("select * from (select * from (select * from (%s))) where o_orderdate = date '1992-01-01'", subQuery); + + String[] expectedPlan = {"numFiles=1, numRowGroups=1, usedMetadataFile=false, columns=\\[`\\*\\*`, `o_orderdate`\\]"}; + String[] excludedPlan = {}; + + PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan); + + testBuilder() + .sqlQuery(query) + .unOrdered() + .sqlBaselineQuery("select * from `%s`.`%s` where o_orderdate = date '1992-01-01'", DFS_TMP_SCHEMA, TABLE_NAME) + .build(); + } + + @Test + public void testFilterPushDownWithSeveralNestedStarSubQueriesWithAdditionalColumns() throws Exception { + String subQuery = String.format("select * from `%s`.`%s`", DFS_TMP_SCHEMA, TABLE_NAME); + String query = String.format("select * from (select * from (select *, o_custkey from (%s))) where o_orderdate = date '1992-01-01'", subQuery); + + String[] expectedPlan = {"numFiles=1, numRowGroups=1, usedMetadataFile=false, columns=\\[`\\*\\*`, `o_custkey`, `o_orderdate`\\]"}; + String[] excludedPlan = {}; + + PlanTestBase.testPlanMatchingPatterns(query, expectedPlan, excludedPlan); + + testBuilder() + .sqlQuery(query) + .unOrdered() + .sqlBaselineQuery("select *, o_custkey from `%s`.`%s` where o_orderdate = date '1992-01-01'", DFS_TMP_SCHEMA, TABLE_NAME) + .build(); + } + }