Repository: drill
Updated Branches:
refs/heads/master a264e7feb -> 9a6cb59b9
DRILL-6259: Support parquet filter push down for complex types
close apache/drill#1173
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/4ee5625d
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/4ee5625d
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/4ee5625d
Branch: refs/heads/master
Commit: 4ee5625d57bd73d3d82b45f687a8574ea6660f8e
Parents: a264e7f
Author: Arina Ielchiieva <arina.yelchiyeva@gmail.com>
Authored: Tue Mar 13 19:54:25 2018 +0200
Committer: Aman Sinha <asinha@maprtech.com>
Committed: Fri Mar 30 22:45:33 2018 -0700
----------------------------------------------------------------------
.../sig/ConstantExpressionIdentifier.java | 6 +
.../exec/expr/ExpressionTreeMaterializer.java | 3 +-
.../exec/expr/stat/ParquetIsPredicates.java | 23 ++++
.../exec/expr/stat/RangeExprEvaluator.java | 27 ++--
.../drill/exec/expr/stat/TypedFieldExpr.java | 63 ----------
.../exec/planner/common/DrillRelOptUtil.java | 14 +--
.../logical/DrillPushFilterPastProjectRule.java | 14 ++-
.../store/parquet/ParquetFilterBuilder.java | 80 ++++--------
.../store/parquet/ParquetPushDownFilter.java | 13 +-
.../parquet/stat/ParquetMetaStatCollector.java | 29 ++---
...estParquetFilterPushDownForComplexTypes.java | 124 +++++++++++++++++++
.../resources/parquet/users/users_1.parquet | Bin 0 -> 657 bytes
.../resources/parquet/users/users_2.parquet | Bin 0 -> 641 bytes
.../resources/parquet/users/users_3.parquet | Bin 0 -> 588 bytes
.../resources/parquet/users/users_4.parquet | Bin 0 -> 653 bytes
.../resources/parquet/users/users_5.parquet | Bin 0 -> 583 bytes
.../resources/parquet/users/users_6.parquet | Bin 0 -> 627 bytes
.../resources/parquet/users/users_7.parquet | Bin 0 -> 662 bytes
.../drill/common/expression/SchemaPath.java | 29 +++--
.../drill/common/expression/TypedFieldExpr.java | 60 +++++++++
.../visitors/AbstractExprVisitor.java | 6 +-
.../expression/visitors/AggregateChecker.java | 5 +
.../expression/visitors/ConstantChecker.java | 5 +
.../common/expression/visitors/ExprVisitor.java | 54 ++++----
.../visitors/ExpressionValidator.java | 5 +
25 files changed, 365 insertions(+), 195 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java
index 1e71773..1a0b7d5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java
@@ -31,6 +31,7 @@ import org.apache.drill.common.expression.IfExpression;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.NullExpression;
import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.expression.TypedFieldExpr;
import org.apache.drill.common.expression.TypedNullConstant;
import org.apache.drill.common.expression.ValueExpressions;
import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
@@ -231,4 +232,9 @@ public class ConstantExpressionIdentifier implements ExprVisitor<Boolean, Identi
public Boolean visitParameter(ValueExpressions.ParameterExpression e, IdentityHashMap<LogicalExpression, Object> value) throws RuntimeException {
return false;
}
+
+ @Override
+ public Boolean visitTypedFieldExpr(TypedFieldExpr e, IdentityHashMap<LogicalExpression, Object> value) throws RuntimeException {
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
index 23df262..f1b50c9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
@@ -44,6 +44,7 @@ import org.apache.drill.common.expression.IfExpression.IfCondition;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.NullExpression;
import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.expression.TypedFieldExpr;
import org.apache.drill.common.expression.TypedNullConstant;
import org.apache.drill.common.expression.ValueExpressions;
import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
@@ -78,7 +79,6 @@ import org.apache.drill.exec.expr.fn.DrillComplexWriterFuncHolder;
import org.apache.drill.exec.expr.fn.DrillFuncHolder;
import org.apache.drill.exec.expr.fn.ExceptionFunction;
import org.apache.drill.exec.expr.fn.FunctionLookupContext;
-import org.apache.drill.exec.expr.stat.TypedFieldExpr;
import org.apache.drill.exec.record.TypedFieldId;
import org.apache.drill.exec.record.VectorAccessible;
import org.apache.drill.exec.resolver.FunctionResolver;
@@ -323,7 +323,6 @@ public class ExpressionTreeMaterializer {
} else {
logger.warn("Unable to find value vector of path {}, returning null-int instance.", path);
return new TypedFieldExpr(path, Types.OPTIONAL_INT);
- // return NullExpression.INSTANCE;
}
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetIsPredicates.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetIsPredicates.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetIsPredicates.java
index a58ce7c..bb8f3ac 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetIsPredicates.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/ParquetIsPredicates.java
@@ -18,6 +18,8 @@ package org.apache.drill.exec.expr.stat;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.LogicalExpressionBase;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.expression.TypedFieldExpr;
import org.apache.drill.common.expression.visitors.ExprVisitor;
import org.apache.parquet.column.statistics.Statistics;
@@ -29,6 +31,7 @@ import java.util.List;
* IS predicates for parquet filter pushdown.
*/
public class ParquetIsPredicates {
+
public static abstract class ParquetIsPredicate extends LogicalExpressionBase implements ParquetFilterPredicate {
protected final LogicalExpression expr;
@@ -54,12 +57,22 @@ public class ParquetIsPredicates {
* IS NULL predicate.
*/
public static class IsNullPredicate extends ParquetIsPredicate {
+ private final boolean isArray;
+
public IsNullPredicate(LogicalExpression expr) {
super(expr);
+ this.isArray = isArray(expr);
}
@Override
public boolean canDrop(RangeExprEvaluator evaluator) {
+
+ // for arrays we are not able to define exact number of nulls
+ // [1,2,3] vs [1,2] -> in second case 3 is absent and thus it's null but statistics shows no nulls
+ if (isArray) {
+ return false;
+ }
+
Statistics exprStat = expr.accept(evaluator, null);
if (!ParquetPredicatesHelper.hasStats(exprStat)) {
@@ -73,6 +86,16 @@ public class ParquetIsPredicates {
return false;
}
}
+
+ private boolean isArray(LogicalExpression expression) {
+ if (expression instanceof TypedFieldExpr) {
+ TypedFieldExpr typedFieldExpr = (TypedFieldExpr) expression;
+ SchemaPath schemaPath = typedFieldExpr.getPath();
+ return schemaPath.isArray();
+ }
+ return false;
+ }
+
}
/**
http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/RangeExprEvaluator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/RangeExprEvaluator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/RangeExprEvaluator.java
index 2d241dc..2cc6a70 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/RangeExprEvaluator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/RangeExprEvaluator.java
@@ -17,11 +17,11 @@
*/
package org.apache.drill.exec.expr.stat;
-import com.google.common.base.Preconditions;
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.expression.FunctionHolderExpression;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.expression.TypedFieldExpr;
import org.apache.drill.common.expression.ValueExpressions;
import org.apache.drill.common.expression.fn.CastFunctions;
import org.apache.drill.common.expression.fn.FuncHolder;
@@ -70,17 +70,20 @@ public class RangeExprEvaluator extends AbstractExprVisitor<Statistics, Void, Ru
@Override
public Statistics visitUnknown(LogicalExpression e, Void value) throws RuntimeException {
- if (e instanceof TypedFieldExpr) {
- TypedFieldExpr fieldExpr = (TypedFieldExpr) e;
- final ColumnStatistics columnStatistics = columnStatMap.get(fieldExpr.getPath());
- if (columnStatistics != null) {
- return columnStatistics.getStatistics();
- } else if (fieldExpr.getMajorType().equals(Types.OPTIONAL_INT)) {
- // field does not exist.
- IntStatistics intStatistics = new IntStatistics();
- intStatistics.setNumNulls(rowCount); // all values are nulls
- return intStatistics;
- }
+ // do nothing for the unknown expression
+ return null;
+ }
+
+ @Override
+ public Statistics visitTypedFieldExpr(TypedFieldExpr typedFieldExpr, Void value) throws RuntimeException {
+ final ColumnStatistics columnStatistics = columnStatMap.get(typedFieldExpr.getPath());
+ if (columnStatistics != null) {
+ return columnStatistics.getStatistics();
+ } else if (typedFieldExpr.getMajorType().equals(Types.OPTIONAL_INT)) {
+ // field does not exist.
+ IntStatistics intStatistics = new IntStatistics();
+ intStatistics.setNumNulls(rowCount); // all values are nulls
+ return intStatistics;
}
return null;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/TypedFieldExpr.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/TypedFieldExpr.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/TypedFieldExpr.java
deleted file mode 100644
index 4287929..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/stat/TypedFieldExpr.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.expr.stat;
-
-import com.google.common.collect.Iterators;
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.expression.LogicalExpressionBase;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.common.expression.visitors.ExprVisitor;
-import org.apache.drill.common.types.TypeProtos;
-
-import java.util.Iterator;
-
-public class TypedFieldExpr extends LogicalExpressionBase {
- TypeProtos.MajorType type;
- SchemaPath path;
-
- public TypedFieldExpr(SchemaPath path, TypeProtos.MajorType type) {
- super(path.getPosition());
- this.path = path;
- this.type = type;
- }
-
- @Override
- public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
- return visitor.visitUnknown(this, value);
- }
-
- @Override
- public Iterator<LogicalExpression> iterator() {
- return Iterators.emptyIterator();
- }
-
- @Override
- public TypeProtos.MajorType getMajorType() {
- return this.type;
- }
-
- @Override
- public String toString() {
- return this.path.getRootSegment().getPath() + "(" + type.getMinorType() + "_" + type.getMode() +")";
- }
-
- public SchemaPath getPath() {
- return this.path;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java
index d5c8d94..dff83f6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillRelOptUtil.java
@@ -18,6 +18,7 @@
package org.apache.drill.exec.planner.common;
import java.util.AbstractList;
+import java.util.Collection;
import java.util.List;
import com.google.common.collect.Lists;
@@ -178,22 +179,20 @@ public abstract class DrillRelOptUtil {
}
/**
- * Travesal RexNode to find the item/flattern operator. Continue search if RexNode has a
+ * Travesal RexNode to find at least one operator in the given collection. Continue search if RexNode has a
* RexInputRef which refers to a RexNode in project expressions.
*
* @param node : RexNode to search
* @param projExprs : the list of project expressions. Empty list means there is No project operator underneath.
+ * @param operators collection of operators to find
* @return : Return null if there is NONE; return the first appearance of item/flatten RexCall.
*/
- public static RexCall findItemOrFlatten(
- final RexNode node,
- final List<RexNode> projExprs) {
+ public static RexCall findOperators(final RexNode node, final List<RexNode> projExprs, final Collection<String> operators) {
try {
RexVisitor<Void> visitor =
new RexVisitorImpl<Void>(true) {
public Void visitCall(RexCall call) {
- if ("item".equals(call.getOperator().getName().toLowerCase()) ||
- "flatten".equals(call.getOperator().getName().toLowerCase())) {
+ if (operators.contains(call.getOperator().getName().toLowerCase())) {
throw new Util.FoundOne(call); /* throw exception to interrupt tree walk (this is similar to
other utility methods in RexUtil.java */
}
@@ -208,8 +207,7 @@ public abstract class DrillRelOptUtil {
RexNode n = projExprs.get(index);
if (n instanceof RexCall) {
RexCall r = (RexCall) n;
- if ("item".equals(r.getOperator().getName().toLowerCase()) ||
- "flatten".equals(r.getOperator().getName().toLowerCase())) {
+ if (operators.contains(r.getOperator().getName().toLowerCase())) {
throw new Util.FoundOne(r);
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushFilterPastProjectRule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushFilterPastProjectRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushFilterPastProjectRule.java
index d24abcc..7b978be 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushFilterPastProjectRule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillPushFilterPastProjectRule.java
@@ -30,13 +30,23 @@ import org.apache.calcite.rex.RexUtil;
import org.apache.calcite.util.Pair;
import org.apache.drill.exec.planner.common.DrillRelOptUtil;
+import java.util.ArrayList;
+import java.util.Collection;
import java.util.List;
public class DrillPushFilterPastProjectRule extends RelOptRule {
public final static RelOptRule INSTANCE = new DrillPushFilterPastProjectRule();
- protected DrillPushFilterPastProjectRule() {
+ private static final Collection<String> BANNED_OPERATORS;
+
+ static {
+ BANNED_OPERATORS = new ArrayList<>(2);
+ BANNED_OPERATORS.add("flatten");
+ BANNED_OPERATORS.add("item");
+ }
+
+ private DrillPushFilterPastProjectRule() {
super(
operand(
LogicalFilter.class,
@@ -60,7 +70,7 @@ public class DrillPushFilterPastProjectRule extends RelOptRule {
for (final RexNode pred : predList) {
- if (DrillRelOptUtil.findItemOrFlatten(pred, projRel.getProjects()) == null) {
+ if (DrillRelOptUtil.findOperators(pred, projRel.getProjects(), BANNED_OPERATORS) == null) {
qualifiedPredList.add(pred);
} else {
unqualifiedPredList.add(pred);
http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFilterBuilder.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFilterBuilder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFilterBuilder.java
index a9e55dd..2d245a1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFilterBuilder.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetFilterBuilder.java
@@ -19,8 +19,7 @@ import com.google.common.collect.ImmutableSet;
import org.apache.drill.common.expression.BooleanOperator;
import org.apache.drill.common.expression.FunctionHolderExpression;
import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.expression.PathSegment;
-import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.expression.TypedFieldExpr;
import org.apache.drill.common.expression.ValueExpressions;
import org.apache.drill.common.expression.fn.CastFunctions;
import org.apache.drill.common.expression.fn.FuncHolder;
@@ -41,7 +40,6 @@ import org.apache.drill.exec.expr.holders.ValueHolder;
import org.apache.drill.exec.expr.stat.ParquetBooleanPredicates;
import org.apache.drill.exec.expr.stat.ParquetComparisonPredicates;
import org.apache.drill.exec.expr.stat.ParquetIsPredicates;
-import org.apache.drill.exec.expr.stat.TypedFieldExpr;
import org.apache.drill.exec.ops.UdfUtilities;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -62,11 +60,12 @@ public class ParquetFilterBuilder extends AbstractExprVisitor<LogicalExpression,
/**
* @param expr materialized filter expression
* @param constantBoundaries set of constant expressions
- * @param udfUtilities
+ * @param udfUtilities udf utilities
+ *
+ * @return logical expression
*/
public static LogicalExpression buildParquetFilterPredicate(LogicalExpression expr, final Set<LogicalExpression> constantBoundaries, UdfUtilities udfUtilities) {
- final LogicalExpression predicate = expr.accept(new ParquetFilterBuilder(udfUtilities), constantBoundaries);
- return predicate;
+ return expr.accept(new ParquetFilterBuilder(udfUtilities), constantBoundaries);
}
private ParquetFilterBuilder(UdfUtilities udfUtilities) {
@@ -75,19 +74,16 @@ public class ParquetFilterBuilder extends AbstractExprVisitor<LogicalExpression,
@Override
public LogicalExpression visitUnknown(LogicalExpression e, Set<LogicalExpression> value) {
- if (e instanceof TypedFieldExpr &&
- ! containsArraySeg(((TypedFieldExpr) e).getPath()) &&
- e.getMajorType().getMode() != TypeProtos.DataMode.REPEATED) {
- // A filter is not qualified for push down, if
- // 1. it contains an array segment : a.b[1], a.b[1].c.d
- // 2. it's repeated type.
- return e;
- }
-
+ // for the unknown expression, do nothing
return null;
}
@Override
+ public LogicalExpression visitTypedFieldExpr(TypedFieldExpr typedFieldExpr, Set<LogicalExpression> value) throws RuntimeException {
+ return typedFieldExpr;
+ }
+
+ @Override
public LogicalExpression visitIntConstant(ValueExpressions.IntExpression intExpr, Set<LogicalExpression> value)
throws RuntimeException {
return intExpr;
@@ -161,18 +157,6 @@ public class ParquetFilterBuilder extends AbstractExprVisitor<LogicalExpression,
}
}
- private boolean containsArraySeg(final SchemaPath schemaPath) {
- PathSegment seg = schemaPath.getRootSegment();
-
- while (seg != null) {
- if (seg.isArray()) {
- return true;
- }
- seg = seg.getChild();
- }
- return false;
- }
-
private LogicalExpression getValueExpressionFromConst(ValueHolder holder, TypeProtos.MinorType type) {
switch (type) {
case INT:
@@ -229,13 +213,9 @@ public class ParquetFilterBuilder extends AbstractExprVisitor<LogicalExpression,
}
if (CastFunctions.isCastFunction(funcName)) {
- List<LogicalExpression> newArgs = new ArrayList();
- for (LogicalExpression arg : funcHolderExpr.args) {
- final LogicalExpression newArg = arg.accept(this, value);
- if (newArg == null) {
- return null;
- }
- newArgs.add(newArg);
+ List<LogicalExpression> newArgs = generateNewExpressions(funcHolderExpr.args, value);
+ if (newArgs == null) {
+ return null;
}
return funcHolderExpr.copy(newArgs);
@@ -244,15 +224,22 @@ public class ParquetFilterBuilder extends AbstractExprVisitor<LogicalExpression,
}
}
- private LogicalExpression handleCompareFunction(FunctionHolderExpression functionHolderExpression, Set<LogicalExpression> value) {
- List<LogicalExpression> newArgs = new ArrayList();
-
- for (LogicalExpression arg : functionHolderExpression.args) {
- LogicalExpression newArg = arg.accept(this, value);
+ private List<LogicalExpression> generateNewExpressions(List<LogicalExpression> expressions, Set<LogicalExpression> value) {
+ List<LogicalExpression> newExpressions = new ArrayList<>();
+ for (LogicalExpression arg : expressions) {
+ final LogicalExpression newArg = arg.accept(this, value);
if (newArg == null) {
return null;
}
- newArgs.add(newArg);
+ newExpressions.add(newArg);
+ }
+ return newExpressions;
+ }
+
+ private LogicalExpression handleCompareFunction(FunctionHolderExpression functionHolderExpression, Set<LogicalExpression> value) {
+ List<LogicalExpression> newArgs = generateNewExpressions(functionHolderExpression.args, value);
+ if (newArgs == null) {
+ return null;
}
String funcName = ((DrillSimpleFuncHolder) functionHolderExpression.getHolder()).getRegisteredNames()[0];
@@ -306,19 +293,6 @@ public class ParquetFilterBuilder extends AbstractExprVisitor<LogicalExpression,
}
}
- private LogicalExpression handleCastFunction(FunctionHolderExpression functionHolderExpression, Set<LogicalExpression> value) {
- for (LogicalExpression arg : functionHolderExpression.args) {
- LogicalExpression newArg = arg.accept(this, value);
- if (newArg == null) {
- return null;
- }
- }
-
- String funcName = ((DrillSimpleFuncHolder) functionHolderExpression.getHolder()).getRegisteredNames()[0];
-
- return null;
- }
-
private static boolean isCompareFunction(String funcName) {
return COMPARE_FUNCTIONS_SET.contains(funcName);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetPushDownFilter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetPushDownFilter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetPushDownFilter.java
index 1ec10d8..4257150 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetPushDownFilter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetPushDownFilter.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -39,6 +39,8 @@ import org.apache.drill.exec.planner.physical.ProjectPrel;
import org.apache.drill.exec.planner.physical.ScanPrel;
import org.apache.drill.exec.store.StoragePluginOptimizerRule;
+import java.util.ArrayList;
+import java.util.Collection;
import java.util.List;
import java.util.concurrent.TimeUnit;
@@ -46,6 +48,13 @@ public abstract class ParquetPushDownFilter extends StoragePluginOptimizerRule {
private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetPushDownFilter.class);
+ private static final Collection<String> BANNED_OPERATORS;
+
+ static {
+ BANNED_OPERATORS = new ArrayList<>(1);
+ BANNED_OPERATORS.add("flatten");
+ }
+
public static RelOptRule getFilterOnProject(OptimizerRulesContext optimizerRulesContext) {
return new ParquetPushDownFilter(
RelOptHelper.some(FilterPrel.class, RelOptHelper.some(ProjectPrel.class, RelOptHelper.any(ScanPrel.class))),
@@ -127,7 +136,7 @@ public abstract class ParquetPushDownFilter extends StoragePluginOptimizerRule {
final List<RexNode> qualifiedPredList = Lists.newArrayList();
for (final RexNode pred : predList) {
- if (DrillRelOptUtil.findItemOrFlatten(pred, ImmutableList.<RexNode>of()) == null) {
+ if (DrillRelOptUtil.findOperators(pred, ImmutableList.of(), BANNED_OPERATORS) == null) {
qualifiedPredList.add(pred);
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/stat/ParquetMetaStatCollector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/stat/ParquetMetaStatCollector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/stat/ParquetMetaStatCollector.java
index 4991a22..a8c1218 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/stat/ParquetMetaStatCollector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/stat/ParquetMetaStatCollector.java
@@ -41,11 +41,11 @@ import java.util.Set;
import java.util.concurrent.TimeUnit;
public class ParquetMetaStatCollector implements ColumnStatCollector{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetMetaStatCollector.class);
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ParquetMetaStatCollector.class);
- private final Metadata.ParquetTableMetadataBase parquetTableMetadata;
- private final List<? extends Metadata.ColumnMetadata> columnMetadataList;
- final Map<String, String> implicitColValues;
+ private final Metadata.ParquetTableMetadataBase parquetTableMetadata;
+ private final List<? extends Metadata.ColumnMetadata> columnMetadataList;
+ private final Map<String, String> implicitColValues;
public ParquetMetaStatCollector(Metadata.ParquetTableMetadataBase parquetTableMetadata,
List<? extends Metadata.ColumnMetadata> columnMetadataList, Map<String, String> implicitColValues) {
@@ -82,11 +82,11 @@ public class ParquetMetaStatCollector implements ColumnStatCollector{
columnMetadataMap.put(schemaPath, columnMetadata);
}
- for (final SchemaPath schemaPath : fields) {
+ for (final SchemaPath field : fields) {
final PrimitiveType.PrimitiveTypeName primitiveType;
final OriginalType originalType;
- final Metadata.ColumnMetadata columnMetadata = columnMetadataMap.get(schemaPath);
+ final Metadata.ColumnMetadata columnMetadata = columnMetadataMap.get(field.getUnIndexed());
if (columnMetadata != null) {
final Object min = columnMetadata.getMinValue();
@@ -95,7 +95,6 @@ public class ParquetMetaStatCollector implements ColumnStatCollector{
primitiveType = this.parquetTableMetadata.getPrimitiveType(columnMetadata.getName());
originalType = this.parquetTableMetadata.getOriginalType(columnMetadata.getName());
- final Integer repetitionLevel = this.parquetTableMetadata.getRepetitionLevel(columnMetadata.getName());
int precision = 0;
int scale = 0;
// ColumnTypeMetadata_v3 stores information about scale and precision
@@ -106,16 +105,16 @@ public class ParquetMetaStatCollector implements ColumnStatCollector{
precision = columnTypeInfo.precision;
}
- statMap.put(schemaPath, getStat(min, max, numNull, primitiveType, originalType, repetitionLevel, scale, precision));
+ statMap.put(field, getStat(min, max, numNull, primitiveType, originalType, scale, precision));
} else {
- final String columnName = schemaPath.getRootSegment().getPath();
+ final String columnName = field.getRootSegment().getPath();
if (implicitColValues.containsKey(columnName)) {
TypeProtos.MajorType type = Types.required(TypeProtos.MinorType.VARCHAR);
Statistics stat = new BinaryStatistics();
stat.setNumNulls(0);
byte[] val = implicitColValues.get(columnName).getBytes();
stat.setMinMaxFromBytes(val, val);
- statMap.put(schemaPath, new ColumnStatistics(stat, type));
+ statMap.put(field, new ColumnStatistics(stat, type));
}
}
}
@@ -128,7 +127,7 @@ public class ParquetMetaStatCollector implements ColumnStatCollector{
}
/**
- * Builds column statistics using given primitiveType, originalType, repetitionLevel, scale,
+ * Builds column statistics using given primitiveType, originalType, scale,
* precision, numNull, min and max values.
*
* @param min min value for statistics
@@ -136,24 +135,18 @@ public class ParquetMetaStatCollector implements ColumnStatCollector{
* @param numNull num_nulls for statistics
* @param primitiveType type that determines statistics class
* @param originalType type that determines statistics class
- * @param repetitionLevel field repetition level
* @param scale scale value (used for DECIMAL type)
* @param precision precision value (used for DECIMAL type)
* @return column statistics
*/
private ColumnStatistics getStat(Object min, Object max, Long numNull,
PrimitiveType.PrimitiveTypeName primitiveType, OriginalType originalType,
- Integer repetitionLevel, int scale, int precision) {
+ int scale, int precision) {
Statistics stat = Statistics.getStatsBasedOnType(primitiveType);
Statistics convertedStat = stat;
TypeProtos.MajorType type = ParquetGroupScan.getType(primitiveType, originalType, scale, precision);
- // Change to repeated if repetitionLevel > 0
- if (repetitionLevel != null && repetitionLevel > 0) {
- type = Types.withScaleAndPrecision(type.getMinorType(), TypeProtos.DataMode.REPEATED, scale, precision);
- }
-
if (numNull != null) {
stat.setNumNulls(numNull);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDownForComplexTypes.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDownForComplexTypes.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDownForComplexTypes.java
new file mode 100644
index 0000000..5cbe5cf
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetFilterPushDownForComplexTypes.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.parquet;
+
+import org.apache.drill.PlanTestBase;
+import org.apache.drill.exec.util.StoragePluginTestUtils;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.nio.file.Paths;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestParquetFilterPushDownForComplexTypes extends PlanTestBase {
+
+ private static final String TABLE_PATH = "parquet/users";
+ private static final String TABLE_NAME = String.format("%s.`%s`", StoragePluginTestUtils.DFS_PLUGIN_NAME, TABLE_PATH);
+
+ @BeforeClass
+ public static void copyData() {
+ /*
+ Parquet schema:
+ message complex_users {
+ required group user {
+ required int32 id;
+ optional int32 age;
+ repeated int32 hobby_ids;
+ optional boolean active;
+ }
+ }
+
+ Data set:
+ users_1.parquet
+ {"id":1,"age":25,"hobby_ids":[1,2,3],"active":true}
+ {"id":2,"age":28,"hobby_ids":[1,2,5],"active":true}
+
+ users_2.parquet
+ {"id":3,"age":31,"hobby_ids":[1,2,3],"active":true}
+ {"id":4,"age":32,"hobby_ids":[4,10,18],"active":false}
+
+ users_3.parquet
+ {"id":5,"hobby_ids":[11,12,13,14,15]}
+
+ users_4.parquet
+ {"id":6,"age":41,"hobby_ids":[20,21,22],"active":true}
+ {"id":7,"hobby_ids":[20,21,22,24]}
+
+ users_5.parquet
+ {"id":8,"age":41,"hobby_ids":[],"active":false}
+
+ users_6.parquet
+ {"id":9,"age":20,"hobby_ids":[],"active":false}
+ {"id":10,"age":21,"hobby_ids":[26,28,29]}
+
+ users_7.parquet
+ {"id":11,"age":23,"hobby_ids":[10,11,12],"active":true}
+ {"id":12,"age":35,"hobby_ids":[22,23,24],"active":false}
+ {"id":13,"age":25,"hobby_ids":[14,22,26]}
+
+ */
+ dirTestWatcher.copyResourceToRoot(Paths.get(TABLE_PATH));
+ }
+
+ @Test
+ public void testPushDownArray() throws Exception {
+ testParquetFilterPushDown("t.`user`.hobby_ids[0] = 1", 3, 2);
+ testParquetFilterPushDown("t.`user`.hobby_ids[0] = 100", 0, 1);
+ testParquetFilterPushDown("t.`user`.hobby_ids[0] <> 1", 8, 6);
+ testParquetFilterPushDown("t.`user`.hobby_ids[2] > 20", 5, 3);
+ testParquetFilterPushDown("t.`user`.hobby_ids[0] between 10 and 20", 5, 4);
+ testParquetFilterPushDown("t.`user`.hobby_ids[4] = 15", 1, 3);
+ testParquetFilterPushDown("t.`user`.hobby_ids[2] is not null", 11, 6);
+ testParquetFilterPushDown("t.`user`.hobby_ids[3] is null", 11, 7);
+ }
+
+ @Test
+ public void testPushDownComplexIntColumn() throws Exception {
+ testParquetFilterPushDown("t.`user`.age = 31", 1, 2);
+ testParquetFilterPushDown("t.`user`.age = 1", 0, 1);
+ testParquetFilterPushDown("t.`user`.age <> 20", 10, 6);
+ testParquetFilterPushDown("t.`user`.age > 30", 5, 4);
+ testParquetFilterPushDown("t.`user`.age between 20 and 30", 6, 3);
+ testParquetFilterPushDown("t.`user`.age is not null", 11, 6);
+ testParquetFilterPushDown("t.`user`.age is null", 2, 2);
+ }
+
+ @Test
+ public void testPushDownComplexBooleanColumn() throws Exception {
+ testParquetFilterPushDown("t.`user`.active is true", 5, 4);
+ testParquetFilterPushDown("t.`user`.active is not true", 8, 6);
+ testParquetFilterPushDown("t.`user`.active is false", 4, 4);
+ testParquetFilterPushDown("t.`user`.active is not false", 9, 6);
+ testParquetFilterPushDown("t.`user`.active is not null", 9, 6);
+ testParquetFilterPushDown("t.`user`.active is null", 4, 4);
+ }
+
+
+ private void testParquetFilterPushDown(String predicate, int expectedRowCount, int expectRowGroupsNumber) throws Exception {
+ String query = String.format("select * from %s t where %s", TABLE_NAME, predicate);
+
+ int actualRowCount = testSql(query);
+ assertEquals("Expected and actual row count should match", expectedRowCount, actualRowCount);
+
+ String expectRowGroupsNumberPattern = "numRowGroups=" + expectRowGroupsNumber;
+ testPlanMatchingPatterns(query, new String[] {expectRowGroupsNumberPattern}, new String[] {});
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/test/resources/parquet/users/users_1.parquet
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/users/users_1.parquet b/exec/java-exec/src/test/resources/parquet/users/users_1.parquet
new file mode 100644
index 0000000..9d9d511
Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/users/users_1.parquet differ
http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/test/resources/parquet/users/users_2.parquet
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/users/users_2.parquet b/exec/java-exec/src/test/resources/parquet/users/users_2.parquet
new file mode 100644
index 0000000..f48c43a
Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/users/users_2.parquet differ
http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/test/resources/parquet/users/users_3.parquet
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/users/users_3.parquet b/exec/java-exec/src/test/resources/parquet/users/users_3.parquet
new file mode 100644
index 0000000..1c1aa8e
Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/users/users_3.parquet differ
http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/test/resources/parquet/users/users_4.parquet
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/users/users_4.parquet b/exec/java-exec/src/test/resources/parquet/users/users_4.parquet
new file mode 100644
index 0000000..66a7288
Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/users/users_4.parquet differ
http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/test/resources/parquet/users/users_5.parquet
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/users/users_5.parquet b/exec/java-exec/src/test/resources/parquet/users/users_5.parquet
new file mode 100644
index 0000000..01aafd1
Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/users/users_5.parquet differ
http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/test/resources/parquet/users/users_6.parquet
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/users/users_6.parquet b/exec/java-exec/src/test/resources/parquet/users/users_6.parquet
new file mode 100644
index 0000000..c12105a
Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/users/users_6.parquet differ
http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/exec/java-exec/src/test/resources/parquet/users/users_7.parquet
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/parquet/users/users_7.parquet b/exec/java-exec/src/test/resources/parquet/users/users_7.parquet
new file mode 100644
index 0000000..00f8016
Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/users/users_7.parquet differ
http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java b/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java
index 583046a..9ea57b1 100644
--- a/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java
+++ b/logical/src/main/java/org/apache/drill/common/expression/SchemaPath.java
@@ -90,19 +90,25 @@ public class SchemaPath extends LogicalExpressionBase {
}
public NamePart getAsNamePart() {
- return getNamePart(rootSegment);
+ return getNamePart(rootSegment, false);
}
- private static NamePart getNamePart(PathSegment s) {
+ private static NamePart getNamePart(PathSegment s, boolean skipArraySegment) {
if (s == null) {
return null;
}
NamePart.Builder b = NamePart.newBuilder();
if (s.getChild() != null) {
- b.setChild(getNamePart(s.getChild()));
+ NamePart namePart = getNamePart(s.getChild(), skipArraySegment);
+ if (namePart != null) {
+ b.setChild(namePart);
+ }
}
if (s.isArray()) {
+ if (skipArraySegment) {
+ return null;
+ }
if (s.getArraySegment().hasIndex()) {
throw new IllegalStateException("You cannot convert a indexed schema path to a NamePart. NameParts can only reference Vectors, not individual records or values.");
}
@@ -129,6 +135,18 @@ public class SchemaPath extends LogicalExpressionBase {
}
/**
+ * Returns schema path with for arrays without index.
+ * Is used to find column statistics in parquet metadata.
+ * Example: a.b.c[0] -> a.b.c
+ *
+ * @return un-indexed schema path
+ */
+ public SchemaPath getUnIndexed() {
+ NamePart namePart = getNamePart(rootSegment, true);
+ return create(namePart);
+ }
+
+ /**
* Parses input string using the same rules which are used for the field in the query.
* If a string contains dot outside back-ticks, or there are no backticks in the string,
* will be created {@link SchemaPath} with the {@link NameSegment}
@@ -255,11 +273,6 @@ public class SchemaPath extends LogicalExpressionBase {
return new SchemaPath(newRoot);
}
- public SchemaPath getUnindexedArrayChild() {
- NameSegment newRoot = rootSegment.cloneWithNewChild(new ArraySegment(null));
- return new SchemaPath(newRoot);
- }
-
public SchemaPath getChild(int index) {
NameSegment newRoot = rootSegment.cloneWithNewChild(new ArraySegment(index));
return new SchemaPath(newRoot);
http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/logical/src/main/java/org/apache/drill/common/expression/TypedFieldExpr.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/TypedFieldExpr.java b/logical/src/main/java/org/apache/drill/common/expression/TypedFieldExpr.java
new file mode 100644
index 0000000..93c7a3c
--- /dev/null
+++ b/logical/src/main/java/org/apache/drill/common/expression/TypedFieldExpr.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.expression;
+
+import org.apache.drill.common.expression.visitors.ExprVisitor;
+import org.apache.drill.common.types.TypeProtos;
+
+import java.util.Collections;
+import java.util.Iterator;
+
+public class TypedFieldExpr extends LogicalExpressionBase {
+ private final TypeProtos.MajorType type;
+ private final SchemaPath path;
+
+ public TypedFieldExpr(SchemaPath path, TypeProtos.MajorType type) {
+ super(path.getPosition());
+ this.path = path;
+ this.type = type;
+ }
+
+ @Override
+ public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor, V value) throws E {
+ return visitor.visitTypedFieldExpr(this, value);
+ }
+
+ @Override
+ public Iterator<LogicalExpression> iterator() {
+ return Collections.emptyIterator();
+ }
+
+ @Override
+ public TypeProtos.MajorType getMajorType() {
+ return this.type;
+ }
+
+ @Override
+ public String toString() {
+ return this.path.getRootSegment().getPath() + "(" + type.getMinorType() + "_" + type.getMode() +")";
+ }
+
+ public SchemaPath getPath() {
+ return this.path;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/logical/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java b/logical/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java
index 189e33d..5356813 100644
--- a/logical/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java
+++ b/logical/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java
@@ -26,6 +26,7 @@ import org.apache.drill.common.expression.IfExpression;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.NullExpression;
import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.expression.TypedFieldExpr;
import org.apache.drill.common.expression.TypedNullConstant;
import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
import org.apache.drill.common.expression.ValueExpressions.DateExpression;
@@ -45,7 +46,6 @@ import org.apache.drill.common.expression.ValueExpressions.TimeExpression;
import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression;
public abstract class AbstractExprVisitor<T, VAL, EXCEP extends Exception> implements ExprVisitor<T, VAL, EXCEP> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractExprVisitor.class);
@Override
public T visitFunctionCall(FunctionCall call, VAL value) throws EXCEP {
@@ -178,4 +178,8 @@ public abstract class AbstractExprVisitor<T, VAL, EXCEP extends Exception> imple
return visitUnknown(e, value);
}
+ @Override
+ public T visitTypedFieldExpr(TypedFieldExpr e, VAL value) throws EXCEP {
+ return visitUnknown(e, value);
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/logical/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java b/logical/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java
index 9a3cdcc..f6fe89d 100644
--- a/logical/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java
+++ b/logical/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java
@@ -28,6 +28,7 @@ import org.apache.drill.common.expression.IfExpression.IfCondition;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.NullExpression;
import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.expression.TypedFieldExpr;
import org.apache.drill.common.expression.TypedNullConstant;
import org.apache.drill.common.expression.ValueExpressions;
import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
@@ -209,4 +210,8 @@ public final class AggregateChecker implements ExprVisitor<Boolean, ErrorCollect
return false;
}
+ @Override
+ public Boolean visitTypedFieldExpr(TypedFieldExpr e, ErrorCollector value) throws RuntimeException {
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/logical/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java b/logical/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java
index 67fe12f..0468bc2 100644
--- a/logical/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java
+++ b/logical/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java
@@ -28,6 +28,7 @@ import org.apache.drill.common.expression.IfExpression.IfCondition;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.NullExpression;
import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.expression.TypedFieldExpr;
import org.apache.drill.common.expression.TypedNullConstant;
import org.apache.drill.common.expression.ValueExpressions;
import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
@@ -213,4 +214,8 @@ final class ConstantChecker implements ExprVisitor<Boolean, ErrorCollector, Runt
return false;
}
+ @Override
+ public Boolean visitTypedFieldExpr(TypedFieldExpr e, ErrorCollector value) throws RuntimeException {
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/logical/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java b/logical/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java
index 7c59f3c..e6198ae 100644
--- a/logical/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java
+++ b/logical/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java
@@ -26,6 +26,7 @@ import org.apache.drill.common.expression.IfExpression;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.NullExpression;
import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.expression.TypedFieldExpr;
import org.apache.drill.common.expression.TypedNullConstant;
import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
import org.apache.drill.common.expression.ValueExpressions.DateExpression;
@@ -45,30 +46,31 @@ import org.apache.drill.common.expression.ValueExpressions.TimeExpression;
import org.apache.drill.common.expression.ValueExpressions.TimeStampExpression;
public interface ExprVisitor<T, VAL, EXCEP extends Exception> {
- public T visitFunctionCall(FunctionCall call, VAL value) throws EXCEP;
- public T visitFunctionHolderExpression(FunctionHolderExpression holder, VAL value) throws EXCEP;
- public T visitIfExpression(IfExpression ifExpr, VAL value) throws EXCEP;
- public T visitBooleanOperator(BooleanOperator call, VAL value) throws EXCEP;
- public T visitSchemaPath(SchemaPath path, VAL value) throws EXCEP;
- public T visitIntConstant(IntExpression intExpr, VAL value) throws EXCEP;
- public T visitFloatConstant(FloatExpression fExpr, VAL value) throws EXCEP;
- public T visitLongConstant(LongExpression intExpr, VAL value) throws EXCEP;
- public T visitDateConstant(DateExpression intExpr, VAL value) throws EXCEP;
- public T visitTimeConstant(TimeExpression intExpr, VAL value) throws EXCEP;
- public T visitTimeStampConstant(TimeStampExpression intExpr, VAL value) throws EXCEP;
- public T visitIntervalYearConstant(IntervalYearExpression intExpr, VAL value) throws EXCEP;
- public T visitIntervalDayConstant(IntervalDayExpression intExpr, VAL value) throws EXCEP;
- public T visitDecimal9Constant(Decimal9Expression decExpr, VAL value) throws EXCEP;
- public T visitDecimal18Constant(Decimal18Expression decExpr, VAL value) throws EXCEP;
- public T visitDecimal28Constant(Decimal28Expression decExpr, VAL value) throws EXCEP;
- public T visitDecimal38Constant(Decimal38Expression decExpr, VAL value) throws EXCEP;
- public T visitDoubleConstant(DoubleExpression dExpr, VAL value) throws EXCEP;
- public T visitBooleanConstant(BooleanExpression e, VAL value) throws EXCEP;
- public T visitQuotedStringConstant(QuotedString e, VAL value) throws EXCEP;
- public T visitNullConstant(TypedNullConstant e, VAL value) throws EXCEP;
- public T visitNullExpression(NullExpression e, VAL value) throws EXCEP;
- public T visitUnknown(LogicalExpression e, VAL value) throws EXCEP;
- public T visitCastExpression(CastExpression e, VAL value) throws EXCEP;
- public T visitConvertExpression(ConvertExpression e, VAL value) throws EXCEP;
- public T visitParameter(ParameterExpression e, VAL value) throws EXCEP;
+ T visitFunctionCall(FunctionCall call, VAL value) throws EXCEP;
+ T visitFunctionHolderExpression(FunctionHolderExpression holder, VAL value) throws EXCEP;
+ T visitIfExpression(IfExpression ifExpr, VAL value) throws EXCEP;
+ T visitBooleanOperator(BooleanOperator call, VAL value) throws EXCEP;
+ T visitSchemaPath(SchemaPath path, VAL value) throws EXCEP;
+ T visitIntConstant(IntExpression intExpr, VAL value) throws EXCEP;
+ T visitFloatConstant(FloatExpression fExpr, VAL value) throws EXCEP;
+ T visitLongConstant(LongExpression intExpr, VAL value) throws EXCEP;
+ T visitDateConstant(DateExpression intExpr, VAL value) throws EXCEP;
+ T visitTimeConstant(TimeExpression intExpr, VAL value) throws EXCEP;
+ T visitTimeStampConstant(TimeStampExpression intExpr, VAL value) throws EXCEP;
+ T visitIntervalYearConstant(IntervalYearExpression intExpr, VAL value) throws EXCEP;
+ T visitIntervalDayConstant(IntervalDayExpression intExpr, VAL value) throws EXCEP;
+ T visitDecimal9Constant(Decimal9Expression decExpr, VAL value) throws EXCEP;
+ T visitDecimal18Constant(Decimal18Expression decExpr, VAL value) throws EXCEP;
+ T visitDecimal28Constant(Decimal28Expression decExpr, VAL value) throws EXCEP;
+ T visitDecimal38Constant(Decimal38Expression decExpr, VAL value) throws EXCEP;
+ T visitDoubleConstant(DoubleExpression dExpr, VAL value) throws EXCEP;
+ T visitBooleanConstant(BooleanExpression e, VAL value) throws EXCEP;
+ T visitQuotedStringConstant(QuotedString e, VAL value) throws EXCEP;
+ T visitNullConstant(TypedNullConstant e, VAL value) throws EXCEP;
+ T visitNullExpression(NullExpression e, VAL value) throws EXCEP;
+ T visitUnknown(LogicalExpression e, VAL value) throws EXCEP;
+ T visitCastExpression(CastExpression e, VAL value) throws EXCEP;
+ T visitConvertExpression(ConvertExpression e, VAL value) throws EXCEP;
+ T visitParameter(ParameterExpression e, VAL value) throws EXCEP;
+ T visitTypedFieldExpr(TypedFieldExpr e, VAL value) throws EXCEP;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/4ee5625d/logical/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidator.java
----------------------------------------------------------------------
diff --git a/logical/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidator.java b/logical/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidator.java
index e8cbc2b..c32825a 100644
--- a/logical/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidator.java
+++ b/logical/src/main/java/org/apache/drill/common/expression/visitors/ExpressionValidator.java
@@ -28,6 +28,7 @@ import org.apache.drill.common.expression.IfExpression.IfCondition;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.expression.NullExpression;
import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.expression.TypedFieldExpr;
import org.apache.drill.common.expression.TypedNullConstant;
import org.apache.drill.common.expression.ValueExpressions;
import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
@@ -236,4 +237,8 @@ public class ExpressionValidator implements ExprVisitor<Void, ErrorCollector, Ru
return null;
}
+ @Override
+ public Void visitTypedFieldExpr(TypedFieldExpr e, ErrorCollector value) throws RuntimeException {
+ return null;
+ }
}
|