Repository: incubator-drill
Updated Branches:
refs/heads/master e5c2da0eb -> caa8b78c5
DRILL-1134 : Fix compilation exception in run-time code, when filter uses two or more array
types.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/2ab3dfab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/2ab3dfab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/2ab3dfab
Branch: refs/heads/master
Commit: 2ab3dfab97a4cd58699919f7c647a7762662759a
Parents: e5c2da0
Author: Jinfeng Ni <jni@maprtech.com>
Authored: Mon Jul 14 16:42:27 2014 -0700
Committer: Jinfeng Ni <jni@maprtech.com>
Committed: Sun Jul 20 22:20:30 2014 -0700
----------------------------------------------------------------------
.../drill/exec/expr/EvaluationVisitor.java | 55 +++++++++++++-------
.../org/apache/drill/TestExampleQueries.java | 12 ++++-
.../complex/writer/TestComplexTypeReader.java | 7 +++
3 files changed, 54 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2ab3dfab/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
index 5511cb2..8bfa149 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
@@ -379,27 +379,30 @@ public class EvaluationVisitor {
} else {
JExpression vector = e.isSuperReader() ? vv1.component(componentVariable) : vv1;
JExpression expr = vector.invoke("getAccessor").invoke("getReader");
- JVar isNull = generator.getEvalBlock().decl(generator.getModel().INT, "isNull", JExpr.lit(0));
+ PathSegment seg = e.getReadPath();
+
+ JVar isNull = null;
+ boolean isNullReaderLikely = isNullReaderLikely(seg, complex || repeated);
+ if (isNullReaderLikely) {
+ isNull = generator.getEvalBlock().decl(generator.getModel().INT, generator.getNextVar("isNull"),
JExpr.lit(0));
+ }
JLabel label = generator.getEvalBlock().label("complex");
JBlock eval = generator.getEvalBlock().block();
// position to the correct value.
eval.add(expr.invoke("setPosition").arg(indexVariable));
- PathSegment seg = e.getReadPath();
int listNum = 0;
- boolean lastWasArray = false;
while (seg != null) {
if (seg.isArray()) {
- lastWasArray = true;
-
+ // stop once we get to the last segment and the final type is neither complex
nor repeated (map, list, repeated list).
+ // In case of non-complex and non-repeated type, we return Holder, in stead of
FieldReader.
if (seg.isLastPath() && !complex && !repeated)
break;
JVar list = generator.declareClassField("list", generator.getModel()._ref(FieldReader.class));
eval.assign(list, expr);
- expr = list;
// if this is an array, set a single position for the expression to
// allow us to read the right data lower down.
@@ -413,7 +416,6 @@ public class EvaluationVisitor {
currentIndex.lt(desiredIndex) //
.cand(list.invoke("next"))).body().assign(currentIndex, currentIndex.plus(JExpr.lit(1)));
- expr = list.invoke("reader");
JBlock ifNoVal = eval._if(desiredIndex.ne(currentIndex))._then().block();
if (out.isOptional()) {
@@ -422,17 +424,13 @@ public class EvaluationVisitor {
ifNoVal.assign(isNull, JExpr.lit(1));
ifNoVal._break(label);
+ expr = list.invoke("reader");
listNum++;
} else {
- lastWasArray = false;
JExpression fieldName = JExpr.lit(seg.getNameSegment().getPath());
expr = expr.invoke("reader").arg(fieldName);
}
seg = seg.getChild();
-
- // stop once we get to last column or when the segment is an array at
- // the end of the reference.
- // if(seg == null || seg.isLastPath() && seg.isArray()) break;
}
if (complex || repeated) {
@@ -440,17 +438,20 @@ public class EvaluationVisitor {
// //
JVar complexReader = generator.declareClassField("reader", generator.getModel()._ref(FieldReader.class));
- JConditional jc = generator.getEvalBlock()._if(isNull.eq(JExpr.lit(0)));
+ if (isNullReaderLikely) {
+ JConditional jc = generator.getEvalBlock()._if(isNull.eq(JExpr.lit(0)));
- JClass nrClass = generator.getModel().ref(org.apache.drill.exec.vector.complex.impl.NullReader.class);
- JExpression nullReader = nrClass.staticRef("INSTANCE");
+ JClass nrClass = generator.getModel().ref(org.apache.drill.exec.vector.complex.impl.NullReader.class);
+ JExpression nullReader = nrClass.staticRef("INSTANCE");
- jc._then().assign(complexReader, expr);
- jc._else().assign(complexReader, nullReader);
+ jc._then().assign(complexReader, expr);
+ jc._else().assign(complexReader, nullReader);
+ } else {
+ eval.assign(complexReader, expr);
+ }
HoldingContainer hc = new HoldingContainer(e.getMajorType(), complexReader, null,
null, false, true);
return hc;
- // //eval.assign(out.getHolder().ref("reader"), expr);
} else {
if (seg != null) {
eval.add(expr.invoke("read").arg(JExpr.lit(seg.getArraySegment().getIndex())).arg(out.getHolder()));
@@ -464,6 +465,24 @@ public class EvaluationVisitor {
return out;
}
+ /* Check if a Path expression could produce a NullReader. A path expression will produce
a null reader, when:
+ * 1) It contains an array segment as non-leaf segment : a.b[2].c. segment [2] might
produce null reader.
+ * 2) It contains an array segment as leaf segment, AND the final output is complex
or repeated : a.b[2], when
+ * the final type of this expression is a map, or releated list, or repeated map.
+ */
+ private boolean isNullReaderLikely(PathSegment seg, boolean complexOrRepeated) {
+ while (seg != null) {
+ if (seg.isArray() && !seg.isLastPath())
+ return true;
+
+ if (seg.isLastPath() && complexOrRepeated)
+ return true;
+
+ seg = seg.getChild();
+ }
+ return false;
+ }
+
private HoldingContainer visitReturnValueExpression(ReturnValueExpression e, ClassGenerator<?>
generator) {
LogicalExpression child = e.getChild();
// Preconditions.checkArgument(child.getMajorType().equals(Types.REQUIRED_BOOLEAN));
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2ab3dfab/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
index 21526ed..cfe2a88 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
@@ -103,6 +103,14 @@ public class TestExampleQueries extends BaseTestQuery{
}
@Test
+ public void testFilterOnArrayTypes() throws Exception {
+ String root = FileUtils.getResourceAsFile("/store/text/data/regions.csv").toURI().toString();
+ String query = String.format("select columns[0] from dfs_test.`%s` " +
+ " where cast(columns[0] as int) > 1 and cast(columns[1] as varchar(20))='ASIA'",
root);
+ test(query);
+ }
+
+ @Test
public void testTextPartitions() throws Exception {
String root = FileUtils.getResourceAsFile("/store/text/data/").toURI().toString();
String query = String.format("select * from dfs_test.`%s`", root);
@@ -116,9 +124,9 @@ public class TestExampleQueries extends BaseTestQuery{
" nations.N_NAME,\n" +
" regions.R_NAME\n" +
"FROM\n" +
- " dfs_test.`[WORKING_PATH]/../../sample-data/nation.parquet` nations\n" +
+ " cp.`tpch/nation.parquet` nations\n" +
"JOIN\n" +
- " dfs_test.`[WORKING_PATH]/../../sample-data/region.parquet` regions\n" +
+ " cp.`tpch/region.parquet` regions\n" +
" on nations.N_REGIONKEY = regions.R_REGIONKEY where 1 = 0");
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2ab3dfab/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
index 3da3543..e459c4c 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
@@ -146,6 +146,13 @@ public class TestComplexTypeReader extends BaseTestQuery{
test(" select convert_to(types[1].minor[0].valueHolder, 'JSON') from cp.`jsoninput/vvtypes.json`;");
}
+
+ @Test
+ //Two complex type functions in SELECT clause : repeated map (map) --> Json,
+ public void testA4() throws Exception{
+ test(" select convert_to(types[1], 'JSON'), convert_to(modes[2], 'JSON') from cp.`jsoninput/vvtypes.json`;");
+ }
+
@Test
//repeated map (map) .
public void testB1() throws Exception{
|