drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jacq...@apache.org
Subject [01/27] git commit: DRILL-1134 : Fix compilation exception in run-time code, when filter uses two or more array types.
Date Sun, 27 Jul 2014 18:46:47 GMT
Repository: incubator-drill
Updated Branches:
  refs/heads/master e5c2da0eb -> caa8b78c5


DRILL-1134 : Fix compilation exception in run-time code, when filter uses two or more array
types.


Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/2ab3dfab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/2ab3dfab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/2ab3dfab

Branch: refs/heads/master
Commit: 2ab3dfab97a4cd58699919f7c647a7762662759a
Parents: e5c2da0
Author: Jinfeng Ni <jni@maprtech.com>
Authored: Mon Jul 14 16:42:27 2014 -0700
Committer: Jinfeng Ni <jni@maprtech.com>
Committed: Sun Jul 20 22:20:30 2014 -0700

----------------------------------------------------------------------
 .../drill/exec/expr/EvaluationVisitor.java      | 55 +++++++++++++-------
 .../org/apache/drill/TestExampleQueries.java    | 12 ++++-
 .../complex/writer/TestComplexTypeReader.java   |  7 +++
 3 files changed, 54 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2ab3dfab/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
index 5511cb2..8bfa149 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/EvaluationVisitor.java
@@ -379,27 +379,30 @@ public class EvaluationVisitor {
       } else {
         JExpression vector = e.isSuperReader() ? vv1.component(componentVariable) : vv1;
         JExpression expr = vector.invoke("getAccessor").invoke("getReader");
-        JVar isNull = generator.getEvalBlock().decl(generator.getModel().INT, "isNull", JExpr.lit(0));
+        PathSegment seg = e.getReadPath();
+
+        JVar isNull = null;
+        boolean isNullReaderLikely = isNullReaderLikely(seg, complex || repeated);
+        if (isNullReaderLikely) {
+          isNull = generator.getEvalBlock().decl(generator.getModel().INT, generator.getNextVar("isNull"),
JExpr.lit(0));
+        }
 
         JLabel label = generator.getEvalBlock().label("complex");
         JBlock eval = generator.getEvalBlock().block();
 
         // position to the correct value.
         eval.add(expr.invoke("setPosition").arg(indexVariable));
-        PathSegment seg = e.getReadPath();
         int listNum = 0;
-        boolean lastWasArray = false;
 
         while (seg != null) {
           if (seg.isArray()) {
-            lastWasArray = true;
-
+            // stop once we get to the last segment and the final type is neither complex
nor repeated (map, list, repeated list).
+            // In case of non-complex and non-repeated type, we return Holder, in stead of
FieldReader.
             if (seg.isLastPath() && !complex && !repeated)
               break;
 
             JVar list = generator.declareClassField("list", generator.getModel()._ref(FieldReader.class));
             eval.assign(list, expr);
-            expr = list;
 
             // if this is an array, set a single position for the expression to
             // allow us to read the right data lower down.
@@ -413,7 +416,6 @@ public class EvaluationVisitor {
                 currentIndex.lt(desiredIndex) //
                     .cand(list.invoke("next"))).body().assign(currentIndex, currentIndex.plus(JExpr.lit(1)));
 
-            expr = list.invoke("reader");
 
             JBlock ifNoVal = eval._if(desiredIndex.ne(currentIndex))._then().block();
             if (out.isOptional()) {
@@ -422,17 +424,13 @@ public class EvaluationVisitor {
             ifNoVal.assign(isNull,  JExpr.lit(1));
             ifNoVal._break(label);
 
+            expr = list.invoke("reader");
             listNum++;
           } else {
-            lastWasArray = false;
             JExpression fieldName = JExpr.lit(seg.getNameSegment().getPath());
             expr = expr.invoke("reader").arg(fieldName);
           }
           seg = seg.getChild();
-
-          // stop once we get to last column or when the segment is an array at
-          // the end of the reference.
-          // if(seg == null || seg.isLastPath() && seg.isArray()) break;
         }
 
         if (complex || repeated) {
@@ -440,17 +438,20 @@ public class EvaluationVisitor {
           // //
           JVar complexReader = generator.declareClassField("reader", generator.getModel()._ref(FieldReader.class));
 
-          JConditional jc = generator.getEvalBlock()._if(isNull.eq(JExpr.lit(0)));
+          if (isNullReaderLikely) {
+            JConditional jc = generator.getEvalBlock()._if(isNull.eq(JExpr.lit(0)));
 
-          JClass nrClass = generator.getModel().ref(org.apache.drill.exec.vector.complex.impl.NullReader.class);
-          JExpression nullReader = nrClass.staticRef("INSTANCE");
+            JClass nrClass = generator.getModel().ref(org.apache.drill.exec.vector.complex.impl.NullReader.class);
+            JExpression nullReader = nrClass.staticRef("INSTANCE");
 
-          jc._then().assign(complexReader, expr);
-          jc._else().assign(complexReader, nullReader);
+            jc._then().assign(complexReader, expr);
+            jc._else().assign(complexReader, nullReader);
+          } else {
+            eval.assign(complexReader, expr);
+          }
 
           HoldingContainer hc = new HoldingContainer(e.getMajorType(), complexReader, null,
null, false, true);
           return hc;
-          // //eval.assign(out.getHolder().ref("reader"), expr);
         } else {
           if (seg != null) {
             eval.add(expr.invoke("read").arg(JExpr.lit(seg.getArraySegment().getIndex())).arg(out.getHolder()));
@@ -464,6 +465,24 @@ public class EvaluationVisitor {
       return out;
     }
 
+    /*  Check if a Path expression could produce a NullReader. A path expression will produce
a null reader, when:
+     *   1) It contains an array segment as non-leaf segment :  a.b[2].c.  segment [2] might
produce null reader.
+     *   2) It contains an array segment as leaf segment, AND the final output is complex
or repeated : a.b[2], when
+     *     the final type of this expression is a map, or releated list, or repeated map.
+     */
+    private boolean isNullReaderLikely(PathSegment seg, boolean complexOrRepeated) {
+      while (seg != null) {
+        if (seg.isArray() && !seg.isLastPath())
+          return true;
+
+        if (seg.isLastPath() && complexOrRepeated)
+          return true;
+
+        seg = seg.getChild();
+      }
+      return false;
+    }
+
     private HoldingContainer visitReturnValueExpression(ReturnValueExpression e, ClassGenerator<?>
generator) {
       LogicalExpression child = e.getChild();
       // Preconditions.checkArgument(child.getMajorType().equals(Types.REQUIRED_BOOLEAN));

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2ab3dfab/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
index 21526ed..cfe2a88 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
@@ -103,6 +103,14 @@ public class TestExampleQueries extends BaseTestQuery{
   }
 
   @Test
+  public void testFilterOnArrayTypes() throws Exception {
+    String root = FileUtils.getResourceAsFile("/store/text/data/regions.csv").toURI().toString();
+    String query = String.format("select columns[0] from dfs_test.`%s` " +
+        " where cast(columns[0] as int) > 1 and cast(columns[1] as varchar(20))='ASIA'",
root);
+    test(query);
+  }
+
+  @Test
   public void testTextPartitions() throws Exception {
     String root = FileUtils.getResourceAsFile("/store/text/data/").toURI().toString();
     String query = String.format("select * from dfs_test.`%s`", root);
@@ -116,9 +124,9 @@ public class TestExampleQueries extends BaseTestQuery{
         "  nations.N_NAME,\n" +
         "  regions.R_NAME\n" +
         "FROM\n" +
-        "  dfs_test.`[WORKING_PATH]/../../sample-data/nation.parquet` nations\n" +
+        "  cp.`tpch/nation.parquet` nations\n" +
         "JOIN\n" +
-        "  dfs_test.`[WORKING_PATH]/../../sample-data/region.parquet` regions\n" +
+        "  cp.`tpch/region.parquet` regions\n" +
         "  on nations.N_REGIONKEY = regions.R_REGIONKEY where 1 = 0");
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2ab3dfab/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
index 3da3543..e459c4c 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
@@ -146,6 +146,13 @@ public class TestComplexTypeReader extends BaseTestQuery{
     test("  select convert_to(types[1].minor[0].valueHolder, 'JSON') from cp.`jsoninput/vvtypes.json`;");
   }
 
+
+  @Test
+  //Two complex type functions in SELECT clause : repeated map (map) --> Json,
+  public void testA4() throws Exception{
+    test("  select convert_to(types[1], 'JSON'), convert_to(modes[2], 'JSON') from cp.`jsoninput/vvtypes.json`;");
+  }
+
   @Test
   //repeated map (map) .
   public void testB1() throws Exception{


Mime
View raw message