drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j..@apache.org
Subject [5/5] drill git commit: DRILL-5164: Equi-join query results in CompileException when inputs have large number of columns.
Date Tue, 24 Jan 2017 06:15:25 GMT
DRILL-5164: Equi-join query results in CompileException when inputs have large number of columns.

close apache/drill#711


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/2af709f4
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/2af709f4
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/2af709f4

Branch: refs/heads/master
Commit: 2af709f43d01f341b2a52c6473ea49d6761fdc61
Parents: 8217697
Author: Serhii-Harnyk <serhii.harnyk@gmail.com>
Authored: Tue Dec 27 16:20:37 2016 +0000
Committer: Jinfeng Ni <jni@apache.org>
Committed: Mon Jan 23 17:08:58 2017 -0800

----------------------------------------------------------------------
 .../exec/physical/impl/join/HashJoinBatch.java  |  4 +-
 .../exec/physical/impl/join/MergeJoinBatch.java |  2 +
 .../physical/impl/join/NestedLoopJoinBatch.java |  4 +-
 .../exec/compile/TestLargeFileCompilation.java  | 94 +++++++++++++++++---
 4 files changed, 87 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/2af709f4/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
index 23741b0..f1f81fb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
@@ -439,7 +439,7 @@ public class HashJoinBatch extends AbstractRecordBatch<HashJoinPOP>
{
             .arg(buildIndex.band(JExpr.lit((int) Character.MAX_VALUE)))
             .arg(outIndex)
             .arg(inVV.component(buildIndex.shrz(JExpr.lit(16)))));
-
+        g.rotateBlock();
         fieldId++;
       }
     }
@@ -475,7 +475,7 @@ public class HashJoinBatch extends AbstractRecordBatch<HashJoinPOP>
{
         final JVar outVV = g.declareVectorValueSetupAndMember("outgoing", new TypedFieldId(outputType,
false, outputFieldId));
 
         g.getEvalBlock().add(outVV.invoke("copyFromSafe").arg(probeIndex).arg(outIndex).arg(inVV));
-
+        g.rotateBlock();
         fieldId++;
         outputFieldId++;
       }

http://git-wip-us.apache.org/repos/asf/drill/blob/2af709f4/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
index a9bb479..c351517 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/MergeJoinBatch.java
@@ -345,6 +345,7 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP>
{
           .arg(copyLeftMapping.getValueReadIndex())
           .arg(copyLeftMapping.getValueWriteIndex())
           .arg(vvIn));
+        cg.rotateBlock();
         ++vectorId;
       }
     }
@@ -373,6 +374,7 @@ public class MergeJoinBatch extends AbstractRecordBatch<MergeJoinPOP>
{
           .arg(copyRightMappping.getValueReadIndex())
           .arg(copyRightMappping.getValueWriteIndex())
           .arg(vvIn));
+        cg.rotateBlock();
         ++vectorId;
       }
     }

http://git-wip-us.apache.org/repos/asf/drill/blob/2af709f4/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java
index 2e92c8d..bdd9f0e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/NestedLoopJoinBatch.java
@@ -247,7 +247,7 @@ public class NestedLoopJoinBatch extends AbstractRecordBatch<NestedLoopJoinPOP>
       JVar outVV = nLJClassGenerator.declareVectorValueSetupAndMember("outgoing", new TypedFieldId(fieldType,
false, outputFieldId));
 
       nLJClassGenerator.getEvalBlock().add(outVV.invoke("copyFromSafe").arg(leftIndex).arg(outIndex).arg(inVV));
-
+      nLJClassGenerator.rotateBlock();
       fieldId++;
       outputFieldId++;
     }
@@ -270,7 +270,7 @@ public class NestedLoopJoinBatch extends AbstractRecordBatch<NestedLoopJoinPOP>
           .arg(recordIndexWithinBatch)
           .arg(outIndex)
           .arg(inVV.component(batchIndex)));
-
+      nLJClassGenerator.rotateBlock();
       fieldId++;
       outputFieldId++;
     }

http://git-wip-us.apache.org/repos/asf/drill/blob/2af709f4/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestLargeFileCompilation.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestLargeFileCompilation.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestLargeFileCompilation.java
index 35bd4c9..8416d73 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestLargeFileCompilation.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestLargeFileCompilation.java
@@ -39,27 +39,33 @@ public class TestLargeFileCompilation extends BaseTestQuery {
 
   private static final String LARGE_QUERY_SELECT_LIST;
 
+  private static final String QUERY_WITH_JOIN;
+
+  private static final String LARGE_TABLE_WRITER;
+
   private static final int ITERATION_COUNT = Integer.valueOf(System.getProperty("TestLargeFileCompilation.iteration",
"1"));
 
-  private static final int NUM_PROJECT_COULMNS = 2000;
+  private static final int NUM_PROJECT_COLUMNS = 2000;
+
+  private static final int NUM_ORDERBY_COLUMNS = 500;
 
-  private static final int NUM_ORDERBY_COULMNS = 500;
+  private static final int NUM_GROUPBY_COLUMNS = 225;
 
-  private static final int NUM_GROUPBY_COULMNS = 225;
+  private static final int NUM_FILTER_COLUMNS = 150;
 
-  private static final int NUM_FILTER_COULMNS = 150;
+  private static final int NUM_JOIN_TABLE_COLUMNS = 500;
 
   static {
     StringBuilder sb = new StringBuilder("select\n\t");
-    for (int i = 0; i < NUM_GROUPBY_COULMNS; i++) {
+    for (int i = 0; i < NUM_GROUPBY_COLUMNS; i++) {
       sb.append("c").append(i).append(", ");
     }
     sb.append("full_name\nfrom (select\n\t");
-    for (int i = 0; i < NUM_GROUPBY_COULMNS; i++) {
+    for (int i = 0; i < NUM_GROUPBY_COLUMNS; i++) {
       sb.append("employee_id+").append(i).append(" as c").append(i).append(", ");
     }
     sb.append("full_name\nfrom cp.`employee.json`)\ngroup by\n\t");
-    for (int i = 0; i < NUM_GROUPBY_COULMNS; i++) {
+    for (int i = 0; i < NUM_GROUPBY_COLUMNS; i++) {
       sb.append("c").append(i).append(", ");
     }
     LARGE_QUERY_GROUP_BY = sb.append("full_name").toString();
@@ -67,7 +73,7 @@ public class TestLargeFileCompilation extends BaseTestQuery {
 
   static {
     StringBuilder sb = new StringBuilder("select\n\t");
-    for (int i = 0; i < NUM_PROJECT_COULMNS; i++) {
+    for (int i = 0; i < NUM_PROJECT_COLUMNS; i++) {
       sb.append("employee_id+").append(i).append(" as col").append(i).append(", ");
     }
     sb.append("full_name\nfrom cp.`employee.json`\n\n\t");
@@ -76,11 +82,11 @@ public class TestLargeFileCompilation extends BaseTestQuery {
 
   static {
     StringBuilder sb = new StringBuilder("select\n\t");
-    for (int i = 0; i < NUM_PROJECT_COULMNS; i++) {
+    for (int i = 0; i < NUM_PROJECT_COLUMNS; i++) {
       sb.append("employee_id+").append(i).append(" as col").append(i).append(", ");
     }
     sb.append("full_name\nfrom cp.`employee.json`\norder by\n\t");
-    for (int i = 0; i < NUM_ORDERBY_COULMNS; i++) {
+    for (int i = 0; i < NUM_ORDERBY_COLUMNS; i++) {
       sb.append(" col").append(i).append(", ");
     }
     LARGE_QUERY_ORDER_BY = sb.append("full_name").toString();
@@ -91,18 +97,24 @@ public class TestLargeFileCompilation extends BaseTestQuery {
     StringBuilder sb = new StringBuilder("select *\n")
       .append("from cp.`employee.json`\n")
       .append("where");
-    for (int i = 0; i < NUM_FILTER_COULMNS; i++) {
+    for (int i = 0; i < NUM_FILTER_COLUMNS; i++) {
       sb.append(" employee_id+").append(i).append(" < employee_id ").append(i%2==0?"OR":"AND");
     }
     LARGE_QUERY_FILTER = sb.append(" true") .toString();
   }
 
   static {
+    LARGE_QUERY_WRITER = createTableWithColsCount(NUM_PROJECT_COLUMNS);
+    LARGE_TABLE_WRITER = createTableWithColsCount(NUM_JOIN_TABLE_COLUMNS);
+    QUERY_WITH_JOIN = "select * from %1$s t1, %1$s t2 where t1.col1 = t2.col1";
+  }
+
+  private static String createTableWithColsCount(int columnsCount) {
     StringBuilder sb = new StringBuilder("create table %s as (select \n");
-    for (int i = 0; i < NUM_PROJECT_COULMNS; i++) {
+    for (int i = 0; i < columnsCount; i++) {
       sb.append("employee_id+").append(i).append(" as col").append(i).append(", ");
     }
-    LARGE_QUERY_WRITER = sb.append("full_name\nfrom cp.`employee.json` limit 1)").toString();
+    return sb.append("full_name\nfrom cp.`employee.json` limit 1)").toString();
   }
 
   @Test
@@ -150,4 +162,60 @@ public class TestLargeFileCompilation extends BaseTestQuery {
     testNoResult("alter session set `%s`='JDK'", ClassCompilerSelector.JAVA_COMPILER_OPTION);
     testNoResult(ITERATION_COUNT, LARGE_QUERY_SELECT_LIST);
   }
+
+  @Test
+  public void testHashJoin() throws Exception {
+    String tableName = "wide_table_hash_join";
+    try {
+      testNoResult("alter session set `%s`='JDK'", ClassCompilerSelector.JAVA_COMPILER_OPTION);
+      testNoResult("alter session set `planner.enable_mergejoin` = false");
+      testNoResult("alter session set `planner.enable_nestedloopjoin` = false");
+      testNoResult("use dfs_test.tmp");
+      testNoResult(LARGE_TABLE_WRITER, tableName);
+      testNoResult(QUERY_WITH_JOIN, tableName);
+    } finally {
+      testNoResult("alter session reset `planner.enable_mergejoin`");
+      testNoResult("alter session reset `planner.enable_nestedloopjoin`");
+      testNoResult("alter session reset `%s`", ClassCompilerSelector.JAVA_COMPILER_OPTION);
+      testNoResult("drop table if exists %s", tableName);
+    }
+  }
+
+  @Test
+  public void testMergeJoin() throws Exception {
+    String tableName = "wide_table_merge_join";
+    try {
+      testNoResult("alter session set `%s`='JDK'", ClassCompilerSelector.JAVA_COMPILER_OPTION);
+      testNoResult("alter session set `planner.enable_hashjoin` = false");
+      testNoResult("alter session set `planner.enable_nestedloopjoin` = false");
+      testNoResult("use dfs_test.tmp");
+      testNoResult(LARGE_TABLE_WRITER, tableName);
+      testNoResult(QUERY_WITH_JOIN, tableName);
+    } finally {
+      testNoResult("alter session reset `planner.enable_hashjoin`");
+      testNoResult("alter session reset `planner.enable_nestedloopjoin`");
+      testNoResult("alter session reset `%s`", ClassCompilerSelector.JAVA_COMPILER_OPTION);
+      testNoResult("drop table if exists %s", tableName);
+    }
+  }
+
+  @Test
+  public void testNestedLoopJoin() throws Exception {
+    String tableName = "wide_table_loop_join";
+    try {
+      testNoResult("alter session set `%s`='JDK'", ClassCompilerSelector.JAVA_COMPILER_OPTION);
+      testNoResult("alter session set `planner.enable_nljoin_for_scalar_only` = false");
+      testNoResult("alter session set `planner.enable_hashjoin` = false");
+      testNoResult("alter session set `planner.enable_mergejoin` = false");
+      testNoResult("use dfs_test.tmp");
+      testNoResult(LARGE_TABLE_WRITER, tableName);
+      testNoResult(QUERY_WITH_JOIN, tableName);
+    } finally {
+      testNoResult("alter session reset `planner.enable_nljoin_for_scalar_only`");
+      testNoResult("alter session reset `planner.enable_hashjoin`");
+      testNoResult("alter session reset `planner.enable_mergejoin`");
+      testNoResult("alter session reset `%s`", ClassCompilerSelector.JAVA_COMPILER_OPTION);
+      testNoResult("drop table if exists %s", tableName);
+    }
+  }
 }


Mime
View raw message