drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j..@apache.org
Subject drill git commit: DRILL-4387: GroupScan or ScanBatchCreator should not use star column in case of skipAll query.
Date Mon, 22 Feb 2016 21:01:34 GMT
Repository: drill
Updated Branches:
  refs/heads/master a45f7fd11 -> c67d070bf


DRILL-4387: GroupScan or ScanBatchCreator should not use star column in case of skipAll query.

The skipAll query should be handled in RecordReader.


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/c67d070b
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/c67d070b
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/c67d070b

Branch: refs/heads/master
Commit: c67d070bf68e9492b7fbe6d9e54d70d587b28a6e
Parents: a45f7fd
Author: Jinfeng Ni <jni@apache.org>
Authored: Fri Feb 12 14:18:59 2016 -0800
Committer: Jinfeng Ni <jni@apache.org>
Committed: Mon Feb 22 11:54:11 2016 -0800

----------------------------------------------------------------------
 .../org/apache/drill/exec/store/hbase/HBaseGroupScan.java     | 6 +++---
 .../java/org/apache/drill/exec/physical/base/GroupScan.java   | 6 ++++++
 .../org/apache/drill/exec/store/AbstractRecordReader.java     | 7 +++++++
 .../org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java   | 2 +-
 .../apache/drill/exec/store/parquet/ParquetRowGroupScan.java  | 2 +-
 .../drill/exec/store/parquet/ParquetScanBatchCreator.java     | 3 ---
 6 files changed, 18 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/c67d070b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java
b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java
index 1d01635..ae243b3 100644
--- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java
+++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java
@@ -114,7 +114,7 @@ public class HBaseGroupScan extends AbstractGroupScan implements DrillHBaseConst
     this.storagePlugin = storagePlugin;
     this.storagePluginConfig = storagePlugin.getConfig();
     this.hbaseScanSpec = scanSpec;
-    this.columns = columns == null || columns.size() == 0? ALL_COLUMNS : columns;
+    this.columns = columns == null ? ALL_COLUMNS : columns;
     init();
   }
 
@@ -124,7 +124,7 @@ public class HBaseGroupScan extends AbstractGroupScan implements DrillHBaseConst
    */
   private HBaseGroupScan(HBaseGroupScan that) {
     super(that);
-    this.columns = that.columns;
+    this.columns = that.columns == null ? ALL_COLUMNS : that.columns;
     this.hbaseScanSpec = that.hbaseScanSpec;
     this.endpointFragmentMapping = that.endpointFragmentMapping;
     this.regionsToScan = that.regionsToScan;
@@ -139,7 +139,7 @@ public class HBaseGroupScan extends AbstractGroupScan implements DrillHBaseConst
   @Override
   public GroupScan clone(List<SchemaPath> columns) {
     HBaseGroupScan newScan = new HBaseGroupScan(this);
-    newScan.columns = columns;
+    newScan.columns = columns == null ? ALL_COLUMNS : columns;;
     newScan.verifyColumns();
     return newScan;
   }

http://git-wip-us.apache.org/repos/asf/drill/blob/c67d070b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java
index 98acb0a..daaa5ac 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java
@@ -35,7 +35,13 @@ import com.google.common.collect.ImmutableList;
  */
 public interface GroupScan extends Scan, HasAffinity{
 
+  /**
+   * columns list in GroupScan : 1) empty_column is for skipAll query.
+   *                             2) NULL is interpreted as ALL_COLUMNS.
+   *  How to handle skipAll query is up to each storage plugin, with different policy in
corresponding RecordReader.
+   */
   public static final List<SchemaPath> ALL_COLUMNS = ImmutableList.of(SchemaPath.getSimplePath("*"));
+
   public static final long NO_COLUMN_STATS = -1;
 
   public abstract void applyAssignments(List<DrillbitEndpoint> endpoints) throws PhysicalOperatorSetupException;

http://git-wip-us.apache.org/repos/asf/drill/blob/c67d070b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractRecordReader.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractRecordReader.java
index 2ddcbaa..16118d9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractRecordReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractRecordReader.java
@@ -54,6 +54,13 @@ public abstract class AbstractRecordReader implements RecordReader {
         + ", isSkipQuery = " + isSkipQuery + "]";
   }
 
+  /**
+   *
+   * @param projected : The column list to be returned from this RecordReader.
+   *                  1) empty column list: this is for skipAll query. It's up to each storage-plugin
to
+   *                  choose different policy of handling skipAll query. By default, it will
use * column.
+   *                  2) NULL : is NOT allowed. It requires the planner's rule, or GroupScan
or ScanBatchCreator to handle NULL.
+   */
   protected final void setColumns(Collection<SchemaPath> projected) {
     Preconditions.checkNotNull(projected, COL_NULL_ERROR);
     isSkipQuery = projected.isEmpty();

http://git-wip-us.apache.org/repos/asf/drill/blob/c67d070b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java
index 791e4f7..ebea2f4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java
@@ -99,7 +99,7 @@ public class EasyGroupScan extends AbstractFileGroupScan{
     super(userName);
     this.selection = Preconditions.checkNotNull(selection);
     this.formatPlugin = Preconditions.checkNotNull(formatPlugin, "Unable to load format plugin
for provided format config.");
-    this.columns = columns == null || columns.size() == 0? ALL_COLUMNS : columns;
+    this.columns = columns == null ? ALL_COLUMNS : columns;
     this.selectionRoot = selectionRoot;
     initFromSelection(selection, formatPlugin);
   }

http://git-wip-us.apache.org/repos/asf/drill/blob/c67d070b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java
index 987f792..cffcdac 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java
@@ -78,7 +78,7 @@ public class ParquetRowGroupScan extends AbstractBase implements SubScan
{
     this.formatPlugin = Preconditions.checkNotNull(formatPlugin);
     this.formatConfig = formatPlugin.getConfig();
     this.rowGroupReadEntries = rowGroupReadEntries;
-    this.columns = columns == null || columns.size() == 0 ? GroupScan.ALL_COLUMNS : columns;
+    this.columns = columns == null ? GroupScan.ALL_COLUMNS : columns;
     this.selectionRoot = selectionRoot;
   }
 

http://git-wip-us.apache.org/repos/asf/drill/blob/c67d070b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
index 485f84a..c730bc9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
@@ -87,9 +87,6 @@ public class ParquetScanBatchCreator implements BatchCreator<ParquetRowGroupScan
           newColumns.add(column);
         }
       }
-      if (newColumns.isEmpty()) {
-        newColumns = GroupScan.ALL_COLUMNS;
-      }
       final int id = rowGroupScan.getOperatorId();
       // Create the new row group scan with the new columns
       rowGroupScan = new ParquetRowGroupScan(rowGroupScan.getUserName(), rowGroupScan.getStorageEngine(),


Mime
View raw message