drill-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jacq...@apache.org
Subject [26/38] git commit: Integrate new SQL changes with Hive storage engine, moving to automatic file detection. Rename storage engine to storage plugin. Separate storage plugins from format plugins, updating Parquet and JSON to format engines. Refactor dist
Date Tue, 04 Mar 2014 08:07:53 GMT
Integrate new SQL changes with Hive storage engine, moving to automatic file detection.  Rename storage engine to storage plugin. Separate storage plugins from format plugins, updating Parquet and JSON to format engines. Refactor distribution logic and enable use within JSON format and all EasyFormatPlugins.


Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/8b90cae3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/8b90cae3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/8b90cae3

Branch: refs/heads/master
Commit: 8b90cae3fe3c5aa48b7d82147452fda043305bd4
Parents: 2dfadc9
Author: Jacques Nadeau <jacques@apache.org>
Authored: Mon Feb 17 12:06:06 2014 -0800
Committer: Jacques Nadeau <jacques@apache.org>
Committed: Mon Mar 3 23:22:17 2014 -0800

----------------------------------------------------------------------
 .gitignore                                      |   3 +
 INSTALL.md                                      |  10 +-
 common/pom.xml                                  |   7 -
 .../org/apache/drill/common/JSONOptions.java    |   6 +-
 .../drill/common/config/CommonConstants.java    |   2 +-
 .../apache/drill/common/config/DrillConfig.java |   7 +-
 .../common/logical/FormatPluginConfig.java      |  27 +
 .../common/logical/FormatPluginConfigBase.java  |  41 +
 .../drill/common/logical/LogicalPlan.java       |  10 +-
 .../common/logical/LogicalPlanBuilder.java      |   4 +-
 .../common/logical/StorageEngineConfig.java     |  28 -
 .../common/logical/StorageEngineConfigBase.java |  41 -
 .../common/logical/StoragePluginConfig.java     |  28 +
 .../common/logical/StoragePluginConfigBase.java |  41 +
 .../drill/common/util/ConstructorChecker.java   |  55 ++
 .../org/apache/drill/common/util/TestTools.java |   6 +-
 .../drill/storage/CheckStorageConfig.java       |   4 +-
 contrib/sqlline/pom.xml                         |   2 -
 distribution/pom.xml                            |   1 +
 distribution/src/resources/storage-engines.json |  44 +-
 exec/java-exec/pom.xml                          |  13 -
 .../org/apache/drill/exec/ops/QueryContext.java |  22 +-
 .../apache/drill/exec/opt/BasicOptimizer.java   |  24 +-
 .../apache/drill/exec/physical/ReadEntry.java   |  33 -
 .../drill/exec/physical/ReadEntryFromHDFS.java  |  54 --
 .../drill/exec/physical/ReadEntryWithPath.java  |  49 --
 .../exec/physical/base/AbstractGroupScan.java   |   2 -
 .../exec/physical/base/AbstractSubScan.java     |  70 ++
 .../drill/exec/physical/base/GroupScan.java     |   3 +-
 .../apache/drill/exec/physical/base/Scan.java   |   1 -
 .../drill/exec/physical/base/SubScan.java       |   1 -
 .../drill/exec/planner/PhysicalPlanReader.java  |   6 +-
 .../drill/exec/planner/logical/DrillTable.java  |  47 +-
 .../exec/planner/logical/DynamicDrillTable.java |  35 +
 .../exec/planner/logical/StorageEngines.java    |  10 +-
 .../exec/planner/sql/DrillSchemaFactory.java    |  96 ---
 .../drill/exec/planner/sql/DrillSqlWorker.java  |  32 +-
 .../exec/planner/sql/FileSystemSchema.java      | 117 ---
 .../drill/exec/server/DrillbitContext.java      |  40 +-
 .../apache/drill/exec/store/AbstractSchema.java | 101 +++
 .../drill/exec/store/AbstractStorageEngine.java |  89 ---
 .../drill/exec/store/AbstractStoragePlugin.java |  54 ++
 .../drill/exec/store/AffinityCalculator.java    | 150 ----
 .../drill/exec/store/ClassPathFileSystem.java   |  21 +-
 .../exec/store/NamedStoragePluginConfig.java    |  27 +
 .../apache/drill/exec/store/SchemaFactory.java  |  27 +
 .../apache/drill/exec/store/SchemaHolder.java   |  44 ++
 .../apache/drill/exec/store/SchemaProvider.java |  25 -
 .../exec/store/SchemaProviderRegistry.java      |  82 --
 .../apache/drill/exec/store/StorageEngine.java  | 122 ---
 .../drill/exec/store/StorageEngineRegistry.java |  84 ---
 .../apache/drill/exec/store/StoragePlugin.java  |  48 ++
 .../drill/exec/store/StoragePluginRegistry.java | 175 +++++
 .../exec/store/dfs/BasicFormatMatcher.java      | 130 ++++
 .../drill/exec/store/dfs/FileSelection.java     | 127 ++++
 .../drill/exec/store/dfs/FileSystemConfig.java  |  35 +
 .../exec/store/dfs/FileSystemFormatConfig.java  |  29 +
 .../drill/exec/store/dfs/FileSystemPlugin.java  | 125 ++++
 .../exec/store/dfs/FileSystemSchemaFactory.java | 116 +++
 .../drill/exec/store/dfs/FormatCreator.java     | 100 +++
 .../drill/exec/store/dfs/FormatMatcher.java     |  28 +
 .../drill/exec/store/dfs/FormatPlugin.java      |  52 ++
 .../drill/exec/store/dfs/FormatSelection.java   |  64 ++
 .../drill/exec/store/dfs/MagicString.java       |  42 ++
 .../exec/store/dfs/NamedFormatPluginConfig.java |  28 +
 .../drill/exec/store/dfs/ReadEntryFromHDFS.java |  44 ++
 .../drill/exec/store/dfs/ReadEntryWithPath.java |  37 +
 .../exec/store/dfs/WorkspaceSchemaFactory.java  | 123 +++
 .../exec/store/dfs/easy/EasyBatchCreator.java   |  38 +
 .../exec/store/dfs/easy/EasyFormatPlugin.java   | 142 ++++
 .../exec/store/dfs/easy/EasyGroupScan.java      | 183 +++++
 .../drill/exec/store/dfs/easy/EasySubScan.java  | 103 +++
 .../drill/exec/store/dfs/easy/FileWork.java     |  27 +
 .../exec/store/dfs/shim/DrillFileSystem.java    |  43 ++
 .../exec/store/dfs/shim/DrillInputStream.java   |  32 +
 .../exec/store/dfs/shim/DrillOutputStream.java  |  29 +
 .../exec/store/dfs/shim/FileSystemCreator.java  |  36 +
 .../dfs/shim/fallback/FallbackFileSystem.java   | 146 ++++
 .../exec/store/easy/json/JSONFormatPlugin.java  |  73 ++
 .../exec/store/easy/json/JSONRecordReader.java  | 541 ++++++++++++++
 .../drill/exec/store/hive/HiveReadEntry.java    |  27 +-
 .../drill/exec/store/hive/HiveReadEntryOld.java |  50 --
 .../apache/drill/exec/store/hive/HiveScan.java  |  32 +-
 .../exec/store/hive/HiveStorageEngine.java      | 194 -----
 .../store/hive/HiveStorageEngineConfig.java     |  71 --
 .../exec/store/hive/HiveStoragePlugin.java      |  81 ++
 .../store/hive/HiveStoragePluginConfig.java     |  71 ++
 .../exec/store/hive/schema/DrillHiveTable.java  | 177 +++++
 .../store/hive/schema/HiveDatabaseSchema.java   |  56 ++
 .../store/hive/schema/HiveSchemaFactory.java    | 274 +++++++
 .../drill/exec/store/json/JSONGroupScan.java    | 162 ----
 .../drill/exec/store/json/JSONRecordReader.java | 541 --------------
 .../exec/store/json/JSONScanBatchCreator.java   |  46 --
 .../exec/store/json/JSONStorageEngine.java      |  78 --
 .../store/json/JSONStorageEngineConfig.java     |  54 --
 .../drill/exec/store/json/JSONSubScan.java      | 122 ---
 .../exec/store/json/JsonSchemaProvider.java     |  62 --
 .../drill/exec/store/mock/MockGroupScanPOP.java |   7 +-
 .../exec/store/mock/MockStorageEngine.java      |  15 +-
 .../store/mock/MockStorageEngineConfig.java     |   4 +-
 .../drill/exec/store/mock/MockSubScanPOP.java   |  26 +-
 .../drill/exec/store/parquet/BitReader.java     |   2 +-
 .../exec/store/parquet/ColumnDataReader.java    |   2 +-
 .../drill/exec/store/parquet/ColumnReader.java  |   2 +-
 .../store/parquet/FixedByteAlignedReader.java   |   2 +-
 .../exec/store/parquet/NullableBitReader.java   |   2 +-
 .../store/parquet/NullableColumnReader.java     |   2 +-
 .../parquet/NullableFixedByteAlignedReader.java |   2 +-
 .../exec/store/parquet/PageReadStatus.java      |   8 +-
 .../exec/store/parquet/ParquetFormatConfig.java |  37 +
 .../exec/store/parquet/ParquetFormatPlugin.java | 178 +++++
 .../exec/store/parquet/ParquetGroupScan.java    | 356 ++++-----
 .../exec/store/parquet/ParquetRecordReader.java |   2 +-
 .../exec/store/parquet/ParquetRowGroupScan.java |  93 +--
 .../store/parquet/ParquetScanBatchCreator.java  |  14 +-
 .../store/parquet/ParquetSchemaProvider.java    |  63 --
 .../store/parquet/ParquetStorageEngine.java     | 123 ---
 .../parquet/ParquetStorageEngineConfig.java     |  68 --
 .../exec/store/parquet/RowGroupReadEntry.java   |  44 ++
 .../exec/store/parquet/VarLenBinaryReader.java  |   4 +
 .../exec/store/schedule/AffinityCreator.java    |  62 ++
 .../exec/store/schedule/AssignmentCreator.java  | 135 ++++
 .../exec/store/schedule/BlockMapBuilder.java    | 215 ++++++
 .../exec/store/schedule/CompleteFileWork.java   | 105 +++
 .../drill/exec/store/schedule/CompleteWork.java |  29 +
 .../exec/store/schedule/EndpointByteMap.java    |  34 +
 .../store/schedule/EndpointByteMapImpl.java     |  61 ++
 .../drill/exec/store/schedule/PartialWork.java  |  43 ++
 .../apache/drill/exec/work/foreman/Foreman.java |   2 +-
 .../src/main/resources/storage-engines.json     |  29 -
 .../java/org/apache/drill/exec/TestPlan.java    |  48 --
 .../exec/physical/impl/TestOptiqPlans.java      |   4 +-
 .../exec/physical/impl/join/TestMergeJoin.java  |   8 +-
 .../exec/store/CachedSingleFileSystem.java      | 171 +++++
 .../drill/exec/store/JSONRecordReaderTest.java  | 390 ----------
 .../exec/store/ParquetRecordReaderTest.java     | 742 -------------------
 .../exec/store/TestAffinityCalculator.java      | 113 +--
 .../drill/exec/store/TestOutputMutator.java     |  74 ++
 .../exec/store/TestParquetPhysicalPlan.java     | 136 ----
 .../exec/store/json/JSONRecordReaderTest.java   | 390 ++++++++++
 .../drill/exec/store/parquet/FieldInfo.java     |  40 +
 .../store/parquet/ParquetRecordReaderTest.java  | 450 ++++++-----
 .../store/parquet/ParquetRecordReaderTest2.java | 742 +++++++++++++++++++
 .../store/parquet/ParquetResultListener.java    | 203 +++++
 .../store/parquet/ParquetTestProperties.java    |  37 +
 .../exec/store/parquet/TestFileGenerator.java   | 142 ++--
 .../store/parquet/TestParquetPhysicalPlan.java  | 136 ++++
 .../exec/store/parquet/WrapAroundCounter.java   |  40 +
 .../join/merge_inner_single_batch.json          |  28 +-
 .../test/resources/join/merge_multi_batch.json  |  30 +-
 .../test/resources/join/merge_single_batch.json |  28 +-
 .../src/test/resources/logical_json_scan.json   |   8 +-
 .../resources/parquet/parquet_scan_screen.json  |  23 +-
 .../parquet_scan_screen_read_entry_replace.json |  15 +-
 .../parquet_scan_union_screen_physical.json     |   6 +-
 .../src/test/resources/parquet_scan_screen.json |  17 +-
 .../parquet_scan_screen_read_entry_replace.json |  15 +-
 .../parquet_selective_column_read.json          |   9 +-
 .../resources/physical_json_scan_test1.json     |  14 +-
 .../resources/sender/broadcast_exchange.json    |  30 +-
 .../src/test/resources/storage-engines.json     |  20 +
 pom.xml                                         |   5 +-
 sqlparser/pom.xml                               |  25 +-
 .../sql/client/full/HiveDatabaseSchema.java     | 314 --------
 .../drill/sql/client/full/HiveSchema.java       | 162 ----
 .../org/apache/drill/jdbc/test/JdbcTest.java    |  10 +-
 .../apache/drill/jdbc/test/TestJdbcQuery.java   |  39 +-
 .../src/test/resources/storage-engines.json     |  47 +-
 168 files changed, 7613 insertions(+), 5511 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 6bda406..c27a41b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,7 +3,10 @@
 .classpath
 .settings/
 .idea/
+*.log
+*.lck
 *.iml
 target/
 *.DS_Store
+*.patch
 *~

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/INSTALL.md
----------------------------------------------------------------------
diff --git a/INSTALL.md b/INSTALL.md
index 3faf7e4..e89da01 100644
--- a/INSTALL.md
+++ b/INSTALL.md
@@ -4,13 +4,9 @@
 
 Currently, the Apache Drill build process is known to work on Linux, Windows and OSX.  To build, you need to have the following software installed on your system to successfully complete a build. 
   * Java 7+
-  * protoc 2.5.x compiler
   * Maven 3.0+
 
 ## Confirm settings
-    # protoc --version
-    libprotoc 2.5.0
-    
     # java -version
     java version "1.7.0_09"
     Java(TM) SE Runtime Environment (build 1.7.0_09-b05)
@@ -30,13 +26,13 @@ Currently, the Apache Drill build process is known to work on Linux, Windows and
     
 ## Start SQLLine
 
-    ./sqlline -u jdbc:drill:schema=parquet-local -n admin -p admin
+    ./sqlline -u jdbc:drill:zk=local -n admin -p admin
 
 ## Run a query
 
     SELECT 
-      _MAP['R_REGIONKEY'] AS region_key, 
-      _MAP['R_NAME'] AS name, _MAP['R_COMMENT'] AS comment
+      region_key, 
+      r_comment
     FROM "sample-data/region.parquet";
     
 ## More information 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/common/pom.xml
----------------------------------------------------------------------
diff --git a/common/pom.xml b/common/pom.xml
index d76f7a1..afcf7a3 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -47,13 +47,6 @@
       <artifactId>optiq-core</artifactId>
     </dependency>
 
-
-    <dependency>
-      <groupId>com.google.protobuf</groupId>
-      <artifactId>protobuf-java</artifactId>
-      <version>2.5.0</version>
-    </dependency>
-
     <dependency>
       <groupId>com.typesafe</groupId>
       <artifactId>config</artifactId>

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/common/src/main/java/org/apache/drill/common/JSONOptions.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/JSONOptions.java b/common/src/main/java/org/apache/drill/common/JSONOptions.java
index c1f9a85..c77412d 100644
--- a/common/src/main/java/org/apache/drill/common/JSONOptions.java
+++ b/common/src/main/java/org/apache/drill/common/JSONOptions.java
@@ -19,6 +19,7 @@ package org.apache.drill.common;
 
 import java.io.IOException;
 import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Type;
 
 import org.apache.drill.common.JSONOptions.De;
 import org.apache.drill.common.JSONOptions.Se;
@@ -87,8 +88,9 @@ public class JSONOptions {
   @SuppressWarnings("unchecked")
   public <T> T getListWith(ObjectMapper mapper, TypeReference<T> t) throws IOException {
     if(opaque != null){
-      ParameterizedType pt = (ParameterizedType) t.getType();
-      if ( pt.getRawType().equals(opaque.getClass())){
+      Type c = t.getType();
+      if(c instanceof ParameterizedType) c = ((ParameterizedType)c).getRawType();
+      if ( c.equals(opaque.getClass())){
         return (T) opaque;
       }else{
         throw new IOException(String.format("Attmpted to retrieve a list with type of %s.  However, the JSON options carried an opaque value of type %s.", t.getType(), opaque.getClass().getName()));

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/common/src/main/java/org/apache/drill/common/config/CommonConstants.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/config/CommonConstants.java b/common/src/main/java/org/apache/drill/common/config/CommonConstants.java
index f36a0e9..234c9b2 100644
--- a/common/src/main/java/org/apache/drill/common/config/CommonConstants.java
+++ b/common/src/main/java/org/apache/drill/common/config/CommonConstants.java
@@ -23,7 +23,7 @@ public interface CommonConstants {
   
   public static final String LOGICAL_OPERATOR_SCAN_PACKAGES = "drill.logical.operator.packages";
   public static final String PHYSICAL_OPERATOR_SCAN_PACKAGES = "drill.physical.operator.packages";
-  public static final String STORAGE_ENGINE_CONFIG_SCAN_PACKAGES = "drill.logical.storage.packages";
+  public static final String STORAGE_PLUGIN_CONFIG_SCAN_PACKAGES = "drill.logical.storage.packages";
   public static final String DRILL_JAR_MARKER_FILE = "drill-module.conf";
   public static final String LOGICAL_FUNCTION_SCAN_PACKAGES = "drill.logical.function.packages";
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/config/DrillConfig.java b/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
index a0e261f..6fc048d 100644
--- a/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
+++ b/common/src/main/java/org/apache/drill/common/config/DrillConfig.java
@@ -27,7 +27,8 @@ import java.util.concurrent.CopyOnWriteArrayList;
 
 import org.apache.drill.common.exceptions.DrillConfigurationException;
 import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.logical.StorageEngineConfigBase;
+import org.apache.drill.common.logical.FormatPluginConfigBase;
+import org.apache.drill.common.logical.StoragePluginConfigBase;
 import org.apache.drill.common.logical.data.LogicalOperatorBase;
 import org.apache.drill.common.util.PathScanner;
 
@@ -61,13 +62,15 @@ public final class DrillConfig extends NestedConfig{
     SimpleModule deserModule = new SimpleModule("LogicalExpressionDeserializationModule")
       .addDeserializer(LogicalExpression.class, new LogicalExpression.De(this));
     
+    
     mapper.registerModule(deserModule);
     mapper.enable(SerializationFeature.INDENT_OUTPUT);
     mapper.configure(Feature.ALLOW_UNQUOTED_FIELD_NAMES, true);
     mapper.configure(JsonGenerator.Feature.QUOTE_FIELD_NAMES, true);
     mapper.configure(Feature.ALLOW_COMMENTS, true);
     mapper.registerSubtypes(LogicalOperatorBase.getSubTypes(this));
-    mapper.registerSubtypes(StorageEngineConfigBase.getSubTypes(this));
+    mapper.registerSubtypes(StoragePluginConfigBase.getSubTypes(this));
+    mapper.registerSubtypes(FormatPluginConfigBase.getSubTypes(this));
     
     RuntimeMXBean bean = ManagementFactory.getRuntimeMXBean();
     this.startupArguments = ImmutableList.copyOf(bean.getInputArguments());

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/common/src/main/java/org/apache/drill/common/logical/FormatPluginConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/FormatPluginConfig.java b/common/src/main/java/org/apache/drill/common/logical/FormatPluginConfig.java
new file mode 100644
index 0000000..20b4d66
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/logical/FormatPluginConfig.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical;
+
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property="type")
+public interface FormatPluginConfig {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FormatPluginConfig.class);
+  
+  
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/common/src/main/java/org/apache/drill/common/logical/FormatPluginConfigBase.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/FormatPluginConfigBase.java b/common/src/main/java/org/apache/drill/common/logical/FormatPluginConfigBase.java
new file mode 100644
index 0000000..e640359
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/logical/FormatPluginConfigBase.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical;
+
+import java.util.List;
+
+import org.apache.drill.common.config.CommonConstants;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.util.PathScanner;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public abstract class FormatPluginConfigBase implements FormatPluginConfig{
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FormatPluginConfigBase.class);
+  
+  
+  public synchronized static Class<?>[] getSubTypes(DrillConfig config){
+    List<String> packages = config.getStringList(CommonConstants.STORAGE_PLUGIN_CONFIG_SCAN_PACKAGES);
+    Class<?>[] sec = PathScanner.scanForImplementationsArr(FormatPluginConfig.class, packages);
+    logger.debug("Adding Format Plugin Configs including {}", (Object) sec );
+    return sec;
+  }
+
+  public abstract boolean equals(Object o);
+  
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/common/src/main/java/org/apache/drill/common/logical/LogicalPlan.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/LogicalPlan.java b/common/src/main/java/org/apache/drill/common/logical/LogicalPlan.java
index a9db251..06ea117 100644
--- a/common/src/main/java/org/apache/drill/common/logical/LogicalPlan.java
+++ b/common/src/main/java/org/apache/drill/common/logical/LogicalPlan.java
@@ -44,15 +44,15 @@ public class LogicalPlan {
   static final Logger logger = LoggerFactory.getLogger(LogicalPlan.class);
 
   private final PlanProperties properties;
-  private final Map<String, StorageEngineConfig> storageEngineMap;
+  private final Map<String, StoragePluginConfig> storageEngineMap;
   private final Graph<LogicalOperator, SinkOperator, SourceOperator> graph;
   
   
   @JsonCreator
   public LogicalPlan(@JsonProperty("head") PlanProperties head,
-      @JsonProperty("storage") Map<String, StorageEngineConfig> storageEngineMap,
+      @JsonProperty("storage") Map<String, StoragePluginConfig> storageEngineMap,
       @JsonProperty("query") List<LogicalOperator> operators) {
-    this.storageEngineMap = storageEngineMap != null ? storageEngineMap : new HashMap<String, StorageEngineConfig>();
+    this.storageEngineMap = storageEngineMap != null ? storageEngineMap : new HashMap<String, StoragePluginConfig>();
     this.properties = head;
     this.graph = Graph.newGraph(operators, SinkOperator.class, SourceOperator.class);
   }
@@ -62,7 +62,7 @@ public class LogicalPlan {
     return GraphAlgos.TopoSorter.sortLogical(graph);
   }
 
-  public StorageEngineConfig getStorageEngineConfig(String name) {
+  public StoragePluginConfig getStorageEngineConfig(String name) {
     return storageEngineMap.get(name);
   }
 
@@ -77,7 +77,7 @@ public class LogicalPlan {
   }
 
   @JsonProperty("storage")
-  public Map<String, StorageEngineConfig> getStorageEngines() {
+  public Map<String, StoragePluginConfig> getStorageEngines() {
     return storageEngineMap;
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/common/src/main/java/org/apache/drill/common/logical/LogicalPlanBuilder.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/LogicalPlanBuilder.java b/common/src/main/java/org/apache/drill/common/logical/LogicalPlanBuilder.java
index 369cd58..1941a92 100644
--- a/common/src/main/java/org/apache/drill/common/logical/LogicalPlanBuilder.java
+++ b/common/src/main/java/org/apache/drill/common/logical/LogicalPlanBuilder.java
@@ -29,7 +29,7 @@ import com.google.common.collect.ImmutableMap;
  */
 public class LogicalPlanBuilder {
   private PlanProperties planProperties;
-  private ImmutableMap.Builder<String, StorageEngineConfig> storageEngines = ImmutableMap.builder();
+  private ImmutableMap.Builder<String, StoragePluginConfig> storageEngines = ImmutableMap.builder();
   private ImmutableList.Builder<LogicalOperator> operators = ImmutableList.builder();
 
   public LogicalPlanBuilder planProperties(PlanProperties planProperties) {
@@ -42,7 +42,7 @@ public class LogicalPlanBuilder {
     return this;
   }
   
-  public LogicalPlanBuilder addStorageEngine(String name, StorageEngineConfig config) {
+  public LogicalPlanBuilder addStorageEngine(String name, StoragePluginConfig config) {
     this.storageEngines.put(name, config);
     return this;
   }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/common/src/main/java/org/apache/drill/common/logical/StorageEngineConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/StorageEngineConfig.java b/common/src/main/java/org/apache/drill/common/logical/StorageEngineConfig.java
deleted file mode 100644
index 343c5db..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/StorageEngineConfig.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical;
-
-
-import com.fasterxml.jackson.annotation.JsonTypeInfo;
-
-
-@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property="type")
-public interface StorageEngineConfig{
-
-  public boolean equals(Object o);
-}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/common/src/main/java/org/apache/drill/common/logical/StorageEngineConfigBase.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/StorageEngineConfigBase.java b/common/src/main/java/org/apache/drill/common/logical/StorageEngineConfigBase.java
deleted file mode 100644
index 21468bc..0000000
--- a/common/src/main/java/org/apache/drill/common/logical/StorageEngineConfigBase.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.common.logical;
-
-import java.util.List;
-
-import org.apache.drill.common.config.CommonConstants;
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.util.PathScanner;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-public abstract class StorageEngineConfigBase implements StorageEngineConfig{
-  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StorageEngineConfigBase.class);
-  
-  
-  public synchronized static Class<?>[] getSubTypes(DrillConfig config){
-    List<String> packages = config.getStringList(CommonConstants.STORAGE_ENGINE_CONFIG_SCAN_PACKAGES);
-    Class<?>[] sec = PathScanner.scanForImplementationsArr(StorageEngineConfig.class, packages);
-    logger.debug("Adding Storage Engine Configs including {}", (Object) sec );
-    return sec;
-  }
-
-  public abstract boolean equals(Object o);
-  
-}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/common/src/main/java/org/apache/drill/common/logical/StoragePluginConfig.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/StoragePluginConfig.java b/common/src/main/java/org/apache/drill/common/logical/StoragePluginConfig.java
new file mode 100644
index 0000000..7d7cf38
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/logical/StoragePluginConfig.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical;
+
+
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+
+
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property="type")
+public interface StoragePluginConfig{
+
+  public boolean equals(Object o);
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/common/src/main/java/org/apache/drill/common/logical/StoragePluginConfigBase.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/logical/StoragePluginConfigBase.java b/common/src/main/java/org/apache/drill/common/logical/StoragePluginConfigBase.java
new file mode 100644
index 0000000..8e01493
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/logical/StoragePluginConfigBase.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.logical;
+
+import java.util.List;
+
+import org.apache.drill.common.config.CommonConstants;
+import org.apache.drill.common.config.DrillConfig;
+import org.apache.drill.common.util.PathScanner;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public abstract class StoragePluginConfigBase implements StoragePluginConfig{
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(StoragePluginConfigBase.class);
+  
+  
+  public synchronized static Class<?>[] getSubTypes(DrillConfig config){
+    List<String> packages = config.getStringList(CommonConstants.STORAGE_PLUGIN_CONFIG_SCAN_PACKAGES);
+    Class<?>[] sec = PathScanner.scanForImplementationsArr(StoragePluginConfig.class, packages);
+    logger.debug("Adding Storage Engine Configs including {}", (Object) sec );
+    return sec;
+  }
+
+  public abstract boolean equals(Object o);
+  
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/common/src/main/java/org/apache/drill/common/util/ConstructorChecker.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/util/ConstructorChecker.java b/common/src/main/java/org/apache/drill/common/util/ConstructorChecker.java
new file mode 100644
index 0000000..a68c5ba
--- /dev/null
+++ b/common/src/main/java/org/apache/drill/common/util/ConstructorChecker.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.common.util;
+
+import java.lang.reflect.Constructor;
+
+public class ConstructorChecker {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ConstructorChecker.class);
+  
+  private final String requirementString;
+  private final Class<?>[] classes;
+ 
+  
+  public ConstructorChecker(Class<?>... classes) {
+    super();
+    this.classes = classes;
+    StringBuffer sb = new StringBuffer();
+    sb.append("The required constructor is (");
+    for(int i =0; i < classes.length; i++){
+      if(i != 0) sb.append(", ");
+      sb.append(classes[i].getName());
+    }
+     
+    this.requirementString = sb.toString();
+  }
+
+  public boolean check(Constructor<?> c){
+    Class<?>[] params = c.getParameterTypes();
+    if(params.length != classes.length) return false;
+    for(int i =0; i < classes.length; i++){
+      if( !classes[i].isAssignableFrom(params[i])) return false;
+    }
+    
+    return true;
+  }
+  
+  public String getRequirementString(){
+    return requirementString;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/common/src/main/java/org/apache/drill/common/util/TestTools.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/util/TestTools.java b/common/src/main/java/org/apache/drill/common/util/TestTools.java
index 0a82826..500be59 100644
--- a/common/src/main/java/org/apache/drill/common/util/TestTools.java
+++ b/common/src/main/java/org/apache/drill/common/util/TestTools.java
@@ -28,7 +28,11 @@ public class TestTools {
 
   
   public static TestRule getTimeoutRule(){
-    return IS_DEBUG ? new TestName() : new Timeout(10000);
+    return getTimeoutRule(10000);
+  }
+
+  public static TestRule getTimeoutRule(int timeout){
+    return IS_DEBUG ? new TestName() : new Timeout(timeout);
   }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/common/src/test/java/org/apache/drill/storage/CheckStorageConfig.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/drill/storage/CheckStorageConfig.java b/common/src/test/java/org/apache/drill/storage/CheckStorageConfig.java
index 5bcdc8a..1dc6c77 100644
--- a/common/src/test/java/org/apache/drill/storage/CheckStorageConfig.java
+++ b/common/src/test/java/org/apache/drill/storage/CheckStorageConfig.java
@@ -23,7 +23,7 @@ import java.util.Collection;
 
 import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.common.logical.LogicalPlan;
-import org.apache.drill.common.logical.StorageEngineConfig;
+import org.apache.drill.common.logical.StoragePluginConfig;
 import org.apache.drill.common.util.FileUtils;
 import org.apache.drill.common.util.PathScanner;
 import org.junit.Test;
@@ -37,7 +37,7 @@ public class CheckStorageConfig {
 
   @Test
   public void ensureStorageEnginePickup() {
-    Collection<?> engines = PathScanner.scanForImplementations(StorageEngineConfig.class, Lists.newArrayList("org"));
+    Collection<?> engines = PathScanner.scanForImplementations(StoragePluginConfig.class, Lists.newArrayList("org"));
     assertEquals(engines.size(), 1);
   }
   

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/contrib/sqlline/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/sqlline/pom.xml b/contrib/sqlline/pom.xml
index 6376e28..0029b61 100644
--- a/contrib/sqlline/pom.xml
+++ b/contrib/sqlline/pom.xml
@@ -42,12 +42,10 @@
     <dependency>
       <groupId>xerces</groupId>
       <artifactId>xercesImpl</artifactId>
-      <version>2.9.1</version>
     </dependency>
     <dependency>
       <groupId>xalan</groupId>
       <artifactId>xalan</artifactId>
-      <version>2.7.1</version>
     </dependency>
   </dependencies>
 </project>

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/distribution/pom.xml
----------------------------------------------------------------------
diff --git a/distribution/pom.xml b/distribution/pom.xml
index 15b2f53..3ad42cb 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -82,6 +82,7 @@
             <exclude>**/control</exclude>
             <exclude>**/*.json</exclude>
             <exclude>**/hadoop-excludes.txt</exclude>
+            <exclude>**/drill_hive_db/*</exclude>
           </excludes>
         </configuration>
       </plugin>

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/distribution/src/resources/storage-engines.json
----------------------------------------------------------------------
diff --git a/distribution/src/resources/storage-engines.json b/distribution/src/resources/storage-engines.json
index d1d0413..a22e7d0 100644
--- a/distribution/src/resources/storage-engines.json
+++ b/distribution/src/resources/storage-engines.json
@@ -1,29 +1,23 @@
 {
   "storage":{
-    "parquet-local" :
-      {
-        "type":"parquet",
-        "dfsName" : "file:///"
-      },
-    "parquet-cp" :
-      {
-        "type":"parquet",
-        "dfsName" : "classpath:///"
-      },
-    "jsonl" :
-      {
-        "type":"json",
-        "dfsName" : "file:///"
-      },
-    "json-cp" :
-      {
-        "type":"json",
-        "dfsName" : "classpath:///"
-      },
-    "parquet" :
-      {
-        "type":"parquet",
-        "dfsName" : "file:///"
+    dfs: {
+      type: "file",
+      connection: "file:///"
+    },
+    cp: {
+      type: "file",
+      connection: "classpath:///"
+    },
+    hive : {
+        type:"hive",
+        config :
+          {
+            "hive.metastore.uris" : "",
+            "javax.jdo.option.ConnectionURL" : "jdbc:derby:;databaseName=../../sample-data/drill_hive_db;create=true",
+            "hive.metastore.warehouse.dir" : "/tmp/drill_hive_wh",
+            "fs.default.name" : "file:///",
+            "hive.metastore.sasl.enabled" : "false"
+          }
       }
   }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index 5c6fcfe..b4c282c 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -272,7 +272,6 @@
     <plugins>
       <plugin>
         <artifactId>maven-resources-plugin</artifactId>
-        <version>2.6</version>
         <executions>
           <execution>
             <id>copy-resources</id>
@@ -369,18 +368,6 @@
         </executions>
       </plugin>
 
-      <!-- <plugin> -->
-      <!-- <groupId>com.github.igor-petruk.protobuf</groupId> -->
-      <!-- <artifactId>protobuf-maven-plugin</artifactId> -->
-      <!-- <version>0.6.2</version> -->
-      <!-- <executions> -->
-      <!-- <execution> -->
-      <!-- <goals> -->
-      <!-- <goal>run</goal> -->
-      <!-- </goals> -->
-      <!-- </execution> -->
-      <!-- </executions> -->
-      <!-- </plugin> -->
     </plugins>
   </build>
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
index 11658e6..36cfb53 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java
@@ -20,18 +20,16 @@ package org.apache.drill.exec.ops;
 import java.util.Collection;
 
 import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.exceptions.ExecutionSetupException;
 import org.apache.drill.common.expression.FunctionRegistry;
-import org.apache.drill.common.logical.StorageEngineConfig;
 import org.apache.drill.exec.cache.DistributedCache;
 import org.apache.drill.exec.planner.PhysicalPlanReader;
-import org.apache.drill.exec.planner.sql.DrillSchemaFactory;
 import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
 import org.apache.drill.exec.proto.UserBitShared.QueryId;
 import org.apache.drill.exec.rpc.control.WorkEventBus;
 import org.apache.drill.exec.rpc.data.DataConnectionCreator;
 import org.apache.drill.exec.server.DrillbitContext;
-import org.apache.drill.exec.store.StorageEngine;
+import org.apache.drill.exec.store.StoragePluginRegistry;
+import org.apache.drill.exec.store.StoragePluginRegistry.DrillSchemaFactory;
 
 public class QueryContext {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(QueryContext.class);
@@ -55,10 +53,11 @@ public class QueryContext {
     return queryId;
   }
 
-  public StorageEngine getStorageEngine(StorageEngineConfig config) throws ExecutionSetupException {
-    return drillbitContext.getStorageEngine(config);
+  public StoragePluginRegistry getStorage(){
+    return drillbitContext.getStorage();
   }
-
+  
+  
   public DistributedCache getCache(){
     return drillbitContext.getCache();
   }
@@ -82,14 +81,13 @@ public class QueryContext {
   public WorkEventBus getWorkBus(){
     return workBus;
   }
-  
-  public DrillSchemaFactory getSchemaFactory(){
-    return drillbitContext.getSchemaFactory();
-  }
 
   public FunctionRegistry getFunctionRegistry(){
     return drillbitContext.getFunctionRegistry();
-    
+  }
+  
+  public DrillSchemaFactory getFactory(){
+    return drillbitContext.getSchemaFactory();
   }
   
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
index cd59428..aed4802 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/opt/BasicOptimizer.java
@@ -30,9 +30,19 @@ import org.apache.drill.common.expression.LogicalExpression;
 import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.common.logical.LogicalPlan;
 import org.apache.drill.common.logical.PlanProperties;
-import org.apache.drill.common.logical.StorageEngineConfig;
-import org.apache.drill.common.logical.data.*;
+import org.apache.drill.common.logical.StoragePluginConfig;
+import org.apache.drill.common.logical.data.CollapsingAggregate;
+import org.apache.drill.common.logical.data.Filter;
+import org.apache.drill.common.logical.data.Join;
+import org.apache.drill.common.logical.data.JoinCondition;
+import org.apache.drill.common.logical.data.NamedExpression;
+import org.apache.drill.common.logical.data.Order;
 import org.apache.drill.common.logical.data.Order.Ordering;
+import org.apache.drill.common.logical.data.Project;
+import org.apache.drill.common.logical.data.Scan;
+import org.apache.drill.common.logical.data.Segment;
+import org.apache.drill.common.logical.data.SinkOperator;
+import org.apache.drill.common.logical.data.Store;
 import org.apache.drill.common.logical.data.visitors.AbstractLogicalVisitor;
 import org.apache.drill.common.types.TypeProtos;
 import org.apache.drill.common.types.TypeProtos.DataMode;
@@ -41,13 +51,13 @@ import org.apache.drill.exec.exception.OptimizerException;
 import org.apache.drill.exec.ops.QueryContext;
 import org.apache.drill.exec.physical.PhysicalPlan;
 import org.apache.drill.exec.physical.base.PhysicalOperator;
+import org.apache.drill.exec.physical.config.Limit;
 import org.apache.drill.exec.physical.config.MergeJoinPOP;
 import org.apache.drill.exec.physical.config.Screen;
 import org.apache.drill.exec.physical.config.SelectionVectorRemover;
 import org.apache.drill.exec.physical.config.Sort;
-import org.apache.drill.exec.physical.config.Limit;
 import org.apache.drill.exec.physical.config.StreamingAggregate;
-import org.apache.drill.exec.store.StorageEngine;
+import org.apache.drill.exec.store.StoragePlugin;
 import org.eigenbase.rel.RelFieldCollation.Direction;
 import org.eigenbase.rel.RelFieldCollation.NullDirection;
 
@@ -192,11 +202,11 @@ public class BasicOptimizer extends Optimizer{
 
     @Override
     public PhysicalOperator visitScan(Scan scan, Object obj) throws OptimizerException {
-      StorageEngineConfig config = logicalPlan.getStorageEngineConfig(scan.getStorageEngine());
+      StoragePluginConfig config = logicalPlan.getStorageEngineConfig(scan.getStorageEngine());
       if(config == null) throw new OptimizerException(String.format("Logical plan referenced the storage engine config %s but the logical plan didn't have that available as a config.", scan.getStorageEngine()));
-      StorageEngine engine;
+      StoragePlugin engine;
       try {
-        engine = context.getStorageEngine(config);
+        engine = context.getStorage().getEngine(config);
         return engine.getPhysicalScan(scan);
       } catch (IOException | ExecutionSetupException e) {
         throw new OptimizerException("Failure while attempting to retrieve storage engine.", e);

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntry.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntry.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntry.java
deleted file mode 100644
index f99ad0e..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntry.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.physical;
-
-import org.apache.drill.exec.physical.base.Size;
-
-import com.fasterxml.jackson.annotation.JsonIgnore;
-
-/**
- * Describes a chunk of read work that will be done.
- */
-public interface ReadEntry {
-  @JsonIgnore
-  public OperatorCost getCost();
-
-  @JsonIgnore
-  public Size getSize();
-}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntryFromHDFS.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntryFromHDFS.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntryFromHDFS.java
deleted file mode 100644
index e748d4c..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntryFromHDFS.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.physical;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.drill.exec.physical.base.Size;
-
-public class ReadEntryFromHDFS extends ReadEntryWithPath {
-
-  private long start;
-  private long length;
-
-  @JsonCreator
-  public ReadEntryFromHDFS(@JsonProperty("path") String path,@JsonProperty("start") long start, @JsonProperty("length") long length) {
-    this.path = path;
-    this.start = start;
-    this.length = length;
-  }
-
-  @Override
-  public OperatorCost getCost() {
-    return new OperatorCost(1, 2, 1, 1);
-  }
-
-  @Override
-  public Size getSize() {
-    // TODO - these values are wrong, I cannot know these until after I read a file
-    return new Size(10, 10);
-  }
-
-  public long getStart() {
-    return start;
-  }
-
-  public long getLength() {
-    return length;
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntryWithPath.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntryWithPath.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntryWithPath.java
deleted file mode 100644
index a24869b..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/ReadEntryWithPath.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.physical;
-
-import org.apache.drill.exec.physical.base.Size;
-
-public class ReadEntryWithPath implements ReadEntry {
-
-  protected String path;
-
-  
-  public ReadEntryWithPath(String path) {
-    super();
-    this.path = path;
-  }
-
-  public ReadEntryWithPath(){}
-  
-  public String getPath(){
-   return path;
-  }
-
-  @Override
-  public OperatorCost getCost() {
-    throw new UnsupportedOperationException(this.getClass().getCanonicalName() + " is only for extracting path data from " +
-        "selections inside a scan node from a logical plan, it cannot be used in an executing plan and has no cost.");
-  }
-
-  @Override
-  public Size getSize() {
-    throw new UnsupportedOperationException(this.getClass().getCanonicalName() + " is only for extracting path data from " +
-        "selections on a scan node from a logical plan, it cannot be used in an executing plan and has no size.");
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScan.java
index 0b486e9..343435b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractGroupScan.java
@@ -19,8 +19,6 @@ package org.apache.drill.exec.physical.base;
 
 import java.util.Iterator;
 
-import org.apache.drill.exec.physical.ReadEntry;
-
 import com.google.common.collect.Iterators;
 
 public abstract class AbstractGroupScan extends AbstractBase implements GroupScan {

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractSubScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractSubScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractSubScan.java
new file mode 100644
index 0000000..c6e11d1
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/AbstractSubScan.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.physical.base;
+
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.graph.GraphVisitor;
+import org.apache.drill.exec.physical.OperatorCost;
+
+import com.google.common.collect.Iterators;
+
+public abstract class AbstractSubScan implements SubScan{
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractSubScan.class);
+
+  @Override
+  public OperatorCost getCost() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public Size getSize() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public boolean isExecutable() {
+    return true;
+  }
+
+  @Override
+  public <T, X, E extends Throwable> T accept(PhysicalVisitor<T, X, E> physicalVisitor, X value) throws E {
+    return physicalVisitor.visitSubScan(this, value);
+  }
+
+  @Override
+  public PhysicalOperator getNewWithChildren(List<PhysicalOperator> children) throws ExecutionSetupException {
+    assert children == null || children.isEmpty();
+    return this;
+  }
+
+  @Override
+  public void accept(GraphVisitor<PhysicalOperator> visitor) {
+    visitor.enter(this);
+    visitor.visit(this);
+    visitor.leave(this);
+  }
+
+  @Override
+  public Iterator<PhysicalOperator> iterator() {
+    return Iterators.emptyIterator();
+  }
+  
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java
index b4a85df..e51ebb6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java
@@ -20,6 +20,7 @@ package org.apache.drill.exec.physical.base;
 import java.util.List;
 
 import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.exceptions.PhysicalOperatorSetupException;
 import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
@@ -30,7 +31,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
  */
 public interface GroupScan extends Scan, HasAffinity{
 
-  public abstract void applyAssignments(List<DrillbitEndpoint> endpoints);
+  public abstract void applyAssignments(List<DrillbitEndpoint> endpoints) throws PhysicalOperatorSetupException;
 
   public abstract SubScan getSpecificScan(int minorFragmentId) throws ExecutionSetupException;
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/Scan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/Scan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/Scan.java
index 63af5eb..9795433 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/Scan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/Scan.java
@@ -17,7 +17,6 @@
  */
 package org.apache.drill.exec.physical.base;
 
-import org.apache.drill.exec.physical.ReadEntry;
 
 public interface Scan extends Leaf {
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/SubScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/SubScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/SubScan.java
index 5fe3b99..e6c58a3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/SubScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/SubScan.java
@@ -17,7 +17,6 @@
  */
 package org.apache.drill.exec.physical.base;
 
-import org.apache.drill.exec.physical.ReadEntry;
 
 /**
  * A SubScan operator represents the data scanned by a particular major/minor fragment.  This is in contrast to

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PhysicalPlanReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PhysicalPlanReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PhysicalPlanReader.java
index e270b79..a100163 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PhysicalPlanReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/PhysicalPlanReader.java
@@ -37,7 +37,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectReader;
 import com.fasterxml.jackson.databind.module.SimpleModule;
 import org.apache.drill.exec.server.DrillbitContext;
-import org.apache.drill.exec.store.StorageEngineRegistry;
+import org.apache.drill.exec.store.StoragePluginRegistry;
 
 public class PhysicalPlanReader {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PhysicalPlanReader.class);
@@ -48,7 +48,7 @@ public class PhysicalPlanReader {
   private final ObjectReader logicalPlanReader;
 
   public PhysicalPlanReader(DrillConfig config, ObjectMapper mapper, final DrillbitEndpoint endpoint,
-                            final StorageEngineRegistry engineRegistry) {
+                            final StoragePluginRegistry engineRegistry) {
 
     // Endpoint serializer/deserializer.
     SimpleModule deserModule = new SimpleModule("PhysicalOperatorModule") //
@@ -61,7 +61,7 @@ public class PhysicalPlanReader {
     mapper.registerModule(deserModule);
     mapper.registerSubtypes(PhysicalOperatorUtil.getSubTypes(config));
     InjectableValues injectables = new InjectableValues.Std() //
-            .addValue(StorageEngineRegistry.class, engineRegistry) //
+            .addValue(StoragePluginRegistry.class, engineRegistry) //
         .addValue(DrillbitEndpoint.class, endpoint); //
 
     this.mapper = mapper;

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java
index 30dd48d..02ca0ad 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillTable.java
@@ -17,42 +17,31 @@
  */
 package org.apache.drill.exec.planner.logical;
 
-import java.util.Collections;
-
 import net.hydromatic.optiq.Schema.TableType;
 import net.hydromatic.optiq.Statistic;
 import net.hydromatic.optiq.Statistics;
 import net.hydromatic.optiq.Table;
 
-import org.apache.drill.common.logical.StorageEngineConfig;
+import org.apache.drill.common.logical.StoragePluginConfig;
 import org.eigenbase.rel.RelNode;
 import org.eigenbase.relopt.RelOptTable;
-import org.eigenbase.reltype.RelDataType;
-import org.eigenbase.reltype.RelDataTypeFactory;
-import org.eigenbase.sql.type.SqlTypeName;
 
-/** Optiq Table used by Drill. */
-public class DrillTable implements Table{
+
+public abstract class DrillTable implements Table{
   
-  private final String name;
   private final String storageEngineName;  
-  public final StorageEngineConfig storageEngineConfig;
+  public final StoragePluginConfig storageEngineConfig;
   private Object selection;
   
   
   /** Creates a DrillTable. */
-  public DrillTable(String name, String storageEngineName, Object selection, StorageEngineConfig storageEngineConfig) {
-    this.name = name;
+  public DrillTable(String storageEngineName, Object selection, StoragePluginConfig storageEngineConfig) {
     this.selection = selection;
     this.storageEngineConfig = storageEngineConfig;
     this.storageEngineName = storageEngineName;
   }
-  
-  public String getName() {
-    return name;
-  }
 
-  public StorageEngineConfig getStorageEngineConfig(){
+  public StoragePluginConfig getStorageEngineConfig(){
     return storageEngineConfig;
   }
   
@@ -76,32 +65,10 @@ public class DrillTable implements Table{
   }
 
   @Override
-  public RelDataType getRowType(RelDataTypeFactory typeFactory) {
-    return new RelDataTypeDrillImpl(typeFactory);
-  }
-
-  @Override
   public TableType getJdbcTableType() {
-    return null;
+    return TableType.TABLE;
   }
 
   
   
-//  /** Factory for custom tables in Optiq schema. */
-//  @SuppressWarnings("UnusedDeclaration")
-//  public static class Factory implements TableFactory<DrillTable> {
-//
-//    @Override
-//    public DrillTable create(Schema schema, String name, Map<String, Object> operand, RelDataType rowType) {
-//      
-//      final ClasspathRSE.ClasspathRSEConfig rseConfig = new ClasspathRSE.ClasspathRSEConfig();
-//      final ClasspathInputConfig inputConfig = new ClasspathInputConfig();
-//      inputConfig.path = "/" + name.toLowerCase() + ".json";
-//      inputConfig.type = DataWriter.ConverterType.JSON;
-//      return createTable(schema.getTypeFactory(), (MutableSchema) schema, name, "donuts-json", rseConfig, inputConfig);
-//    }
-//  }
-
-  
-  
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DynamicDrillTable.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DynamicDrillTable.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DynamicDrillTable.java
new file mode 100644
index 0000000..c50e67c
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DynamicDrillTable.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.planner.logical;
+
+import org.apache.drill.common.logical.StoragePluginConfig;
+import org.eigenbase.reltype.RelDataType;
+import org.eigenbase.reltype.RelDataTypeFactory;
+
+public class DynamicDrillTable extends DrillTable{
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DynamicDrillTable.class);
+
+  public DynamicDrillTable(String storageEngineName, Object selection, StoragePluginConfig storageEngineConfig) {
+    super(storageEngineName, selection, storageEngineConfig);
+  }
+
+  @Override
+  public RelDataType getRowType(RelDataTypeFactory typeFactory) {
+    return new RelDataTypeDrillImpl(typeFactory);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/StorageEngines.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/StorageEngines.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/StorageEngines.java
index a020ab3..d298040 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/StorageEngines.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/StorageEngines.java
@@ -23,19 +23,19 @@ import java.util.Map;
 import java.util.Map.Entry;
 
 import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.logical.StorageEngineConfig;
+import org.apache.drill.common.logical.StoragePluginConfig;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.Charsets;
 import com.google.common.io.Resources;
 
-public class StorageEngines implements Iterable<Map.Entry<String, StorageEngineConfig>>{
+public class StorageEngines implements Iterable<Map.Entry<String, StoragePluginConfig>>{
   
-  private Map<String, StorageEngineConfig> storage;
+  private Map<String, StoragePluginConfig> storage;
   
   @JsonCreator
-  public StorageEngines(@JsonProperty("storage") Map<String, StorageEngineConfig> storage){
+  public StorageEngines(@JsonProperty("storage") Map<String, StoragePluginConfig> storage){
     this.storage = storage;
   }
   
@@ -53,7 +53,7 @@ public class StorageEngines implements Iterable<Map.Entry<String, StorageEngineC
   }
 
   @Override
-  public Iterator<Entry<String, StorageEngineConfig>> iterator() {
+  public Iterator<Entry<String, StoragePluginConfig>> iterator() {
     return storage.entrySet().iterator();
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSchemaFactory.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSchemaFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSchemaFactory.java
deleted file mode 100644
index 23d03b8..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSchemaFactory.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.planner.sql;
-
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import net.hydromatic.linq4j.function.Function1;
-import net.hydromatic.optiq.Schema;
-import net.hydromatic.optiq.SchemaPlus;
-
-import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.logical.StorageEngineConfig;
-import org.apache.drill.exec.exception.SetupException;
-import org.apache.drill.exec.planner.logical.StorageEngines;
-import org.apache.drill.exec.store.SchemaProvider;
-import org.apache.drill.exec.store.SchemaProviderRegistry;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-public class DrillSchemaFactory  implements Function1<SchemaPlus, Schema>{
-  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillSchemaFactory.class);
-
-  private final SchemaProviderRegistry registry;
-  private final Map<String, StorageEngineEntry> preEntries = Maps.newHashMap();
-  
-  public static DrillSchemaFactory createEmpty(){
-    return new DrillSchemaFactory();
-  }
-  
-  private DrillSchemaFactory(){
-    this.registry = null;
-  }
-  
-  public DrillSchemaFactory(StorageEngines engines, DrillConfig config) throws SetupException {
-    super();
-    this.registry = new SchemaProviderRegistry(config);
-    
-    for (Map.Entry<String, StorageEngineConfig> entry : engines) {
-      SchemaProvider provider = registry.getSchemaProvider(entry.getValue());
-      preEntries.put(entry.getKey(), new StorageEngineEntry(entry.getValue(), provider));
-    }
-    
-  }
-
-  public Schema apply(SchemaPlus root) {
-    List<String> schemaNames = Lists.newArrayList();
-    Schema defaultSchema = null;
-    for(Entry<String, StorageEngineEntry> e : preEntries.entrySet()){
-      FileSystemSchema schema = new FileSystemSchema(e.getValue().getConfig(), e.getValue().getProvider(), root, e.getKey());
-      if(defaultSchema == null) defaultSchema = schema;
-      root.add(schema);
-      schemaNames.add(e.getKey());
-    }
-    logger.debug("Registered schemas for {}", schemaNames);
-    return defaultSchema;
-  }
-  
-  
-  private class StorageEngineEntry{
-    StorageEngineConfig config;
-    SchemaProvider provider;
-    
-    
-    public StorageEngineEntry(StorageEngineConfig config, SchemaProvider provider) {
-      super();
-      this.config = config;
-      this.provider = provider;
-    }
-    
-    public StorageEngineConfig getConfig() {
-      return config;
-    }
-    public SchemaProvider getProvider() {
-      return provider;
-    }
-    
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
index 818e892..0fdff3c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
@@ -24,7 +24,6 @@ import net.hydromatic.optiq.tools.RelConversionException;
 import net.hydromatic.optiq.tools.RuleSet;
 import net.hydromatic.optiq.tools.ValidationException;
 
-import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.common.expression.FunctionRegistry;
 import org.apache.drill.common.logical.LogicalPlan;
 import org.apache.drill.common.logical.PlanProperties.Generator.ResultMode;
@@ -34,7 +33,7 @@ import org.apache.drill.exec.planner.logical.DrillRel;
 import org.apache.drill.exec.planner.logical.DrillRuleSets;
 import org.apache.drill.exec.planner.logical.DrillScreenRel;
 import org.apache.drill.exec.planner.logical.DrillStoreRel;
-import org.apache.drill.exec.planner.logical.StorageEngines;
+import org.apache.drill.exec.store.StoragePluginRegistry.DrillSchemaFactory;
 import org.eigenbase.rel.RelNode;
 import org.eigenbase.sql.SqlExplain;
 import org.eigenbase.sql.SqlKind;
@@ -43,9 +42,6 @@ import org.eigenbase.sql.SqlNode;
 import org.eigenbase.sql.fun.SqlStdOperatorTable;
 import org.eigenbase.sql.parser.SqlParseException;
 
-import com.google.common.base.Charsets;
-import com.google.common.io.Resources;
-
 public class DrillSqlWorker {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillSqlWorker.class);
 
@@ -55,9 +51,7 @@ public class DrillSqlWorker {
   public DrillSqlWorker(DrillSchemaFactory schemaFactory, FunctionRegistry functionRegistry) throws Exception {
     this.registry = functionRegistry;
     this.planner = Frameworks.getPlanner(ConnectionConfig.Lex.MYSQL, schemaFactory, SqlStdOperatorTable.instance(), new RuleSet[]{DrillRuleSets.DRILL_BASIC_RULES});
-    
   }
-
   
   public LogicalPlan getPlan(String sql) throws SqlParseException, ValidationException, RelConversionException{
     SqlNode sqlNode = planner.parse(sql);
@@ -77,8 +71,6 @@ public class DrillSqlWorker {
         break;
       default:
       }
-
-      
     }
     
     SqlNode validatedNode = planner.validate(sqlNode);
@@ -96,28 +88,6 @@ public class DrillSqlWorker {
     return implementor.getPlan();
     
   }
-  private void x() throws Exception {
-    String sqlAgg = "select a, count(1) from parquet.`/Users/jnadeau/region.parquet` group by a";
-    String sql = "select * from parquet.`/Users/jnadeau/region.parquet`";
-
-    
-    System.out.println(sql);
-    System.out.println(getPlan(sql).toJsonString(DrillConfig.create()));
-    System.out.println("///////////");
-    System.out.println(sqlAgg);
-    System.out.println(getPlan(sqlAgg).toJsonString(DrillConfig.create()));
-  }
-
-  public static void main(String[] args) throws Exception {
-    DrillConfig config = DrillConfig.create();
-    
-    String enginesData = Resources.toString(Resources.getResource("storage-engines.json"), Charsets.UTF_8);
-    StorageEngines engines = config.getMapper().readValue(enginesData, StorageEngines.class);
-    FunctionRegistry fr = new FunctionRegistry(config);
-    DrillSchemaFactory schemaFactory = new DrillSchemaFactory(engines, config);
-    DrillSqlWorker worker = new DrillSqlWorker(schemaFactory, fr);
-    worker.x();
-  }
 
   
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/FileSystemSchema.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/FileSystemSchema.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/FileSystemSchema.java
deleted file mode 100644
index ac29fc3..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/FileSystemSchema.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.planner.sql;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Set;
-
-import net.hydromatic.linq4j.expressions.DefaultExpression;
-import net.hydromatic.linq4j.expressions.Expression;
-import net.hydromatic.optiq.Schema;
-import net.hydromatic.optiq.SchemaPlus;
-import net.hydromatic.optiq.TableFunction;
-
-import org.apache.drill.common.logical.StorageEngineConfig;
-import org.apache.drill.exec.planner.logical.DrillTable;
-import org.apache.drill.exec.store.SchemaProvider;
-
-public class FileSystemSchema implements Schema, ExpandingConcurrentMap.MapValueFactory<String, DrillTable>{
-  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FileSystemSchema.class);
-  
-  private ExpandingConcurrentMap<String, DrillTable> tables = new ExpandingConcurrentMap<String, DrillTable>(this);
-  
-  private final SchemaPlus parentSchema;
-  private final String name;
-  private final Expression expression = new DefaultExpression(Object.class);
-  private final SchemaProvider schemaProvider;
-  private final StorageEngineConfig config;
-  
-  public FileSystemSchema(StorageEngineConfig config, SchemaProvider schemaProvider, SchemaPlus parentSchema, String name) {
-    super();
-    this.parentSchema = parentSchema;
-    this.name = name;
-    this.schemaProvider = schemaProvider;
-    this.config = config;
-  }
-
-  @Override
-  public Schema getSubSchema(String name) {
-    return null;
-  }
-
-  @Override
-  public String getName() {
-    return name;
-  }
-
-  @Override
-  public Expression getExpression() {
-    return expression;
-  }
-
-  @Override
-  public Collection<TableFunction> getTableFunctions(String name) {
-    return Collections.emptyList();
-  }
-  
-  @Override
-  public SchemaPlus getParentSchema() {
-    return parentSchema;
-  }
-
-  @Override
-  public Set<String> getTableNames() {
-    return tables.keySet();
-  }
-
-  @Override
-  public Set<String> getTableFunctionNames() {
-    return Collections.emptySet();
-  }
-
-  @Override
-  public Set<String> getSubSchemaNames() {
-    return Collections.emptySet();
-  }
-
-  @Override
-  public boolean isMutable() {
-    return true;
-  }
-  
-  @Override
-  public DrillTable getTable(String name) {
-    return tables.get(name);
-  }
-
-  @Override
-  public DrillTable create(String key) {
-    Object selection = schemaProvider.getSelectionBaseOnName(key);
-    if(selection == null) return null;
-    
-    return new DrillTable(name, this.name, selection, config);
-  }
-
-  @Override
-  public void destroy(DrillTable value) {
-  }
-
-  
-  
-}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/8b90cae3/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
index 60a7d5c..0e6b7bf 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
@@ -24,26 +24,22 @@ import java.util.Collection;
 import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.common.exceptions.ExecutionSetupException;
 import org.apache.drill.common.expression.FunctionRegistry;
-import org.apache.drill.common.logical.StorageEngineConfig;
 import org.apache.drill.exec.cache.DistributedCache;
 import org.apache.drill.exec.coord.ClusterCoordinator;
 import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
 import org.apache.drill.exec.memory.BufferAllocator;
 import org.apache.drill.exec.physical.impl.OperatorCreatorRegistry;
 import org.apache.drill.exec.planner.PhysicalPlanReader;
-import org.apache.drill.exec.planner.logical.StorageEngines;
-import org.apache.drill.exec.planner.sql.DrillSchemaFactory;
 import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
 import org.apache.drill.exec.rpc.control.Controller;
 import org.apache.drill.exec.rpc.control.WorkEventBus;
 import org.apache.drill.exec.rpc.data.DataConnectionCreator;
-import org.apache.drill.exec.store.StorageEngine;
-import org.apache.drill.exec.store.StorageEngineRegistry;
+import org.apache.drill.exec.store.StoragePluginRegistry;
+import org.apache.drill.exec.store.StoragePluginRegistry.DrillSchemaFactory;
+import org.apache.drill.exec.store.StoragePlugin;
 
 import com.codahale.metrics.MetricRegistry;
-import com.google.common.base.Charsets;
 import com.google.common.base.Preconditions;
-import com.google.common.io.Resources;
 
 public class DrillbitContext {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillbitContext.class);
@@ -55,13 +51,12 @@ public class DrillbitContext {
   private final DataConnectionCreator connectionsPool;
   private final DistributedCache cache;
   private final DrillbitEndpoint endpoint;
-  private final StorageEngineRegistry storageEngineRegistry;
+  private final StoragePluginRegistry storagePlugins;
   private final OperatorCreatorRegistry operatorCreatorRegistry;
   private final Controller controller;
   private final WorkEventBus workBus;
   private final FunctionImplementationRegistry functionRegistry;
   private final FunctionRegistry functionRegistryX;
-  private final DrillSchemaFactory factory;
   
   public DrillbitContext(DrillbitEndpoint endpoint, BootStrapContext context, ClusterCoordinator coord, Controller controller, DataConnectionCreator connectionsPool, DistributedCache cache, WorkEventBus workBus) {
     super();
@@ -76,27 +71,11 @@ public class DrillbitContext {
     this.connectionsPool = connectionsPool;
     this.cache = cache;
     this.endpoint = endpoint;
-    this.storageEngineRegistry = new StorageEngineRegistry(this);
-    this.reader = new PhysicalPlanReader(context.getConfig(), context.getConfig().getMapper(), endpoint, storageEngineRegistry);
+    this.storagePlugins = new StoragePluginRegistry(this);
+    this.reader = new PhysicalPlanReader(context.getConfig(), context.getConfig().getMapper(), endpoint, storagePlugins);
     this.operatorCreatorRegistry = new OperatorCreatorRegistry(context.getConfig());
     this.functionRegistry = new FunctionImplementationRegistry(context.getConfig());
-    
-    DrillSchemaFactory factory = null;
-    try{
-      String enginesData = Resources.toString(Resources.getResource("storage-engines.json"), Charsets.UTF_8);
-      StorageEngines engines = context.getConfig().getMapper().readValue(enginesData, StorageEngines.class);
-      factory = new DrillSchemaFactory(engines, context.getConfig());
-    }catch(Exception e){
-      logger.error("Failure reading storage engines data. Creating empty list of schemas.", e);
-      factory = DrillSchemaFactory.createEmpty();
-    }
-    this.factory = factory;
     this.functionRegistryX = new FunctionRegistry(context.getConfig());
-
-  }
-  
-  public DrillSchemaFactory getSchemaFactory(){
-    return factory;
   }
 
   public FunctionRegistry getFunctionRegistry(){
@@ -131,8 +110,8 @@ public class DrillbitContext {
     return operatorCreatorRegistry;
   }
 
-  public StorageEngine getStorageEngine(StorageEngineConfig config) throws ExecutionSetupException {
-    return storageEngineRegistry.getEngine(config);
+  public StoragePluginRegistry getStorage(){
+    return this.storagePlugins;
   }
   
   public NioEventLoopGroup getBitLoopGroup(){
@@ -160,5 +139,8 @@ public class DrillbitContext {
     return reader;
   }
   
+  public DrillSchemaFactory getSchemaFactory(){
+    return storagePlugins.getSchemaFactory();
+  }
   
 }


Mime
View raw message