drill-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jacq...@apache.org
Subject [01/38] DRILL-357: Hive Storage Engine phase 2 - hive record reader
Date Tue, 04 Mar 2014 08:07:28 GMT
Repository: incubator-drill
Updated Branches:
  refs/heads/master b5ab447de -> 4871fd0de


http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveTextRecordReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveTextRecordReader.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveTextRecordReader.java
new file mode 100644
index 0000000..1e47684
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/hive/HiveTextRecordReader.java
@@ -0,0 +1,172 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.hive;
+
+import com.google.common.collect.Lists;
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+import org.apache.drill.common.exceptions.ExecutionSetupException;
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.vector.*;
+import org.apache.drill.exec.vector.allocator.VectorAllocator;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapred.InputSplit;
+
+import java.io.IOException;
+import java.util.List;
+
+public class HiveTextRecordReader extends HiveRecordReader {
+
+  public final byte delimiter;
+  public final List<Integer> columnIds;
+  private final int numCols;
+
+  public HiveTextRecordReader(Table table, Partition partition, InputSplit inputSplit, List<FieldReference>
columns, FragmentContext context) throws ExecutionSetupException {
+    super(table, partition, inputSplit, columns, context);
+    String d = table.getSd().getSerdeInfo().getParameters().get("field.delim");
+    if (d != null) {
+      delimiter = d.getBytes()[0];
+    } else {
+      delimiter = (byte) 1;
+    }
+    assert delimiter > 0;
+    List<Integer> ids = Lists.newArrayList();
+    for (int i = 0; i < tableColumns.size(); i++) {
+      if (columnNames.contains(tableColumns.get(i))) {
+        ids.add(i);
+      }
+    }
+    columnIds = ids;
+    numCols = tableColumns.size();
+  }
+
+  public boolean setValue(PrimitiveObjectInspector.PrimitiveCategory pCat, ValueVector vv,
int index, byte[] bytes, int start) {
+    switch(pCat) {
+      case BINARY:
+        throw new UnsupportedOperationException();
+      case BOOLEAN:
+        throw new UnsupportedOperationException();
+      case BYTE:
+        throw new UnsupportedOperationException();
+      case DECIMAL:
+        throw new UnsupportedOperationException();
+      case DOUBLE:
+        throw new UnsupportedOperationException();
+      case FLOAT:
+        throw new UnsupportedOperationException();
+      case INT: {
+        int value = 0;
+        byte b;
+        for (int i = start; (b = bytes[i]) != delimiter; i++) {
+          value = (value * 10) + b - 48;
+        }
+        ((IntVector) vv).getMutator().set(index, value); // No need to use setSafe for fixed
length vectors
+        return true;
+      }
+      case LONG: {
+        long value = 0;
+        byte b;
+        for (int i = start; (b = bytes[i]) != delimiter; i++) {
+          value = (value * 10) + b - 48;
+        }
+        ((BigIntVector) vv).getMutator().set(index, value); // No need to use setSafe for
fixed length vectors
+        return true;
+      }
+      case SHORT:
+        throw new UnsupportedOperationException();
+      case STRING: {
+        int end = start;
+        for (int i = start; i < bytes.length; i++) {
+          if (bytes[i] == delimiter) {
+            end = i;
+            break;
+          }
+          end = bytes.length;
+        }
+        return ((VarCharVector) vv).getMutator().setSafe(index, bytes, start, end - start);
+      }
+      case TIMESTAMP:
+        throw new UnsupportedOperationException();
+
+      default:
+        throw new UnsupportedOperationException("Could not determine type");
+    }
+  }
+
+
+  @Override
+  public int next() {
+    for (ValueVector vv : vectors) {
+      VectorAllocator.getAllocator(vv, 50).alloc(TARGET_RECORD_COUNT);
+    }
+    try {
+      int recordCount = 0;
+      if (redoRecord != null) {
+        int length = ((Text) value).getLength();
+        byte[] bytes = ((Text) value).getBytes();
+        int[] delimPositions = new int[numCols];
+        delimPositions[0] = -1;
+        int p = 0;
+        for (int i = 0; i < length; i++) {
+          if (bytes[i] == delimiter) {
+            delimPositions[p++] = i;
+          }
+        }
+        for (int id : columnIds) {
+          boolean success = setValue(primitiveCategories.get(id), vectors.get(id), recordCount,
bytes, delimPositions[id]);
+          if (!success) {
+            throw new DrillRuntimeException(String.format("Failed to write value for column
%s", columnNames.get(id)));
+          }
+
+        }
+        redoRecord = null;
+      }
+      while (recordCount < TARGET_RECORD_COUNT && reader.next(key, value)) {
+        int length = ((Text) value).getLength();
+        byte[] bytes = ((Text) value).getBytes();
+        int[] delimPositions = new int[numCols + 1];
+        delimPositions[0] = -1;
+        int p = 1;
+        for (int i = 0; i < length; i++) {
+          if (bytes[i] == delimiter) {
+            delimPositions[p++] = i;
+          }
+        }
+        for (int i = 0; i < columnIds.size(); i++) {
+          int id = columnIds.get(i);
+          boolean success = setValue(primitiveCategories.get(i), vectors.get(i), recordCount,
bytes, delimPositions[id] + 1);
+          if (!success) {
+            redoRecord = value;
+            if (partition != null) populatePartitionVectors(recordCount);
+            return recordCount;
+          }
+        }
+        recordCount++;
+      }
+      if (partition != null) populatePartitionVectors(recordCount);
+      return recordCount;
+    } catch (IOException e) {
+      throw new DrillRuntimeException(e);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java
index 209961d..86be49e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetSchemaProvider.java
@@ -25,7 +25,6 @@ import org.apache.drill.exec.store.ClassPathFileSystem;
 import org.apache.drill.exec.store.SchemaProvider;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
 
 import com.beust.jcommander.internal.Lists;
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java
new file mode 100644
index 0000000..aa68752
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/BatchPrinter.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.util;
+
+import com.beust.jcommander.internal.Lists;
+import org.apache.commons.lang.StringUtils;
+import org.apache.drill.exec.exception.SchemaChangeException;
+import org.apache.drill.exec.record.VectorAccessible;
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.record.selection.SelectionVector4;
+import org.apache.drill.exec.rpc.RpcException;
+import org.apache.drill.exec.vector.ValueVector;
+
+import java.util.List;
+
+/**
+ * This is a tool for printing the content of record batches to screen. Used for debugging.
+ */
+public class BatchPrinter {
+  public static void printHyperBatch(VectorAccessible batch) {
+    List<String> columns = Lists.newArrayList();
+    List<ValueVector> vectors = Lists.newArrayList();
+    int numBatches = 0;
+    for (VectorWrapper vw : batch) {
+      columns.add(vw.getValueVectors()[0].getField().getName());
+      numBatches = vw.getValueVectors().length;
+    }
+    int width = columns.size();
+      for (int i = 0; i < numBatches; i++) {
+        int rows = batch.iterator().next().getValueVectors()[i].getMetadata().getValueCount();
+        for (int j = 0; j < rows; j++) {
+          for (VectorWrapper vw : batch) {
+            Object o = vw.getValueVectors()[i].getAccessor().getObject(j);
+            if (o instanceof byte[]) {
+              String value = new String((byte[]) o);
+              System.out.printf("| %-15s",value.length() <= 15 ? value : value.substring(0,
14));
+            } else {
+              String value = o.toString();
+              System.out.printf("| %-15s",value.length() <= 15 ? value : value.substring(0,14));
+            }
+          }
+          System.out.printf("|\n");
+        }
+      }
+      System.out.printf("|\n");
+  }
+  public static void printBatch(VectorAccessible batch) {
+    List<String> columns = Lists.newArrayList();
+    List<ValueVector> vectors = Lists.newArrayList();
+    for (VectorWrapper vw : batch) {
+      columns.add(vw.getValueVector().getField().getName());
+      vectors.add(vw.getValueVector());
+    }
+    int width = columns.size();
+    int rows = vectors.get(0).getMetadata().getValueCount();
+    for (int row = 0; row < rows; row++) {
+      if (row%50 == 0) {
+        System.out.println(StringUtils.repeat("-", width * 17 + 1));
+        for (String column : columns) {
+          System.out.printf("| %-15s", width <= 15 ? column : column.substring(0, 14));
+        }
+        System.out.printf("|\n");
+        System.out.println(StringUtils.repeat("-", width*17 + 1));
+      }
+      for (ValueVector vv : vectors) {
+        Object o = vv.getAccessor().getObject(row);
+        if (o instanceof byte[]) {
+          String value = new String((byte[]) o);
+          System.out.printf("| %-15s",value.length() <= 15 ? value : value.substring(0,
14));
+        } else {
+          String value = o.toString();
+          System.out.printf("| %-15s",value.length() <= 15 ? value : value.substring(0,14));
+        }
+      }
+      System.out.printf("|\n");
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/exec/java-exec/src/main/resources/drill-module.conf
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/drill-module.conf b/exec/java-exec/src/main/resources/drill-module.conf
index c801163..4dd08c1 100644
--- a/exec/java-exec/src/main/resources/drill-module.conf
+++ b/exec/java-exec/src/main/resources/drill-module.conf
@@ -82,5 +82,6 @@ drill.exec: {
       delete: false,
       size: 100000000
     }
-  }
+  },
+  cache.hazel.subnets: ["*.*.*.*"]
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/exec/java-exec/src/test/java/org/apache/drill/exec/TestPlan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/TestPlan.java b/exec/java-exec/src/test/java/org/apache/drill/exec/TestPlan.java
new file mode 100644
index 0000000..71e6283
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/TestPlan.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec;
+
+import org.apache.drill.exec.client.QuerySubmitter;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.io.IOException;
+
+/**
+ * Created with IntelliJ IDEA.
+ * User: sphillips
+ * Date: 1/24/14
+ * Time: 3:46 PM
+ * To change this template use File | Settings | File Templates.
+ */
+public class TestPlan {
+
+  String location = "/Users/sphillips/hive-lineitem-orderkey";
+  String type = "physical";
+  String zkQuorum = null;
+  boolean local = true;
+  int bits = 1;
+
+
+  @Test
+  @Ignore
+  public void testSubmitPlan() throws Exception {
+    QuerySubmitter submitter = new QuerySubmitter();
+    submitter.submitQuery(location, type, zkQuorum, local, bits);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/TestHiveScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/TestHiveScan.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/TestHiveScan.java
new file mode 100644
index 0000000..c6edc20
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/hive/TestHiveScan.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.hive;
+
+import com.google.common.base.Charsets;
+import com.google.common.io.Files;
+import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.exec.client.DrillClient;
+import org.apache.drill.exec.pop.PopUnitTestBase;
+import org.apache.drill.exec.proto.UserProtos;
+import org.apache.drill.exec.rpc.user.QueryResultBatch;
+import org.apache.drill.exec.server.Drillbit;
+import org.apache.drill.exec.server.RemoteServiceSet;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Created with IntelliJ IDEA.
+ * User: sphillips
+ * Date: 1/23/14
+ * Time: 5:22 AM
+ * To change this template use File | Settings | File Templates.
+ */
+public class TestHiveScan extends PopUnitTestBase {
+  @Ignore
+  @Test
+  public void twoBitTwoExchangeTwoEntryRun() throws Exception {
+    RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
+
+    try(Drillbit bit1 = new Drillbit(CONFIG, serviceSet);
+        Drillbit bit2 = new Drillbit(CONFIG, serviceSet);
+        DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator());) {
+
+      bit1.run();
+      bit2.run();
+      client.connect();
+      List<QueryResultBatch> results = client.runQuery(UserProtos.QueryType.PHYSICAL,
+              Files.toString(FileUtils.getResourceAsFile("/hive/test.json"),
+                      Charsets.UTF_8));
+      int count = 0;
+      for(QueryResultBatch b : results) {
+        if (b.getHeader().getRowCount() != 0)
+          count += b.getHeader().getRowCount();
+      }
+      assertEquals(100, count);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/exec/java-exec/src/test/java/org/apache/drill/exec/util/BatchPrinter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/util/BatchPrinter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/util/BatchPrinter.java
deleted file mode 100644
index aa68752..0000000
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/util/BatchPrinter.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.util;
-
-import com.beust.jcommander.internal.Lists;
-import org.apache.commons.lang.StringUtils;
-import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.record.VectorAccessible;
-import org.apache.drill.exec.record.VectorWrapper;
-import org.apache.drill.exec.record.selection.SelectionVector4;
-import org.apache.drill.exec.rpc.RpcException;
-import org.apache.drill.exec.vector.ValueVector;
-
-import java.util.List;
-
-/**
- * This is a tool for printing the content of record batches to screen. Used for debugging.
- */
-public class BatchPrinter {
-  public static void printHyperBatch(VectorAccessible batch) {
-    List<String> columns = Lists.newArrayList();
-    List<ValueVector> vectors = Lists.newArrayList();
-    int numBatches = 0;
-    for (VectorWrapper vw : batch) {
-      columns.add(vw.getValueVectors()[0].getField().getName());
-      numBatches = vw.getValueVectors().length;
-    }
-    int width = columns.size();
-      for (int i = 0; i < numBatches; i++) {
-        int rows = batch.iterator().next().getValueVectors()[i].getMetadata().getValueCount();
-        for (int j = 0; j < rows; j++) {
-          for (VectorWrapper vw : batch) {
-            Object o = vw.getValueVectors()[i].getAccessor().getObject(j);
-            if (o instanceof byte[]) {
-              String value = new String((byte[]) o);
-              System.out.printf("| %-15s",value.length() <= 15 ? value : value.substring(0,
14));
-            } else {
-              String value = o.toString();
-              System.out.printf("| %-15s",value.length() <= 15 ? value : value.substring(0,14));
-            }
-          }
-          System.out.printf("|\n");
-        }
-      }
-      System.out.printf("|\n");
-  }
-  public static void printBatch(VectorAccessible batch) {
-    List<String> columns = Lists.newArrayList();
-    List<ValueVector> vectors = Lists.newArrayList();
-    for (VectorWrapper vw : batch) {
-      columns.add(vw.getValueVector().getField().getName());
-      vectors.add(vw.getValueVector());
-    }
-    int width = columns.size();
-    int rows = vectors.get(0).getMetadata().getValueCount();
-    for (int row = 0; row < rows; row++) {
-      if (row%50 == 0) {
-        System.out.println(StringUtils.repeat("-", width * 17 + 1));
-        for (String column : columns) {
-          System.out.printf("| %-15s", width <= 15 ? column : column.substring(0, 14));
-        }
-        System.out.printf("|\n");
-        System.out.println(StringUtils.repeat("-", width*17 + 1));
-      }
-      for (ValueVector vv : vectors) {
-        Object o = vv.getAccessor().getObject(row);
-        if (o instanceof byte[]) {
-          String value = new String((byte[]) o);
-          System.out.printf("| %-15s",value.length() <= 15 ? value : value.substring(0,
14));
-        } else {
-          String value = o.toString();
-          System.out.printf("| %-15s",value.length() <= 15 ? value : value.substring(0,14));
-        }
-      }
-      System.out.printf("|\n");
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/exec/java-exec/src/test/resources/hive/test.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/hive/test.json b/exec/java-exec/src/test/resources/hive/test.json
new file mode 100644
index 0000000..a039d9e
--- /dev/null
+++ b/exec/java-exec/src/test/resources/hive/test.json
@@ -0,0 +1,75 @@
+{
+    head:{
+        type:"APACHE_DRILL_PHYSICAL",
+        version:"1",
+        generator:{
+            type:"manual"
+        }
+    },
+    graph:[
+        {
+            @id:1,
+            pop:"hive-scan",
+            storageengine: { type: "hive"},
+            hive-table: {
+                          "tableName" : "nation",
+                          "dbName" : "default",
+                          "owner" : "root",
+                          "createTime" : 1386876893,
+                          "lastAccessTime" : 0,
+                          "retention" : 0,
+                          "sd" : {
+                            "cols" : [ {
+                              "name" : "n_nationkey",
+                              "type" : "bigint",
+                              "comment" : null
+                            }, {
+                              "name" : "n_name",
+                              "type" : "string",
+                              "comment" : null
+                            }, {
+                              "name" : "n_regionkey",
+                              "type" : "bigint",
+                              "comment" : null
+                            }, {
+                              "name" : "n_comment",
+                              "type" : "string",
+                              "comment" : null
+                            } ],
+                            "location" : "maprfs:/user/hive/warehouse/nation",
+                            "inputFormat" : "org.apache.hadoop.mapred.TextInputFormat",
+                            "outputFormat" : "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
+                            "compressed" : false,
+                            "numBuckets" : 0,
+                            "serDeInfo" : {
+                              "name" : null,
+                              "serializationLib" : "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
+                              "parameters" : {
+                                "serialization.format" : "|",
+                                "field.delim" : "|"
+                              }
+                            },
+                            "sortCols" : [ ],
+                            "parameters" : { }
+                          },
+                          "partitionKeys" : [ ],
+                          "parameters" : {
+                            "numPartitions" : "0",
+                            "numFiles" : "1",
+                            "transient_lastDdlTime" : "1386877487",
+                            "totalSize" : "2224",
+                            "numRows" : "0",
+                            "rawDataSize" : "0"
+                          },
+                          "viewOriginalText" : null,
+                          "viewExpandedText" : null,
+                          "tableType" : "MANAGED_TABLE"
+                        }
+        },
+        {
+            @id: 2,
+            child: 1,
+            pop: "screen"
+        }
+    ]
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 2121adc..f8ddbf2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -356,6 +356,52 @@
       <version>${dep.slf4j.version}</version>
     </dependency>
 
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-exec</artifactId>
+      <version>0.12.0</version>
+      <exclusions>
+        <exclusion>
+          <groupId>commons-logging</groupId>
+          <artifactId>commons-logging</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>commons-logging</groupId>
+          <artifactId>commons-logging-api</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>jline</groupId>
+          <artifactId>jline</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>log4j</groupId>
+          <artifactId>log4j</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.google.guava</groupId>
+          <artifactId>guava</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-hbase-handler</artifactId>
+      <version>0.12.0</version>
+      <exclusions>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.apache.hbase</groupId>
+          <artifactId>hbase</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
     <!-- Test Dependencies -->
     <dependency>
       <groupId>com.googlecode.jmockit</groupId>
@@ -594,6 +640,11 @@
             <version>1.0.3-mapr-3.0.0</version>
           </dependency>
           <dependency>
+            <groupId>org.apache.hbase</groupId>
+            <artifactId>hbase</artifactId>
+            <version>0.94.13-mapr-1401-m7-3.0.2</version>
+          </dependency>
+          <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-core</artifactId>
             <version>1.0.3-mapr-3.0.0</version>

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java
index fefe7bc..4712302 100644
--- a/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java
+++ b/sqlparser/src/main/java/org/apache/drill/jdbc/DrillHandler.java
@@ -41,7 +41,7 @@ import org.apache.drill.exec.server.Drillbit;
 import org.apache.drill.exec.server.RemoteServiceSet;
 import org.apache.drill.exec.store.SchemaProvider;
 import org.apache.drill.exec.store.SchemaProviderRegistry;
-import org.apache.drill.exec.store.hive.HiveSchemaProvider;
+import org.apache.drill.exec.store.hive.HiveStorageEngine.HiveSchemaProvider;
 import org.apache.drill.exec.store.json.JsonSchemaProvider;
 import org.apache.drill.exec.store.parquet.ParquetSchemaProvider;
 import org.apache.drill.sql.client.full.FileSystemSchema;

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/sqlparser/src/main/java/org/apache/drill/sql/client/full/BatchLoaderMap.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/sql/client/full/BatchLoaderMap.java
b/sqlparser/src/main/java/org/apache/drill/sql/client/full/BatchLoaderMap.java
index d444348..1b58805 100644
--- a/sqlparser/src/main/java/org/apache/drill/sql/client/full/BatchLoaderMap.java
+++ b/sqlparser/src/main/java/org/apache/drill/sql/client/full/BatchLoaderMap.java
@@ -23,7 +23,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import jline.internal.Preconditions;
+import com.google.common.base.Preconditions;
 
 import org.apache.drill.exec.client.DrillClient;
 import org.apache.drill.exec.exception.SchemaChangeException;

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveDatabaseSchema.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveDatabaseSchema.java
b/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveDatabaseSchema.java
index c12057a..c02a478 100644
--- a/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveDatabaseSchema.java
+++ b/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveDatabaseSchema.java
@@ -31,12 +31,15 @@ import net.hydromatic.optiq.impl.java.JavaTypeFactory;
 
 import org.apache.drill.exec.client.DrillClient;
 import org.apache.drill.exec.store.SchemaProvider;
+import org.apache.drill.exec.store.hive.HiveReadEntry;
+import org.apache.drill.exec.store.hive.HiveStorageEngine;
 import org.apache.drill.exec.store.hive.HiveStorageEngineConfig;
 import org.apache.drill.jdbc.DrillTable;
 
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Multimap;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
@@ -53,11 +56,11 @@ public class HiveDatabaseSchema implements Schema{
   private final String name;
   private final Expression expression;
   private final QueryProvider queryProvider;
-  private final SchemaProvider schemaProvider;
+  private final HiveStorageEngine.HiveSchemaProvider schemaProvider;
   private final DrillClient client;
   private final HiveStorageEngineConfig config;
 
-  public HiveDatabaseSchema(DrillClient client, HiveStorageEngineConfig config, SchemaProvider
schemaProvider,
+  public HiveDatabaseSchema(DrillClient client, HiveStorageEngineConfig config, HiveStorageEngine.HiveSchemaProvider
schemaProvider,
                     JavaTypeFactory typeFactory, HiveSchema parentSchema, String name,
                     Expression expression, QueryProvider queryProvider) {
     super();
@@ -121,7 +124,54 @@ public class HiveDatabaseSchema implements Schema{
     return Collections.EMPTY_LIST;
   }
 
-  static Map<PrimitiveObjectInspector.PrimitiveCategory, SqlTypeName> mapPrimHive2Sql
= new HashMap<>();
+  private RelDataType getRelDataTypeFromHiveTypeString(String type) {
+    switch(type) {
+      case "boolean":
+        return typeFactory.createSqlType(SqlTypeName.BOOLEAN);
+
+      case "tinyint":
+        return typeFactory.createSqlType(SqlTypeName.TINYINT);
+
+      case "smallint":
+        return typeFactory.createSqlType(SqlTypeName.SMALLINT);
+
+      case "int":
+        return typeFactory.createSqlType(SqlTypeName.INTEGER);
+
+      case "bigint":
+        return typeFactory.createSqlType(SqlTypeName.BIGINT);
+
+      case "float":
+        return typeFactory.createSqlType(SqlTypeName.FLOAT);
+
+      case "double":
+        return typeFactory.createSqlType(SqlTypeName.DOUBLE);
+
+      case "date":
+        return typeFactory.createSqlType(SqlTypeName.DATE);
+
+      case "timestamp":
+        return typeFactory.createSqlType(SqlTypeName.TIMESTAMP);
+
+      case "binary":
+        return typeFactory.createSqlType(SqlTypeName.BINARY);
+
+      case "decimal":
+        return typeFactory.createSqlType(SqlTypeName.DECIMAL);
+
+      case "string":
+      case "varchar": {
+        return typeFactory.createTypeWithCharsetAndCollation(
+                typeFactory.createSqlType(SqlTypeName.VARCHAR), /*input type*/
+                Charset.forName("ISO-8859-1"), /*unicode char set*/
+                SqlCollation.IMPLICIT /* TODO: need to decide if implicit is the correct
one */
+        );
+      }
+
+      default:
+        throw new RuntimeException("Unknown or unsupported hive type: " + type);
+    }
+  }
 
   private RelDataType getRelDataTypeFromHivePrimitiveType(PrimitiveObjectInspector poi) {
     switch(poi.getPrimitiveCategory()) {
@@ -161,7 +211,7 @@ public class HiveDatabaseSchema implements Schema{
       case VARCHAR: {
         return typeFactory.createTypeWithCharsetAndCollation(
           typeFactory.createSqlType(SqlTypeName.VARCHAR), /*input type*/
-          Charset.forName("UTF-16"), /*unicode char set*/
+          Charset.forName("ISO-8859-1"), /*unicode char set*/
           SqlCollation.IMPLICIT /* TODO: need to decide if implicit is the correct one */
         );
       }
@@ -189,22 +239,20 @@ public class HiveDatabaseSchema implements Schema{
   @SuppressWarnings("unchecked")
   @Override
   public <E> Table<E> getTable(String name, Class<E> elementType) {
-    try {
-      org.apache.hadoop.hive.ql.metadata.Table hiveTable =
-        parentSchema.getHiveDb().getTable(getName(), name, false /*throwException*/);
-
-      if (hiveTable == null) {
-        logger.debug("Table name {} is invalid", name);
-        return null;
-      }
+    Object selection = schemaProvider.getSelectionBaseOnName(String.format("%s.%s",this.name,
name));
+    if(selection == null) return null;
+    org.apache.hadoop.hive.metastore.api.Table t = ((HiveReadEntry) selection).getTable();
+    if (t == null) {
+      logger.debug("Table name {} is invalid", name);
+      return null;
+    }
+    org.apache.hadoop.hive.ql.metadata.Table hiveTable = new org.apache.hadoop.hive.ql.metadata.Table(t);
 
-      Object selection = schemaProvider.getSelectionBaseOnName(name);
-      if(selection == null) return null;
 
-      final MethodCallExpression call = Expressions.call(getExpression(), //
-        BuiltinMethod.DATA_CONTEXT_GET_TABLE.method, //
-        Expressions.constant(name), //
-        Expressions.constant(Object.class));
+    final MethodCallExpression call = Expressions.call(getExpression(), //
+      BuiltinMethod.DATA_CONTEXT_GET_TABLE.method, //
+      Expressions.constant(name), //
+      Expressions.constant(Object.class));
 
       ArrayList<RelDataType> typeList = new ArrayList<>();
       ArrayList<String> fieldNameList = new ArrayList<>();
@@ -215,21 +263,22 @@ public class HiveDatabaseSchema implements Schema{
         typeList.add(getRelDataTypeFromHiveType(hiveField.getFieldObjectInspector()));
       }
 
-      final RelDataType rowType = typeFactory.createStructType(typeList, fieldNameList);
-      return (Table<E>) new DrillTable(
-        client,
-        this,
-        Object.class,
-        call,
-        rowType,
-        name,
-        null /*storageEngineName*/,
-        selection,
-        config /*storageEngineConfig*/);
-    } catch (HiveException ex) {
-      logger.error("getTable failed", ex);
-      return null;
-    }
+      for (FieldSchema field : hiveTable.getPartitionKeys()) {
+        fieldNameList.add(field.getName());
+        typeList.add(getRelDataTypeFromHiveTypeString(field.getType()));
+      }
+
+    final RelDataType rowType = typeFactory.createStructType(typeList, fieldNameList);
+    return (Table<E>) new DrillTable(
+      client,
+      this,
+      Object.class,
+      call,
+      rowType,
+      name,
+      null /*storageEngineName*/,
+      selection,
+      config /*storageEngineConfig*/);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveSchema.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveSchema.java b/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveSchema.java
index ffd6a6e..007c6d9 100644
--- a/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveSchema.java
+++ b/sqlparser/src/main/java/org/apache/drill/sql/client/full/HiveSchema.java
@@ -30,6 +30,7 @@ import org.apache.drill.common.logical.StorageEngineConfig;
 import org.apache.drill.exec.client.DrillClient;
 import org.apache.drill.exec.store.SchemaProvider;
 import org.apache.drill.exec.store.hive.HiveStorageEngineConfig;
+import org.apache.drill.exec.store.hive.HiveStorageEngine.HiveSchemaProvider;
 
 import org.apache.drill.jdbc.DrillTable;
 import org.apache.hadoop.hive.ql.metadata.Hive;
@@ -47,7 +48,7 @@ public class HiveSchema implements Schema{
   private final String name;
   private final Expression expression;
   private final QueryProvider queryProvider;
-  private final SchemaProvider schemaProvider;
+  private final HiveSchemaProvider schemaProvider;
   private final DrillClient client;
   private final HiveStorageEngineConfig config;
   private Hive hiveDb;
@@ -62,7 +63,7 @@ public class HiveSchema implements Schema{
     this.name = name;
     this.expression = expression;
     this.queryProvider = queryProvider;
-    this.schemaProvider = schemaProvider;
+    this.schemaProvider = (HiveSchemaProvider) schemaProvider;
     this.config = (HiveStorageEngineConfig) config;
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/cdf46fd3/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java
----------------------------------------------------------------------
diff --git a/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java b/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java
index 4f9d210..5e5812c 100644
--- a/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java
+++ b/sqlparser/src/test/java/org/apache/drill/jdbc/test/FullEngineTest.java
@@ -54,6 +54,7 @@ public class FullEngineTest {
    * @throws Exception
    */
   @Test(timeout=100000) // derby initialization is slow
+  @Ignore
   public void listHiveTables() throws Exception {
     JdbcAssert.withFull("hive-derby")
       .sql("select * from \"metadata\".\"TABLES\"")


Mime
View raw message