drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ve...@apache.org
Subject [1/3] drill git commit: DRILL-745: Add a test case for Hive avro tables.
Date Wed, 17 Jun 2015 21:31:04 GMT
Repository: drill
Updated Branches:
  refs/heads/master 4e2df00e7 -> d32acdb90


DRILL-745: Add a test case for Hive avro tables.


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/b46c53eb
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/b46c53eb
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/b46c53eb

Branch: refs/heads/master
Commit: b46c53eb8e82d19fd5243ff2a9050ca8aae1346b
Parents: ae62407
Author: vkorukanti <venki.korukanti@gmail.com>
Authored: Fri Jun 12 14:18:08 2015 -0700
Committer: vkorukanti <venki.korukanti@gmail.com>
Committed: Wed Jun 17 13:33:15 2015 -0700

----------------------------------------------------------------------
 contrib/storage-hive/core/pom.xml               |  5 ++
 .../apache/drill/exec/hive/TestHiveStorage.java | 10 ++++
 .../exec/hive/TestInfoSchemaOnHiveStorage.java  |  1 +
 .../exec/store/hive/HiveTestDataGenerator.java  | 23 ++++++++
 .../src/test/resources/avro_test_schema.json    |  9 +++
 pom.xml                                         | 62 +++++++++++++++++---
 6 files changed, 102 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/b46c53eb/contrib/storage-hive/core/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/pom.xml b/contrib/storage-hive/core/pom.xml
index 7bf644a..fb96a0d 100644
--- a/contrib/storage-hive/core/pom.xml
+++ b/contrib/storage-hive/core/pom.xml
@@ -78,6 +78,11 @@
       <classifier>tests</classifier>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-api</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>

http://git-wip-us.apache.org/repos/asf/drill/blob/b46c53eb/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
index 27ba9fe..2898f91 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
@@ -176,4 +176,14 @@ public class TestHiveStorage extends HiveTestBase {
         .baselineValues(1, " key_1")
         .go();
   }
+
+  @Test // DRILL-745
+  public void queryingHiveAvroTable() throws Exception {
+    testBuilder()
+        .sqlQuery("SELECT * FROM hive.db1.avro ORDER BY key DESC LIMIT 1")
+        .unOrdered()
+        .baselineColumns("key", "value")
+        .baselineValues(5, " key_5")
+        .go();
+  }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/b46c53eb/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java
b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java
index 0f578ad..b684538 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java
@@ -45,6 +45,7 @@ public class TestInfoSchemaOnHiveStorage extends HiveTestBase {
         .unOrdered()
         .baselineColumns("TABLE_SCHEMA", "TABLE_NAME")
         .baselineValues("hive.db1", "kv_db1")
+        .baselineValues("hive.db1", "avro")
         .go();
   }
 

http://git-wip-us.apache.org/repos/asf/drill/blob/b46c53eb/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
index d47d035..5b60c05 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
@@ -26,10 +26,12 @@ import java.util.Map;
 
 import com.google.common.io.Files;
 import org.apache.commons.io.FileUtils;
+import org.apache.drill.BaseTestQuery;
 import org.apache.drill.common.exceptions.DrillException;
 import org.apache.drill.exec.store.StoragePluginRegistry;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.ql.CommandNeedRetryException;
 import org.apache.hadoop.hive.ql.Driver;
 import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
@@ -118,6 +120,8 @@ public class HiveTestDataGenerator {
     conf.set("javax.jdo.option.ConnectionURL", String.format("jdbc:derby:;databaseName=%s;create=true",
dbDir));
     conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///");
     conf.set("hive.metastore.warehouse.dir", whDir);
+    conf.set("mapred.job.tracker", "local");
+    conf.set("hive.exec.scratchdir",  Files.createTempDir().getAbsolutePath() + File.separator
+ "scratch_dir");
 
     SessionState ss = new SessionState(conf);
     SessionState.start(ss);
@@ -130,6 +134,16 @@ public class HiveTestDataGenerator {
     executeQuery(hiveDriver, "CREATE DATABASE IF NOT EXISTS db1");
     createTableAndLoadData(hiveDriver, "db1", "kv_db1", testDataFile);
 
+    // Create an Avro format based table backed by schema in a separate file
+    final String avroCreateQuery = String.format("CREATE TABLE db1.avro " +
+        "ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' " +
+        "STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat'
" +
+        "OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' " +
+        "TBLPROPERTIES ('avro.schema.url'='file:///%s')", getSchemaFile("avro_test_schema.json"));
+
+    executeQuery(hiveDriver, avroCreateQuery);
+    executeQuery(hiveDriver, "INSERT INTO TABLE db1.avro SELECT * FROM default.kv");
+
     executeQuery(hiveDriver, "USE default");
 
     // create a table with no data
@@ -302,6 +316,15 @@ public class HiveTestDataGenerator {
     return file.getPath();
   }
 
+  private String getSchemaFile(final String resource) throws Exception {
+    final File file = getTempFile();
+    PrintWriter printWriter = new PrintWriter(file);
+    printWriter.write(BaseTestQuery.getFile(resource));
+    printWriter.close();
+
+    return file.getPath();
+  }
+
   private String generateTestDataFileWithDate() throws Exception {
     final File file = getTempFile();
 

http://git-wip-us.apache.org/repos/asf/drill/blob/b46c53eb/contrib/storage-hive/core/src/test/resources/avro_test_schema.json
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/resources/avro_test_schema.json b/contrib/storage-hive/core/src/test/resources/avro_test_schema.json
new file mode 100644
index 0000000..d91d0fd
--- /dev/null
+++ b/contrib/storage-hive/core/src/test/resources/avro_test_schema.json
@@ -0,0 +1,9 @@
+{
+  "namespace": "drill.test",
+  "name": "test_serializer",
+  "type": "record",
+  "fields": [
+    { "name" : "key", "type" : "int" },
+    { "name" : "value", "type" : "string" }
+  ]
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/drill/blob/b46c53eb/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 2d7a9cd..1b14ae1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -41,6 +41,7 @@
       Apache Hive 1.0.0. If the version is changed, make sure the jars and their dependencies
are updated.
     -->
     <hive.version>1.0.0</hive.version>
+    <hadoop.version>2.4.1</hadoop.version>
   </properties>
 
   <scm>
@@ -561,6 +562,26 @@
   <dependencyManagement>
     <dependencies>
       <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-yarn-api</artifactId>
+        <version>${hadoop.version}</version>
+        <scope>test</scope>
+        <exclusions>
+          <exclusion>
+            <groupId>commons-logging</groupId>
+            <artifactId>commons-logging</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+          </exclusion>
+          <exclusion>
+            <groupId>com.google.protobuf</groupId>
+            <artifactId>protobuf-java</artifactId>
+          </exclusion>
+        </exclusions>
+      </dependency>
+      <dependency>
         <groupId>org.apache.hive</groupId>
         <artifactId>hive-exec</artifactId>
         <version>${hive.version}</version>
@@ -755,7 +776,7 @@
           <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>
-            <version>2.4.1</version>
+            <version>${hadoop.version}</version>
             <exclusions>
               <exclusion>
                 <groupId>org.mortbay.jetty</groupId>
@@ -846,7 +867,7 @@
           <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>
-            <version>2.4.1</version>
+            <version>${hadoop.version}</version>
             <scope>test</scope>
             <classifier>tests</classifier>
             <exclusions>
@@ -939,7 +960,7 @@
           <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-client</artifactId>
-            <version>2.4.1</version>
+            <version>${hadoop.version}</version>
             <exclusions>
               <exclusion>
                 <groupId>javax.servlet</groupId>
@@ -1219,7 +1240,7 @@
           <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-hdfs</artifactId>
-            <version>2.4.1</version>
+            <version>${hadoop.version}</version>
             <scope>test</scope>
             <exclusions>
               <exclusion>
@@ -1240,7 +1261,7 @@
           <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-hdfs</artifactId>
-            <version>2.4.1</version>
+            <version>${hadoop.version}</version>
             <scope>test</scope>
             <classifier>tests</classifier>
             <exclusions>
@@ -1271,13 +1292,14 @@
         <alt-hadoop>mapr</alt-hadoop>
         <rat.excludeSubprojects>true</rat.excludeSubprojects>
         <hive.version>1.0.0-mapr-1504</hive.version>
+        <hadoop.version>2.5.1-mapr-1503</hadoop.version>
       </properties>
       <dependencyManagement>
         <dependencies>
           <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>
-            <version>2.5.1-mapr-1503</version>
+            <version>${hadoop.version}</version>
             <exclusions>
               <exclusion>
                 <artifactId>commons-logging</artifactId>
@@ -1369,7 +1391,7 @@
           <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-client</artifactId>
-            <version>2.5.1-mapr-1503</version>
+            <version>${hadoop.version}</version>
             <exclusions>
               <exclusion>
                 <artifactId>commons-logging</artifactId>
@@ -1521,7 +1543,23 @@
           <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-hdfs</artifactId>
-            <version>2.5.1-mapr-1503</version>
+            <version>${hadoop.version}</version>
+            <scope>test</scope>
+            <exclusions>
+              <exclusion>
+                <groupId>commons-logging</groupId>
+                <artifactId>commons-logging</artifactId>
+              </exclusion>
+              <exclusion>
+                <groupId>javax.servlet</groupId>
+                <artifactId>servlet-api</artifactId>
+              </exclusion>
+            </exclusions>
+          </dependency>
+          <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-yarn-api</artifactId>
+            <version>${hadoop.version}</version>
             <scope>test</scope>
             <exclusions>
               <exclusion>
@@ -1529,9 +1567,17 @@
                 <artifactId>commons-logging</artifactId>
               </exclusion>
               <exclusion>
+                <groupId>org.mortbay.jetty</groupId>
+                <artifactId>servlet-api</artifactId>
+              </exclusion>
+              <exclusion>
                 <groupId>javax.servlet</groupId>
                 <artifactId>servlet-api</artifactId>
               </exclusion>
+              <exclusion>
+                <groupId>log4j</groupId>
+                <artifactId>log4j</artifactId>
+              </exclusion>
             </exclusions>
           </dependency>
           <dependency>


Mime
View raw message