Repository: drill
Updated Branches:
refs/heads/master 4e2df00e7 -> d32acdb90
DRILL-745: Add a test case for Hive avro tables.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/b46c53eb
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/b46c53eb
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/b46c53eb
Branch: refs/heads/master
Commit: b46c53eb8e82d19fd5243ff2a9050ca8aae1346b
Parents: ae62407
Author: vkorukanti <venki.korukanti@gmail.com>
Authored: Fri Jun 12 14:18:08 2015 -0700
Committer: vkorukanti <venki.korukanti@gmail.com>
Committed: Wed Jun 17 13:33:15 2015 -0700
----------------------------------------------------------------------
contrib/storage-hive/core/pom.xml | 5 ++
.../apache/drill/exec/hive/TestHiveStorage.java | 10 ++++
.../exec/hive/TestInfoSchemaOnHiveStorage.java | 1 +
.../exec/store/hive/HiveTestDataGenerator.java | 23 ++++++++
.../src/test/resources/avro_test_schema.json | 9 +++
pom.xml | 62 +++++++++++++++++---
6 files changed, 102 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/b46c53eb/contrib/storage-hive/core/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/pom.xml b/contrib/storage-hive/core/pom.xml
index 7bf644a..fb96a0d 100644
--- a/contrib/storage-hive/core/pom.xml
+++ b/contrib/storage-hive/core/pom.xml
@@ -78,6 +78,11 @@
<classifier>tests</classifier>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/drill/blob/b46c53eb/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
index 27ba9fe..2898f91 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
@@ -176,4 +176,14 @@ public class TestHiveStorage extends HiveTestBase {
.baselineValues(1, " key_1")
.go();
}
+
+ @Test // DRILL-745
+ public void queryingHiveAvroTable() throws Exception {
+ testBuilder()
+ .sqlQuery("SELECT * FROM hive.db1.avro ORDER BY key DESC LIMIT 1")
+ .unOrdered()
+ .baselineColumns("key", "value")
+ .baselineValues(5, " key_5")
+ .go();
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/b46c53eb/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java
b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java
index 0f578ad..b684538 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java
@@ -45,6 +45,7 @@ public class TestInfoSchemaOnHiveStorage extends HiveTestBase {
.unOrdered()
.baselineColumns("TABLE_SCHEMA", "TABLE_NAME")
.baselineValues("hive.db1", "kv_db1")
+ .baselineValues("hive.db1", "avro")
.go();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/b46c53eb/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
index d47d035..5b60c05 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
@@ -26,10 +26,12 @@ import java.util.Map;
import com.google.common.io.Files;
import org.apache.commons.io.FileUtils;
+import org.apache.drill.BaseTestQuery;
import org.apache.drill.common.exceptions.DrillException;
import org.apache.drill.exec.store.StoragePluginRegistry;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.CommandNeedRetryException;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
@@ -118,6 +120,8 @@ public class HiveTestDataGenerator {
conf.set("javax.jdo.option.ConnectionURL", String.format("jdbc:derby:;databaseName=%s;create=true",
dbDir));
conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///");
conf.set("hive.metastore.warehouse.dir", whDir);
+ conf.set("mapred.job.tracker", "local");
+ conf.set("hive.exec.scratchdir", Files.createTempDir().getAbsolutePath() + File.separator
+ "scratch_dir");
SessionState ss = new SessionState(conf);
SessionState.start(ss);
@@ -130,6 +134,16 @@ public class HiveTestDataGenerator {
executeQuery(hiveDriver, "CREATE DATABASE IF NOT EXISTS db1");
createTableAndLoadData(hiveDriver, "db1", "kv_db1", testDataFile);
+ // Create an Avro format based table backed by schema in a separate file
+ final String avroCreateQuery = String.format("CREATE TABLE db1.avro " +
+ "ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' " +
+ "STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat'
" +
+ "OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' " +
+ "TBLPROPERTIES ('avro.schema.url'='file:///%s')", getSchemaFile("avro_test_schema.json"));
+
+ executeQuery(hiveDriver, avroCreateQuery);
+ executeQuery(hiveDriver, "INSERT INTO TABLE db1.avro SELECT * FROM default.kv");
+
executeQuery(hiveDriver, "USE default");
// create a table with no data
@@ -302,6 +316,15 @@ public class HiveTestDataGenerator {
return file.getPath();
}
+ private String getSchemaFile(final String resource) throws Exception {
+ final File file = getTempFile();
+ PrintWriter printWriter = new PrintWriter(file);
+ printWriter.write(BaseTestQuery.getFile(resource));
+ printWriter.close();
+
+ return file.getPath();
+ }
+
private String generateTestDataFileWithDate() throws Exception {
final File file = getTempFile();
http://git-wip-us.apache.org/repos/asf/drill/blob/b46c53eb/contrib/storage-hive/core/src/test/resources/avro_test_schema.json
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/resources/avro_test_schema.json b/contrib/storage-hive/core/src/test/resources/avro_test_schema.json
new file mode 100644
index 0000000..d91d0fd
--- /dev/null
+++ b/contrib/storage-hive/core/src/test/resources/avro_test_schema.json
@@ -0,0 +1,9 @@
+{
+ "namespace": "drill.test",
+ "name": "test_serializer",
+ "type": "record",
+ "fields": [
+ { "name" : "key", "type" : "int" },
+ { "name" : "value", "type" : "string" }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/b46c53eb/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 2d7a9cd..1b14ae1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -41,6 +41,7 @@
Apache Hive 1.0.0. If the version is changed, make sure the jars and their dependencies
are updated.
-->
<hive.version>1.0.0</hive.version>
+ <hadoop.version>2.4.1</hadoop.version>
</properties>
<scm>
@@ -561,6 +562,26 @@
<dependencyManagement>
<dependencies>
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-exec</artifactId>
<version>${hive.version}</version>
@@ -755,7 +776,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <version>2.4.1</version>
+ <version>${hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>org.mortbay.jetty</groupId>
@@ -846,7 +867,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <version>2.4.1</version>
+ <version>${hadoop.version}</version>
<scope>test</scope>
<classifier>tests</classifier>
<exclusions>
@@ -939,7 +960,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
- <version>2.4.1</version>
+ <version>${hadoop.version}</version>
<exclusions>
<exclusion>
<groupId>javax.servlet</groupId>
@@ -1219,7 +1240,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
- <version>2.4.1</version>
+ <version>${hadoop.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
@@ -1240,7 +1261,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
- <version>2.4.1</version>
+ <version>${hadoop.version}</version>
<scope>test</scope>
<classifier>tests</classifier>
<exclusions>
@@ -1271,13 +1292,14 @@
<alt-hadoop>mapr</alt-hadoop>
<rat.excludeSubprojects>true</rat.excludeSubprojects>
<hive.version>1.0.0-mapr-1504</hive.version>
+ <hadoop.version>2.5.1-mapr-1503</hadoop.version>
</properties>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <version>2.5.1-mapr-1503</version>
+ <version>${hadoop.version}</version>
<exclusions>
<exclusion>
<artifactId>commons-logging</artifactId>
@@ -1369,7 +1391,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
- <version>2.5.1-mapr-1503</version>
+ <version>${hadoop.version}</version>
<exclusions>
<exclusion>
<artifactId>commons-logging</artifactId>
@@ -1521,7 +1543,23 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
- <version>2.5.1-mapr-1503</version>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <version>${hadoop.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
@@ -1529,9 +1567,17 @@
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
|