DRILL-1192: Hive Scalar UDFs: Add Date, TimeStamp and Decimal type support
Also following refactoring:
+ Minimize the number of variables in HiveTypes.tdd
+ Make use of Hive TypeEntries and Hive AbstractPrimitiveObjectInspector to simplify Drill ObjectInspectors implementations.
Test:
+ Add Hive UDF test implementations and testcases to cover all supported types (passing data into Hive UDF and reading data returned from Hive UDF).
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/1e9930fb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/1e9930fb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/1e9930fb
Branch: refs/heads/master
Commit: 1e9930fbae8279afe3eed9e7f18392fb1a08688a
Parents: 9b2eae2
Author: vkorukanti <venki.korukanti@gmail.com>
Authored: Wed Jul 23 09:33:15 2014 -0700
Committer: Jacques Nadeau <jacques@apache.org>
Committed: Fri Jul 25 14:34:36 2014 -0700
----------------------------------------------------------------------
contrib/storage-hive/core/pom.xml | 8 +
.../core/src/main/codegen/data/HiveTypes.tdd | 58 +++--
.../templates/ObjectInspectorHelper.java | 45 +++-
.../codegen/templates/ObjectInspectors.java | 223 +++++++++++++----
.../drill/exec/expr/fn/HiveFuncHolder.java | 6 +-
.../exec/expr/fn/HiveFunctionRegistry.java | 11 +-
.../AbstractDrillPrimitiveObjectInspector.java | 39 +++
.../hive/AbstractPrimitiveObjectInspector.java | 70 ------
.../drill/exec/TestHiveProjectPushDown.java | 8 +-
.../drill/exec/fn/hive/HiveTestUDFImpls.java | 237 +++++++++++++++++++
.../drill/exec/fn/hive/TestSampleHiveUDFs.java | 154 ++++++++++++
.../exec/store/hive/HiveTestDataGenerator.java | 1 +
.../core/src/test/resources/drill-module.conf | 5 +
.../org/apache/drill/exec/util/VectorUtil.java | 35 +++
.../java/org/apache/drill/BaseTestQuery.java | 19 ++
15 files changed, 751 insertions(+), 168 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1e9930fb/contrib/storage-hive/core/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/pom.xml b/contrib/storage-hive/core/pom.xml
index 38e4a41..9d54444 100644
--- a/contrib/storage-hive/core/pom.xml
+++ b/contrib/storage-hive/core/pom.xml
@@ -123,6 +123,14 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <forkCount>1</forkCount>
+ <reuseForks>false</reuseForks>
+ </configuration>
+ </plugin>
</plugins>
</build>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1e9930fb/contrib/storage-hive/core/src/main/codegen/data/HiveTypes.tdd
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/codegen/data/HiveTypes.tdd b/contrib/storage-hive/core/src/main/codegen/data/HiveTypes.tdd
index c23f981..2b4338b 100644
--- a/contrib/storage-hive/core/src/main/codegen/data/HiveTypes.tdd
+++ b/contrib/storage-hive/core/src/main/codegen/data/HiveTypes.tdd
@@ -19,82 +19,80 @@
{
hiveType: "BOOLEAN",
hiveOI: "BooleanObjectInspector",
- serdeConstant: "BOOLEAN_TYPE_NAME",
javaType: "boolean",
- minorType: "BIT",
- holder: "Bit"
+ drillType: "Bit"
},
{
hiveType: "BYTE",
hiveOI: "ByteObjectInspector",
- serdeConstant: "TINYINT_TYPE_NAME",
javaType: "byte",
- minorType: "TINYINT",
- holder: "TinyInt"
+ drillType: "TinyInt"
},
{
hiveType: "SHORT",
hiveOI: "ShortObjectInspector",
- serdeConstant: "SMALLINT_TYPE_NAME",
javaType: "short",
- minorType: "SMALLINT",
- holder: "SmallInt"
+ drillType: "SmallInt"
},
{
hiveType: "INT",
hiveOI: "IntObjectInspector",
- serdeConstant: "INT_TYPE_NAME",
javaType: "int",
- minorType: "INT",
- holder: "Int"
+ drillType: "Int"
},
{
hiveType: "LONG",
hiveOI: "LongObjectInspector",
- serdeConstant: "BIGINT_TYPE_NAME",
javaType: "long",
- minorType: "BIGINT",
- holder: "BigInt"
+ drillType: "BigInt"
},
{
hiveType: "FLOAT",
hiveOI: "FloatObjectInspector",
- serdeConstant: "FLOAT_TYPE_NAME",
javaType: "float",
- minorType: "FLOAT4",
- holder: "Float4"
+ drillType: "Float4"
},
{
hiveType: "DOUBLE",
hiveOI: "DoubleObjectInspector",
- serdeConstant: "DOUBLE_TYPE_NAME",
javaType: "double",
- minorType: "FLOAT8",
- holder: "Float8"
+ drillType: "Float8"
},
{
hiveType: "VARCHAR",
hiveOI: "HiveVarcharObjectInspector",
- serdeConstant: "VARCHAR_TYPE_NAME",
javaType: "",
- minorType: "VARCHAR",
- holder: "VarChar"
+ drillType: "VarChar"
},
{
hiveType: "STRING",
hiveOI: "StringObjectInspector",
- serdeConstant: "STRING_TYPE_NAME",
javaType: "",
- minorType: "VAR16CHAR",
- holder: "Var16Char"
+ drillType: "Var16Char"
},
{
hiveType: "BINARY",
hiveOI: "BinaryObjectInspector",
- serdeConstant: "BINARY_TYPE_NAME",
javaType: "",
- minorType: "VARBINARY",
- holder: "VarBinary"
+ drillType: "VarBinary"
+ },
+ {
+ hiveType: "TIMESTAMP",
+ hiveOI: "TimestampObjectInspector",
+ javaType: "java.sql.Timestamp",
+ drillType: "TimeStamp"
+ },
+ {
+ hiveType: "DECIMAL",
+ hiveOI: "HiveDecimalObjectInspector",
+ javaType: "org.apache.hadoop.hive.common.type.HiveDecimal",
+ drillType: "Decimal38Sparse"
+ },
+ {
+ hiveType: "DATE",
+ hiveOI: "DateObjectInspector",
+ javaType: "java.sql.Date",
+ drillType: "Date"
}
]
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1e9930fb/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectorHelper.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectorHelper.java b/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectorHelper.java
index 22a9eb2..091f027 100644
--- a/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectorHelper.java
+++ b/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectorHelper.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
<@pp.dropOutputFile />
<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/hive/ObjectInspectorHelper.java" />
@@ -25,38 +26,51 @@ package org.apache.drill.exec.expr.fn.impl.hive;
import com.sun.codemodel.*;
import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MinorType;
import org.apache.drill.exec.expr.DirectExpression;
import org.apache.drill.exec.expr.TypeHelper;
-import org.apache.drill.exec.expr.holders.*;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.*;
+import java.lang.UnsupportedOperationException;
+import java.sql.Timestamp;
import java.util.HashMap;
import java.util.Map;
public class ObjectInspectorHelper {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ObjectInspectorHelper.class);
- private static Map<MinorType, Class> OIMAP = new HashMap<>();
+ private static Map<MinorType, Class> OIMAP_REQUIRED = new HashMap<>();
+ private static Map<MinorType, Class> OIMAP_OPTIONAL = new HashMap<>();
static {
<#list drillOI.map as entry>
- OIMAP.put(MinorType.${entry.minorType}, Drill${entry.holder}ObjectInspector.class);
+ OIMAP_REQUIRED.put(MinorType.${entry.drillType?upper_case}, Drill${entry.drillType}ObjectInspector.Required.class);
+ OIMAP_OPTIONAL.put(MinorType.${entry.drillType?upper_case}, Drill${entry.drillType}ObjectInspector.Optional.class);
</#list>
}
- public static ObjectInspector getDrillObjectInspector(MinorType drillType) {
- if (OIMAP.containsKey(drillType)) {
- try {
- return (ObjectInspector)OIMAP.get(drillType).newInstance();
- } catch(InstantiationException | IllegalAccessException e) {
- throw new RuntimeException("Failed to instantiate ObjectInspector", e);
+ public static ObjectInspector getDrillObjectInspector(DataMode mode, MinorType minorType) {
+ try {
+ if (mode == DataMode.REQUIRED) {
+ if (OIMAP_REQUIRED.containsKey(minorType)) {
+ return (ObjectInspector) OIMAP_REQUIRED.get(minorType).newInstance();
+ }
+ } else if (mode == DataMode.OPTIONAL) {
+ if (OIMAP_OPTIONAL.containsKey(minorType)) {
+ return (ObjectInspector) OIMAP_OPTIONAL.get(minorType).newInstance();
+ }
+ } else {
+ throw new UnsupportedOperationException("Repeated types are not supported as arguement to Hive UDFs");
}
+ } catch(InstantiationException | IllegalAccessException e) {
+ throw new RuntimeException("Failed to instantiate ObjectInspector", e);
}
- throw new UnsupportedOperationException(drillType.toString());
+ throw new UnsupportedOperationException(
+ String.format("Type %s[%s] not supported as arguement to Hive UDFs", minorType.toString(), mode.toString()));
}
public static JBlock initReturnValueHolder(JCodeModel m, JVar returnValueHolder, ObjectInspector oi, MinorType returnType) {
@@ -95,7 +109,7 @@ public class ObjectInspectorHelper {
private static Map<PrimitiveCategory, MinorType> TYPE_HIVE2DRILL = new HashMap<>();
static {
<#list drillOI.map as entry>
- TYPE_HIVE2DRILL.put(PrimitiveCategory.${entry.hiveType}, MinorType.${entry.minorType});
+ TYPE_HIVE2DRILL.put(PrimitiveCategory.${entry.hiveType}, MinorType.${entry.drillType?upper_case});
</#list>
}
@@ -168,7 +182,14 @@ public class ObjectInspectorHelper {
.invoke("setBytes").arg(JExpr.lit(0)).arg(data));
jc._else().assign(returnValueHolder.ref("start"), JExpr.lit(0));
jc._else().assign(returnValueHolder.ref("end"), data.ref("length"));
-
+ <#elseif entry.hiveType == "TIMESTAMP">
+ JVar tsVar = jc._else().decl(m.directClass(java.sql.Timestamp.class.getCanonicalName()), "ts",
+ castedOI.invoke("getPrimitiveJavaObject").arg(returnValue));
+ jc._else().assign(returnValueHolder.ref("value"), tsVar.invoke("getTime"));
+ <#elseif entry.hiveType == "DATE">
+ JVar dVar = jc._else().decl(m.directClass(java.sql.Date.class.getCanonicalName()), "d",
+ castedOI.invoke("getPrimitiveJavaObject").arg(returnValue));
+ jc._else().assign(returnValueHolder.ref("value"), dVar.invoke("getTime"));
<#else>
jc._else().assign(returnValueHolder.ref("value"),
castedOI.invoke("get").arg(returnValue));
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1e9930fb/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java b/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java
index 9a8c837..2267ad5 100644
--- a/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java
+++ b/contrib/storage-hive/core/src/main/codegen/templates/ObjectInspectors.java
@@ -18,28 +18,40 @@
<@pp.dropOutputFile />
<#list drillOI.map as entry>
-<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/hive/Drill${entry.holder}ObjectInspector.java" />
+<@pp.changeOutputFile name="/org/apache/drill/exec/expr/fn/impl/hive/Drill${entry.drillType}ObjectInspector.java" />
<#include "/@includes/license.ftl" />
package org.apache.drill.exec.expr.fn.impl.hive;
+import org.apache.drill.common.util.DecimalUtility;
import org.apache.drill.exec.expr.holders.*;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.common.type.HiveVarchar;
import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.io.ByteWritable;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
+import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable;
+import org.apache.hadoop.hive.serde2.io.ShortWritable;
+import org.apache.hadoop.hive.serde2.io.TimestampWritable;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.*;
+import org.apache.hadoop.io.BooleanWritable;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
-public class Drill${entry.holder}ObjectInspector extends AbstractPrimitiveObjectInspector
+public abstract class Drill${entry.drillType}ObjectInspector extends AbstractDrillPrimitiveObjectInspector
implements ${entry.hiveOI} {
- @Override
- public String getTypeName() {
- return serdeConstants.${entry.serdeConstant};
+ public Drill${entry.drillType}ObjectInspector() {
+ super(PrimitiveObjectInspectorUtils.${entry.hiveType?lower_case}TypeEntry);
}
-<#if entry.minorType == "VARCHAR">
+<#if entry.drillType == "VarChar">
@Override
public HiveVarcharWritable getPrimitiveWritableObject(Object o) {
HiveVarcharWritable valW = new HiveVarcharWritable();
@@ -47,67 +59,182 @@ public class Drill${entry.holder}ObjectInspector extends AbstractPrimitiveObject
return valW;
}
- @Override
- public HiveVarchar getPrimitiveJavaObject(Object o) {
- String val = ((VarCharHolder)o).toString();
- return new HiveVarchar(val, HiveVarchar.MAX_VARCHAR_LENGTH);
+ public static class Required extends Drill${entry.drillType}ObjectInspector {
+ @Override
+ public HiveVarchar getPrimitiveJavaObject(Object o) {
+ return new HiveVarchar(((VarCharHolder)o).toString(), HiveVarchar.MAX_VARCHAR_LENGTH);
+ }
+ }
+
+ public static class Optional extends Drill${entry.drillType}ObjectInspector {
+ @Override
+ public HiveVarchar getPrimitiveJavaObject(Object o) {
+ return new HiveVarchar(((NullableVarCharHolder)o).toString(), HiveVarchar.MAX_VARCHAR_LENGTH);
+ }
}
-<#elseif entry.minorType == "VAR16CHAR">
+
+<#elseif entry.drillType == "Var16Char">
@Override
public Text getPrimitiveWritableObject(Object o) {
throw new UnsupportedOperationException();
}
- @Override
- public String getPrimitiveJavaObject(Object o) {
- if (o instanceof Var16CharHolder)
- return ((Var16CharHolder)o).toString();
- else
- return ((NullableVar16CharHolder)o).toString();
+ public static class Required extends Drill${entry.drillType}ObjectInspector {
+ @Override
+ public String getPrimitiveJavaObject(Object o){
+ return((Var16CharHolder)o).toString();
+ }
}
-<#elseif entry.minorType == "VARBINARY">
-@Override
-public org.apache.hadoop.io.BytesWritable getPrimitiveWritableObject(Object o) {
- throw new UnsupportedOperationException();
-}
-@Override
-public byte[] getPrimitiveJavaObject(Object o) {
- if (o instanceof VarBinaryHolder){
- VarBinaryHolder h = (VarBinaryHolder)o;
- byte[] buf = new byte[h.end-h.start];
- h.buffer.getBytes(h.start, buf, 0, h.end-h.start);
- return buf;
- }else{
- NullableVarBinaryHolder h = (NullableVarBinaryHolder)o;
- byte[] buf = new byte[h.end-h.start];
- h.buffer.getBytes(h.start, buf, 0, h.end-h.start);
- return buf;
-
+ public static class Optional extends Drill${entry.drillType}ObjectInspector {
+ @Override
+ public String getPrimitiveJavaObject(Object o){
+ return((NullableVar16CharHolder)o).toString();
+ }
}
-}
-<#elseif entry.minorType == "BIT">
+<#elseif entry.drillType == "VarBinary">
@Override
- public boolean get(Object o) {
- if (o instanceof BitHolder)
- return ((BitHolder)o).value == 0 ? false : true;
- else
+ public BytesWritable getPrimitiveWritableObject(Object o) {
+ return new BytesWritable(getPrimitiveJavaObject(o));
+ }
+
+ public static class Required extends Drill${entry.drillType}ObjectInspector {
+ @Override
+ public byte[] getPrimitiveJavaObject(Object o) {
+ VarBinaryHolder h = (VarBinaryHolder)o;
+ byte[] buf = new byte[h.end-h.start];
+ h.buffer.getBytes(h.start, buf, 0, h.end-h.start);
+ return buf;
+ }
+ }
+
+ public static class Optional extends Drill${entry.drillType}ObjectInspector {
+ @Override
+ public byte[] getPrimitiveJavaObject(Object o) {
+ NullableVarBinaryHolder h = (NullableVarBinaryHolder)o;
+ byte[] buf = new byte[h.end-h.start];
+ h.buffer.getBytes(h.start, buf, 0, h.end-h.start);
+ return buf;
+ }
+ }
+
+<#elseif entry.drillType == "Bit">
+ public static class Required extends Drill${entry.drillType}ObjectInspector {
+ @Override
+ public boolean get(Object o) {
+ return ((BitHolder)o).value == 0 ? false : true;
+ }
+ }
+
+ public static class Optional extends Drill${entry.drillType}ObjectInspector {
+ @Override
+ public boolean get(Object o) {
return ((NullableBitHolder)o).value == 0 ? false : true;
+ }
+ }
+
+ @Override
+ public BooleanWritable getPrimitiveWritableObject(Object o) {
+ return new BooleanWritable(get(o));
+ }
+
+ @Override
+ public Boolean getPrimitiveJavaObject(Object o) {
+ return new Boolean(get(o));
+ }
+
+<#elseif entry.drillType == "Decimal38Sparse">
+ public HiveDecimalWritable getPrimitiveWritableObject(Object o) {
+ return new HiveDecimalWritable(getPrimitiveJavaObject(o));
+ }
+
+ public static class Required extends Drill${entry.drillType}ObjectInspector{
+ @Override
+ public HiveDecimal getPrimitiveJavaObject(Object o){
+ Decimal38SparseHolder h = (Decimal38SparseHolder) o;
+ return new HiveDecimal(DecimalUtility.getBigDecimalFromSparse(h.buffer, h.start, h.nDecimalDigits, h.scale));
+ }
+ }
+
+ public static class Optional extends Drill${entry.drillType}ObjectInspector{
+ @Override
+ public HiveDecimal getPrimitiveJavaObject(Object o){
+ NullableDecimal38SparseHolder h = (NullableDecimal38SparseHolder) o;
+ return new HiveDecimal(DecimalUtility.getBigDecimalFromSparse(h.buffer, h.start, h.nDecimalDigits, h.scale));
+ }
+ }
+
+<#elseif entry.drillType == "TimeStamp">
+ @Override
+ public TimestampWritable getPrimitiveWritableObject(Object o) {
+ return new TimestampWritable(getPrimitiveJavaObject(o));
+ }
+
+ public static class Required extends Drill${entry.drillType}ObjectInspector{
+ @Override
+ public java.sql.Timestamp getPrimitiveJavaObject(Object o){
+ return new java.sql.Timestamp(((TimeStampHolder)o).value);
+ }
+ }
+
+ public static class Optional extends Drill${entry.drillType}ObjectInspector{
+ @Override
+ public java.sql.Timestamp getPrimitiveJavaObject(Object o){
+ return new java.sql.Timestamp(((NullableTimeStampHolder)o).value);
+ }
+ }
+
+<#elseif entry.drillType == "Date">
+ @Override
+ public DateWritable getPrimitiveWritableObject(Object o) {
+ return new DateWritable(getPrimitiveJavaObject(o));
+ }
+
+ public static class Required extends Drill${entry.drillType}ObjectInspector{
+ @Override
+ public java.sql.Date getPrimitiveJavaObject(Object o){
+ return new java.sql.Date(((DateHolder)o).value);
+ }
+ }
+
+ public static class Optional extends Drill${entry.drillType}ObjectInspector{
+ @Override
+ public java.sql.Date getPrimitiveJavaObject(Object o){
+ return new java.sql.Date(((NullableDateHolder)o).value);
+ }
}
<#else>
+<#if entry.drillType == "Int">
@Override
- public ${entry.javaType} get(Object o) {
- if (o instanceof ${entry.holder}Holder)
- return ((${entry.holder}Holder)o).value;
- else
- return ((Nullable${entry.holder}Holder)o).value;
+ public Integer getPrimitiveJavaObject(Object o) {
+ return new Integer(get(o));
+ }
+<#else>
+ @Override
+ public ${entry.javaType?cap_first} getPrimitiveJavaObject(Object o) {
+ return new ${entry.javaType?cap_first}(get(o));
}
</#if>
@Override
- public PrimitiveCategory getPrimitiveCategory() {
- return PrimitiveCategory.${entry.hiveType};
+ public ${entry.javaType?cap_first}Writable getPrimitiveWritableObject(Object o) {
+ return new ${entry.javaType?cap_first}Writable(get(o));
+ }
+
+ public static class Required extends Drill${entry.drillType}ObjectInspector{
+ @Override
+ public ${entry.javaType} get(Object o){
+ return((${entry.drillType}Holder)o).value;
+ }
}
+
+ public static class Optional extends Drill${entry.drillType}ObjectInspector{
+ @Override
+ public ${entry.javaType} get(Object o){
+ return((Nullable${entry.drillType}Holder)o).value;
+ }
+ }
+</#if>
}
</#list>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1e9930fb/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java
index 813d4c5..68a9b97 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFuncHolder.java
@@ -30,6 +30,7 @@ import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.FunctionHolderExpression;
import org.apache.drill.common.expression.LogicalExpression;
import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MajorType;
import org.apache.drill.exec.expr.ClassGenerator;
import org.apache.drill.exec.expr.ClassGenerator.HoldingContainer;
@@ -170,12 +171,13 @@ public class HiveFuncHolder extends AbstractFuncHolder {
JClass oih = m.directClass(ObjectInspectorHelper.class.getCanonicalName());
JClass mt = m.directClass(TypeProtos.MinorType.class.getCanonicalName());
+ JClass mode = m.directClass(DataMode.class.getCanonicalName());
for(int i=0; i<argTypes.length; i++) {
sub.assign(
oiArray.component(JExpr.lit(i)),
oih.staticInvoke("getDrillObjectInspector")
- .arg(mt.staticInvoke("valueOf")
- .arg(JExpr.lit(argTypes[i].getMinorType().getNumber()))));
+ .arg(mode.staticInvoke("valueOf").arg(JExpr.lit(argTypes[i].getMode().getNumber())))
+ .arg(mt.staticInvoke("valueOf").arg(JExpr.lit(argTypes[i].getMinorType().getNumber()))));
}
// declare and instantiate DeferredObject array
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1e9930fb/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java
index b9369ed..119c93b 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/HiveFunctionRegistry.java
@@ -99,8 +99,15 @@ public class HiveFunctionRegistry implements PluggableFunctionRegistry{
MajorType[] argTypes = new MajorType[call.args.size()];
ObjectInspector[] argOIs = new ObjectInspector[call.args.size()];
for(int i=0; i<call.args.size(); i++) {
- argTypes[i] = call.args.get(i).getMajorType();
- argOIs[i] = ObjectInspectorHelper.getDrillObjectInspector(argTypes[i].getMinorType());
+ try {
+ argTypes[i] = call.args.get(i).getMajorType();
+ argOIs[i] = ObjectInspectorHelper.getDrillObjectInspector(argTypes[i].getMode(), argTypes[i].getMinorType());
+ } catch(Exception e) {
+ // Hive throws errors if there are unsupported types. Consider there is no hive UDF supporting the
+ // given argument types
+ logger.trace("Failed to find a hive function for given FunctionCall: '{}'", call.toString(), e);
+ return null;
+ }
}
String funcName = call.getName().toLowerCase();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1e9930fb/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/AbstractDrillPrimitiveObjectInspector.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/AbstractDrillPrimitiveObjectInspector.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/AbstractDrillPrimitiveObjectInspector.java
new file mode 100644
index 0000000..2a98398
--- /dev/null
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/AbstractDrillPrimitiveObjectInspector.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr.fn.impl.hive;
+
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry;
+
+
+public abstract class AbstractDrillPrimitiveObjectInspector extends AbstractPrimitiveObjectInspector {
+
+ public AbstractDrillPrimitiveObjectInspector(PrimitiveTypeEntry typeEntry) {
+ super(typeEntry);
+ }
+
+ @Override
+ public Object copyObject(Object o) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean preferWritable() {
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1e9930fb/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/AbstractPrimitiveObjectInspector.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/AbstractPrimitiveObjectInspector.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/AbstractPrimitiveObjectInspector.java
deleted file mode 100644
index 04b552e..0000000
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/expr/fn/impl/hive/AbstractPrimitiveObjectInspector.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.expr.fn.impl.hive;
-
-import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
-import org.apache.hadoop.hive.serde2.typeinfo.BaseTypeParams;
-
-
-public abstract class AbstractPrimitiveObjectInspector implements PrimitiveObjectInspector {
-
- @Override
- public Class<?> getPrimitiveWritableClass() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Object getPrimitiveWritableObject(Object o) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Class<?> getJavaPrimitiveClass() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Object getPrimitiveJavaObject(Object o) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Object copyObject(Object o) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public boolean preferWritable() {
- return false;
- }
-
- @Override
- public BaseTypeParams getTypeParams() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void setTypeParams(BaseTypeParams baseTypeParams) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Category getCategory() {
- return Category.PRIMITIVE;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1e9930fb/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHiveProjectPushDown.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHiveProjectPushDown.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHiveProjectPushDown.java
index 6ced636..a985cae 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHiveProjectPushDown.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/TestHiveProjectPushDown.java
@@ -52,7 +52,7 @@ public class TestHiveProjectPushDown extends PlanTestBase {
String query = "SELECT boolean_field as b_f, tinyint_field as ti_f FROM hive.`default`.readtest";
String expectedColNames = " \"columns\" : [ \"`boolean_field`\", \"`tinyint_field`\" ]";
- testHelper(query, expectedColNames, 1);
+ testHelper(query, expectedColNames, 2);
}
@Test
@@ -60,7 +60,7 @@ public class TestHiveProjectPushDown extends PlanTestBase {
String query = "SELECT double_part as dbl_p FROM hive.`default`.readtest";
String expectedColNames = " \"columns\" : [ \"`double_part`\" ]";
- testHelper(query, expectedColNames, 1);
+ testHelper(query, expectedColNames, 2);
}
@Test
@@ -68,7 +68,7 @@ public class TestHiveProjectPushDown extends PlanTestBase {
String query = "SELECT double_part as dbl_p, decimal_part as dec_p FROM hive.`default`.readtest";
String expectedColNames = " \"columns\" : [ \"`double_part`\", \"`decimal_part`\" ]";
- testHelper(query, expectedColNames, 1);
+ testHelper(query, expectedColNames, 2);
}
@Test
@@ -78,7 +78,7 @@ public class TestHiveProjectPushDown extends PlanTestBase {
String expectedColNames = " \"columns\" : [ \"`boolean_field`\", \"`tinyint_field`\", " +
"\"`double_part`\", \"`decimal_part`\" ]";
- testHelper(query, expectedColNames, 1);
+ testHelper(query, expectedColNames, 2);
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1e9930fb/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/HiveTestUDFImpls.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/HiveTestUDFImpls.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/HiveTestUDFImpls.java
new file mode 100644
index 0000000..8fc059b
--- /dev/null
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/HiveTestUDFImpls.java
@@ -0,0 +1,237 @@
+
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.fn.hive;
+
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveVarcharObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector;
+
+/**
+ * Contains test Hive UDF that take a particular data type and return the same data type and value.
+ * These test UDFs exercise the code paths in Drill ObjectInspectors and parsing the return value from Hive UDF
+ * for all supported data types in Hive UDF.
+ */
+public class HiveTestUDFImpls {
+ public static abstract class GenericUDFTestBase extends GenericUDF {
+ protected final String udfName;
+ protected final PrimitiveCategory inputType;
+ protected final PrimitiveCategory outputType;
+ protected ObjectInspector argumentOI;
+
+ public GenericUDFTestBase(String udfName, PrimitiveCategory type) {
+ this(udfName, type, type);
+ }
+
+ public GenericUDFTestBase(String udfName, PrimitiveCategory inputType, PrimitiveCategory outputType) {
+ this.udfName = udfName;
+ this.inputType = inputType;
+ this.outputType = outputType;
+ }
+
+ @Override
+ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
+ if (arguments.length != 1) {
+ throw new UDFArgumentLengthException(String.format("%s needs 1 argument, got %d", udfName, arguments.length));
+ }
+
+ if (arguments[0].getCategory() != Category.PRIMITIVE ||
+ ((PrimitiveObjectInspector) arguments[0]).getPrimitiveCategory() != inputType) {
+ String actual = arguments[0].getCategory() + (arguments[0].getCategory() == Category.PRIMITIVE ?
+ "[" + ((PrimitiveObjectInspector) arguments[0]).getPrimitiveCategory() + "]" : "");
+ throw new UDFArgumentException(
+ String.format("%s only takes primitive type %s, got %s", udfName, inputType, actual));
+ }
+ argumentOI = arguments[0];
+ return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(outputType);
+ }
+
+ @Override
+ public String getDisplayString(String[] children) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(udfName);
+ sb.append("(");
+ if (children.length > 0) {
+ sb.append(children[0]);
+ for (int i = 1; i < children.length; i++) {
+ sb.append(",");
+ sb.append(children[i]);
+ }
+ }
+ sb.append(")");
+ return sb.toString();
+ }
+
+ @Override
+ public Object evaluate(DeferredObject[] arguments) throws HiveException {
+ if (arguments[0] == null) {
+ return null;
+ }
+
+ Object input = arguments[0].get();
+ switch(inputType) {
+ case BOOLEAN:
+ return ((BooleanObjectInspector)argumentOI).get(input) ? Boolean.TRUE : Boolean.FALSE;
+ case BYTE:
+ return new Byte(((ByteObjectInspector)argumentOI).get(input));
+ case SHORT:
+ return new Short(((ShortObjectInspector)argumentOI).get(input));
+ case INT:
+ return new Integer(((IntObjectInspector)argumentOI).get(input));
+ case LONG:
+ return new Long(((LongObjectInspector)argumentOI).get(input));
+ case FLOAT:
+ return new Float(((FloatObjectInspector)argumentOI).get(input));
+ case DOUBLE:
+ return new Double(((DoubleObjectInspector)argumentOI).get(input));
+ case STRING:
+ return PrimitiveObjectInspectorUtils.getString(input, (StringObjectInspector)argumentOI);
+ case BINARY:
+ return PrimitiveObjectInspectorUtils.getBinary(input, (BinaryObjectInspector) argumentOI).getBytes();
+ case VARCHAR:
+ return PrimitiveObjectInspectorUtils.getHiveVarchar(input, (HiveVarcharObjectInspector)argumentOI);
+ case DATE:
+ return PrimitiveObjectInspectorUtils.getDate(input, (DateObjectInspector) argumentOI);
+ case TIMESTAMP:
+ return PrimitiveObjectInspectorUtils.getTimestamp(input, (TimestampObjectInspector) argumentOI);
+ case DECIMAL:
+ // return type is a HiveVarchar
+ HiveDecimal decimalValue =
+ PrimitiveObjectInspectorUtils.getHiveDecimal(input, (HiveDecimalObjectInspector) argumentOI);
+ return new HiveVarchar(decimalValue.toString(), HiveVarchar.MAX_VARCHAR_LENGTH);
+ }
+
+ throw new UnsupportedOperationException(String.format("Unexpected input type '%s' in Test UDF", inputType));
+ }
+ }
+
+ @Description(name = "testHiveUDFBOOLEAN", value = "_FUNC_(BOOLEAN) - Tests boolean data as input and output")
+ public static class GenericUDFTestBOOLEAN extends GenericUDFTestBase {
+ public GenericUDFTestBOOLEAN() {
+ super("testHiveUDFBOOLEAN", PrimitiveCategory.BOOLEAN);
+ }
+ }
+
+ @Description(name = "testHiveUDFBYTE", value = "_FUNC_(BYTE) - Tests byte data as input and output")
+ public static class GenericUDFTestBYTE extends GenericUDFTestBase {
+ public GenericUDFTestBYTE() {
+ super("testHiveUDFBYTE", PrimitiveCategory.BYTE);
+ }
+ }
+
+ @Description(name = "testHiveUDFSHORT", value = "_FUNC_(SHORT) - Tests short data as input and output")
+ public static class GenericUDFTestSHORT extends GenericUDFTestBase {
+ public GenericUDFTestSHORT() {
+ super("testHiveUDFSHORT", PrimitiveCategory.SHORT);
+ }
+ }
+
+ @Description(name = "testHiveUDFINT", value = "_FUNC_(INT) - Tests int data as input and output")
+ public static class GenericUDFTestINT extends GenericUDFTestBase {
+ public GenericUDFTestINT() {
+ super("testHiveUDFINT", PrimitiveCategory.INT);
+ }
+ }
+
+ @Description(name = "testHiveUDFLONG", value = "_FUNC_(LONG) - Tests long data as input and output")
+ public static class GenericUDFTestLONG extends GenericUDFTestBase {
+ public GenericUDFTestLONG() {
+ super("testHiveUDFLONG", PrimitiveCategory.LONG);
+ }
+ }
+
+ @Description(name = "testHiveUDFFLOAT", value = "_FUNC_(FLOAT) - Tests float data as input and output")
+ public static class GenericUDFTestFLOAT extends GenericUDFTestBase {
+ public GenericUDFTestFLOAT() {
+ super("testHiveUDFFLOAT", PrimitiveCategory.FLOAT);
+ }
+ }
+
+ @Description(name = "testHiveUDFDOUBLE", value = "_FUNC_(DOUBLE) - Tests double data as input and output")
+ public static class GenericUDFTestDOUBLE extends GenericUDFTestBase {
+ public GenericUDFTestDOUBLE() {
+ super("testHiveUDFDOUBLE", PrimitiveCategory.DOUBLE);
+ }
+ }
+
+ @Description(name = "testHiveUDFVARCHAR", value = "_FUNC_(VARCHAR) - Tests varchar data as input and output")
+ public static class GenericUDFTestVARCHAR extends GenericUDFTestBase {
+ public GenericUDFTestVARCHAR() {
+ super("testHiveUDFVARCHAR", PrimitiveCategory.VARCHAR);
+ }
+ }
+
+ @Description(name = "testHiveUDFSTRING", value = "_FUNC_(STRING) - Tests string data as input and output")
+ public static class GenericUDFTestSTRING extends GenericUDFTestBase {
+ public GenericUDFTestSTRING() {
+ super("testHiveUDFSTRING", PrimitiveCategory.STRING);
+ }
+ }
+
+ @Description(name = "testHiveUDFBINARY", value = "_FUNC_(BINARY) - Tests binary data as input and output")
+ public static class GenericUDFTestBINARY extends GenericUDFTestBase {
+ public GenericUDFTestBINARY() {
+ super("testHiveUDFBINARY", PrimitiveCategory.BINARY);
+ }
+ }
+
+ @Description(name = "testHiveUDFTIMESTAMP", value = "_FUNC_(TIMESTAMP) - Tests timestamp data as input and output")
+ public static class GenericUDFTestTIMESTAMP extends GenericUDFTestBase {
+ public GenericUDFTestTIMESTAMP() {
+ super("testHiveUDFTIMESTAMP", PrimitiveCategory.TIMESTAMP);
+ }
+ }
+
+ @Description(name = "testHiveUDFDATE", value = "_FUNC_(DATE) - Tests date data as input and output")
+ public static class GenericUDFTestDATE extends GenericUDFTestBase {
+ public GenericUDFTestDATE() {
+ super("testHiveUDFDATE", PrimitiveCategory.DATE);
+ }
+ }
+
+ @Description(name = "testHiveUDFDECIMAL", value = "_FUNC_(DECIMAL) - Tests decimal data as input and output")
+ public static class GenericUDFTestDECIMAL extends GenericUDFTestBase {
+ public GenericUDFTestDECIMAL() {
+ super("testHiveUDFDECIMAL", PrimitiveCategory.DECIMAL, PrimitiveCategory.VARCHAR);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1e9930fb/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestSampleHiveUDFs.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestSampleHiveUDFs.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestSampleHiveUDFs.java
new file mode 100644
index 0000000..2881a18
--- /dev/null
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestSampleHiveUDFs.java
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.fn.hive;
+
+import org.apache.drill.BaseTestQuery;
+import org.apache.drill.exec.rpc.user.QueryResultBatch;
+import org.apache.drill.exec.store.hive.HiveTestDataGenerator;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.junit.Assert.assertTrue;
+
+public class TestSampleHiveUDFs extends BaseTestQuery {
+
+ @BeforeClass
+ public static void addStoragePlugins() throws Exception{
+ new HiveTestDataGenerator().createAndAddHiveTestPlugin(bit.getContext().getStorage());
+ }
+
+ private void helper(String query, String expected) throws Exception {
+ List<QueryResultBatch> results = testSqlWithResults(query);
+ String actual = getResultString(results, ",");
+ assertTrue(String.format("Result:\n%s\ndoes not match:\n%s", actual, expected), expected.equals(actual));
+ }
+
+ @Test
+ public void booleanInOut() throws Exception{
+ String query = "SELECT " +
+ "testHiveUDFBoolean(true) as col1," +
+ "testHiveUDFBoolean(false) as col2," +
+ "testHiveUDFBoolean(cast(null as boolean)) as col3 " +
+ "FROM hive.kv LIMIT 1";
+
+ String expected = "col1,col2,col3\n" + "true,false,null\n";
+ helper(query, expected);
+ }
+
+ @Test
+ public void byteInOut() throws Exception{
+ String query = "SELECT testHiveUDFByte(tinyint_field) as col1 FROM hive.readtest";
+ String expected = "col1\n" + "34\n" + "null\n";
+ helper(query, expected);
+ }
+
+ @Test
+ public void shortInOut() throws Exception{
+ String query = "SELECT testHiveUDFShort(smallint_field) as col1 FROM hive.readtest";
+ String expected = "col1\n" + "3455\n" + "null\n";
+ helper(query, expected);
+ }
+
+ @Test
+ public void intInOut() throws Exception{
+ String query = "SELECT testHiveUDFInt(int_field) as col1 FROM hive.readtest";
+ String expected = "col1\n" + "123456\n" + "null\n";
+ helper(query, expected);
+ }
+
+ @Test
+ public void longInOut() throws Exception{
+ String query = "SELECT testHiveUDFLong(bigint_field) as col1 FROM hive.readtest";
+ String expected = "col1\n" + "234235\n" + "null\n";
+ helper(query, expected);
+ }
+
+ @Test
+ public void floatInOut() throws Exception{
+ String query = "SELECT testHiveUDFFloat(float_field) as col1 FROM hive.readtest";
+ String expected = "col1\n" + "4.67\n" + "null\n"; helper(query, expected);
+ }
+
+ @Test
+ public void doubleInOut() throws Exception{
+ String query = "SELECT testHiveUDFDouble(double_field) as col1 FROM hive.readtest";
+ String expected = "col1\n" + "8.345\n" + "null\n";
+ helper(query, expected);
+ }
+
+ @Test
+ @Ignore // TODO: need var16char input. String type field read from Hive is converted to VarChar type
+ public void stringInOut() throws Exception{
+ String query = "SELECT testHiveUDFString(string_field) as col1 FROM hive.readtest";
+ String expected = "col1\n" + "8.345\n" + "null\n";
+ helper(query, expected);
+ }
+
+ @Test
+ public void binaryInOut() throws Exception{
+ String query = "SELECT testHiveUDFBinary(binary_field) as col1 FROM hive.readtest";
+ String expected = "col1\n" + "binaryfield\n" + "\n";
+ helper(query, expected); helper(query, expected);
+ }
+
+ @Test
+ public void varcharInOut() throws Exception{
+ String query = "SELECT " +
+ "testHiveUDFVarChar('This is a varchar') as col1," +
+ "testHiveUDFVarChar(cast(null as varchar)) as col2 " +
+ "FROM hive.kv LIMIT 1";
+
+ String expected = "col1,col2\n" + "This is a varchar,null\n";
+ helper(query, expected);
+ }
+
+ @Test
+ public void dateInOut() throws Exception{
+ String query = "SELECT " +
+ "testHiveUDFDate(cast('1970-01-02 10:20:33' as date)) as col1," +
+ "testHiveUDFDate(cast(null as date)) as col2 " +
+ "FROM hive.kv LIMIT 1";
+
+ String expected = "col1,col2\n" + "1970-01-01T08:00:00.000-08:00,null\n";
+ helper(query, expected);
+ }
+
+ @Test
+ public void timestampInOut() throws Exception{
+ String query = "SELECT " +
+ "testHiveUDFTimeStamp(cast('1970-01-02 10:20:33' as timestamp)) as col1," +
+ "testHiveUDFTimeStamp(cast(null as timestamp)) as col2 " +
+ "FROM hive.kv LIMIT 1";
+
+ String expected = "col1,col2\n" + "1970-01-02T10:20:33.000-08:00,null\n";
+ helper(query, expected);
+ }
+
+ @Test
+ public void decimalInOut() throws Exception{
+ String query = "SELECT " +
+ "testHiveUDFDecimal(cast('1234567891234567891234567891234567891.4' as decimal(38, 1))) as col1 " +
+ "FROM hive.kv LIMIT 1";
+
+ String expected = "col1\n" + "1234567891234567891234567891234567891.4\n";
+ helper(query, expected);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1e9930fb/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
index adb007d..2f500b8 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
@@ -261,6 +261,7 @@ public class HiveTestDataGenerator {
PrintWriter printWriter = new PrintWriter(file);
printWriter.println("YmluYXJ5ZmllbGQ=,false,34,3489423929323435243,8.345,4.67,123456,234235,3455,stringfield,varcharfield,2013-07-05 17:01:00,2013-07-05");
+ printWriter.println(",,,,,,,,,,,,");
printWriter.close();
return file.getPath();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1e9930fb/contrib/storage-hive/core/src/test/resources/drill-module.conf
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/resources/drill-module.conf b/contrib/storage-hive/core/src/test/resources/drill-module.conf
new file mode 100644
index 0000000..271395e
--- /dev/null
+++ b/contrib/storage-hive/core/src/test/resources/drill-module.conf
@@ -0,0 +1,5 @@
+// This file tells Drill to consider this module when class path scanning.
+// This file can also include any supplementary configuration information.
+// This file is in HOCON format, see https://github.com/typesafehub/config/blob/master/HOCON.md for more information.
+
+drill.logical.function.packages += "org.apache.hadoop.hive"
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1e9930fb/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
index 98585c4..81fce5d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
@@ -19,6 +19,7 @@ package org.apache.drill.exec.util;
import java.util.List;
+import com.google.common.base.Joiner;
import org.apache.commons.lang.StringUtils;
import org.apache.drill.common.util.DrillStringUtils;
import org.apache.drill.exec.record.MaterializedField;
@@ -69,6 +70,40 @@ public class VectorUtil {
}
}
+ public static void appendVectorAccessibleContent(VectorAccessible va, StringBuilder formattedResults,
+ final String delimiter, boolean includeHeader) {
+ if (includeHeader) {
+ List<String> columns = Lists.newArrayList();
+ for (VectorWrapper<?> vw : va) {
+ columns.add(vw.getValueVector().getField().getPath().getAsUnescapedPath());
+ }
+
+ formattedResults.append(Joiner.on(delimiter).join(columns));
+ formattedResults.append("\n");
+ }
+
+ int rows = va.getRecordCount();
+ for (int row = 0; row < rows; row++) {
+ List<String> rowValues = Lists.newArrayList();
+ for (VectorWrapper<?> vw : va) {
+ Object o = vw.getValueVector().getAccessor().getObject(row);
+ if (o == null) {
+ rowValues.add("null");
+ } else if (o instanceof byte[]) {
+ rowValues.add(new String((byte[]) o));
+ } else {
+ rowValues.add(o.toString());
+ }
+ }
+ formattedResults.append(Joiner.on(delimiter).join(rowValues));
+ formattedResults.append("\n");
+ }
+
+ for (VectorWrapper<?> vw : va) {
+ vw.clear();
+ }
+ }
+
public static void showVectorAccessibleContent(VectorAccessible va) {
showVectorAccessibleContent(va, DEFAULT_COLUMN_WIDTH);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1e9930fb/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
index 9aa76ec..5e52e82 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
@@ -275,4 +275,23 @@ public class BaseTestQuery extends ExecTest{
return rowCount;
}
+ protected String getResultString(List<QueryResultBatch> results, String delimiter) throws SchemaChangeException {
+ StringBuilder formattedResults = new StringBuilder();
+ boolean includeHeader = true;
+ RecordBatchLoader loader = new RecordBatchLoader(getAllocator());
+ for(QueryResultBatch result : results){
+ loader.load(result.getHeader().getDef(), result.getData());
+ if (loader.getRecordCount() <= 0) {
+ break;
+ }
+ VectorUtil.appendVectorAccessibleContent(loader, formattedResults, delimiter, includeHeader);
+ if (!includeHeader) {
+ includeHeader = false;
+ }
+ loader.clear();
+ result.release();
+ }
+
+ return formattedResults.toString();
+ }
}
|