DRILL-364: Fix VarChar casting
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/153c727a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/153c727a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/153c727a
Branch: refs/heads/master
Commit: 153c727a6d393203501f3cfed2535a0fb0601061
Parents: ee9eaa1
Author: Jinfeng Ni <jni@maprtech.com>
Authored: Sun Feb 23 16:02:26 2014 -0800
Committer: Jacques Nadeau <jacques@apache.org>
Committed: Mon Mar 3 23:22:17 2014 -0800
----------------------------------------------------------------------
.../common/expression/OutputTypeDeterminer.java | 20 +++++
.../common/expression/fn/CastFunctionDefs.java | 14 ++--
exec/java-exec/src/main/codegen/data/Casts.tdd | 4 +-
.../exec/physical/impl/TestCastFunctions.java | 80 +++++++++++++++++++-
.../functions/cast/testCastVarCharNull.json | 37 +++++++++
.../src/test/resources/jsoninput/input1.json | 6 ++
6 files changed, 148 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/153c727a/common/src/main/java/org/apache/drill/common/expression/OutputTypeDeterminer.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/OutputTypeDeterminer.java
b/common/src/main/java/org/apache/drill/common/expression/OutputTypeDeterminer.java
index 66523c4..69acf12 100644
--- a/common/src/main/java/org/apache/drill/common/expression/OutputTypeDeterminer.java
+++ b/common/src/main/java/org/apache/drill/common/expression/OutputTypeDeterminer.java
@@ -74,4 +74,24 @@ public interface OutputTypeDeterminer {
}
}
+ public static class NullIfNullType implements OutputTypeDeterminer{
+ public MinorType outputMinorType;
+
+
+ public NullIfNullType(MinorType outputType) {
+ super();
+ this.outputMinorType = outputType;
+ }
+
+ @Override
+ public MajorType getOutputType(List<LogicalExpression> expressions) {
+ for(LogicalExpression e : expressions){
+ if(e.getMajorType().getMode() == DataMode.OPTIONAL){
+ return Types.optional(outputMinorType);
+ }
+ }
+ return Types.required(outputMinorType);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/153c727a/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctionDefs.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctionDefs.java
b/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctionDefs.java
index 6a98f94..4be3820 100644
--- a/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctionDefs.java
+++ b/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctionDefs.java
@@ -25,13 +25,13 @@ import org.apache.drill.common.types.TypeProtos.MinorType;
public class CastFunctionDefs implements CallProvider{
- private static final FunctionDefinition CAST_BIG_INT = FunctionDefinition.simple("castBIGINT",
new ArgumentValidators.AnyTypeAllowed(1,3), OutputTypeDeterminer.FIXED_BIGINT);
- private static final FunctionDefinition CAST_INT = FunctionDefinition.simple("castINT",
new ArgumentValidators.AnyTypeAllowed(1,3), OutputTypeDeterminer.FIXED_INT);
- private static final FunctionDefinition CAST_FLOAT4 = FunctionDefinition.simple("castFLOAT4",
new ArgumentValidators.AnyTypeAllowed(1,3), OutputTypeDeterminer.FIXED_FLOAT4);
- private static final FunctionDefinition CAST_FLOAT8 = FunctionDefinition.simple("castFLOAT8",
new ArgumentValidators.AnyTypeAllowed(1,3), OutputTypeDeterminer.FIXED_FLOAT8);
- private static final FunctionDefinition CAST_VARCHAR = FunctionDefinition.simple("castVARCHAR",
new ArgumentValidators.AnyTypeAllowed(1,4), OutputTypeDeterminer.FIXED_VARCHAR);
- private static final FunctionDefinition CAST_VARBINARY = FunctionDefinition.simple("castVARBINARY",
new ArgumentValidators.AnyTypeAllowed(1,4), OutputTypeDeterminer.FIXED_VARBINARY);
- private static final FunctionDefinition CAST_VAR16CHAR = FunctionDefinition.simple("castVAR16CHAR",
new ArgumentValidators.AnyTypeAllowed(1,4), OutputTypeDeterminer.FIXED_VAR16CHAR);
+ private static final FunctionDefinition CAST_BIG_INT = FunctionDefinition.simple("castBIGINT",
new ArgumentValidators.AnyTypeAllowed(1,3), new OutputTypeDeterminer.NullIfNullType(MinorType.BIGINT));
+ private static final FunctionDefinition CAST_INT = FunctionDefinition.simple("castINT",
new ArgumentValidators.AnyTypeAllowed(1,3), new OutputTypeDeterminer.NullIfNullType(MinorType.INT));
+ private static final FunctionDefinition CAST_FLOAT4 = FunctionDefinition.simple("castFLOAT4",
new ArgumentValidators.AnyTypeAllowed(1,3), new OutputTypeDeterminer.NullIfNullType(MinorType.FLOAT4));
+ private static final FunctionDefinition CAST_FLOAT8 = FunctionDefinition.simple("castFLOAT8",
new ArgumentValidators.AnyTypeAllowed(1,3), new OutputTypeDeterminer.NullIfNullType(MinorType.FLOAT8));
+ private static final FunctionDefinition CAST_VARCHAR = FunctionDefinition.simple("castVARCHAR",
new ArgumentValidators.AnyTypeAllowed(1,4), new OutputTypeDeterminer.NullIfNullType(MinorType.VARCHAR));
+ private static final FunctionDefinition CAST_VARBINARY = FunctionDefinition.simple("castVARBINARY",
new ArgumentValidators.AnyTypeAllowed(1,4), new OutputTypeDeterminer.NullIfNullType(MinorType.VARBINARY));
+ private static final FunctionDefinition CAST_VAR16CHAR = FunctionDefinition.simple("castVAR16CHAR",
new ArgumentValidators.AnyTypeAllowed(1,4), new OutputTypeDeterminer.NullIfNullType(MinorType.VAR16CHAR));
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/153c727a/exec/java-exec/src/main/codegen/data/Casts.tdd
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/data/Casts.tdd b/exec/java-exec/src/main/codegen/data/Casts.tdd
index ceb9cde..fde63cf 100644
--- a/exec/java-exec/src/main/codegen/data/Casts.tdd
+++ b/exec/java-exec/src/main/codegen/data/Casts.tdd
@@ -39,11 +39,11 @@
{from: "VarBinary", to: "Float8", major: "SrcVarlen", javaType:"Double", parse:"Double"},
{from: "BigInt", to: "VarChar", major: "TargetVarlen", javaType: "Long", bufferLength:"20"},
- {from: "Int", to: "VarChar", major: "TargetVarlen", javaType: "Integer", bufferLength:"10"},
+ {from: "Int", to: "VarChar", major: "TargetVarlen", javaType: "Integer", bufferLength:"11"},
{from: "Float4", to: "VarChar", major: "TargetVarlen", javaType: "Float", bufferLength:"100"},
{from: "Float8", to: "VarChar", major: "TargetVarlen", javaType: "Double", bufferLength:"100"},
{from: "BigInt", to: "VarBinary", major: "TargetVarlen", javaType: "Long", bufferLength:"20"},
- {from: "Int", to: "VarBinary", major: "TargetVarlen", javaType: "Integer", bufferLength:"10"},
+ {from: "Int", to: "VarBinary", major: "TargetVarlen", javaType: "Integer", bufferLength:"11"},
{from: "Float4", to: "VarBinary", major: "TargetVarlen", javaType: "Float", bufferLength:"100"},
{from: "Float8", to: "VarBinary", major: "TargetVarlen", javaType: "Double", bufferLength:"100"},
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/153c727a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java
index 3010c37..4d18815 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java
@@ -19,6 +19,9 @@ package org.apache.drill.exec.physical.impl;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+
+import java.util.List;
+
import mockit.Injectable;
import mockit.NonStrictExpectations;
@@ -26,6 +29,7 @@ import org.apache.drill.common.config.DrillConfig;
import org.apache.drill.common.expression.ExpressionPosition;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.expr.holders.BigIntHolder;
import org.apache.drill.exec.expr.holders.Float4Holder;
@@ -38,34 +42,44 @@ import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.PhysicalPlan;
import org.apache.drill.exec.physical.base.FragmentRoot;
import org.apache.drill.exec.planner.PhysicalPlanReader;
+import org.apache.drill.exec.pop.PopUnitTestBase;
import org.apache.drill.exec.proto.BitControl.PlanFragment;
import org.apache.drill.exec.proto.CoordinationProtos;
-import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
+import org.apache.drill.exec.proto.UserProtos;
+import org.apache.drill.exec.record.RecordBatchLoader;
+import org.apache.drill.exec.record.VectorAccessible;
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.rpc.user.QueryResultBatch;
import org.apache.drill.exec.rpc.user.UserServer;
+import org.apache.drill.exec.server.Drillbit;
import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.server.RemoteServiceSet;
import org.apache.drill.exec.vector.BigIntVector;
import org.apache.drill.exec.vector.Float4Vector;
import org.apache.drill.exec.vector.Float8Vector;
import org.apache.drill.exec.vector.IntVector;
+import org.apache.drill.exec.vector.NullableVarCharVector;
+import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.VarBinaryVector;
import org.apache.drill.exec.vector.VarCharVector;
import org.junit.After;
+import org.junit.Ignore;
import org.junit.Test;
import com.codahale.metrics.MetricRegistry;
import com.google.common.base.Charsets;
import com.google.common.io.Files;
-public class TestCastFunctions {
+public class TestCastFunctions extends PopUnitTestBase{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestSimpleFunctions.class);
DrillConfig c = DrillConfig.create();
+
@Test
// cast to bigint.
public void testCastBigInt(@Injectable final DrillbitContext bitContext,
@Injectable UserServer.UserClientConnection connection) throws
Throwable{
-
new NonStrictExpectations(){{
bitContext.getMetrics(); result = new MetricRegistry();
bitContext.getAllocator(); result = new TopLevelAllocator();
@@ -342,7 +356,6 @@ public class TestCastFunctions {
}
@Test(expected = NumberFormatException.class)
- //nested: cast is nested in another cast, or another function.
public void testCastNumException(@Injectable final DrillbitContext bitContext,
@Injectable UserServer.UserClientConnection connection) throws
Throwable{
@@ -370,6 +383,65 @@ public class TestCastFunctions {
}
+ @Test
+ public void testCastFromNullablCol() throws Throwable {
+ RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
+
+ try(Drillbit bit = new Drillbit(CONFIG, serviceSet);
+ DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator())) {
+ bit.run();
+
+ client.connect();
+ List<QueryResultBatch> results = client.runQuery(UserProtos.QueryType.PHYSICAL,
+ Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastVarCharNull.json"),
Charsets.UTF_8).replace("#{TEST_FILE}", FileUtils.getResourceAsFile("/jsoninput/input1.json").toURI().toString()));
+
+ QueryResultBatch batch = results.get(0);
+
+ RecordBatchLoader batchLoader = new RecordBatchLoader(bit.getContext().getAllocator());
+ batchLoader.load(batch.getHeader().getDef(), batch.getData());
+
+ Object [][] result = getRunResult(batchLoader);
+
+ Object [][] expected = new Object[2][2];
+
+ expected[0][0] = new String("2001");
+ expected[0][1] = new String("1.2");
+
+ expected[1][0] = new String("-2002");
+ expected[1][1] = new String("-1.2");
+
+ assertEquals(result.length, expected.length);
+ assertEquals(result[0].length, expected[0].length);
+
+ for (int i = 0; i<result.length; i++ ) {
+ for (int j = 0; j<result[0].length; j++) {
+ assertEquals(String.format("Column %s at row %s have wrong result", j, i), result[i][j],
expected[i][j]);
+ }
+ }
+ }
+ }
+
+ private Object[][] getRunResult(VectorAccessible va) {
+ int size = 0;
+ for (VectorWrapper v : va) {
+ size++;
+ }
+
+ Object[][] res = new Object [va.getRecordCount()][size];
+ for (int j = 0; j < va.getRecordCount(); j++) {
+ int i = 0;
+ for (VectorWrapper v : va) {
+ Object o = v.getValueVector().getAccessor().getObject(j);
+ if (o instanceof byte[]) {
+ res[j][i++] = new String((byte[]) o);
+ } else {
+ res[j][i++] = o;
+ }
+ }
+ }
+ return res;
+ }
+
@After
public void tearDown() throws Exception{
// pause to get logger to catch up.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/153c727a/exec/java-exec/src/test/resources/functions/cast/testCastVarCharNull.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/functions/cast/testCastVarCharNull.json b/exec/java-exec/src/test/resources/functions/cast/testCastVarCharNull.json
new file mode 100644
index 0000000..030b696
--- /dev/null
+++ b/exec/java-exec/src/test/resources/functions/cast/testCastVarCharNull.json
@@ -0,0 +1,37 @@
+{
+ head:{
+ type:"APACHE_DRILL_PHYSICAL",
+ version:"1",
+ generator:{
+ type:"manual"
+ }
+ },
+ graph:[
+ {
+ @id:1,
+ pop:"json-scan",
+ entries: [
+ {
+ path : "#{TEST_FILE}"
+ }
+ ],
+ storageengine: {
+ "type": "json",
+ "dfsName": "file:///"
+ }
+ }, {
+ @id:2,
+ child: 1,
+ pop:"project",
+ exprs: [
+ { ref: "int2varchar", expr:"cast(integer as varchar(20))" },
+ { ref: "float2varchar", expr:"cast(float as varchar(20))" }
+ ]
+ },
+ {
+ @id: 3,
+ child: 2,
+ pop: "screen"
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/153c727a/exec/java-exec/src/test/resources/jsoninput/input1.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/jsoninput/input1.json b/exec/java-exec/src/test/resources/jsoninput/input1.json
new file mode 100644
index 0000000..e9bde7e
--- /dev/null
+++ b/exec/java-exec/src/test/resources/jsoninput/input1.json
@@ -0,0 +1,6 @@
+{ "integer" : 2001,
+ "float" : 1.2
+}
+{ "integer" : -2002,
+ "float" : -1.2
+}
|