drill-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jacq...@apache.org
Subject [27/38] git commit: DRILL-364: Fix VarChar casting
Date Tue, 04 Mar 2014 08:07:54 GMT
DRILL-364: Fix VarChar casting


Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/153c727a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/153c727a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/153c727a

Branch: refs/heads/master
Commit: 153c727a6d393203501f3cfed2535a0fb0601061
Parents: ee9eaa1
Author: Jinfeng Ni <jni@maprtech.com>
Authored: Sun Feb 23 16:02:26 2014 -0800
Committer: Jacques Nadeau <jacques@apache.org>
Committed: Mon Mar 3 23:22:17 2014 -0800

----------------------------------------------------------------------
 .../common/expression/OutputTypeDeterminer.java | 20 +++++
 .../common/expression/fn/CastFunctionDefs.java  | 14 ++--
 exec/java-exec/src/main/codegen/data/Casts.tdd  |  4 +-
 .../exec/physical/impl/TestCastFunctions.java   | 80 +++++++++++++++++++-
 .../functions/cast/testCastVarCharNull.json     | 37 +++++++++
 .../src/test/resources/jsoninput/input1.json    |  6 ++
 6 files changed, 148 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/153c727a/common/src/main/java/org/apache/drill/common/expression/OutputTypeDeterminer.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/OutputTypeDeterminer.java
b/common/src/main/java/org/apache/drill/common/expression/OutputTypeDeterminer.java
index 66523c4..69acf12 100644
--- a/common/src/main/java/org/apache/drill/common/expression/OutputTypeDeterminer.java
+++ b/common/src/main/java/org/apache/drill/common/expression/OutputTypeDeterminer.java
@@ -74,4 +74,24 @@ public interface OutputTypeDeterminer {
     }
   }
   
+  public static class NullIfNullType implements OutputTypeDeterminer{
+    public MinorType outputMinorType;
+    
+    
+    public NullIfNullType(MinorType outputType) {
+      super();
+      this.outputMinorType = outputType;
+    }
+
+    @Override
+    public MajorType getOutputType(List<LogicalExpression> expressions) {
+      for(LogicalExpression e : expressions){
+        if(e.getMajorType().getMode() == DataMode.OPTIONAL){
+          return Types.optional(outputMinorType);
+        }
+      }
+      return Types.required(outputMinorType);
+    }
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/153c727a/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctionDefs.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctionDefs.java
b/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctionDefs.java
index 6a98f94..4be3820 100644
--- a/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctionDefs.java
+++ b/common/src/main/java/org/apache/drill/common/expression/fn/CastFunctionDefs.java
@@ -25,13 +25,13 @@ import org.apache.drill.common.types.TypeProtos.MinorType;
 
 public class CastFunctionDefs implements CallProvider{
   
-  private static final FunctionDefinition CAST_BIG_INT = FunctionDefinition.simple("castBIGINT",
new ArgumentValidators.AnyTypeAllowed(1,3), OutputTypeDeterminer.FIXED_BIGINT);
-  private static final FunctionDefinition CAST_INT = FunctionDefinition.simple("castINT",
new ArgumentValidators.AnyTypeAllowed(1,3), OutputTypeDeterminer.FIXED_INT);
-  private static final FunctionDefinition CAST_FLOAT4 = FunctionDefinition.simple("castFLOAT4",
new ArgumentValidators.AnyTypeAllowed(1,3), OutputTypeDeterminer.FIXED_FLOAT4); 
-  private static final FunctionDefinition CAST_FLOAT8 = FunctionDefinition.simple("castFLOAT8",
new ArgumentValidators.AnyTypeAllowed(1,3), OutputTypeDeterminer.FIXED_FLOAT8);
-  private static final FunctionDefinition CAST_VARCHAR = FunctionDefinition.simple("castVARCHAR",
new ArgumentValidators.AnyTypeAllowed(1,4), OutputTypeDeterminer.FIXED_VARCHAR);
-  private static final FunctionDefinition CAST_VARBINARY = FunctionDefinition.simple("castVARBINARY",
new ArgumentValidators.AnyTypeAllowed(1,4), OutputTypeDeterminer.FIXED_VARBINARY);
-  private static final FunctionDefinition CAST_VAR16CHAR = FunctionDefinition.simple("castVAR16CHAR",
new ArgumentValidators.AnyTypeAllowed(1,4), OutputTypeDeterminer.FIXED_VAR16CHAR);
+  private static final FunctionDefinition CAST_BIG_INT = FunctionDefinition.simple("castBIGINT",
new ArgumentValidators.AnyTypeAllowed(1,3), new OutputTypeDeterminer.NullIfNullType(MinorType.BIGINT));
+  private static final FunctionDefinition CAST_INT = FunctionDefinition.simple("castINT",
new ArgumentValidators.AnyTypeAllowed(1,3), new OutputTypeDeterminer.NullIfNullType(MinorType.INT));
+  private static final FunctionDefinition CAST_FLOAT4 = FunctionDefinition.simple("castFLOAT4",
new ArgumentValidators.AnyTypeAllowed(1,3), new OutputTypeDeterminer.NullIfNullType(MinorType.FLOAT4));

+  private static final FunctionDefinition CAST_FLOAT8 = FunctionDefinition.simple("castFLOAT8",
new ArgumentValidators.AnyTypeAllowed(1,3), new OutputTypeDeterminer.NullIfNullType(MinorType.FLOAT8));
+  private static final FunctionDefinition CAST_VARCHAR = FunctionDefinition.simple("castVARCHAR",
new ArgumentValidators.AnyTypeAllowed(1,4), new OutputTypeDeterminer.NullIfNullType(MinorType.VARCHAR));
+  private static final FunctionDefinition CAST_VARBINARY = FunctionDefinition.simple("castVARBINARY",
new ArgumentValidators.AnyTypeAllowed(1,4), new OutputTypeDeterminer.NullIfNullType(MinorType.VARBINARY));
+  private static final FunctionDefinition CAST_VAR16CHAR = FunctionDefinition.simple("castVAR16CHAR",
new ArgumentValidators.AnyTypeAllowed(1,4), new OutputTypeDeterminer.NullIfNullType(MinorType.VAR16CHAR));
 
   
   @Override

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/153c727a/exec/java-exec/src/main/codegen/data/Casts.tdd
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/data/Casts.tdd b/exec/java-exec/src/main/codegen/data/Casts.tdd
index ceb9cde..fde63cf 100644
--- a/exec/java-exec/src/main/codegen/data/Casts.tdd
+++ b/exec/java-exec/src/main/codegen/data/Casts.tdd
@@ -39,11 +39,11 @@
     {from: "VarBinary", to: "Float8", major: "SrcVarlen", javaType:"Double", parse:"Double"},

 
     {from: "BigInt", to: "VarChar", major: "TargetVarlen", javaType: "Long", bufferLength:"20"},
-    {from: "Int", to: "VarChar", major: "TargetVarlen", javaType: "Integer", bufferLength:"10"},
+    {from: "Int", to: "VarChar", major: "TargetVarlen", javaType: "Integer", bufferLength:"11"},
     {from: "Float4", to: "VarChar", major: "TargetVarlen", javaType: "Float", bufferLength:"100"},
     {from: "Float8", to: "VarChar", major: "TargetVarlen", javaType: "Double", bufferLength:"100"},
     {from: "BigInt", to: "VarBinary", major: "TargetVarlen", javaType: "Long", bufferLength:"20"},
-    {from: "Int", to: "VarBinary", major: "TargetVarlen", javaType: "Integer", bufferLength:"10"},
+    {from: "Int", to: "VarBinary", major: "TargetVarlen", javaType: "Integer", bufferLength:"11"},
     {from: "Float4", to: "VarBinary", major: "TargetVarlen", javaType: "Float", bufferLength:"100"},
     {from: "Float8", to: "VarBinary", major: "TargetVarlen", javaType: "Double", bufferLength:"100"},
    
     

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/153c727a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java
index 3010c37..4d18815 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestCastFunctions.java
@@ -19,6 +19,9 @@ package org.apache.drill.exec.physical.impl;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+
+import java.util.List;
+
 import mockit.Injectable;
 import mockit.NonStrictExpectations;
 
@@ -26,6 +29,7 @@ import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.common.expression.ExpressionPosition;
 import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.exec.client.DrillClient;
 import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
 import org.apache.drill.exec.expr.holders.BigIntHolder;
 import org.apache.drill.exec.expr.holders.Float4Holder;
@@ -38,34 +42,44 @@ import org.apache.drill.exec.ops.FragmentContext;
 import org.apache.drill.exec.physical.PhysicalPlan;
 import org.apache.drill.exec.physical.base.FragmentRoot;
 import org.apache.drill.exec.planner.PhysicalPlanReader;
+import org.apache.drill.exec.pop.PopUnitTestBase;
 import org.apache.drill.exec.proto.BitControl.PlanFragment;
 import org.apache.drill.exec.proto.CoordinationProtos;
-import org.apache.drill.exec.proto.ExecProtos.FragmentHandle;
+import org.apache.drill.exec.proto.UserProtos;
+import org.apache.drill.exec.record.RecordBatchLoader;
+import org.apache.drill.exec.record.VectorAccessible;
+import org.apache.drill.exec.record.VectorWrapper;
+import org.apache.drill.exec.rpc.user.QueryResultBatch;
 import org.apache.drill.exec.rpc.user.UserServer;
+import org.apache.drill.exec.server.Drillbit;
 import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.server.RemoteServiceSet;
 import org.apache.drill.exec.vector.BigIntVector;
 import org.apache.drill.exec.vector.Float4Vector;
 import org.apache.drill.exec.vector.Float8Vector;
 import org.apache.drill.exec.vector.IntVector;
+import org.apache.drill.exec.vector.NullableVarCharVector;
+import org.apache.drill.exec.vector.ValueVector;
 import org.apache.drill.exec.vector.VarBinaryVector;
 import org.apache.drill.exec.vector.VarCharVector;
 import org.junit.After;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import com.codahale.metrics.MetricRegistry;
 import com.google.common.base.Charsets;
 import com.google.common.io.Files;
 
-public class TestCastFunctions {
+public class TestCastFunctions extends PopUnitTestBase{
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestSimpleFunctions.class);
 
   DrillConfig c = DrillConfig.create();
 
+
   @Test
   // cast to bigint.
   public void testCastBigInt(@Injectable final DrillbitContext bitContext,
                             @Injectable UserServer.UserClientConnection connection) throws
Throwable{
-
     new NonStrictExpectations(){{
       bitContext.getMetrics(); result = new MetricRegistry();
       bitContext.getAllocator(); result = new TopLevelAllocator();
@@ -342,7 +356,6 @@ public class TestCastFunctions {
   }
 
   @Test(expected = NumberFormatException.class)
-  //nested: cast is nested in another cast, or another function.
   public void testCastNumException(@Injectable final DrillbitContext bitContext,
                             @Injectable UserServer.UserClientConnection connection) throws
Throwable{
 
@@ -370,6 +383,65 @@ public class TestCastFunctions {
 
   }
   
+  @Test
+  public void testCastFromNullablCol() throws Throwable {
+    RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
+
+    try(Drillbit bit = new Drillbit(CONFIG, serviceSet);
+        DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator())) {
+      bit.run();
+      
+      client.connect();
+      List<QueryResultBatch> results = client.runQuery(UserProtos.QueryType.PHYSICAL,
+          Files.toString(FileUtils.getResourceAsFile("/functions/cast/testCastVarCharNull.json"),
Charsets.UTF_8).replace("#{TEST_FILE}", FileUtils.getResourceAsFile("/jsoninput/input1.json").toURI().toString()));
     
+    
+      QueryResultBatch batch = results.get(0);
+
+      RecordBatchLoader batchLoader = new RecordBatchLoader(bit.getContext().getAllocator());
+      batchLoader.load(batch.getHeader().getDef(), batch.getData());
+     
+      Object [][] result = getRunResult(batchLoader);
+      
+      Object [][] expected = new Object[2][2];
+      
+      expected[0][0] = new String("2001");
+      expected[0][1] = new String("1.2");
+      
+      expected[1][0] = new String("-2002");
+      expected[1][1] = new String("-1.2");
+ 
+      assertEquals(result.length, expected.length);
+      assertEquals(result[0].length, expected[0].length);
+      
+      for (int i = 0; i<result.length; i++ ) {
+        for (int j = 0; j<result[0].length; j++) {
+          assertEquals(String.format("Column %s at row %s have wrong result",  j, i), result[i][j],
expected[i][j]);
+        }
+      }     
+    }  
+  }
+
+  private Object[][] getRunResult(VectorAccessible va) {
+    int size = 0;
+    for (VectorWrapper v : va) {
+      size++;     
+    }   
+  
+    Object[][] res = new Object [va.getRecordCount()][size];
+    for (int j = 0; j < va.getRecordCount(); j++) {
+      int i = 0;
+      for (VectorWrapper v : va) {
+        Object o =  v.getValueVector().getAccessor().getObject(j);
+        if (o instanceof byte[]) {
+          res[j][i++] =  new String((byte[]) o);
+        } else { 
+          res[j][i++] = o;
+        }
+      }
+    }
+    return res;
+ }  
+
   @After
   public void tearDown() throws Exception{
     // pause to get logger to catch up.

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/153c727a/exec/java-exec/src/test/resources/functions/cast/testCastVarCharNull.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/functions/cast/testCastVarCharNull.json b/exec/java-exec/src/test/resources/functions/cast/testCastVarCharNull.json
new file mode 100644
index 0000000..030b696
--- /dev/null
+++ b/exec/java-exec/src/test/resources/functions/cast/testCastVarCharNull.json
@@ -0,0 +1,37 @@
+{
+    head:{
+        type:"APACHE_DRILL_PHYSICAL",
+        version:"1",
+        generator:{
+            type:"manual"
+        }
+    },
+    graph:[
+        {
+        	@id:1,
+          pop:"json-scan",
+          entries: [
+           {
+              path : "#{TEST_FILE}"
+           }
+         ],
+         storageengine: {
+         	"type": "json",
+          "dfsName": "file:///"
+         }
+       }, {
+            @id:2,
+            child: 1,
+            pop:"project",
+            exprs: [
+							{ ref: "int2varchar", expr:"cast(integer as varchar(20))" },
+							{ ref: "float2varchar", expr:"cast(float as varchar(20))" }
+	        ]
+        },
+        {
+            @id: 3,
+            child: 2,
+            pop: "screen"
+        }
+    ]
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/153c727a/exec/java-exec/src/test/resources/jsoninput/input1.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/jsoninput/input1.json b/exec/java-exec/src/test/resources/jsoninput/input1.json
new file mode 100644
index 0000000..e9bde7e
--- /dev/null
+++ b/exec/java-exec/src/test/resources/jsoninput/input1.json
@@ -0,0 +1,6 @@
+{ "integer" : 2001, 
+  "float"   : 1.2
+}
+{ "integer" : -2002,
+  "float"   : -1.2 
+}


Mime
View raw message