drill-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jacq...@apache.org
Subject [30/38] git commit: DRILL-363: Custom null handling in hash functions
Date Tue, 04 Mar 2014 08:07:57 GMT
DRILL-363: Custom null handling in hash functions

Added custom null handling for hash functions, to override the privies NULL_IF_NULL handling.
Added Integer and Float literals to the logical expression package and corresponding methods
in the various visitor implementations. Also added new Hash functions for double and float
values based on the doubleToLongBits and floatToIntBits methods in the corresponding boxed
primitive classes together with the previously used murmur3_128 hash algorithm used for plain
Ints and Longs.

Signed-off-by: Jacques Nadeau <jacques@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/b7f9e600
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/b7f9e600
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/b7f9e600

Branch: refs/heads/master
Commit: b7f9e600fa31ac7b9365b31396730719b8334469
Parents: 153c727
Author: Jason Altekruse <altekrusejason@gmial.com>
Authored: Sun Feb 9 13:43:42 2014 -0600
Committer: Jacques Nadeau <jacques@apache.org>
Committed: Mon Mar 3 23:22:17 2014 -0800

----------------------------------------------------------------------
 .../common/expression/ExpressionValidator.java  |  10 ++
 .../common/expression/ValueExpressions.java     |  63 +++++++++
 .../visitors/AbstractExprVisitor.java           |  12 ++
 .../expression/visitors/AggregateChecker.java   |  12 ++
 .../expression/visitors/ConstantChecker.java    |  12 ++
 .../common/expression/visitors/ExprVisitor.java |   7 +-
 .../expression/visitors/SimpleExprVisitor.java  |  14 ++
 .../sig/ConstantExpressionIdentifier.java       |  14 +-
 .../exec/expr/ExpressionTreeMaterializer.java   |  10 ++
 .../drill/exec/expr/fn/impl/HashFunctions.java  | 128 +++++++++++++++++--
 .../apache/drill/exec/expr/fn/impl/IsNull.java  |   2 +-
 .../exec/resolver/ResolverTypePrecedence.java   |  56 ++++----
 .../drill/exec/resolver/TypeCastRules.java      |  30 +++--
 .../exec/physical/impl/TestSimpleFunctions.java | 105 ++++++++++++++-
 14 files changed, 426 insertions(+), 49 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b7f9e600/common/src/main/java/org/apache/drill/common/expression/ExpressionValidator.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/ExpressionValidator.java
b/common/src/main/java/org/apache/drill/common/expression/ExpressionValidator.java
index 11d97e8..076c150 100644
--- a/common/src/main/java/org/apache/drill/common/expression/ExpressionValidator.java
+++ b/common/src/main/java/org/apache/drill/common/expression/ExpressionValidator.java
@@ -75,6 +75,16 @@ public class ExpressionValidator implements ExprVisitor<Void, ErrorCollector,
Ru
   }
 
   @Override
+  public Void visitIntConstant(ValueExpressions.IntExpression intExpr, ErrorCollector value)
throws RuntimeException {
+    return null;
+  }
+
+  @Override
+  public Void visitFloatConstant(ValueExpressions.FloatExpression fExpr, ErrorCollector value)
throws RuntimeException {
+    return null;
+  }
+
+  @Override
   public Void visitLongConstant(LongExpression intExpr, ErrorCollector errors) throws RuntimeException
{
     return null;
   }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b7f9e600/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
b/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
index 49f3773..5156786 100644
--- a/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
+++ b/common/src/main/java/org/apache/drill/common/expression/ValueExpressions.java
@@ -120,6 +120,69 @@ public class ValueExpressions {
 
   }
 
+  public static class FloatExpression extends LogicalExpressionBase {
+    private float f;
+
+    private static final MajorType FLOAT_CONSTANT = Types.required(MinorType.FLOAT4);
+
+    public FloatExpression(float f, ExpressionPosition pos) {
+      super(pos);
+      this.f = f;
+    }
+
+    public float getFloat() {
+      return f;
+    }
+
+    @Override
+    public MajorType getMajorType() {
+      return FLOAT_CONSTANT;
+    }
+
+    @Override
+    public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor,
V value) throws E {
+      return visitor.visitFloatConstant(this, value);
+    }
+
+    @Override
+    public Iterator<LogicalExpression> iterator() {
+      return Iterators.emptyIterator();
+    }
+
+  }
+
+  public static class IntExpression extends LogicalExpressionBase {
+
+    private static final MajorType INT_CONSTANT = Types.required(MinorType.INT);
+
+    private int i;
+
+    public IntExpression(int i, ExpressionPosition pos) {
+      super(pos);
+      this.i = i;
+    }
+
+    public int getInt() {
+      return i;
+    }
+
+    @Override
+    public MajorType getMajorType() {
+      return INT_CONSTANT;
+    }
+
+    @Override
+    public <T, V, E extends Exception> T accept(ExprVisitor<T, V, E> visitor,
V value) throws E {
+      return visitor.visitIntConstant(this, value);
+    }
+
+    @Override
+    public Iterator<LogicalExpression> iterator() {
+      return Iterators.emptyIterator();
+    }
+
+  }
+
   public static class DoubleExpression extends LogicalExpressionBase {
     private double d;
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b7f9e600/common/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java
b/common/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java
index 711bdb8..469cedd 100644
--- a/common/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java
+++ b/common/src/main/java/org/apache/drill/common/expression/visitors/AbstractExprVisitor.java
@@ -22,6 +22,8 @@ import org.apache.drill.common.expression.IfExpression;
 import org.apache.drill.common.expression.LogicalExpression;
 import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
+import org.apache.drill.common.expression.ValueExpressions.FloatExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntExpression;
 import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
 import org.apache.drill.common.expression.ValueExpressions.LongExpression;
 import org.apache.drill.common.expression.ValueExpressions.QuotedString;
@@ -45,6 +47,16 @@ public abstract class AbstractExprVisitor<T, VAL, EXCEP extends Exception>
imple
   }
 
   @Override
+  public T visitFloatConstant(FloatExpression fExpr, VAL value) throws EXCEP {
+    return visitUnknown(fExpr, value);
+  }
+
+  @Override
+  public T visitIntConstant(IntExpression intExpr, VAL value) throws EXCEP {
+    return visitUnknown(intExpr, value);
+  }
+
+  @Override
   public T visitLongConstant(LongExpression intExpr, VAL value) throws EXCEP {
     return visitUnknown(intExpr, value);
   }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b7f9e600/common/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java
b/common/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java
index 128f284..36e6a52 100644
--- a/common/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java
+++ b/common/src/main/java/org/apache/drill/common/expression/visitors/AggregateChecker.java
@@ -25,6 +25,8 @@ import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
 import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
 import org.apache.drill.common.expression.ValueExpressions.LongExpression;
+import org.apache.drill.common.expression.ValueExpressions.FloatExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntExpression;
 import org.apache.drill.common.expression.ValueExpressions.QuotedString;
 
 public final class AggregateChecker extends SimpleExprVisitor<Boolean>{
@@ -66,6 +68,16 @@ public final class AggregateChecker extends SimpleExprVisitor<Boolean>{
   }
 
   @Override
+  public Boolean visitIntConstant(IntExpression intExpr) {
+    return false;
+  }
+
+  @Override
+  public Boolean visitFloatConstant(FloatExpression fExpr) {
+    return false;
+  }
+
+  @Override
   public Boolean visitLongConstant(LongExpression intExpr) {
     return false;
   }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b7f9e600/common/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java
b/common/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java
index 02ce231..a951a60 100644
--- a/common/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java
+++ b/common/src/main/java/org/apache/drill/common/expression/visitors/ConstantChecker.java
@@ -25,6 +25,8 @@ import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
 import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
 import org.apache.drill.common.expression.ValueExpressions.LongExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntExpression;
+import org.apache.drill.common.expression.ValueExpressions.FloatExpression;
 import org.apache.drill.common.expression.ValueExpressions.QuotedString;
 
 public final class ConstantChecker extends SimpleExprVisitor<Boolean>{
@@ -60,6 +62,16 @@ public final class ConstantChecker extends SimpleExprVisitor<Boolean>{
   }
 
   @Override
+  public Boolean visitIntConstant(IntExpression intExpr) {
+    return true;
+  }
+
+  @Override
+  public Boolean visitFloatConstant(FloatExpression fExpr) {
+    return true;
+  }
+
+  @Override
   public Boolean visitLongConstant(LongExpression intExpr) {
     return true;
   }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b7f9e600/common/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java
b/common/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java
index db34ab7..87d146a 100644
--- a/common/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java
+++ b/common/src/main/java/org/apache/drill/common/expression/visitors/ExprVisitor.java
@@ -25,11 +25,16 @@ import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
 import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
 import org.apache.drill.common.expression.ValueExpressions.LongExpression;
 import org.apache.drill.common.expression.ValueExpressions.QuotedString;
+import org.apache.drill.common.expression.ValueExpressions.FloatExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntExpression;
+
 
 public interface ExprVisitor<T, VAL, EXCEP extends Exception> {
 	public T visitFunctionCall(FunctionCall call, VAL value) throws EXCEP;
 	public T visitIfExpression(IfExpression ifExpr, VAL value) throws EXCEP;
-	public T visitSchemaPath(SchemaPath path, VAL value) throws EXCEP;	
+	public T visitSchemaPath(SchemaPath path, VAL value) throws EXCEP;
+  public T visitIntConstant(IntExpression intExpr, VAL value) throws EXCEP;
+  public T visitFloatConstant(FloatExpression fExpr, VAL value) throws EXCEP;
 	public T visitLongConstant(LongExpression intExpr, VAL value) throws EXCEP;
 	public T visitDoubleConstant(DoubleExpression dExpr, VAL value) throws EXCEP;
 	public T visitBooleanConstant(BooleanExpression e, VAL value) throws EXCEP;

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b7f9e600/common/src/main/java/org/apache/drill/common/expression/visitors/SimpleExprVisitor.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/visitors/SimpleExprVisitor.java
b/common/src/main/java/org/apache/drill/common/expression/visitors/SimpleExprVisitor.java
index 65a939b..71266f7 100644
--- a/common/src/main/java/org/apache/drill/common/expression/visitors/SimpleExprVisitor.java
+++ b/common/src/main/java/org/apache/drill/common/expression/visitors/SimpleExprVisitor.java
@@ -21,6 +21,8 @@ import org.apache.drill.common.expression.FunctionCall;
 import org.apache.drill.common.expression.IfExpression;
 import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.common.expression.ValueExpressions.BooleanExpression;
+import org.apache.drill.common.expression.ValueExpressions.IntExpression;
+import org.apache.drill.common.expression.ValueExpressions.FloatExpression;
 import org.apache.drill.common.expression.ValueExpressions.DoubleExpression;
 import org.apache.drill.common.expression.ValueExpressions.LongExpression;
 import org.apache.drill.common.expression.ValueExpressions.QuotedString;
@@ -43,6 +45,16 @@ public abstract class SimpleExprVisitor<T> implements ExprVisitor<T,
Void, Runti
   }
 
   @Override
+  public T visitIntConstant(IntExpression intExpr, Void value) throws RuntimeException {
+    return visitIntConstant(intExpr);
+  }
+
+  @Override
+  public T visitFloatConstant(FloatExpression fExpr, Void value) throws RuntimeException
{
+    return visitFloatConstant(fExpr);
+  }
+
+  @Override
   public T visitLongConstant(LongExpression intExpr, Void value) throws RuntimeException
{
     return visitLongConstant(intExpr);
   }
@@ -66,6 +78,8 @@ public abstract class SimpleExprVisitor<T> implements ExprVisitor<T,
Void, Runti
   public abstract T visitFunctionCall(FunctionCall call);
   public abstract T visitIfExpression(IfExpression ifExpr);
   public abstract T visitSchemaPath(SchemaPath path);
+  public abstract T visitIntConstant(IntExpression intExpr);
+  public abstract T visitFloatConstant(FloatExpression fExpr);
   public abstract T visitLongConstant(LongExpression intExpr);
   public abstract T visitDoubleConstant(DoubleExpression dExpr);
   public abstract T visitBooleanConstant(BooleanExpression e);

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b7f9e600/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java
index 46dbbe9..4d07a8c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/sig/ConstantExpressionIdentifier.java
@@ -98,7 +98,19 @@ public class ConstantExpressionIdentifier implements ExprVisitor<Boolean,
Identi
   public Boolean visitSchemaPath(SchemaPath path, IdentityHashMap<LogicalExpression, Object>
value){
     return false;
   }
-  
+
+  @Override
+  public Boolean visitIntConstant(ValueExpressions.IntExpression intExpr, IdentityHashMap<LogicalExpression,
Object> value) throws RuntimeException {
+    value.put(intExpr, true);
+    return true;
+  }
+
+  @Override
+  public Boolean visitFloatConstant(ValueExpressions.FloatExpression fExpr, IdentityHashMap<LogicalExpression,
Object> value) throws RuntimeException {
+    value.put(fExpr, true);
+    return true;
+  }
+
   @Override
   public Boolean visitLongConstant(LongExpression intExpr, IdentityHashMap<LogicalExpression,
Object> value){
     value.put(intExpr, true);

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b7f9e600/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
index cbd722c..c0f34f4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/ExpressionTreeMaterializer.java
@@ -107,6 +107,16 @@ public class ExpressionTreeMaterializer {
     }
 
     @Override
+    public LogicalExpression visitIntConstant(ValueExpressions.IntExpression intExpr) {
+      return intExpr;
+    }
+
+    @Override
+    public LogicalExpression visitFloatConstant(ValueExpressions.FloatExpression fExpr) {
+      return fExpr;
+    }
+
+    @Override
     public LogicalExpression visitLongConstant(ValueExpressions.LongExpression intExpr) {
       return intExpr;
     }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b7f9e600/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java
index 76683f3..83ae35e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/HashFunctions.java
@@ -26,9 +26,7 @@ import org.apache.drill.exec.expr.annotations.FunctionTemplate;
 import org.apache.drill.exec.expr.annotations.FunctionTemplate.FunctionScope;
 import org.apache.drill.exec.expr.annotations.Output;
 import org.apache.drill.exec.expr.annotations.Param;
-import org.apache.drill.exec.expr.holders.BigIntHolder;
-import org.apache.drill.exec.expr.holders.IntHolder;
-import org.apache.drill.exec.expr.holders.VarBinaryHolder;
+import org.apache.drill.exec.expr.holders.*;
 import org.apache.drill.exec.record.RecordBatch;
 
 public class HashFunctions {
@@ -44,8 +42,122 @@ public class HashFunctions {
     }
 
   }
-  
-  @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL)
+
+  @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL
)
+  public static class NullableFloatHash implements DrillSimpleFunc {
+
+    @Param NullableFloat4Holder in;
+    @Output IntHolder out;
+
+    public void setup(RecordBatch incoming) {
+    }
+
+    public void eval() {
+      if (in.isSet == 0)
+        out.value = 0;
+      else
+        out.value = com.google.common.hash.Hashing.murmur3_128().hashInt(Float.floatToIntBits(in.value)).asInt();
+    }
+  }
+
+  @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL
)
+  public static class FloatHash implements DrillSimpleFunc {
+
+    @Param Float4Holder in;
+    @Output IntHolder out;
+
+    public void setup(RecordBatch incoming) {
+    }
+
+    public void eval() {
+      out.value = com.google.common.hash.Hashing.murmur3_128().hashInt(Float.floatToIntBits(in.value)).asInt();
+    }
+  }
+
+  @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL
)
+  public static class NullableDoubleHash implements DrillSimpleFunc {
+
+    @Param NullableFloat8Holder in;
+    @Output IntHolder out;
+
+    public void setup(RecordBatch incoming) {
+    }
+
+    public void eval() {
+      if (in.isSet == 0)
+        out.value = 0;
+      else
+        out.value = com.google.common.hash.Hashing.murmur3_128().hashLong(Double.doubleToLongBits(in.value)).asInt();
+    }
+  }
+
+  @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL
)
+  public static class DoubleHash implements DrillSimpleFunc {
+
+    @Param Float8Holder in;
+    @Output IntHolder out;
+
+    public void setup(RecordBatch incoming) {
+    }
+
+    public void eval() {
+      out.value = com.google.common.hash.Hashing.murmur3_128().hashLong(Double.doubleToLongBits(in.value)).asInt();
+    }
+  }
+
+  @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL
)
+  public static class NullableVarBinaryHash implements DrillSimpleFunc {
+
+    @Param NullableVarBinaryHolder in;
+    @Output IntHolder out;
+
+    public void setup(RecordBatch incoming) {
+    }
+
+    public void eval() {
+      if (in.isSet == 0)
+        out.value = 0;
+      else
+        out.value = org.apache.drill.exec.expr.fn.impl.HashHelper.hash(in.buffer.nioBuffer(),
0);
+    }
+  }
+
+  @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+  public static class NullableBigIntHash implements DrillSimpleFunc {
+
+    @Param NullableBigIntHolder in;
+    @Output IntHolder out;
+
+    public void setup(RecordBatch incoming) {
+    }
+
+    public void eval() {
+      // TODO: implement hash function for other types
+      if (in.isSet == 0)
+        out.value = 0;
+      else
+        out.value = com.google.common.hash.Hashing.murmur3_128().hashLong(in.value).asInt();
+    }
+  }
+
+  @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
+  public static class NullableIntHash implements DrillSimpleFunc {
+    @Param NullableIntHolder in;
+    @Output IntHolder out;
+
+    public void setup(RecordBatch incoming) {
+    }
+
+    public void eval() {
+      // TODO: implement hash function for other types
+      if (in.isSet == 0)
+        out.value = 0;
+      else
+        out.value = com.google.common.hash.Hashing.murmur3_128().hashInt(in.value).asInt();
+    }
+  }
+
+  @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
   public static class VarBinaryHash implements DrillSimpleFunc {
 
     @Param VarBinaryHolder in;
@@ -59,7 +171,7 @@ public class HashFunctions {
     }
   }
   
-  @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL)
+  @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
   public static class HashBigInt implements DrillSimpleFunc {
 
     @Param BigIntHolder in;
@@ -74,7 +186,7 @@ public class HashFunctions {
     }
   }
 
-  @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.NULL_IF_NULL)
+  @FunctionTemplate(name = "hash", scope = FunctionScope.SIMPLE, nulls = FunctionTemplate.NullHandling.INTERNAL)
   public static class IntHash implements DrillSimpleFunc {
     @Param IntHolder in;
     @Output IntHolder out;
@@ -87,5 +199,5 @@ public class HashFunctions {
       out.value = com.google.common.hash.Hashing.murmur3_128().hashInt(in.value).asInt();
     }
   }
-  
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b7f9e600/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNull.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNull.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNull.java
index a042a5a..a1d3368 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNull.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/IsNull.java
@@ -29,7 +29,7 @@ import org.apache.drill.exec.expr.holders.BitHolder;
 import org.apache.drill.exec.expr.holders.NullableFloat8Holder;
 import org.apache.drill.exec.record.RecordBatch;
 
-@FunctionTemplate(name = "isNull", scope = FunctionTemplate.FunctionScope.SIMPLE)
+@FunctionTemplate(name = "isNull", scope = FunctionTemplate.FunctionScope.SIMPLE, nulls =
FunctionTemplate.NullHandling.INTERNAL)
 public class IsNull implements DrillSimpleFunc {
 
   @Param NullableFloat8Holder input;

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b7f9e600/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/ResolverTypePrecedence.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/ResolverTypePrecedence.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/ResolverTypePrecedence.java
index 26ac961..f35215b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/ResolverTypePrecedence.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/ResolverTypePrecedence.java
@@ -41,34 +41,34 @@ public static final Map<MinorType, Integer> precedenceMap;
      */
     int i = 0;
     precedenceMap = new HashMap<MinorType, Integer>();
-   	precedenceMap.put(MinorType.NULL, i++);       // NULL is legal to implicitly be promoted
to any other type	 
-  	precedenceMap.put(MinorType.FIXEDBINARY, i++); // Fixed-length is promoted to var length
-  	precedenceMap.put(MinorType.VARBINARY, i++);
-    precedenceMap.put(MinorType.FIXEDCHAR, i++);
-   	precedenceMap.put(MinorType.VARCHAR, i++);
-    precedenceMap.put(MinorType.FIXED16CHAR, i++);
-   	precedenceMap.put(MinorType.VAR16CHAR, i++);
-   	precedenceMap.put(MinorType.BIT, i++);
-   	precedenceMap.put(MinorType.TINYINT, i++);   //type with few bytes is promoted to type
with more bytes ==> no data loss. 
-   	precedenceMap.put(MinorType.UINT1, i++);     //signed is legal to implicitly be promoted
to unsigned.   
-   	precedenceMap.put(MinorType.SMALLINT, i++);
-   	precedenceMap.put(MinorType.UINT2, i++); 
-  	precedenceMap.put(MinorType.INT, i++);
-  	precedenceMap.put(MinorType.UINT4, i++); 
-  	precedenceMap.put(MinorType.BIGINT, i++);
-  	precedenceMap.put(MinorType.UINT8, i++);
-  	precedenceMap.put(MinorType.MONEY, i++);
-  	precedenceMap.put(MinorType.DECIMAL4, i++);
-  	precedenceMap.put(MinorType.DECIMAL8, i++);
-  	precedenceMap.put(MinorType.DECIMAL12, i++);
-  	precedenceMap.put(MinorType.DECIMAL16, i++);
-  	precedenceMap.put(MinorType.FLOAT4, i++);
-  	precedenceMap.put(MinorType.FLOAT8, i++);
-  	precedenceMap.put(MinorType.TIME, i++);
-  	precedenceMap.put(MinorType.DATE, i++);
-  	precedenceMap.put(MinorType.DATETIME, i++);
-    precedenceMap.put(MinorType.TIMETZ, i++);
-    precedenceMap.put(MinorType.TIMESTAMP, i++);  	
+   	precedenceMap.put(MinorType.NULL, i += 2);       // NULL is legal to implicitly be promoted
to any other type
+  	precedenceMap.put(MinorType.FIXEDBINARY, i += 2); // Fixed-length is promoted to var length
+  	precedenceMap.put(MinorType.VARBINARY, i += 2);
+    precedenceMap.put(MinorType.FIXEDCHAR, i += 2);
+   	precedenceMap.put(MinorType.VARCHAR, i += 2);
+    precedenceMap.put(MinorType.FIXED16CHAR, i += 2);
+   	precedenceMap.put(MinorType.VAR16CHAR, i += 2);
+   	precedenceMap.put(MinorType.BIT, i += 2);
+   	precedenceMap.put(MinorType.TINYINT, i += 2);   //type with few bytes is promoted to
type with more bytes ==> no data loss.
+   	precedenceMap.put(MinorType.UINT1, i += 2);     //signed is legal to implicitly be promoted
to unsigned.
+   	precedenceMap.put(MinorType.SMALLINT, i += 2);
+   	precedenceMap.put(MinorType.UINT2, i += 2);
+  	precedenceMap.put(MinorType.INT, i += 2);
+  	precedenceMap.put(MinorType.UINT4, i += 2);
+  	precedenceMap.put(MinorType.BIGINT, i += 2);
+  	precedenceMap.put(MinorType.UINT8, i += 2);
+  	precedenceMap.put(MinorType.MONEY, i += 2);
+  	precedenceMap.put(MinorType.DECIMAL4, i += 2);
+  	precedenceMap.put(MinorType.DECIMAL8, i += 2);
+  	precedenceMap.put(MinorType.DECIMAL12, i += 2);
+  	precedenceMap.put(MinorType.DECIMAL16, i += 2);
+  	precedenceMap.put(MinorType.FLOAT4, i += 2);
+  	precedenceMap.put(MinorType.FLOAT8, i += 2);
+  	precedenceMap.put(MinorType.TIME, i += 2);
+  	precedenceMap.put(MinorType.DATE, i += 2);
+  	precedenceMap.put(MinorType.DATETIME, i += 2);
+    precedenceMap.put(MinorType.TIMETZ, i += 2);
+    precedenceMap.put(MinorType.TIMESTAMP, i += 2);
     
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b7f9e600/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java
index b16d1ff..12ca75d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/resolver/TypeCastRules.java
@@ -534,14 +534,28 @@ public class TypeCastRules {
       // Check null vs non-null, using same logic as that in Types.softEqual()
       // Only when the function uses NULL_IF_NULL, nullable and non-nullable are inter-changable.
       // Otherwise, the function implementation is not a match. 
-      if (argType.getMode() != parmType.getMode())
-        if (!((holder.getNullHandling() == NullHandling.NULL_IF_NULL) &&
-            (argType.getMode() == DataMode.OPTIONAL ||
-             argType.getMode() == DataMode.REQUIRED ||
-             parmType.getMode() == DataMode.OPTIONAL ||
-             parmType.getMode() == DataMode.REQUIRED )))
-          return -1;
-     
+      if (argType.getMode() != parmType.getMode()) {
+        // TODO - this does not seem to do what it is intended to
+//        if (!((holder.getNullHandling() == NullHandling.NULL_IF_NULL) &&
+//            (argType.getMode() == DataMode.OPTIONAL ||
+//             argType.getMode() == DataMode.REQUIRED ||
+//             parmType.getMode() == DataMode.OPTIONAL ||
+//             parmType.getMode() == DataMode.REQUIRED )))
+//          return -1;
+        // if the function is designed to take optional with custom null handling, and a
required
+        // is being passed, increase the cost to account for a null check
+        // this allows for a non-nullable implementation to be preferred
+        if (holder.getNullHandling() == NullHandling.INTERNAL) {
+          // a function that expects required output, but nullable was provided
+          if (parmType.getMode() == DataMode.REQUIRED && argType.getMode() == DataMode.OPTIONAL)
{
+            return -1;
+          }
+          else if (parmType.getMode() == DataMode.OPTIONAL && argType.getMode() ==
DataMode.REQUIRED) {
+            cost++;
+          }
+        }
+      }
+
       cost += (parmVal - argVal); 
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b7f9e600/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
index 25daa7a..f6a8096 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFunctions.java
@@ -19,13 +19,23 @@ package org.apache.drill.exec.physical.impl;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+
+import com.google.common.collect.Lists;
+import com.sun.codemodel.JClassAlreadyExistsException;
+import com.sun.codemodel.JVar;
 import mockit.Injectable;
 import mockit.NonStrictExpectations;
 
 import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.expression.ExpressionPosition;
-import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.expression.*;
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.Types;
 import org.apache.drill.common.util.FileUtils;
+import org.apache.drill.exec.compile.sig.GeneratorMapping;
+import org.apache.drill.exec.compile.sig.MappingSet;
+import org.apache.drill.exec.expr.ClassGenerator;
+import org.apache.drill.exec.expr.CodeGenerator;
+import org.apache.drill.exec.expr.fn.DrillFuncHolder;
 import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
 import org.apache.drill.exec.expr.holders.NullableVarBinaryHolder;
 import org.apache.drill.exec.expr.holders.NullableVarCharHolder;
@@ -34,10 +44,14 @@ import org.apache.drill.exec.memory.TopLevelAllocator;
 import org.apache.drill.exec.ops.FragmentContext;
 import org.apache.drill.exec.physical.PhysicalPlan;
 import org.apache.drill.exec.physical.base.FragmentRoot;
+import org.apache.drill.exec.physical.impl.aggregate.Aggregator;
+import org.apache.drill.exec.physical.impl.project.Projector;
 import org.apache.drill.exec.planner.PhysicalPlanReader;
 import org.apache.drill.exec.proto.CoordinationProtos;
 import org.apache.drill.exec.proto.ExecProtos;
 import org.apache.drill.exec.proto.BitControl.PlanFragment;
+import org.apache.drill.exec.resolver.FunctionResolver;
+import org.apache.drill.exec.resolver.FunctionResolverFactory;
 import org.apache.drill.exec.rpc.user.UserServer;
 import org.apache.drill.exec.server.DrillbitContext;
 import org.apache.drill.exec.vector.NullableVarBinaryVector;
@@ -49,12 +63,99 @@ import com.google.common.base.Charsets;
 import com.google.common.io.Files;
 import com.codahale.metrics.MetricRegistry;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
 public class TestSimpleFunctions {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestSimpleFunctions.class);
 
   DrillConfig c = DrillConfig.create();
 
   @Test
+  public void testHashFunctionResolution(@Injectable DrillConfig config) throws JClassAlreadyExistsException,
IOException {
+    FunctionImplementationRegistry registry = new FunctionImplementationRegistry(config);
+    // test required vs nullable Int input
+    resolveHash(config,
+        new TypedNullConstant(Types.optional(TypeProtos.MinorType.INT)),
+        Types.optional(TypeProtos.MinorType.INT),
+        Types.required(TypeProtos.MinorType.INT),
+        TypeProtos.DataMode.OPTIONAL,
+        registry);
+
+    resolveHash(config,
+        new ValueExpressions.IntExpression(1, ExpressionPosition.UNKNOWN),
+        Types.required(TypeProtos.MinorType.INT),
+        Types.required(TypeProtos.MinorType.INT),
+        TypeProtos.DataMode.REQUIRED,
+        registry);
+
+    // test required vs nullable float input
+    resolveHash(config,
+        new TypedNullConstant(Types.optional(TypeProtos.MinorType.FLOAT4)),
+        Types.optional(TypeProtos.MinorType.FLOAT4),
+        Types.required(TypeProtos.MinorType.FLOAT4),
+        TypeProtos.DataMode.OPTIONAL,
+        registry);
+
+    resolveHash(config,
+        new ValueExpressions.FloatExpression(5.0f, ExpressionPosition.UNKNOWN),
+        Types.required(TypeProtos.MinorType.FLOAT4),
+        Types.required(TypeProtos.MinorType.FLOAT4),
+        TypeProtos.DataMode.REQUIRED,
+        registry);
+
+    // test required vs nullable long input
+    resolveHash(config,
+        new TypedNullConstant(Types.optional(TypeProtos.MinorType.BIGINT)),
+        Types.optional(TypeProtos.MinorType.BIGINT),
+        Types.required(TypeProtos.MinorType.BIGINT),
+        TypeProtos.DataMode.OPTIONAL,
+        registry);
+
+    resolveHash(config,
+        new ValueExpressions.LongExpression(100L, ExpressionPosition.UNKNOWN),
+        Types.required(TypeProtos.MinorType.BIGINT),
+        Types.required(TypeProtos.MinorType.BIGINT),
+        TypeProtos.DataMode.REQUIRED,
+        registry);
+
+    // test required vs nullable double input
+    resolveHash(config,
+        new TypedNullConstant(Types.optional(TypeProtos.MinorType.FLOAT8)),
+        Types.optional(TypeProtos.MinorType.FLOAT8),
+        Types.required(TypeProtos.MinorType.FLOAT8),
+        TypeProtos.DataMode.OPTIONAL,
+        registry);
+
+    resolveHash(config,
+        new ValueExpressions.DoubleExpression(100.0, ExpressionPosition.UNKNOWN),
+        Types.required(TypeProtos.MinorType.FLOAT8),
+        Types.required(TypeProtos.MinorType.FLOAT8),
+        TypeProtos.DataMode.REQUIRED,
+        registry);
+  }
+
+  public void resolveHash(DrillConfig config, LogicalExpression arg, TypeProtos.MajorType
expectedArg,
+                                    TypeProtos.MajorType expectedOut, TypeProtos.DataMode
expectedBestInputMode,
+                                    FunctionImplementationRegistry registry) throws JClassAlreadyExistsException,
IOException {
+    List<LogicalExpression> args = new ArrayList<>();
+    args.add(arg);
+    String[] registeredNames = { "hash" };
+    FunctionCall call = new FunctionCall(
+        FunctionDefinition.simple("hash",
+          new BasicArgumentValidator(new Arg[]{ new Arg(expectedArg)}),
+          new OutputTypeDeterminer.FixedType(expectedOut),
+          registeredNames),
+        args,
+        ExpressionPosition.UNKNOWN
+    );
+    FunctionResolver resolver = FunctionResolverFactory.getResolver(call);
+    DrillFuncHolder matchedFuncHolder = resolver.getBestMatch(registry.getMethods().get(call.getDefinition().getName()),
call);
+    assertEquals( expectedBestInputMode, matchedFuncHolder.getParmMajorType(0).getMode());
+  }
+
+  @Test
   public void testIsNull(@Injectable final DrillbitContext bitContext,
                          @Injectable UserServer.UserClientConnection connection) throws Throwable
{
 


Mime
View raw message