carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [1/2] incubator-carbondata git commit: added check for null value in expression result
Date Thu, 04 May 2017 06:15:07 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 35a286950 -> ccd8175f8


added check for null value in expression result


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/58751fd5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/58751fd5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/58751fd5

Branch: refs/heads/master
Commit: 58751fd573be920193f4256c6a204b4c4986aabf
Parents: 35a2869
Author: kunal642 <kunal.kapoor@knoldus.in>
Authored: Tue May 2 17:17:28 2017 +0530
Committer: ravipesala <ravi.pesala@gmail.com>
Committed: Thu May 4 11:43:21 2017 +0530

----------------------------------------------------------------------
 .../expression/conditional/InExpression.java    |  14 +-
 .../expression/conditional/NotInExpression.java |  12 +-
 .../src/test/resources/filter/datawithnull.csv  |   3 +
 .../test/resources/filter/datawithoutnull.csv   |   3 +
 .../ExpressionWithNullTestCase.scala            | 150 +++++++++++++++++++
 5 files changed, 180 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/58751fd5/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/InExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/InExpression.java
b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/InExpression.java
index fe0cb13..130063d 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/InExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/InExpression.java
@@ -82,7 +82,19 @@ public class InExpression extends BinaryConditionalExpression {
         setOfExprResult.add(val);
       }
     }
-    leftRsult.set(DataType.BOOLEAN, setOfExprResult.contains(leftRsult));
+    // Only left result needs to be checked for null because InExpression is basically
+    // an OR Operation on the list of predicates that are provided.
+    // Example: x in (1,2,null) would be converted to x=1 OR x=2 OR x=null.
+    // If any of the predicates is null then the result is unknown thus we will return false
+    // for x=null.
+    // Left check will cover both the cases when left and right is null therefore no need
+    // for a check on the right result.
+    // Example: (null==null) -> Left null return false, (1==null) would automatically
be false.
+    if (leftRsult.isNull()) {
+      leftRsult.set(DataType.BOOLEAN, false);
+    } else {
+      leftRsult.set(DataType.BOOLEAN, setOfExprResult.contains(leftRsult));
+    }
     return leftRsult;
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/58751fd5/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpression.java
b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpression.java
index 2c86724..2552f96 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpression.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpression.java
@@ -80,8 +80,18 @@ public class NotInExpression extends BinaryConditionalExpression {
         setOfExprResult.add(val);
       }
     }
+    // Both left and right results need to be checked for null because NotInExpression is
basically
+    // an And Operation on the list of predicates that are provided.
+    // Example: x in (1,2,null) would be converted to x=1 AND x=2 AND x=null.
+    // If any of the predicates is null then the result is unknown for all the predicates
thus
+    // we will return false for each of them.
+    for (ExpressionResult expressionResult: setOfExprResult) {
+      if (expressionResult.isNull() || leftRsult.isNull()) {
+        leftRsult.set(DataType.BOOLEAN, false);
+        return leftRsult;
+      }
+    }
     leftRsult.set(DataType.BOOLEAN, !setOfExprResult.contains(leftRsult));
-
     return leftRsult;
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/58751fd5/integration/spark-common-test/src/test/resources/filter/datawithnull.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/filter/datawithnull.csv b/integration/spark-common-test/src/test/resources/filter/datawithnull.csv
new file mode 100644
index 0000000..cf9f0b7
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/filter/datawithnull.csv
@@ -0,0 +1,3 @@
+1,emp1,1234
+2,emp2,4321
+3,emp3,xyz
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/58751fd5/integration/spark-common-test/src/test/resources/filter/datawithoutnull.csv
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/resources/filter/datawithoutnull.csv b/integration/spark-common-test/src/test/resources/filter/datawithoutnull.csv
new file mode 100644
index 0000000..e4ec272
--- /dev/null
+++ b/integration/spark-common-test/src/test/resources/filter/datawithoutnull.csv
@@ -0,0 +1,3 @@
+1,emp1,1234
+2,emp2,4321
+3,emp3,22
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/58751fd5/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ExpressionWithNullTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ExpressionWithNullTestCase.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ExpressionWithNullTestCase.scala
new file mode 100644
index 0000000..a421c7e
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ExpressionWithNullTestCase.scala
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.spark.testsuite.detailquery
+
+import org.apache.spark.sql.common.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+class ExpressionWithNullTestCase extends QueryTest with BeforeAndAfterAll {
+
+  override def beforeAll() {
+    sql("drop table if exists expression_test")
+    sql("drop table if exists expression_test_hive")
+    sql("drop table if exists expression")
+    sql("drop table if exists expression_hive")
+    sql("create table expression_test (id int, name string, number int) stored by 'carbondata'")
+    sql(s"load data local inpath '$resourcesPath/filter/datawithnull.csv' into table expression_test
options('FILEHEADER'='id,name,number')")
+    sql("""create table expression_test_hive (id int, name string, number int) row format
delimited fields terminated by ','""")
+    sql(s"load data local inpath '$resourcesPath/filter/datawithnull.csv' into table expression_test_hive")
+
+    sql("""create table expression (id int, name string, number int) row format delimited
fields terminated by ','""")
+    sql(s"load data local inpath '$resourcesPath/filter/datawithoutnull.csv' into table expression")
+    sql("""create table expression_hive (id int, name string, number int) row format delimited
fields terminated by ','""")
+    sql(s"load data local inpath '$resourcesPath/filter/datawithoutnull.csv' into table expression_hive")
+  }
+
+  test("test to check in expression with null values") {
+    checkAnswer(sql("select * from expression_test where id in (1,2,'')"), sql("select *
from expression_test_hive where id in (1,2,'')"))
+    checkAnswer(sql("select * from expression_test where id in ('')"), sql("select * from
expression_test_hive where id in ('')"))
+    checkAnswer(sql("select * from expression_test where number in (null)"), sql("select
* from expression_test_hive where number in (null)"))
+    checkAnswer(sql("select * from expression_test where number in (2)"), sql("select * from
expression_test_hive where number in (2)"))
+    checkAnswer(sql("select * from expression_test where number in (1,null)"), sql("select
* from expression_test_hive where number in (1,null)"))
+    checkAnswer(sql("select * from expression where number in (1,null)"), sql("select * from
expression_hive where number in (1,null)"))
+    checkAnswer(sql("select * from expression where id in (3)"), sql("select * from expression_hive
where id in (3)"))
+    checkAnswer(sql("select * from expression where id in ('2')"), sql("select * from expression_hive
where id in ('2')"))
+    checkAnswer(sql("select * from expression where id in (cast('2' as int))"), sql("select
* from expression_hive where id in (cast('2' as int))"))
+    checkAnswer(sql("select * from expression_test where id in (3)"), sql("select * from
expression_test_hive where id in (3)"))
+    checkAnswer(sql("select * from expression_test where id in ('2')"), sql("select * from
expression_test_hive where id in ('2')"))
+    checkAnswer(sql("select * from expression_test where id in (cast('2' as int))"), sql("select
* from expression_test_hive where id in (cast('2' as int))"))
+    checkAnswer(sql("select * from expression_test where id in (cast('null' as int))"), sql("select
* from expression_test_hive where id in (cast('null' as int))"))
+  }
+
+  test("test to check not in expression with null values") {
+    checkAnswer(sql("select * from expression_test where id not in (1,2,'')"), sql("select
* from expression_test_hive where id not in (1,2,'')"))
+    checkAnswer(sql("select * from expression_test where id not in ('')"), sql("select *
from expression_test_hive where id not in ('')"))
+    checkAnswer(sql("select * from expression_test where number not in (null)"), sql("select
* from expression_test_hive where number not in (null)"))
+    checkAnswer(sql("select * from expression_test where number not in (1,null)"), sql("select
* from expression_test_hive where number not in (1,null)"))
+    checkAnswer(sql("select * from expression where number not in (1,null)"), sql("select
* from expression_hive where number not in (1,null)"))
+    checkAnswer(sql("select * from expression where id not in (3)"), sql("select * from expression_hive
where id not in (3)"))
+    checkAnswer(sql("select * from expression where id not in ('2')"), sql("select * from
expression_hive where id not in ('2')"))
+    checkAnswer(sql("select * from expression where id not in (cast('2' as int))"), sql("select
* from expression_hive where id not in (cast('2' as int))"))
+    checkAnswer(sql("select * from expression_test where id not in (3)"), sql("select * from
expression_test_hive where id not in (3)"))
+    checkAnswer(sql("select * from expression_test where id not in ('2')"), sql("select *
from expression_test_hive where id not in ('2')"))
+    checkAnswer(sql("select * from expression_test where id not in (cast('2' as int))"),
sql("select * from expression_test_hive where id not in (cast('2' as int))"))
+    checkAnswer(sql("select * from expression_test where id not in (cast('null' as int))"),
sql("select * from expression_test_hive where id not in (cast('null' as int))"))
+  }
+
+  test("test to check equals expression with null values") {
+    checkAnswer(sql("select * from expression_test where id=''"), sql("select * from expression_test_hive
where id=''"))
+    checkAnswer(sql("select * from expression_test where id=' '"), sql("select * from expression_test_hive
where id=' '"))
+    checkAnswer(sql("select * from expression_test where number=null"), sql("select * from
expression_test_hive where number=null"))
+    checkAnswer(sql("select * from expression_test where id=3"), sql("select * from expression_test_hive
where id=3"))
+    checkAnswer(sql("select * from expression where number=null"), sql("select * from expression_hive
where number=null"))
+    checkAnswer(sql("select * from expression where id=2"), sql("select * from expression_hive
where id=2"))
+    checkAnswer(sql("select * from expression_test where id='2'"), sql("select * from expression_test_hive
where id='2'"))
+    checkAnswer(sql("select * from expression_test where cast(id as int)='2'"), sql("select
* from expression_test_hive where cast(id as int)='2'"))
+    checkAnswer(sql("select * from expression where id='2'"), sql("select * from expression_hive
where id='2'"))
+    checkAnswer(sql("select * from expression where cast(id as int)='null'"), sql("select
* from expression_hive where cast(id as int)='null'"))
+  }
+
+  test("test to check not equals expression with null values") {
+    checkAnswer(sql("select * from expression_test where name != ''"), sql("select * from
expression_test_hive where name != ''"))
+    checkAnswer(sql("select * from expression_test where name != ' '"), sql("select * from
expression_test_hive where name != ' '"))
+    checkAnswer(sql("select * from expression_test where id=3"), sql("select * from expression_test_hive
where id=3"))
+    checkAnswer(sql("select * from expression_test where number is not null"), sql("select
* from expression_test_hive where number is not null"))
+    checkAnswer(sql("select * from expression where number is not null"), sql("select * from
expression_hive where number is not null"))
+    checkAnswer(sql("select * from expression where id!=2"), sql("select * from expression_hive
where id!=2"))
+    checkAnswer(sql("select * from expression_test where id!='2'"), sql("select * from expression_test_hive
where id!='2'"))
+    checkAnswer(sql("select * from expression_test where cast(id as int)!='2'"), sql("select
* from expression_test_hive where cast(id as int)!='2'"))
+    checkAnswer(sql("select * from expression where id!='2'"), sql("select * from expression_hive
where id!='2'"))
+    checkAnswer(sql("select * from expression where cast(id as int)!='2'"), sql("select *
from expression_hive where cast(id as int)!='2'"))
+    checkAnswer(sql("select * from expression where cast(id as int)!='null'"), sql("select
* from expression_hive where cast(id as int)!='null'"))
+  }
+
+  test("test to check greater than equals to expression with null values") {
+    checkAnswer(sql("select * from expression_test where id >= ''"), sql("select * from
expression_test_hive where id >= ''"))
+    checkAnswer(sql("select * from expression_test where id >= ' '"), sql("select * from
expression_test_hive where id >= ' '"))
+    checkAnswer(sql("select * from expression_test where number >= null"), sql("select
* from expression_test_hive where number >= null"))
+    checkAnswer(sql("select * from expression where number >= null"), sql("select * from
expression_hive where number >= null"))
+    checkAnswer(sql("select * from expression where id>=2"), sql("select * from expression_hive
where id>=2"))
+    checkAnswer(sql("select * from expression_test where id>='2'"), sql("select * from
expression_test_hive where id>='2'"))
+    checkAnswer(sql("select * from expression_test where cast(id as int)>='2'"), sql("select
* from expression_test_hive where cast(id as int)>='2'"))
+    checkAnswer(sql("select * from expression where id>='2'"), sql("select * from expression_hive
where id>='2'"))
+    checkAnswer(sql("select * from expression where cast(id as int)>='2'"), sql("select
* from expression_hive where cast(id as int)>='2'"))
+    checkAnswer(sql("select * from expression where cast(id as int)>='null'"), sql("select
* from expression_hive where cast(id as int)>='null'"))
+  }
+
+  test("test to check less than equals to expression with null values") {
+    checkAnswer(sql("select * from expression_test where id <= ''"), sql("select * from
expression_test_hive where id <= ''"))
+    checkAnswer(sql("select * from expression_test where id <= ' '"), sql("select * from
expression_test_hive where id <= ' '"))
+    checkAnswer(sql("select * from expression_test where number <= null"), sql("select
* from expression_test_hive where number <= null"))
+    checkAnswer(sql("select * from expression where number <= null"), sql("select * from
expression_hive where number <= null"))
+    checkAnswer(sql("select * from expression where id<=2"), sql("select * from expression_hive
where id<=2"))
+    checkAnswer(sql("select * from expression_test where id<='2'"), sql("select * from
expression_test_hive where id<='2'"))
+    checkAnswer(sql("select * from expression_test where cast(id as int)<='2'"), sql("select
* from expression_test_hive where cast(id as int)<='2'"))
+    checkAnswer(sql("select * from expression where id<='2'"), sql("select * from expression_hive
where id<='2'"))
+    checkAnswer(sql("select * from expression where cast(id as int)<='2'"), sql("select
* from expression_hive where cast(id as int)<='2'"))
+    checkAnswer(sql("select * from expression where cast(id as int)<='null'"), sql("select
* from expression_hive where cast(id as int)<='null'"))
+  }
+
+  test("test to check greater than expression with null values") {
+    checkAnswer(sql("select * from expression_test where id > ''"), sql("select * from
expression_test_hive where id > ''"))
+    checkAnswer(sql("select * from expression_test where id > ' '"), sql("select * from
expression_test_hive where id > ' '"))
+    checkAnswer(sql("select * from expression_test where number > null"), sql("select
* from expression_test_hive where number > null"))
+    checkAnswer(sql("select * from expression where number > null"), sql("select * from
expression_hive where number > null"))
+    checkAnswer(sql("select * from expression where id>2"), sql("select * from expression_hive
where id>2"))
+    checkAnswer(sql("select * from expression_test where id>'2'"), sql("select * from
expression_test_hive where id>'2'"))
+    checkAnswer(sql("select * from expression_test where cast(id as int)>'2'"), sql("select
* from expression_test_hive where cast(id as int)>'2'"))
+    checkAnswer(sql("select * from expression where id>'2'"), sql("select * from expression_hive
where id>'2'"))
+    checkAnswer(sql("select * from expression where cast(id as int)>'2'"), sql("select
* from expression_hive where cast(id as int)>'2'"))
+    checkAnswer(sql("select * from expression where cast(id as int)>'null'"), sql("select
* from expression_hive where cast(id as int)>'null'"))
+  }
+
+  test("test to check less than expression with null values") {
+    checkAnswer(sql("select * from expression_test where id < ''"), sql("select * from
expression_test_hive where id < ''"))
+    checkAnswer(sql("select * from expression_test where id < ' '"), sql("select * from
expression_test_hive where id < ' '"))
+    checkAnswer(sql("select * from expression_test where number < null"), sql("select
* from expression_test_hive where number < null"))
+    checkAnswer(sql("select * from expression where number < null"), sql("select * from
expression_hive where number < null"))
+    checkAnswer(sql("select * from expression where id<2"), sql("select * from expression_hive
where id<2"))
+    checkAnswer(sql("select * from expression_test where id<'2'"), sql("select * from
expression_test_hive where id<'2'"))
+    checkAnswer(sql("select * from expression_test where cast(id as int)<'2'"), sql("select
* from expression_test_hive where cast(id as int)<'2'"))
+    checkAnswer(sql("select * from expression where id<'2'"), sql("select * from expression_hive
where id<'2'"))
+    checkAnswer(sql("select * from expression where cast(id as int)<'2'"), sql("select
* from expression_hive where cast(id as int)<'2'"))
+    checkAnswer(sql("select * from expression where cast(id as int)<'null'"), sql("select
* from expression_hive where cast(id as int)<'null'"))
+  }
+
+}


Mime
View raw message