carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [25/47] incubator-carbondata git commit: Problem: Null values getting display in not equals filter for timestamp datatype column (#878)
Date Mon, 01 Aug 2016 10:05:23 GMT
Problem: Null values getting display in not equals filter for timestamp datatype column (#878)

Analysis: While setting surrogates for exclude filter in custom dictionary visitor, surrogate
for null value is not getting added to surrogate list due to which the query displays null
value in case of dictionary column filters.

Impact: Not equals, equals, not null and is null query on timestamp datatype column

Fix: Add surrogate value for null member in exclude filter list

Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/e367e12b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/e367e12b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/e367e12b

Branch: refs/heads/master
Commit: e367e12b1c2523f12887af03d2a87547d4e897ae
Parents: 82b7407
Author: manishgupta88 <tomanishgupta18@gmail.com>
Authored: Wed Jul 27 20:30:49 2016 +0530
Committer: sujith71955 <sujithchacko.2010@gmail.com>
Committed: Wed Jul 27 20:30:49 2016 +0530

----------------------------------------------------------------------
 .../visitor/CustomTypeDictionaryVisitor.java    |  8 ++++++
 .../visitor/DictionaryColumnVisitor.java        |  4 ++-
 .../spark/src/test/resources/datasample.csv     |  1 +
 .../dataload/TestLoadDataWithNoMeasure.scala    |  2 +-
 ...estampDataTypeDirectDictionaryTestCase.scala | 29 ++++++++++++++++++--
 ...TypeDirectDictionaryWithNoDictTestCase.scala |  3 +-
 6 files changed, 42 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e367e12b/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
b/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
index fa0f0e9..a2e5e21 100644
--- a/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
+++ b/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
@@ -64,6 +64,14 @@ public class CustomTypeDictionaryVisitor implements ResolvedFilterInfoVisitorInt
     resolvedFilterObject = getDirectDictionaryValKeyMemberForFilter(metadata.getTableIdentifier(),
         metadata.getColumnExpression(), evaluateResultListFinal, metadata.isIncludeFilter(),
         isNotTimestampType);
+    if (!metadata.isIncludeFilter() && null != resolvedFilterObject && !resolvedFilterObject
+        .getFilterList().contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY))
{
+      // Adding default surrogate key of null member inorder to not display the same while
+      // displaying the report as per hive compatibility.
+      resolvedFilterObject.getFilterList()
+          .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
+      Collections.sort(resolvedFilterObject.getFilterList());
+    }
     visitableObj.setFilterValues(resolvedFilterObject);
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e367e12b/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
b/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
index 333a254..8488ad9 100644
--- a/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
+++ b/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
@@ -23,6 +23,7 @@ import java.util.List;
 
 import org.carbondata.common.logging.LogService;
 import org.carbondata.common.logging.LogServiceFactory;
+import org.carbondata.core.constants.CarbonCommonConstants;
 import org.carbondata.query.carbon.executor.exception.QueryExecutionException;
 import org.carbondata.query.expression.exception.FilterIllegalMemberException;
 import org.carbondata.query.expression.exception.FilterUnsupportedException;
@@ -61,7 +62,8 @@ public class DictionaryColumnVisitor implements ResolvedFilterInfoVisitorIntf
{
       if (!metadata.isIncludeFilter() && null != resolvedFilterObject) {
         // Adding default surrogate key of null member inorder to not display the same while
         // displaying the report as per hive compatibility.
-        resolvedFilterObject.getFilterList().add(1);
+        resolvedFilterObject.getFilterList()
+            .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
         Collections.sort(resolvedFilterObject.getFilterList());
       }
     } catch (QueryExecutionException e) {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e367e12b/integration/spark/src/test/resources/datasample.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/datasample.csv b/integration/spark/src/test/resources/datasample.csv
index fdf80e6..2f8ef35 100644
--- a/integration/spark/src/test/resources/datasample.csv
+++ b/integration/spark/src/test/resources/datasample.csv
@@ -1,3 +1,4 @@
 empno,doj,salary
 11,2016-04-14 15:00:09,5040.56
 12,2016-03-14 15:00:09,1040.56
+13,,1040.56

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e367e12b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithNoMeasure.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithNoMeasure.scala
b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithNoMeasure.scala
index f88e74c..9aa4189 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithNoMeasure.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/dataload/TestLoadDataWithNoMeasure.scala
@@ -48,7 +48,7 @@ class TestLoadDataWithNoMeasure extends QueryTest with BeforeAndAfterAll
{
 
     checkAnswer(
       sql("select empno from nomeasureTest"),
-      Seq(Row("11"), Row("12"))
+      Seq(Row("11"), Row("12"), Row("13"))
     )
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e367e12b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
index baff810..f325254 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
@@ -70,15 +70,40 @@ class TimestampDataTypeDirectDictionaryTest extends QueryTest with BeforeAndAfte
     }
   }
 
-  test("select doj from directDictionaryCube") {
+  test("test direct dictionary for not null condition") {
     checkAnswer(
-      sql("select doj from directDictionaryCube"),
+      sql("select doj from directDictionaryCube where doj is not null"),
       Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09.0")),
         Row(Timestamp.valueOf("2016-04-14 15:00:09.0"))
       )
     )
   }
 
+  test("test direct dictionary for getting all the values") {
+    checkAnswer(
+      sql("select doj from directDictionaryCube"),
+      Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09.0")),
+        Row(Timestamp.valueOf("2016-04-14 15:00:09.0")),
+        Row(null)
+      )
+    )
+  }
+
+  test("test direct dictionary for not equals condition") {
+    checkAnswer(
+      sql("select doj from directDictionaryCube where doj != '2016-04-14 15:00:09.0'"),
+      Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09.0"))
+      )
+    )
+  }
+
+  test("test direct dictionary for null condition") {
+    checkAnswer(
+      sql("select doj from directDictionaryCube where doj is null"),
+      Seq(Row(null)
+      )
+    )
+  }
 
   test("select doj from directDictionaryCube with equals filter") {
     checkAnswer(

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/e367e12b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala
b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala
index a62c632..c654e95 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala
@@ -74,7 +74,8 @@ class TimestampDataTypeDirectDictionaryWithNoDictTestCase extends QueryTest
with
     checkAnswer(
       sql("select doj from directDictionaryTable"),
       Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09.0")),
-        Row(Timestamp.valueOf("2016-04-14 15:00:09.0"))
+        Row(Timestamp.valueOf("2016-04-14 15:00:09.0")),
+        Row(null)
       )
     )
   }


Mime
View raw message