carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vimal...@apache.org
Subject [1/2] incubator-carbondata git commit: [CARBONDATA-177] Greater than and Less than filter returning wrong result
Date Wed, 31 Aug 2016 02:43:34 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master be642c972 -> 0d7fa0383


[CARBONDATA-177] Greater than and Less than filter returning wrong result


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/7a8ca539
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/7a8ca539
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/7a8ca539

Branch: refs/heads/master
Commit: 7a8ca5393bf636b2f1b7841f935de6ce9ede4a65
Parents: be642c9
Author: mohammadshahidkhan <mohdshahidkhan1987@gmail.com>
Authored: Thu Aug 25 01:32:02 2016 +0530
Committer: Vimal Das Kammath <vkammath@VIMALDK-M01.vmware.com>
Committed: Wed Aug 31 08:01:01 2016 +0530

----------------------------------------------------------------------
 .../RowLevelRangeGrtThanFiterExecuterImpl.java  |  7 +-
 .../RowLevelRangeLessThanFiterExecuterImpl.java |  7 +-
 .../src/test/resources/filter/datagrtlrt.csv    |  3 +
 .../filterexpr/TestGrtLessFilter.scala          | 96 ++++++++++++++++++++
 4 files changed, 109 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7a8ca539/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
b/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
index 8a215dc..10b83d7 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
@@ -178,8 +178,11 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
         start = CarbonUtil.getFirstIndexUsingBinarySearch(
             (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, startIndex, numerOfRows
- 1,
             filterValues[k], true);
-        start = CarbonUtil.nextGreaterValueToTarget(start,
-            (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, filterValues[k], numerOfRows);
+        if(start >= 0){
+          start = CarbonUtil.nextGreaterValueToTarget(start,
+              (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, filterValues[k],
+              numerOfRows);
+        }
         if (start < 0) {
           start = -(start + 1);
           if (start == numerOfRows) {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7a8ca539/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
b/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
index af7f135..31760e6 100644
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
@@ -222,10 +222,13 @@ public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecut
         start = CarbonUtil.getFirstIndexUsingBinarySearch(
             (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, startIndex, numerOfRows
- 1,
             filterValues[k], false);
-        start = CarbonUtil.nextLesserValueToTarget(start,
-            (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, filterValues[k]);
+        if(start >= 0) {
+          start = CarbonUtil.nextLesserValueToTarget(start,
+              (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, filterValues[k]);
+        }
         if (start < 0) {
           start = -(start + 1);
+
           if (start >= numerOfRows) {
             start = numerOfRows - 1;
           }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7a8ca539/integration/spark/src/test/resources/filter/datagrtlrt.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/filter/datagrtlrt.csv b/integration/spark/src/test/resources/filter/datagrtlrt.csv
new file mode 100644
index 0000000..e7acc28
--- /dev/null
+++ b/integration/spark/src/test/resources/filter/datagrtlrt.csv
@@ -0,0 +1,3 @@
+2015-7-23 12:07:28,china,15000
+2018-7-24 12:07:28,china,15001
+2017-7-25 12:07:28,china,15002
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/7a8ca539/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/TestGrtLessFilter.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/TestGrtLessFilter.scala
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/TestGrtLessFilter.scala
new file mode 100644
index 0000000..ea28252
--- /dev/null
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/TestGrtLessFilter.scala
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.spark.testsuite.filterexpr
+
+import java.io.File
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.spark.sql.common.util.CarbonHiveContext._
+import org.apache.spark.sql.common.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+/**
+ * Test cases for testing columns having \N or \null values for non numeric columns
+ */
+class TestGrtLessFilter extends QueryTest with BeforeAndAfterAll {
+
+  override def beforeAll {
+    sql("drop table if exists carbonTable")
+    sql("drop table if exists hiveTable")
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+        CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT
+      )
+    val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
+      .getCanonicalPath
+    val csvFilePath = currentDirectory + "/src/test/resources/filter/datagrtlrt.csv"
+    sql(
+      "CREATE TABLE IF NOT EXISTS carbonTable(date Timestamp, country String, salary Int)
STORED " +
+      "BY " +
+      "'carbondata'"
+    )
+    sql(
+      "create table if not exists hiveTable(date Timestamp, country String, salary Int)row
format" +
+      " delimited fields " +
+      "terminated by ','"
+    )
+    sql(
+      "LOAD DATA LOCAL INPATH '" + csvFilePath + "' into table carbonTable OPTIONS " +
+      "('FILEHEADER'='date,country,salary')"
+    )
+    sql(
+      "LOAD DATA local inpath '" + csvFilePath + "' INTO table hiveTable"
+    )
+  }
+
+
+  test("select * from carbonTable where date > cast('2017-7-25 12:07:29' as timestamp)")
{
+    checkAnswer(
+      sql("select * from carbonTable where date > cast('2017-7-25 12:07:29' as timestamp)"),
+      sql("select * from hiveTable where date > cast('2017-7-25 12:07:29' as timestamp)")
+    )
+  }
+  test("select * from carbonTable where date < cast('2017-7-25 12:07:29' as timestamp)")
{
+    checkAnswer(
+      sql("select * from carbonTable where date < cast('2017-7-25 12:07:29' as timestamp)"),
+      sql("select * from hiveTable where date < cast('2017-7-25 12:07:29' as timestamp)")
+    )
+  }
+  test("select * from carbonTable where date > cast('2018-7-24 12:07:28' as timestamp)")
{
+    checkAnswer(
+      sql("select * from carbonTable where date > cast('2018-7-24 12:07:28' as timestamp)"),
+      sql("select * from hiveTable where date > cast('2018-7-24 12:07:28' as timestamp)")
+    )
+  }
+  test("select * from carbonTable where date < cast('2018-7-24 12:07:28' as timestamp)")
{
+    checkAnswer(
+      sql("select * from carbonTable where date < cast('2018-7-24 12:07:28' as timestamp)"),
+      sql("select * from hiveTable where date < cast('2018-7-24 12:07:28' as timestamp)")
+    )
+  }
+
+  override def afterAll {
+    sql("drop table if exists carbonTable")
+    sql("drop table if exists hiveTable")
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+  }
+}


Mime
View raw message