carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chenliang...@apache.org
Subject [17/50] [abbrv] incubator-carbondata git commit: [Bug] Timestamp with different format compare to spark date format will fail to provide filter result. this is because while (#754)
Date Thu, 30 Jun 2016 17:42:04 GMT
[Bug] Timestamp with different format compare to spark date format will fail to provide filter
result. this is because while (#754)

querying the engine is trying to get the direct surrogate based in user defined time format
in properties file.

Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/5b6081d7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/5b6081d7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/5b6081d7

Branch: refs/heads/master
Commit: 5b6081d7b203411fcc2e4b941c6f67f62d8c2878
Parents: 73975b0
Author: sujith71955 <sujithchacko.2010@gmail.com>
Authored: Sun Jun 26 22:09:17 2016 +0800
Committer: Ravindra Pesala <ravi.pesala@gmail.com>
Committed: Sun Jun 26 19:39:17 2016 +0530

----------------------------------------------------------------------
 .../DirectDictionaryGenerator.java              | 13 ++++++++
 .../TimeStampDirectDictionaryGenerator.java     | 32 ++++++++++++++++----
 .../query/expression/ExpressionResult.java      |  9 ++++--
 .../visitor/CustomTypeDictionaryVisitor.java    |  4 ++-
 .../src/test/resources/data2_DiffTimeFormat.csv |  4 +++
 .../filterexpr/FilterProcessorTestCase.scala    | 25 +++++++++++++++
 6 files changed, 77 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5b6081d7/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/DirectDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/DirectDictionaryGenerator.java
b/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/DirectDictionaryGenerator.java
index 476d09f..145c89a 100644
--- a/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/DirectDictionaryGenerator.java
+++ b/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/DirectDictionaryGenerator.java
@@ -40,4 +40,17 @@ public interface DirectDictionaryGenerator {
    */
   Object getValueFromSurrogate(int key);
 
+  /**
+   * The method generate and returns the dictionary / surrogate key for direct dictionary
column
+   * This Method is called while executing filter queries for getting direct surrogate members.
+   * Currently the query engine layer only supports yyyy-MM-dd HH:mm:ss date format no matter
+   * in which format the data is been stored, so while retrieving the direct surrogate value
for
+   * filter member first it should be converted in date form as per above format and needs
to
+   * retrieve time stamp.
+   *
+   * @param member The member string value
+   * @return returns dictionary/ surrogate value
+   */
+  int generateDirectSurrogateKey(String memberStr, String format);
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5b6081d7/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
b/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
index 8493991..97baf3d 100644
--- a/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
+++ b/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
@@ -40,12 +40,6 @@ import static org.carbondata.core.keygenerator.directdictionary.timestamp.TimeSt
 public class TimeStampDirectDictionaryGenerator implements DirectDictionaryGenerator {
 
   /**
-   * Logger instance
-   */
-  private static final LogService LOGGER =
-      LogServiceFactory.getLogService(TimeStampDirectDictionaryGenerator.class.getName());
-
-  /**
    * The value of 1 unit of the SECOND, MINUTE, HOUR, or DAY in millis.
    */
   public static final long granularityFactor;
@@ -55,6 +49,11 @@ public class TimeStampDirectDictionaryGenerator implements DirectDictionaryGener
    * customized the start of position. for example "January 1, 2000"
    */
   public static final long cutOffTimeStamp;
+  /**
+   * Logger instance
+   */
+  private static final LogService LOGGER =
+      LogServiceFactory.getLogService(TimeStampDirectDictionaryGenerator.class.getName());
 
   /**
    * initialization block for granularityFactor and cutOffTimeStamp
@@ -119,6 +118,27 @@ public class TimeStampDirectDictionaryGenerator implements DirectDictionaryGener
         .equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
       return 1;
     }
+    return getDirectSurrogateForMember(memberStr, timeParser);
+  }
+
+  /**
+   * The method take member String as input and converts
+   * and returns the dictionary key
+   *
+   * @param memberStr date format string
+   * @return dictionary value
+   */
+  public int generateDirectSurrogateKey(String memberStr, String format) {
+    SimpleDateFormat timeParser = new SimpleDateFormat(format);
+    timeParser.setLenient(false);
+    if (null == memberStr || memberStr.trim().isEmpty() || memberStr
+        .equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
+      return 1;
+    }
+    return getDirectSurrogateForMember(memberStr, timeParser);
+  }
+
+  private int getDirectSurrogateForMember(String memberStr, SimpleDateFormat timeParser)
{
     Date dateToStr = null;
     try {
       dateToStr = timeParser.parse(memberStr);

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5b6081d7/core/src/main/java/org/carbondata/query/expression/ExpressionResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/expression/ExpressionResult.java b/core/src/main/java/org/carbondata/query/expression/ExpressionResult.java
index 1f258e2..067f2d3 100644
--- a/core/src/main/java/org/carbondata/query/expression/ExpressionResult.java
+++ b/core/src/main/java/org/carbondata/query/expression/ExpressionResult.java
@@ -243,9 +243,12 @@ public class ExpressionResult implements Comparable<ExpressionResult>
{
     try {
       switch (this.getDataType()) {
         case StringType:
-          SimpleDateFormat parser = new SimpleDateFormat(CarbonProperties.getInstance()
-              .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
-                  CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
+          // Currently the query engine layer only supports yyyy-MM-dd HH:mm:ss date format
+          // no matter in which format the data is been stored, so while retrieving the direct
+          // surrogate value for filter member first it should be converted in date form
as per
+          // above format and needs to retrieve time stamp.
+          SimpleDateFormat parser =
+              new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT);
           Date dateToStr;
           try {
             dateToStr = parser.parse(value.toString());

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5b6081d7/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
b/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
index 0aec1c1..81808e8 100644
--- a/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
+++ b/core/src/main/java/org/carbondata/query/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
@@ -25,6 +25,7 @@ import java.util.List;
 import org.carbondata.common.logging.LogService;
 import org.carbondata.common.logging.LogServiceFactory;
 import org.carbondata.core.carbon.AbsoluteTableIdentifier;
+import org.carbondata.core.constants.CarbonCommonConstants;
 import org.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
 import org.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
 import org.carbondata.query.expression.ColumnExpression;
@@ -71,7 +72,8 @@ public class CustomTypeDictionaryVisitor implements ResolvedFilterInfoVisitorInt
         .getDirectDictionaryGenerator(columnExpression.getDimension().getDataType());
     // Reading the dictionary value direct
     for (String filterMember : evaluateResultListFinal) {
-      surrogates.add(directDictionaryGenerator.generateDirectSurrogateKey(filterMember));
+      surrogates.add(directDictionaryGenerator.generateDirectSurrogateKey(filterMember,
+          CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
     }
     Collections.sort(surrogates);
     DimColumnFilterInfo columnFilterInfo = null;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5b6081d7/integration/spark/src/test/resources/data2_DiffTimeFormat.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/data2_DiffTimeFormat.csv b/integration/spark/src/test/resources/data2_DiffTimeFormat.csv
new file mode 100644
index 0000000..c5fe230
--- /dev/null
+++ b/integration/spark/src/test/resources/data2_DiffTimeFormat.csv
@@ -0,0 +1,4 @@
+ID,date,country,name,phonetype,serialname,salary
+4,07-10-2014 00:00:00,china,aaa4,phone2435,ASD66902,4
+8,07-20-2014 00:00:00,china,aaa5,phone2441,ASD90633,10
+6,07-25-2014 00:00:00,china,aaa6,phone294,ASD59961,15005
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5b6081d7/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/FilterProcessorTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/FilterProcessorTestCase.scala
b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/FilterProcessorTestCase.scala
index 325800b..41930df 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/FilterProcessorTestCase.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/filterexpr/FilterProcessorTestCase.scala
@@ -40,10 +40,34 @@ class FilterProcessorTestCase extends QueryTest with BeforeAndAfterAll
{
     sql("drop table if exists filtertestTables")
     sql("drop table if exists filtertestTablesWithDecimal")
     sql("drop table if exists filtertestTablesWithNull")
+    sql("drop table if exists filterWithTimeStamp")
     sql("CREATE TABLE filtertestTables (ID int, date Timestamp, country String, " +
       "name String, phonetype String, serialname String, salary int) " +
         "STORED BY 'org.apache.carbondata.format'"
     )
+    
+     CarbonProperties.getInstance()
+        .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "MM-dd-yyyy HH:mm:ss")
+        
+    sql("CREATE TABLE filterWithTimeStamp (ID int, date Timestamp, country String, " +
+      "name String, phonetype String, serialname String, salary int) " +
+        "STORED BY 'org.apache.carbondata.format'"
+    )
+    sql(
+      s"LOAD DATA LOCAL INPATH './src/test/resources/data2_DiffTimeFormat.csv' INTO TABLE
" +
+        s"filterWithTimeStamp " +
+        s"OPTIONS('DELIMITER'= ',', " +
+        s"'FILEHEADER'= '')"
+    )
+    
+     test("Time stamp filter with diff time format for load ") {
+    checkAnswer(
+      sql("select date  from filterWithTimeStamp where date > '2014-07-10 00:00:00'"),
+      Seq(Row(Timestamp.valueOf("2014-07-20 00:00:00.0")),
+        Row(Timestamp.valueOf("2014-07-25 00:00:00.0"))
+      )
+    )
+  }
     CarbonProperties.getInstance()
       .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/MM/dd")
     sql(
@@ -79,6 +103,7 @@ class FilterProcessorTestCase extends QueryTest with BeforeAndAfterAll
{
     )
   }
 
+    
   test("Is not null filter") {
     checkAnswer(
       sql("select id from filtertestTablesWithNull " + "where id is not null"),


Mime
View raw message