carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbono...@apache.org
Subject [14/56] [abbrv] incubator-carbondata git commit: [Issue - 685] Support null or empty value for timestamp column (#686)
Date Thu, 23 Jun 2016 14:16:02 GMT
[Issue - 685] Support null or empty value for timestamp column (#686)



Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/91951acf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/91951acf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/91951acf

Branch: refs/heads/master
Commit: 91951acf2f89a0f5d0cf340a273f5ede8a08afb2
Parents: f479e7e
Author: Mohammad Shahid Khan <mohdshahidkhan1987@gmail.com>
Authored: Fri Jun 17 23:43:25 2016 +0530
Committer: Venkata Ramana G <g.ramana.v@gmail.com>
Committed: Fri Jun 17 23:43:25 2016 +0530

----------------------------------------------------------------------
 .../TimeStampDirectDictionaryGenerator.java     | 21 +++--
 .../DirectDictionaryDimensionAggregator.java    | 21 +++--
 .../resolver/ConditionalFilterResolverImpl.java |  9 +-
 .../spark/src/test/resources/datasamplenull.csv |  3 +
 .../TimestampDataTypeNullDataTest.scala         | 92 ++++++++++++++++++++
 5 files changed, 127 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/91951acf/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
b/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
index d37dcf1..060adcb 100644
--- a/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
+++ b/core/src/main/java/org/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
@@ -1,4 +1,3 @@
-
 /*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
@@ -115,22 +114,27 @@ public class TimeStampDirectDictionaryGenerator implements DirectDictionaryGener
         .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
             CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
     timeParser.setLenient(false);
+    if (null == memberStr || memberStr.trim().isEmpty() || memberStr
+        .equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
+      return 1;
+    }
     Date dateToStr = null;
     try {
       dateToStr = timeParser.parse(memberStr);
     } catch (ParseException e) {
-      LOGGER.error("Cannot convert" + TIMESTAMP.toString() + " to Time/Long type value"
-          + e.getMessage());
+      LOGGER.error(
+          "Cannot convert" + TIMESTAMP.toString() + " to Time/Long type value" + e.getMessage());
     }
+    //adding +2 to reserve the first cuttOffDiff value for null or empty date
     if (null == dateToStr) {
       return -1;
     } else {
       if (cutOffTimeStamp >= 0) {
         int keyValue = (int) ((dateToStr.getTime() - cutOffTimeStamp) / granularityFactor);
-        return keyValue < 0 ? -1 : keyValue;
+        return keyValue < 0 ? -1 : keyValue + 2;
       } else {
         int keyValue = (int) (dateToStr.getTime() / granularityFactor);
-        return keyValue < 0 ? -1 : keyValue;
+        return keyValue < 0 ? -1 : keyValue + 2;
       }
     }
   }
@@ -142,11 +146,14 @@ public class TimeStampDirectDictionaryGenerator implements DirectDictionaryGener
    * @return member value/actual value Date
    */
   @Override public Object getValueFromSurrogate(int key) {
+    if (key == 1) {
+      return null;
+    }
     long timeStamp = 0;
     if (cutOffTimeStamp >= 0) {
-      timeStamp = (key * granularityFactor + cutOffTimeStamp);
+      timeStamp = ((key - 2) * granularityFactor + cutOffTimeStamp);
     } else {
-      timeStamp = key * granularityFactor;
+      timeStamp = (key - 2) * granularityFactor;
     }
     return timeStamp * 1000L;
   }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/91951acf/core/src/main/java/org/carbondata/query/carbon/aggregator/dimension/impl/DirectDictionaryDimensionAggregator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/carbon/aggregator/dimension/impl/DirectDictionaryDimensionAggregator.java
b/core/src/main/java/org/carbondata/query/carbon/aggregator/dimension/impl/DirectDictionaryDimensionAggregator.java
index 971e4cc..8794021 100644
--- a/core/src/main/java/org/carbondata/query/carbon/aggregator/dimension/impl/DirectDictionaryDimensionAggregator.java
+++ b/core/src/main/java/org/carbondata/query/carbon/aggregator/dimension/impl/DirectDictionaryDimensionAggregator.java
@@ -19,6 +19,7 @@
 package org.carbondata.query.carbon.aggregator.dimension.impl;
 
 import java.nio.ByteBuffer;
+import java.sql.Timestamp;
 
 import org.carbondata.core.constants.CarbonCommonConstants;
 import org.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
@@ -96,17 +97,19 @@ public class DirectDictionaryDimensionAggregator implements DimensionDataAggrega
       MeasureAggregator[] aggeragtor) {
     byte[] dimensionData = scannedResult.getDimensionKey(blockIndex);
     int surrogateKey = CarbonUtil.getSurrogateKey(dimensionData, buffer);
-    Object dataBasedOnDataType =
-        (long) directDictionaryGenerator.getValueFromSurrogate(surrogateKey) / 1000;
+    Object valueFromSurrogate = directDictionaryGenerator.getValueFromSurrogate(surrogateKey);
+    if (null != valueFromSurrogate) {
+      Timestamp dataBasedOnDataType = new Timestamp((long) valueFromSurrogate / 1000);
 
-    if (actualTypeAggregatorIndex.length > 0) {
-      for (int j = 0; j < actualTypeAggregatorIndex.length; j++) {
-        aggeragtor[aggregatorStartIndex + actualTypeAggregatorIndex[j]].agg(dataBasedOnDataType);
+      if (actualTypeAggregatorIndex.length > 0) {
+        for (int j = 0; j < actualTypeAggregatorIndex.length; j++) {
+          aggeragtor[aggregatorStartIndex + actualTypeAggregatorIndex[j]].agg(dataBasedOnDataType);
+        }
       }
-    }
-    if (numberTypeAggregatorIndex.length > 0) {
-      for (int j = 0; j < numberTypeAggregatorIndex.length; j++) {
-        aggeragtor[aggregatorStartIndex + numberTypeAggregatorIndex[j]].agg(dataBasedOnDataType);
+      if (numberTypeAggregatorIndex.length > 0) {
+        for (int j = 0; j < numberTypeAggregatorIndex.length; j++) {
+          aggeragtor[aggregatorStartIndex + numberTypeAggregatorIndex[j]].agg(dataBasedOnDataType);
+        }
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/91951acf/core/src/main/java/org/carbondata/query/filter/resolver/ConditionalFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/filter/resolver/ConditionalFilterResolverImpl.java
b/core/src/main/java/org/carbondata/query/filter/resolver/ConditionalFilterResolverImpl.java
index 4897736..59ef1be 100644
--- a/core/src/main/java/org/carbondata/query/filter/resolver/ConditionalFilterResolverImpl.java
+++ b/core/src/main/java/org/carbondata/query/filter/resolver/ConditionalFilterResolverImpl.java
@@ -24,6 +24,7 @@ import java.util.SortedMap;
 import org.carbondata.core.carbon.AbsoluteTableIdentifier;
 import org.carbondata.core.carbon.datastore.block.SegmentProperties;
 import org.carbondata.core.carbon.metadata.encoder.Encoding;
+import org.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension;
 import org.carbondata.query.carbon.executor.exception.QueryExecutionException;
 import org.carbondata.query.carbonfilterinterface.FilterExecuterType;
 import org.carbondata.query.expression.ColumnExpression;
@@ -80,9 +81,11 @@ public class ConditionalFilterResolverImpl implements FilterResolverIntf
{
         // column expression.
         // we need to check if the other expression contains column
         // expression or not in depth.
-        if (FilterUtil.checkIfExpressionContainsColumn(rightExp)||
-            FilterUtil.isExpressionNeedsToResolved(rightExp,isIncludeFilter) &&
-            columnExpression.getDimension().hasEncoding(Encoding.DICTIONARY)){
+        CarbonDimension dimension = columnExpression.getDimension();
+        if (FilterUtil.checkIfExpressionContainsColumn(rightExp)
+            || FilterUtil.isExpressionNeedsToResolved(rightExp, isIncludeFilter) &&
+            dimension.hasEncoding(Encoding.DICTIONARY) && !dimension
+            .hasEncoding(Encoding.DIRECT_DICTIONARY)) {
           isExpressionResolve = true;
         } else {
           //Visitor pattern is been used in this scenario inorder to populate the

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/91951acf/integration/spark/src/test/resources/datasamplenull.csv
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/resources/datasamplenull.csv b/integration/spark/src/test/resources/datasamplenull.csv
new file mode 100644
index 0000000..475bbe1
--- /dev/null
+++ b/integration/spark/src/test/resources/datasamplenull.csv
@@ -0,0 +1,3 @@
+ID,date,country,name,phonetype,serialname,salary
+1,2015/7/23,china,aaa1,phone197,ASD69643,15000
+2,,china,aaa2,phone756,ASD42892,15001
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/91951acf/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala
b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala
new file mode 100644
index 0000000..c8c1f81
--- /dev/null
+++ b/integration/spark/src/test/scala/org/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.carbondata.spark.testsuite.directdictionary
+
+import java.io.File
+import java.sql.Timestamp
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.common.util.CarbonHiveContext._
+import org.apache.spark.sql.common.util.QueryTest
+import org.apache.spark.sql.hive.HiveContext
+import org.carbondata.core.constants.CarbonCommonConstants
+import org.carbondata.core.keygenerator.directdictionary.timestamp.TimeStampGranularityConstants
+import org.carbondata.core.util.CarbonProperties
+import org.scalatest.BeforeAndAfterAll
+
+
+/**
+  * Test Class for detailed query on timestamp datatypes
+  *
+  *
+  */
+class TimestampDataTypeNullDataTest extends QueryTest with BeforeAndAfterAll {
+  var oc: HiveContext = _
+
+  override def beforeAll {
+    try {
+      CarbonProperties.getInstance()
+        .addProperty(TimeStampGranularityConstants.CARBON_CUTOFF_TIMESTAMP, "2000-12-13 02:10.00.0")
+      CarbonProperties.getInstance()
+        .addProperty(TimeStampGranularityConstants.CARBON_TIME_GRANULARITY,
+          TimeStampGranularityConstants.TIME_GRAN_SEC.toString
+        )
+      sql(
+        """CREATE TABLE IF NOT EXISTS timestampTyeNullData
+                     (ID Int, date Timestamp, country String,
+                     name String, phonetype String, serialname String, salary Int)
+                    STORED BY 'org.apache.carbondata.format'"""
+      )
+
+      CarbonProperties.getInstance()
+        .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy/mm/dd")
+      val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../")
+        .getCanonicalPath
+      var csvFilePath = currentDirectory + "/src/test/resources/datasamplenull.csv"
+      sql("LOAD DATA LOCAL INPATH '" + csvFilePath + "' INTO TABLE timestampTyeNullData").collect();
+
+    } catch {
+      case x: Throwable => CarbonProperties.getInstance()
+        .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+    }
+  }
+
+  test("SELECT max(date) FROM timestampTyeNullData where date is not null") {
+    checkAnswer(
+      sql("SELECT max(date) FROM timestampTyeNullData where date is not null"),
+      Seq(Row(Timestamp.valueOf("2015-01-23 00:07:00.0"))
+      )
+    )
+  }
+    test("SELECT * FROM timestampTyeNullData where date is null") {
+      checkAnswer(
+        sql("SELECT date FROM timestampTyeNullData where date is null"),
+        Seq(Row(null)
+        ))
+  }
+
+  override def afterAll {
+    sql("drop table timestampTyeNullData")
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
+    CarbonProperties.getInstance().addProperty("carbon.direct.dictionary", "false")
+  }
+
+}
\ No newline at end of file


Mime
View raw message