carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From manishgupt...@apache.org
Subject carbondata git commit: [CARBONDATA-1258] CarbonData should not allow loading Date Type values violating the boundary condition ("0001-01-01" through "9999-12-31")
Date Thu, 21 Dec 2017 06:49:24 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master 71959dad6 -> 425f820bd


[CARBONDATA-1258] CarbonData should not allow loading Date Type values violating the boundary
condition ("0001-01-01" through "9999-12-31")

Spark date type, supporting "0001-01-01" through "9999-12-31". As carbon pushes the data to
spark layer so can support beyond this range.

This closes #1126


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/425f820b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/425f820b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/425f820b

Branch: refs/heads/master
Commit: 425f820bded52998e2da3bc8712a52a1544e7138
Parents: 71959da
Author: mohammadshahidkhan <mohdshahidkhan1987@gmail.com>
Authored: Mon Jul 3 15:48:40 2017 +0530
Committer: manishgupta88 <tomanishgupta18@gmail.com>
Committed: Thu Dec 21 12:22:09 2017 +0530

----------------------------------------------------------------------
 .../DateDirectDictionaryGenerator.java          | 28 +++++++
 .../DateDirectDictionaryGeneratorTest.java      | 84 ++++++++++++++++++++
 2 files changed, 112 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/425f820b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
index cd32e97..c49af9c 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
@@ -43,11 +43,33 @@ public class DateDirectDictionaryGenerator implements DirectDictionaryGenerator
   private String dateFormat;
 
   /**
+   * min value supported for date type column
+   */
+  private static final long MIN_VALUE;
+  /**
+   * MAx value supported for date type column
+   */
+  private static final long MAX_VALUE;
+  /**
    * Logger instance
    */
   private static final LogService LOGGER =
       LogServiceFactory.getLogService(DateDirectDictionaryGenerator.class.getName());
 
+  static {
+    SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd");
+    df.setTimeZone(TimeZone.getTimeZone("GMT"));
+    long minValue = 0;
+    long maxValue = 0;
+    try {
+      minValue = df.parse("0001-01-01").getTime();
+      maxValue = df.parse("9999-12-31").getTime();
+    } catch (ParseException e) {
+      // the Exception will not occur as constant value is being parsed
+    }
+    MIN_VALUE = minValue;
+    MAX_VALUE = maxValue;
+  }
   public DateDirectDictionaryGenerator(String dateFormat) {
     this.dateFormat = dateFormat;
     initialize();
@@ -142,6 +164,12 @@ public class DateDirectDictionaryGenerator implements DirectDictionaryGenerator
   }
 
   private int generateKey(long timeValue) {
+    if (timeValue < MIN_VALUE || timeValue > MAX_VALUE) {
+      if (LOGGER.isDebugEnabled()) {
+        LOGGER.debug("Value for date type column is not in valid range. Value considered
as null.");
+      }
+      return 1;
+    }
     return (int) Math.floor((double) timeValue / MILLIS_PER_DAY) + cutOffDate;
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/425f820b/core/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/DateDirectDictionaryGeneratorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/DateDirectDictionaryGeneratorTest.java
b/core/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/DateDirectDictionaryGeneratorTest.java
new file mode 100644
index 0000000..7390556
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/keygenerator/directdictionary/DateDirectDictionaryGeneratorTest.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.keygenerator.directdictionary;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.keygenerator.directdictionary.timestamp.TimeStampDirectDictionaryGenerator;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Unit test case for the DateDirectDictionaryGenerator
+ */
+public class DateDirectDictionaryGeneratorTest {
+  private DirectDictionaryGenerator dictionaryGenerator = null;
+
+  @Before public void setUp() throws Exception {
+    TimeStampDirectDictionaryGenerator generator = new TimeStampDirectDictionaryGenerator();
+    dictionaryGenerator = DirectDictionaryKeyGeneratorFactory
+        .getDirectDictionaryGenerator(DataTypes.DATE,
+            CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT);
+  }
+
+  /**
+   * The generated surrogateKey must be greater than 1
+   *
+   * @throws Exception
+   */
+  @Test public void lowerBoundaryValueTest() {
+    int surrogateKey = dictionaryGenerator.generateDirectSurrogateKey("0001-01-01");
+    Assert.assertTrue(surrogateKey > 1);
+  }
+  /**
+   * The generated surrogateKey must be greater than 1
+   *
+   * @throws Exception
+   */
+  @Test public void lowerBoundaryInvalidValueTest() {
+    int surrogateKey = dictionaryGenerator.generateDirectSurrogateKey("0001-01-00");
+    Assert.assertTrue(surrogateKey == 1);
+  }
+
+  /**
+   * The generated surrogateKey must be greater than 1
+   *
+   * @throws Exception
+   */
+  @Test public void upperBoundaryValueTest() {
+    int surrogateKey = dictionaryGenerator.generateDirectSurrogateKey("9999-12-31");
+    Assert.assertTrue(surrogateKey > 1);
+  }
+
+  /**
+   * The generated surrogateKey must be greater than 1
+   *
+   * @throws Exception
+   */
+  @Test public void upperBoundaryInvalidValueTest() {
+    int surrogateKey = dictionaryGenerator.generateDirectSurrogateKey("10000-12-31");
+    Assert.assertTrue(surrogateKey == 1);
+  }
+
+  @After public void tearDown() throws Exception {
+    dictionaryGenerator = null;
+  }
+}


Mime
View raw message