orc-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject [6/8] orc git commit: HIVE-13648. ORC schema evolution doesn't support the same type conversions for varchar, char, or decimal when max length, precision, or scale are different.
Date Fri, 01 Jul 2016 17:56:21 GMT
HIVE-13648. ORC schema evolution doesn't support the same type conversions for
varchar, char, or decimal when max length, precision, or scale are different.

Signed-off-by: Owen O'Malley <omalley@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/orc/repo
Commit: http://git-wip-us.apache.org/repos/asf/orc/commit/775bc2ad
Tree: http://git-wip-us.apache.org/repos/asf/orc/tree/775bc2ad
Diff: http://git-wip-us.apache.org/repos/asf/orc/diff/775bc2ad

Branch: refs/heads/branch-1.1
Commit: 775bc2ad91f83fbf1e99d1af10567767be1119af
Parents: 6b61f42
Author: Owen O'Malley <omalley@apache.org>
Authored: Thu Jun 30 10:26:24 2016 -0700
Committer: Owen O'Malley <omalley@apache.org>
Committed: Fri Jul 1 09:33:08 2016 -0700

----------------------------------------------------------------------
 .../orc/impl/ConvertTreeReaderFactory.java      | 28 +++++++++++---------
 .../org/apache/orc/impl/SchemaEvolution.java    |  6 ++---
 .../org/apache/orc/impl/TreeReaderFactory.java  |  2 +-
 3 files changed, 18 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/orc/blob/775bc2ad/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java b/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
index 753e5bc..eda47d3 100644
--- a/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
+++ b/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
@@ -257,7 +257,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
           bytesColVector.vector[elementNum],
           bytesColVector.start[elementNum], bytesColVector.length[elementNum],
           StandardCharsets.UTF_8);
- 
+
       return string;
     }
 
@@ -1323,6 +1323,8 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
 
     private DecimalTreeReader decimalTreeReader;
 
+    private final TypeDescription fileType;
+    private final TypeDescription readerType;
     private DecimalColumnVector fileDecimalColVector;
     private int filePrecision;
     private int fileScale;
@@ -1333,8 +1335,10 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
     DecimalFromDecimalTreeReader(int columnId, TypeDescription fileType, TypeDescription
readerType)
         throws IOException {
       super(columnId);
+      this.fileType = fileType;
       filePrecision = fileType.getPrecision();
       fileScale = fileType.getScale();
+      this.readerType = readerType;
       readerPrecision = readerType.getPrecision();
       readerScale = readerType.getScale();
       decimalTreeReader = new DecimalTreeReader(columnId, filePrecision, fileScale);
@@ -2248,7 +2252,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
       return new TimestampFromDecimalTreeReader(columnId, fileType, skipCorrupt);
 
     case DECIMAL:
-      // UNDONE: Decimal to Decimal conversion????
+      return new DecimalFromDecimalTreeReader(columnId, fileType, readerType);
 
     // Not currently supported conversion(s):
     case BINARY:
@@ -2354,8 +2358,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
       return new StringGroupFromStringGroupTreeReader(columnId, fileType, readerType);
 
     case CHAR:
-      throw new IllegalArgumentException("No conversion of type " +
-          readerType.getCategory() + " to self needed");
+      return new StringGroupFromStringGroupTreeReader(columnId, fileType, readerType);
 
     case BINARY:
       return new BinaryFromStringGroupTreeReader(columnId, fileType);
@@ -2411,8 +2414,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
       return new StringGroupFromStringGroupTreeReader(columnId, fileType, readerType);
 
     case VARCHAR:
-      throw new IllegalArgumentException("No conversion of type " +
-          readerType.getCategory() + " to self needed");
+      return new StringGroupFromStringGroupTreeReader(columnId, fileType, readerType);
 
     case BINARY:
       return new BinaryFromStringGroupTreeReader(columnId, fileType);
@@ -2628,11 +2630,11 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
    *   StringGroupFromFloatTreeReader (written)
    *   StringGroupFromDoubleTreeReader (written)
    *   StringGroupFromDecimalTreeReader (written)
-   *  
+   *
    *   String from Char/Varchar conversion
    *   Char from String/Varchar conversion
    *   Varchar from String/Char conversion
-   *  
+   *
    *   StringGroupFromTimestampTreeReader (written)
    *   StringGroupFromDateTreeReader (written)
    *   StringGroupFromBinaryTreeReader *****
@@ -2650,7 +2652,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
    *   TimestampFromDecimalTreeeReader (written)
    *   TimestampFromStringGroupTreeReader (written)
    *   TimestampFromDateTreeReader
-   * 
+   *
    *
    * To DATE:
    *   Convert from (STRING, CHAR, VARCHAR) using string conversion.
@@ -2780,7 +2782,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
       // Fall through.
     }
 
-    // Now look for the few cases we don't convert from 
+    // Now look for the few cases we don't convert from
     switch (fileType.getCategory()) {
 
     case BOOLEAN:
@@ -2799,8 +2801,8 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
       default:
         return true;
       }
-   
-    
+
+
     case STRING:
     case CHAR:
     case VARCHAR:
@@ -2836,7 +2838,7 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
       default:
         return true;
       }
-    
+
     case BINARY:
       switch (readerType.getCategory()) {
       // Not currently supported conversion(s):

http://git-wip-us.apache.org/repos/asf/orc/blob/775bc2ad/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java b/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java
index 68000d6..07b527d 100644
--- a/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java
+++ b/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java
@@ -100,12 +100,10 @@ public class SchemaEvolution {
           break;
         case CHAR:
         case VARCHAR:
-          // HIVE-13648: Look at ORC data type conversion edge cases (CHAR, VARCHAR, DECIMAL)
-          isOk = fileType.getMaxLength() == readerType.getMaxLength();
+          // We do conversion when same CHAR/VARCHAR type but different maxLength.
           break;
         case DECIMAL:
-          // HIVE-13648: Look at ORC data type conversion edge cases (CHAR, VARCHAR, DECIMAL)
-          // TODO we don't enforce scale and precision checks, but probably should
+          // We do conversion when same DECIMAL type but different precision/scale.
           break;
         case UNION:
         case MAP:

http://git-wip-us.apache.org/repos/asf/orc/blob/775bc2ad/java/core/src/java/org/apache/orc/impl/TreeReaderFactory.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/TreeReaderFactory.java b/java/core/src/java/org/apache/orc/impl/TreeReaderFactory.java
index d390ff4..3371e1e 100644
--- a/java/core/src/java/org/apache/orc/impl/TreeReaderFactory.java
+++ b/java/core/src/java/org/apache/orc/impl/TreeReaderFactory.java
@@ -2036,7 +2036,7 @@ public class TreeReaderFactory {
       return new NullTreeReader(0);
     }
     TypeDescription.Category readerTypeCategory = readerType.getCategory();
-    if (!fileType.getCategory().equals(readerTypeCategory) &&
+    if (!fileType.equals(readerType) &&
         (readerTypeCategory != TypeDescription.Category.STRUCT &&
          readerTypeCategory != TypeDescription.Category.MAP &&
          readerTypeCategory != TypeDescription.Category.LIST &&


Mime
View raw message