orc-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject [1/2] orc git commit: HIVE-14333. ORC schema evolution from float to double break predicate pushdown. (Prasanth Jayachandran reviewed by Matt McCline)
Date Mon, 01 Aug 2016 23:48:18 GMT
Repository: orc
Updated Branches:
  refs/heads/master 7ac867d13 -> 14b818adc


HIVE-14333. ORC schema evolution from float to double break predicate pushdown.
(Prasanth Jayachandran reviewed by Matt McCline)

Signed-off-by: Owen O'Malley <omalley@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/orc/repo
Commit: http://git-wip-us.apache.org/repos/asf/orc/commit/90a0cf8a
Tree: http://git-wip-us.apache.org/repos/asf/orc/tree/90a0cf8a
Diff: http://git-wip-us.apache.org/repos/asf/orc/diff/90a0cf8a

Branch: refs/heads/master
Commit: 90a0cf8adc262c00eb592d0977e6f03edb6c9966
Parents: 7ac867d
Author: Owen O'Malley <omalley@apache.org>
Authored: Mon Aug 1 13:59:40 2016 -0700
Committer: Owen O'Malley <omalley@apache.org>
Committed: Mon Aug 1 14:00:44 2016 -0700

----------------------------------------------------------------------
 .../orc/impl/ConvertTreeReaderFactory.java      | 28 +++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/orc/blob/90a0cf8a/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java b/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
index eda47d3..87f379f 100644
--- a/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
+++ b/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java
@@ -982,8 +982,34 @@ public class ConvertTreeReaderFactory extends TreeReaderFactory {
     public void nextVector(ColumnVector previousVector,
                            boolean[] isNull,
                            final int batchSize) throws IOException {
-      // The DoubleColumnVector produced by FloatTreeReader is what we want.
+      // we get the DoubleColumnVector produced by float tree reader first, then iterate
through
+      // the elements and make double -> float -> string -> double conversion to
preserve the
+      // precision. When float tree reader reads float and assign it to double, java's widening
+      // conversion adds more precision which will break all comparisons.
+      // Example: float f = 74.72
+      // double d = f ---> 74.72000122070312
+      // Double.parseDouble(String.valueOf(f)) ---> 74.72
       floatTreeReader.nextVector(previousVector, isNull, batchSize);
+
+      DoubleColumnVector doubleColumnVector = (DoubleColumnVector) previousVector;
+      if (doubleColumnVector.isRepeating) {
+        if (doubleColumnVector.noNulls || !doubleColumnVector.isNull[0]) {
+          final float f = (float) doubleColumnVector.vector[0];
+          doubleColumnVector.vector[0] = Double.parseDouble(String.valueOf(f));
+        }
+      } else if (doubleColumnVector.noNulls){
+        for (int i = 0; i < batchSize; i++) {
+          final float f = (float) doubleColumnVector.vector[i];
+          doubleColumnVector.vector[i] = Double.parseDouble(String.valueOf(f));
+        }
+      } else {
+        for (int i = 0; i < batchSize; i++) {
+          if (!doubleColumnVector.isNull[i]) {
+            final float f = (float) doubleColumnVector.vector[i];
+            doubleColumnVector.vector[i] = Double.parseDouble(String.valueOf(f));
+          }
+        }
+      }
     }
   }
 


Mime
View raw message