orc-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject orc git commit: ORC-367: Fix incorrect reads of boolean columns after a seek.
Date Tue, 22 May 2018 22:03:51 GMT
Repository: orc
Updated Branches:
  refs/heads/master 5120baaef -> b0504f55e


ORC-367: Fix incorrect reads of boolean columns after a seek.

Fix #272

Signed-off-by: Owen O'Malley <omalley@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/orc/repo
Commit: http://git-wip-us.apache.org/repos/asf/orc/commit/b0504f55
Tree: http://git-wip-us.apache.org/repos/asf/orc/tree/b0504f55
Diff: http://git-wip-us.apache.org/repos/asf/orc/diff/b0504f55

Branch: refs/heads/master
Commit: b0504f55e4a68de0c73b89ab5bccb2a673efe9b4
Parents: 5120baa
Author: Owen O'Malley <omalley@apache.org>
Authored: Tue May 22 14:43:06 2018 -0700
Committer: Owen O'Malley <omalley@apache.org>
Committed: Tue May 22 14:45:30 2018 -0700

----------------------------------------------------------------------
 .../org/apache/orc/impl/BitFieldReader.java     |  8 ++---
 .../org/apache/orc/impl/TestBitFieldReader.java | 35 ++++++++++++++++++++
 2 files changed, 38 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/orc/blob/b0504f55/java/core/src/java/org/apache/orc/impl/BitFieldReader.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/BitFieldReader.java b/java/core/src/java/org/apache/orc/impl/BitFieldReader.java
index 9102555..5daa204 100644
--- a/java/core/src/java/org/apache/orc/impl/BitFieldReader.java
+++ b/java/core/src/java/org/apache/orc/impl/BitFieldReader.java
@@ -24,9 +24,8 @@ import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector;
 
 public final class BitFieldReader {
   private final RunLengthByteReader input;
-  /** The number of bits in one item. Non-test code always uses 1. */
   private int current;
-  private byte currentIdx;
+  private byte currentIdx = 8;
 
   public BitFieldReader(InStream input) {
     this.input = new RunLengthByteReader(input);
@@ -42,8 +41,7 @@ public final class BitFieldReader {
   }
 
   public int next() throws IOException {
-    // mod 8
-    if ((currentIdx & 7) == 0) {
+    if (currentIdx > 7) {
       readByte();
     }
 
@@ -86,7 +84,7 @@ public final class BitFieldReader {
       readByte();
       currentIdx = (byte) consumed;
     } else {
-      currentIdx = 0;
+      currentIdx = 8;
     }
   }
 

http://git-wip-us.apache.org/repos/asf/orc/blob/b0504f55/java/core/src/test/org/apache/orc/impl/TestBitFieldReader.java
----------------------------------------------------------------------
diff --git a/java/core/src/test/org/apache/orc/impl/TestBitFieldReader.java b/java/core/src/test/org/apache/orc/impl/TestBitFieldReader.java
index 156f10a..f7a2a5c 100644
--- a/java/core/src/test/org/apache/orc/impl/TestBitFieldReader.java
+++ b/java/core/src/test/org/apache/orc/impl/TestBitFieldReader.java
@@ -111,4 +111,39 @@ public class TestBitFieldReader {
       in.skip(0);
     }
   }
+
+  @Test
+  public void testSeekSkip() throws Exception {
+    TestInStream.OutputCollector collect = new TestInStream.OutputCollector();
+    BitFieldWriter out = new BitFieldWriter(
+        new OutStream("test", 100, null, collect), 1);
+    final int COUNT = 256;
+    TestInStream.PositionCollector posn = new TestInStream.PositionCollector();
+    for(int i=0; i < COUNT; ++i) {
+      if (i == 200) {
+        out.getPosition(posn);
+      }
+      if (i < COUNT/2) {
+        out.write(i & 1);
+      } else {
+        out.write((i/3) & 1);
+      }
+    }
+    out.flush();
+    ByteBuffer inBuf = ByteBuffer.allocate(collect.buffer.size());
+    collect.buffer.setByteBuffer(inBuf, 0, collect.buffer.size());
+    inBuf.flip();
+    BitFieldReader in = new BitFieldReader(InStream.create("test", new ByteBuffer[]{inBuf},
+        new long[]{0}, inBuf.remaining(), null, 100));
+    in.seek(posn);
+    in.skip(10);
+    for(int r = 210; r < COUNT; ++r) {
+      int x = (int) in.next();
+      if (r < COUNT/2) {
+        assertEquals(r & 1, x);
+      } else {
+        assertEquals((r/3) & 1, x);
+      }
+    }
+  }
 }


Mime
View raw message