pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From knogu...@apache.org
Subject svn commit: r1855282 - in /pig/trunk: CHANGES.txt src/org/apache/pig/impl/util/hive/HiveUtils.java test/org/apache/pig/builtin/TestOrcStorage.java
Date Tue, 12 Mar 2019 04:27:49 GMT
Author: knoguchi
Date: Tue Mar 12 04:27:48 2019
New Revision: 1855282

URL: http://svn.apache.org/viewvc?rev=1855282&view=rev
Log:
PIG-5383: OrcStorage fails when "bytearray" represents unknown type (knoguchi)

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/impl/util/hive/HiveUtils.java
    pig/trunk/test/org/apache/pig/builtin/TestOrcStorage.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1855282&r1=1855281&r2=1855282&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Mar 12 04:27:48 2019
@@ -87,6 +87,9 @@ PIG-5251: Bump joda-time to 2.9.9 (dbist
 OPTIMIZATIONS
  
 BUG FIXES
+
+PIG-5383: OrcStorage fails when "bytearray" represents unknown type (knoguchi)
+
 PIG-5372: SAMPLE/RANDOM(udf) before skewed join failing with NPE (knoguchi)
 
 PIG-5374: Use CircularFifoBuffer in InterRecordReader (szita)

Modified: pig/trunk/src/org/apache/pig/impl/util/hive/HiveUtils.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/util/hive/HiveUtils.java?rev=1855282&r1=1855281&r2=1855282&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/util/hive/HiveUtils.java (original)
+++ pig/trunk/src/org/apache/pig/impl/util/hive/HiveUtils.java Tue Mar 12 04:27:48 2019
@@ -658,9 +658,13 @@ public class HiveUtils {
 
         @Override
         public BytesWritable getPrimitiveWritableObject(Object o) {
-            return o == null ? null : (o instanceof DataByteArray
+            try {
+                return o == null ? null : (o instanceof DataByteArray
                             ? new BytesWritable(((DataByteArray) o).get())
-                            : new BytesWritable((byte[]) o));
+                            : new BytesWritable((byte[]) DataType.toBytes(o)));
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
         }
 
         @Override

Modified: pig/trunk/test/org/apache/pig/builtin/TestOrcStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/builtin/TestOrcStorage.java?rev=1855282&r1=1855281&r2=1855282&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/builtin/TestOrcStorage.java (original)
+++ pig/trunk/test/org/apache/pig/builtin/TestOrcStorage.java Tue Mar 12 04:27:48 2019
@@ -20,6 +20,8 @@ package org.apache.pig.builtin;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
+import static org.apache.pig.builtin.mock.Storage.resetData;
+import static org.apache.pig.builtin.mock.Storage.tuple;
 
 import java.io.File;
 import java.io.FileNotFoundException;
@@ -57,6 +59,7 @@ import org.apache.hadoop.io.LongWritable
 import org.apache.hadoop.io.Text;
 import org.apache.pig.PigServer;
 import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
+import org.apache.pig.builtin.mock.Storage.Data;
 import org.apache.pig.data.BinSedesTuple;
 import org.apache.pig.data.DataByteArray;
 import org.apache.pig.data.DataType;
@@ -283,6 +286,28 @@ public class TestOrcStorage {
         assertTrue(t.toString().endsWith(",12345678.6547456)"));
     }
 
+    @Test
+    // See PIG-5383
+    public void testByteArrayStore() throws Exception {
+        Data data = resetData(pigServer);
+        data.set("foo", "intButTypeByteArray:bytearray",
+            tuple(1),
+            tuple(2),
+            tuple(3)
+        );
+
+        // Emunlating the case when "bytearray" represents an unknown type
+
+        pigServer.registerQuery("A = LOAD 'foo' USING mock.Storage();");
+        pigServer.registerQuery("store A into '" + OUTPUT1 + "' using OrcStorage();");
+        pigServer.registerQuery("B = load '" + OUTPUT1 + "' using OrcStorage();");
+        Iterator <Tuple> iter = pigServer.openIterator("B");
+        assertEquals(Integer.valueOf(1), DataType.toInteger(iter.next().get(0), DataType.BYTEARRAY));
+        assertEquals(Integer.valueOf(2), DataType.toInteger(iter.next().get(0), DataType.BYTEARRAY));
+        assertEquals(Integer.valueOf(3), DataType.toInteger(iter.next().get(0), DataType.BYTEARRAY));
+        assertFalse(iter.hasNext());
+    }
+
     private void verifyData(Path orcFile, Iterator<Tuple> iter, FileSystem fs, int
expectedTotalRows) throws Exception {
 
         int expectedRows = 0;



Mime
View raw message