Author: cutting
Date: Fri Dec 13 23:32:56 2013
New Revision: 1550850

URL: http://svn.apache.org/r1550850
Log:
AVRO-1397. Java: Binary fragment tools can now read multiple objects from their input.  Contributed by Rob Turner.

Modified:
    avro/trunk/CHANGES.txt
    avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/BinaryFragmentToJsonTool.java
    avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/JsonToBinaryFragmentTool.java
    avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestJsonToFromBinaryFragmentTools.java

Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1550850&r1=1550849&r2=1550850&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Fri Dec 13 23:32:56 2013
@@ -15,6 +15,9 @@ Trunk (not yet released)
     AVRO-1400. Java: Add AvroDefault reflect annotation to specify
     default values. (cutting)
 
+    AVRO-1397. Java: Binary fragment tools can now read multiple
+    objects from their input.  (Rob Turner via cutting)
+
   IMPROVEMENTS
 
     AVRO-1355. Java: Reject schemas with duplicate field

Modified: avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/BinaryFragmentToJsonTool.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/BinaryFragmentToJsonTool.java?rev=1550850&r1=1550849&r2=1550850&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/BinaryFragmentToJsonTool.java (original)
+++ avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/BinaryFragmentToJsonTool.java Fri Dec 13 23:32:56 2013
@@ -22,15 +22,14 @@ import java.io.PrintStream;
 import java.util.List;
 
 import org.apache.avro.Schema;
+import org.apache.avro.io.BinaryDecoder;
 import org.apache.avro.io.DecoderFactory;
 import org.apache.avro.io.DatumReader;
 import org.apache.avro.io.DatumWriter;
 import org.apache.avro.io.EncoderFactory;
+import org.apache.avro.io.JsonEncoder;
 import org.apache.avro.generic.GenericDatumReader;
 import org.apache.avro.generic.GenericDatumWriter;
-import org.codehaus.jackson.JsonEncoding;
-import org.codehaus.jackson.JsonFactory;
-import org.codehaus.jackson.JsonGenerator;
 
 /** Converts an input file from Avro binary into JSON. */
 public class BinaryFragmentToJsonTool implements Tool {
@@ -47,14 +46,16 @@ public class BinaryFragmentToJsonTool im
 
     try {
       DatumReader<Object> reader = new GenericDatumReader<Object>(schema);
-      Object datum = reader.read(null,
-          DecoderFactory.get().binaryDecoder(input, null));
+      BinaryDecoder binaryDecoder =
+        DecoderFactory.get().binaryDecoder(input, null);
       DatumWriter<Object> writer = new GenericDatumWriter<Object>(schema);
-      JsonGenerator g =
-        new JsonFactory().createJsonGenerator(out, JsonEncoding.UTF8);
-      g.useDefaultPrettyPrinter();
-      writer.write(datum, EncoderFactory.get().jsonEncoder(schema, g));
-      g.flush();
+      JsonEncoder jsonEncoder = EncoderFactory.get().jsonEncoder(schema, out);
+      Object datum = null;
+      while (!binaryDecoder.isEnd()){
+        datum = reader.read(datum, binaryDecoder);
+        writer.write(datum, jsonEncoder);
+        jsonEncoder.flush();
+      }
       out.println();
       out.flush();
     } finally {

Modified: avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/JsonToBinaryFragmentTool.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/JsonToBinaryFragmentTool.java?rev=1550850&r1=1550849&r2=1550850&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/JsonToBinaryFragmentTool.java (original)
+++ avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/JsonToBinaryFragmentTool.java Fri Dec 13 23:32:56 2013
@@ -17,6 +17,7 @@
  */
 package org.apache.avro.tool;
 
+import java.io.EOFException;
 import java.io.InputStream;
 import java.io.PrintStream;
 import java.util.List;
@@ -27,6 +28,7 @@ import org.apache.avro.generic.GenericDa
 import org.apache.avro.io.Encoder;
 import org.apache.avro.io.EncoderFactory;
 import org.apache.avro.io.DecoderFactory;
+import org.apache.avro.io.JsonDecoder;
 
 /** Tool to convert JSON data into the binary form. */
 public class JsonToBinaryFragmentTool implements Tool {
@@ -43,14 +45,20 @@ public class JsonToBinaryFragmentTool im
     try {
     GenericDatumReader<Object> reader = 
         new GenericDatumReader<Object>(schema);
-      Object datum = reader.read(null,
-          DecoderFactory.get().jsonDecoder(schema, input));
     
+    JsonDecoder jsonDecoder = 
+      DecoderFactory.get().jsonDecoder(schema, input);
     GenericDatumWriter<Object> writer = 
         new GenericDatumWriter<Object>(schema);
     Encoder e = EncoderFactory.get().binaryEncoder(out, null);
-    writer.write(datum, e);
-    e.flush();
+    Object datum = null;
+    try {
+      while(true) {
+        datum = reader.read(datum, jsonDecoder);
+        writer.write(datum, e);
+        e.flush();
+      }
+    } catch (EOFException eofException) {}
     } finally {
       Util.close(input);
     }

Modified: avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestJsonToFromBinaryFragmentTools.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestJsonToFromBinaryFragmentTools.java?rev=1550850&r1=1550849&r2=1550850&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestJsonToFromBinaryFragmentTools.java (original)
+++ avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestJsonToFromBinaryFragmentTools.java Fri Dec 13 23:32:56 2013
@@ -26,6 +26,7 @@ import java.io.PrintStream;
 import java.util.Arrays;
 
 import org.apache.avro.Schema;
+import org.apache.avro.Schema.Type;
 import org.junit.Test;
 
 /**
@@ -33,35 +34,54 @@ import org.junit.Test;
  * and {@link BinaryFragmentToJsonTool}.
  */
 public class TestJsonToFromBinaryFragmentTools {
-  private static final Schema STRING_SCHEMA = Schema.parse("\"string\"");
+  private static final Schema STRING_SCHEMA = Schema.create(Type.STRING);
   private static final String UTF8 = "utf-8";
   private static final String AVRO = 
     "ZLong string implies readable length encoding.";
   private static final String JSON = 
-    "\"Long string implies readable length encoding.\"";
+    "\"Long string implies readable length encoding.\"\n";
 
   @Test
   public void testBinaryToJson() throws Exception {
+    binaryToJson(AVRO, JSON);
+  }
+  
+  @Test
+    public void testJsonToBinary() throws Exception {
+    jsonToBinary(JSON, AVRO);
+  }
+
+  @Test
+    public void testMultiBinaryToJson() throws Exception {
+    binaryToJson(AVRO + AVRO + AVRO, JSON + JSON + JSON);
+  }
+
+  @Test
+    public void testMultiJsonToBinary() throws Exception {
+    jsonToBinary(JSON + JSON + JSON, AVRO + AVRO + AVRO);
+  }
+
+  private void binaryToJson(String avro, String json) throws Exception {
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
     PrintStream p = new PrintStream(new BufferedOutputStream(baos));
     
     new BinaryFragmentToJsonTool().run(
-        new ByteArrayInputStream(AVRO.getBytes(UTF8)), // stdin
+        new ByteArrayInputStream(avro.getBytes(UTF8)), // stdin
         p, // stdout
         null, // stderr
         Arrays.asList(STRING_SCHEMA.toString(), "-"));
-    assertEquals(JSON + "\n", baos.toString(UTF8).replace("\r", ""));
+    assertEquals(json, baos.toString(UTF8).replace("\r", ""));
   }
   
-  @Test
-  public void testJsonToBinary() throws Exception {
+  private void jsonToBinary(String json, String avro) throws Exception {
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
     PrintStream p = new PrintStream(new BufferedOutputStream(baos));
+
     new JsonToBinaryFragmentTool().run(
-        new ByteArrayInputStream(JSON.getBytes(UTF8)), // stdin
+        new ByteArrayInputStream(json.getBytes(UTF8)), // stdin
         p, // stdout
         null, // stderr
         Arrays.asList(STRING_SCHEMA.toString(), "-"));
-    assertEquals(AVRO, baos.toString(UTF8));
+    assertEquals(avro, baos.toString(UTF8));
   }
 }