avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r1551338 - in /avro/trunk: ./ lang/java/avro/src/main/java/org/apache/avro/io/ lang/java/tools/src/main/java/org/apache/avro/tool/ lang/java/tools/src/test/java/org/apache/avro/tool/
Date Mon, 16 Dec 2013 20:30:38 GMT
Author: cutting
Date: Mon Dec 16 20:30:38 2013
New Revision: 1551338

URL: http://svn.apache.org/r1551338
Log:
AVRO-1396. Java: Enable tojson command-line tool to pretty print output. Contributed by Rob
Turner.

Modified:
    avro/trunk/CHANGES.txt
    avro/trunk/lang/java/avro/src/main/java/org/apache/avro/io/EncoderFactory.java
    avro/trunk/lang/java/avro/src/main/java/org/apache/avro/io/JsonEncoder.java
    avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/BinaryFragmentToJsonTool.java
    avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/DataFileReadTool.java
    avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/JsonToBinaryFragmentTool.java
    avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java
    avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestJsonToFromBinaryFragmentTools.java

Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1551338&r1=1551337&r2=1551338&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Mon Dec 16 20:30:38 2013
@@ -18,6 +18,9 @@ Trunk (not yet released)
     AVRO-1397. Java: Binary fragment tools can now read multiple
     objects from their input.  (Rob Turner via cutting)
 
+    AVRO-1396. Java: Enable tojson command-line tool to pretty print output.
+    (Rob Turner via cutting)
+
   IMPROVEMENTS
 
     AVRO-1355. Java: Reject schemas with duplicate field

Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/io/EncoderFactory.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/io/EncoderFactory.java?rev=1551338&r1=1551337&r2=1551338&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/io/EncoderFactory.java (original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/io/EncoderFactory.java Mon Dec
16 20:30:38 2013
@@ -283,6 +283,29 @@ public class EncoderFactory {
   }
 
   /**
+   * Creates a {@link JsonEncoder} using the OutputStream provided for writing
+   * data conforming to the Schema provided with optional pretty printing.
+   * <p/>
+   * {@link JsonEncoder} buffers its output. Data may not appear on the
+   * underlying OutputStream until {@link Encoder#flush()} is called.
+   * <p/>
+   * {@link JsonEncoder} is not thread-safe.
+   * 
+   * @param schema
+   *          The Schema for data written to this JsonEncoder. Cannot be null.
+   * @param out
+   *          The OutputStream to write to. Cannot be null.
+   * @param pretty
+   *          Pretty print encoding.
+   * @return A JsonEncoder configured with <i>out</i>, <i>schema</i>
and <i>pretty</i>
+   * @throws IOException
+   */
+  public JsonEncoder jsonEncoder(Schema schema, OutputStream out, boolean pretty)
+      throws IOException {
+    return new JsonEncoder(schema, out, pretty);
+  }
+
+  /**
    * Creates a {@link JsonEncoder} using the {@link JsonGenerator} provided for
    * output of data conforming to the Schema provided.
    * <p/>

Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/io/JsonEncoder.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/io/JsonEncoder.java?rev=1551338&r1=1551337&r2=1551338&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/io/JsonEncoder.java (original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/io/JsonEncoder.java Mon Dec 16
20:30:38 2013
@@ -31,6 +31,7 @@ import org.apache.avro.util.Utf8;
 import org.codehaus.jackson.JsonEncoding;
 import org.codehaus.jackson.JsonFactory;
 import org.codehaus.jackson.JsonGenerator;
+import org.codehaus.jackson.util.DefaultPrettyPrinter;
 import org.codehaus.jackson.util.MinimalPrettyPrinter;
 
 /** An {@link Encoder} for Avro's JSON data encoding. 
@@ -43,6 +44,7 @@ import org.codehaus.jackson.util.Minimal
  * JsonEncoder is not thread-safe.
  * */
 public class JsonEncoder extends ParsingEncoder implements Parser.ActionHandler {
+  private static final String LINE_SEPARATOR = System.getProperty("line.separator");
   final Parser parser;
   private JsonGenerator out;
   /**
@@ -51,7 +53,11 @@ public class JsonEncoder extends Parsing
   protected BitSet isEmpty = new BitSet();
 
   JsonEncoder(Schema sc, OutputStream out) throws IOException {
-    this(sc, getJsonGenerator(out));
+    this(sc, getJsonGenerator(out, false));
+  }
+
+  JsonEncoder(Schema sc, OutputStream out, boolean pretty) throws IOException {
+    this(sc, getJsonGenerator(out, pretty));
   }
 
   JsonEncoder(Schema sc, JsonGenerator out) throws IOException {
@@ -68,16 +74,29 @@ public class JsonEncoder extends Parsing
     }
   }
 
-  // by default, one object per line
-  private static JsonGenerator getJsonGenerator(OutputStream out)
+  // by default, one object per line.
+  // with pretty option use default pretty printer with root line separator.
+  private static JsonGenerator getJsonGenerator(OutputStream out, boolean pretty)
       throws IOException {
     if (null == out)
       throw new NullPointerException("OutputStream cannot be null"); 
     JsonGenerator g
       = new JsonFactory().createJsonGenerator(out, JsonEncoding.UTF8);
-    MinimalPrettyPrinter pp = new MinimalPrettyPrinter();
-    pp.setRootValueSeparator(System.getProperty("line.separator"));
-    g.setPrettyPrinter(pp);
+    if (pretty) {
+      DefaultPrettyPrinter pp = new DefaultPrettyPrinter() {
+        //@Override
+        public void writeRootValueSeparator(JsonGenerator jg)
+            throws IOException
+        {
+          jg.writeRaw(LINE_SEPARATOR);
+        }
+      };
+      g.setPrettyPrinter(pp);
+    } else {
+      MinimalPrettyPrinter pp = new MinimalPrettyPrinter();
+      pp.setRootValueSeparator(LINE_SEPARATOR);
+      g.setPrettyPrinter(pp);
+    }
     return g;
   }
   
@@ -96,7 +115,7 @@ public class JsonEncoder extends Parsing
    * @return this JsonEncoder
    */
   public JsonEncoder configure(OutputStream out) throws IOException {
-    this.configure(getJsonGenerator(out));
+    this.configure(getJsonGenerator(out, false));
     return this;
   }
   

Modified: avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/BinaryFragmentToJsonTool.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/BinaryFragmentToJsonTool.java?rev=1551338&r1=1551337&r2=1551338&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/BinaryFragmentToJsonTool.java
(original)
+++ avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/BinaryFragmentToJsonTool.java
Mon Dec 16 20:30:38 2013
@@ -21,6 +21,10 @@ import java.io.InputStream;
 import java.io.PrintStream;
 import java.util.List;
 
+import joptsimple.OptionParser;
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+
 import org.apache.avro.Schema;
 import org.apache.avro.io.BinaryDecoder;
 import org.apache.avro.io.DecoderFactory;
@@ -36,20 +40,43 @@ public class BinaryFragmentToJsonTool im
   @Override
   public int run(InputStream stdin, PrintStream out, PrintStream err,
       List<String> args) throws Exception {
-    if (args.size() != 2) {
-      err.println("Expected 2 arguments: schema binary_data_file");
-      err.println("Use '-' as binary_data_file for stdin.");
+    OptionParser optionParser = new OptionParser();
+    OptionSpec<Void> noPrettyOption = optionParser
+        .accepts("no-pretty", "Turns off pretty printing.");
+    OptionSpec<String> schemaFileOption = optionParser
+        .accepts("schema-file", "File containing schema, must not occur with inline schema.")
+        .withOptionalArg()
+        .ofType(String.class);
+    
+    OptionSet optionSet = optionParser.parse(args.toArray(new String[0]));
+    Boolean noPretty = optionSet.has(noPrettyOption);
+    List<String> nargs = optionSet.nonOptionArguments();
+    String schemaFile = schemaFileOption.value(optionSet);
+    
+    if (nargs.size() != (schemaFile == null ? 2 : 1)) {
+      err.println("fragtojson --no-pretty --schema-file <file> [inline-schema] input-file");
+      err.println("   converts Avro fragments to JSON.");
+      optionParser.printHelpOn(err);
+      err.println("   A dash '-' for input-file means stdin.");
       return 1;
     }
-    Schema schema = new Schema.Parser().parse(args.get(0));
-    InputStream input = Util.fileOrStdin(args.get(1), stdin);
+    Schema schema;
+    String inputFile;
+    if (schemaFile == null) {
+      schema = new Schema.Parser().parse(nargs.get(0));
+      inputFile = nargs.get(1);
+    } else {
+      schema = new Schema.Parser().parse(Util.openFromFS(schemaFile));
+      inputFile = nargs.get(0);
+    }
+    InputStream input = Util.fileOrStdin(inputFile, stdin);
 
     try {
       DatumReader<Object> reader = new GenericDatumReader<Object>(schema);
       BinaryDecoder binaryDecoder =
         DecoderFactory.get().binaryDecoder(input, null);
       DatumWriter<Object> writer = new GenericDatumWriter<Object>(schema);
-      JsonEncoder jsonEncoder = EncoderFactory.get().jsonEncoder(schema, out);
+      JsonEncoder jsonEncoder = EncoderFactory.get().jsonEncoder(schema, out, !noPretty);
       Object datum = null;
       while (!binaryDecoder.isEnd()){
         datum = reader.read(datum, binaryDecoder);

Modified: avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/DataFileReadTool.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/DataFileReadTool.java?rev=1551338&r1=1551337&r2=1551338&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/DataFileReadTool.java (original)
+++ avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/DataFileReadTool.java Mon
Dec 16 20:30:38 2013
@@ -21,6 +21,10 @@ import java.io.InputStream;
 import java.io.PrintStream;
 import java.util.List;
 
+import joptsimple.OptionParser;
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+
 import org.apache.avro.Schema;
 import org.apache.avro.file.FileReader;
 import org.apache.avro.file.DataFileReader;
@@ -40,26 +44,36 @@ public class DataFileReadTool implements
 
   @Override
   public String getShortDescription() {
-    return "Dumps an Avro data file as JSON, one record per line.";
+    return "Dumps an Avro data file as JSON, record per line or pretty.";
   }
 
   @Override
   public int run(InputStream stdin, PrintStream out, PrintStream err,
       List<String> args) throws Exception {
-    if (args.size() != 1) {
+    OptionParser optionParser = new OptionParser();
+    OptionSpec<Void> prettyOption = optionParser
+        .accepts("pretty", "Turns on pretty printing.");
+
+    OptionSet optionSet = optionParser.parse(args.toArray(new String[0]));
+    Boolean pretty = optionSet.has(prettyOption);
+    List<String> nargs = optionSet.nonOptionArguments();
+
+    if (nargs.size() != 1) {
       // Unlike other commands, "-" can't be used for stdin, because
       // we can only use seekable files.
-      err.println("Expected 1 argument: input_file.");
+      err.println("tojson --pretty input-file");
+      err.println("   converts Avro data file to JSON.");
+      optionParser.printHelpOn(err);
       return 1;
     }
 
     GenericDatumReader<Object> reader = new GenericDatumReader<Object>();
     FileReader<Object> fileReader =
-      DataFileReader.openReader(Util.openSeekableFromFS(args.get(0)), reader);
+      DataFileReader.openReader(Util.openSeekableFromFS(nargs.get(0)), reader);
     try {
       Schema schema = fileReader.getSchema();
       DatumWriter<Object> writer = new GenericDatumWriter<Object>(schema);
-      JsonEncoder encoder = EncoderFactory.get().jsonEncoder(schema, out);
+      JsonEncoder encoder = EncoderFactory.get().jsonEncoder(schema, out, pretty);
       for (Object datum : fileReader)
         writer.write(datum, encoder);
       encoder.flush();

Modified: avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/JsonToBinaryFragmentTool.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/JsonToBinaryFragmentTool.java?rev=1551338&r1=1551337&r2=1551338&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/JsonToBinaryFragmentTool.java
(original)
+++ avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/JsonToBinaryFragmentTool.java
Mon Dec 16 20:30:38 2013
@@ -22,6 +22,10 @@ import java.io.InputStream;
 import java.io.PrintStream;
 import java.util.List;
 
+import joptsimple.OptionParser;
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+
 import org.apache.avro.Schema;
 import org.apache.avro.generic.GenericDatumReader;
 import org.apache.avro.generic.GenericDatumWriter;
@@ -35,30 +39,53 @@ public class JsonToBinaryFragmentTool im
   @Override
   public int run(InputStream stdin, PrintStream out, PrintStream err,
       List<String> args) throws Exception {
-    if (args.size() != 2) {
-      err.println("Expected 2 arguments: schema json_data_file");
-      err.println("Use '-' as json_data_file for stdin.");
+    OptionParser optionParser = new OptionParser();
+    OptionSpec<String> schemaFileOption = optionParser
+        .accepts("schema-file", "File containing schema, must not occur with inline schema.")
+        .withOptionalArg()
+        .ofType(String.class);
+    
+    OptionSet optionSet = optionParser.parse(args.toArray(new String[0]));
+    List<String> nargs = optionSet.nonOptionArguments();
+    String schemaFile = schemaFileOption.value(optionSet);
+    
+    if (nargs.size() != (schemaFile == null ? 2 : 1)) {
+      err.println("jsontofrag --schema-file <file> [inline-schema] input-file");
+      err.println("   converts JSON to Avro fragments.");
+      optionParser.printHelpOn(err);
+      err.println("   A dash '-' for input-file means stdin.");
       return 1;
     }
-    Schema schema = new Schema.Parser().parse(args.get(0));
-    InputStream input = Util.fileOrStdin(args.get(1), stdin);
+    Schema schema;
+    String inputFile;
+    if (schemaFile == null) {
+      schema = new Schema.Parser().parse(nargs.get(0));
+      inputFile = nargs.get(1);
+    } else {
+      schema = new Schema.Parser().parse(Util.openFromFS(schemaFile));
+      inputFile = nargs.get(0);
+    }
+    InputStream input = Util.fileOrStdin(inputFile, stdin);
+
     try {
-    GenericDatumReader<Object> reader = 
-        new GenericDatumReader<Object>(schema);
+      GenericDatumReader<Object> reader = 
+          new GenericDatumReader<Object>(schema);
     
-    JsonDecoder jsonDecoder = 
+      JsonDecoder jsonDecoder = 
       DecoderFactory.get().jsonDecoder(schema, input);
-    GenericDatumWriter<Object> writer = 
-        new GenericDatumWriter<Object>(schema);
-    Encoder e = EncoderFactory.get().binaryEncoder(out, null);
-    Object datum = null;
-    try {
+      GenericDatumWriter<Object> writer = 
+          new GenericDatumWriter<Object>(schema);
+      Encoder e = EncoderFactory.get().binaryEncoder(out, null);
+      Object datum = null;
       while(true) {
-        datum = reader.read(datum, jsonDecoder);
+        try {
+          datum = reader.read(datum, jsonDecoder);
+        } catch (EOFException eofException) {
+          break;
+        }
         writer.write(datum, e);
         e.flush();
       }
-    } catch (EOFException eofException) {}
     } finally {
       Util.close(input);
     }

Modified: avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java?rev=1551338&r1=1551337&r2=1551338&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java (original)
+++ avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java Mon
Dec 16 20:30:38 2013
@@ -100,6 +100,12 @@ public class TestDataFileTools {
   }
   
   @Test
+  public void testReadToJsonPretty() throws Exception {
+    assertEquals(jsonData.toString(),
+        run(new DataFileReadTool(), "--pretty", sampleFile.getPath()));
+  }
+  
+  @Test
   public void testGetMeta() throws Exception {
     String output = run(new DataFileGetMetaTool(), sampleFile.getPath());
     assertTrue(output, output.contains("avro.schema\t"+schema.toString()+"\n"));

Modified: avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestJsonToFromBinaryFragmentTools.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestJsonToFromBinaryFragmentTools.java?rev=1551338&r1=1551337&r2=1551338&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestJsonToFromBinaryFragmentTools.java
(original)
+++ avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestJsonToFromBinaryFragmentTools.java
Mon Dec 16 20:30:38 2013
@@ -22,11 +22,18 @@ import static org.junit.Assert.assertEqu
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
 import java.io.PrintStream;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
 
+import org.apache.avro.AvroTestUtil;
 import org.apache.avro.Schema;
 import org.apache.avro.Schema.Type;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 /**
@@ -34,7 +41,7 @@ import org.junit.Test;
  * and {@link BinaryFragmentToJsonTool}.
  */
 public class TestJsonToFromBinaryFragmentTools {
-  private static final Schema STRING_SCHEMA = Schema.create(Type.STRING);
+  private static final String STRING_SCHEMA = Schema.create(Type.STRING).toString();
   private static final String UTF8 = "utf-8";
   private static final String AVRO = 
     "ZLong string implies readable length encoding.";
@@ -43,45 +50,80 @@ public class TestJsonToFromBinaryFragmen
 
   @Test
   public void testBinaryToJson() throws Exception {
-    binaryToJson(AVRO, JSON);
+    binaryToJson(AVRO, JSON, STRING_SCHEMA);
   }
   
   @Test
     public void testJsonToBinary() throws Exception {
-    jsonToBinary(JSON, AVRO);
+    jsonToBinary(JSON, AVRO, STRING_SCHEMA);
   }
 
   @Test
     public void testMultiBinaryToJson() throws Exception {
-    binaryToJson(AVRO + AVRO + AVRO, JSON + JSON + JSON);
+    binaryToJson(AVRO + AVRO + AVRO, JSON + JSON + JSON, STRING_SCHEMA);
   }
 
   @Test
     public void testMultiJsonToBinary() throws Exception {
-    jsonToBinary(JSON + JSON + JSON, AVRO + AVRO + AVRO);
+    jsonToBinary(JSON + JSON + JSON, AVRO + AVRO + AVRO, STRING_SCHEMA);
   }
 
-  private void binaryToJson(String avro, String json) throws Exception {
+  @Test
+  public void testBinaryToNoPrettyJson() throws Exception {
+    binaryToJson(AVRO, JSON, "--no-pretty", STRING_SCHEMA);
+  }
+
+  @Test
+    public void testMultiBinaryToNoPrettyJson() throws Exception {
+    binaryToJson(AVRO + AVRO + AVRO, JSON + JSON + JSON, "--no-pretty", STRING_SCHEMA);
+  }
+
+  @Test
+  public void testBinaryToJsonSchemaFile() throws Exception {
+    binaryToJson(AVRO, JSON, "--schema-file", schemaFile());
+  }
+  
+  @Test
+    public void testJsonToBinarySchemaFile() throws Exception {
+    jsonToBinary(JSON, AVRO, "--schema-file", schemaFile());
+  }
+
+  private void binaryToJson(String avro, String json, String... options) throws Exception
{
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
     PrintStream p = new PrintStream(new BufferedOutputStream(baos));
     
+    List<String> args = new ArrayList<String>();
+    args.addAll(Arrays.asList(options));
+    args.add("-");
     new BinaryFragmentToJsonTool().run(
         new ByteArrayInputStream(avro.getBytes(UTF8)), // stdin
         p, // stdout
         null, // stderr
-        Arrays.asList(STRING_SCHEMA.toString(), "-"));
+        args);
+    System.out.println(baos.toString(UTF8).replace("\r", ""));
     assertEquals(json, baos.toString(UTF8).replace("\r", ""));
   }
-  
-  private void jsonToBinary(String json, String avro) throws Exception {
+
+  private void jsonToBinary(String json, String avro, String... options) throws Exception
{
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
     PrintStream p = new PrintStream(new BufferedOutputStream(baos));
 
+    List<String> args = new ArrayList<String>();
+    args.addAll(Arrays.asList(options));
+    args.add("-");
     new JsonToBinaryFragmentTool().run(
         new ByteArrayInputStream(json.getBytes(UTF8)), // stdin
         p, // stdout
         null, // stderr
-        Arrays.asList(STRING_SCHEMA.toString(), "-"));
+        args);
     assertEquals(avro, baos.toString(UTF8));
   }
+
+  private static String schemaFile() throws IOException {
+    File schemaFile = AvroTestUtil.tempFile(TestJsonToFromBinaryFragmentTools.class, "String.avsc");
+    FileWriter fw = new FileWriter(schemaFile);
+    fw.append(STRING_SCHEMA);
+    fw.close();
+    return schemaFile.toString();
+  }
 }



Mime
View raw message