avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r1480808 - in /avro/trunk: ./ lang/java/avro/src/main/java/org/apache/avro/ lang/java/avro/src/test/java/org/apache/avro/ lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/ lang/java/compiler/src/main/velocity/org/apache/av...
Date Thu, 09 May 2013 22:15:17 GMT
Author: cutting
Date: Thu May  9 22:15:16 2013
New Revision: 1480808

URL: http://svn.apache.org/r1480808
Log:
AVRO-1316. Java: Fix compiler to split long schema string constants so javac can compile them.
 Contributed by Jeremy Kahn.

Added:
    avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestSchema.java   (with props)
Modified:
    avro/trunk/CHANGES.txt
    avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Protocol.java
    avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Schema.java
    avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestProtocol.java
    avro/trunk/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java
    avro/trunk/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/protocol.vm
    avro/trunk/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm
    avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java
    avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestSpecificCompilerTool.java

Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1480808&r1=1480807&r2=1480808&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Thu May  9 22:15:16 2013
@@ -80,6 +80,9 @@ Trunk (not yet released)
     AVRO-1322. Java: Add Hadoop version classifier to trevni-avro
     Maven artifacts.  (massie)
 
+    AVRO-1316. Java: Fix compiler to split long schema string
+    constants so javac can compile them. (Jeremy Kahn via cutting)
+
 Avro 1.7.4 (22 February 2012)
 
   NEW FEATURES

Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Protocol.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Protocol.java?rev=1480808&r1=1480807&r2=1480808&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Protocol.java (original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Protocol.java Thu May  9 22:15:16
2013
@@ -366,6 +366,14 @@ public class Protocol extends JsonProper
     return parse(Schema.FACTORY.createJsonParser(stream));
   }
 
+  /** Read a protocol from one or more json strings */
+  public static Protocol parse(String string, String... more) {
+    StringBuilder b = new StringBuilder(string);
+    for (String part : more)
+      b.append(part);
+    return parse(b.toString());
+  }
+      
   /** Read a protocol from a Json string. */
   public static Protocol parse(String string) {
     try {

Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Schema.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Schema.java?rev=1480808&r1=1480807&r2=1480808&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Schema.java (original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Schema.java Thu May  9 22:15:16
2013
@@ -915,6 +915,14 @@ public abstract class Schema extends Jso
       return parse(FACTORY.createJsonParser(in));
     }
 
+    /** Read a schema from one or more json strings */
+    public Schema parse(String s, String... more) {
+      StringBuilder b = new StringBuilder(s);
+      for (String part : more)
+        b.append(part);
+      return parse(b.toString());
+    }
+      
     /** Parse a schema from the provided string.
      * If named, the schema is added to the names known to this parser. */
     public Schema parse(String s) {

Modified: avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestProtocol.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestProtocol.java?rev=1480808&r1=1480807&r2=1480808&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestProtocol.java (original)
+++ avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestProtocol.java Thu May  9 22:15:16
2013
@@ -17,8 +17,11 @@
  */
 package org.apache.avro;
 
-import org.junit.Test;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+
+import org.junit.Test;
 
 public class TestProtocol {
 
@@ -29,5 +32,26 @@ public class TestProtocol {
     p2.addProp("a","2");
     assertFalse(p1.equals(p2));
   }
+  
+  @Test
+  public void testSplitProtocolBuild() {
+    Protocol p = new Protocol("P", null, "foo");
+    p.addProp("property", "some value");
+     
+    String protocolString = p.toString();
+    final int mid = protocolString.length() / 2;
+    String[] parts = {
+      protocolString.substring(0, mid),
+      protocolString.substring(mid),
+    }; 
+    
+    Protocol parsedStringProtocol = org.apache.avro.Protocol.parse(protocolString);
+    Protocol parsedArrayOfStringProtocol =
+      org.apache.avro.Protocol.parse(protocolString.substring(0, mid),
+                                     protocolString.substring(mid));
 
+    assertNotNull(parsedStringProtocol);
+    assertNotNull(parsedArrayOfStringProtocol);
+    assertEquals(parsedStringProtocol.toString(), parsedArrayOfStringProtocol.toString());
+  }
 }

Added: avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestSchema.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestSchema.java?rev=1480808&view=auto
==============================================================================
--- avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestSchema.java (added)
+++ avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestSchema.java Thu May  9 22:15:16
2013
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.avro;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+import org.junit.Test;
+
+public class TestSchema {  
+  @Test
+  public void testSplitSchemaBuild() {
+    Schema s = SchemaBuilder
+       .recordType("HandshakeRequest")
+       .namespace("org.apache.avro.ipc")
+       .unionType("clientProtocol", SchemaBuilder.unionType(
+           SchemaBuilder.NULL,
+           SchemaBuilder.STRING)
+           .build())
+       .unionType("meta", SchemaBuilder.unionType(
+           SchemaBuilder.NULL,
+           SchemaBuilder.mapType(SchemaBuilder.BYTES)
+             .build())
+           .build())
+       .build();
+    
+    String schemaString = s.toString();
+    final int mid = schemaString.length() / 2;
+    
+    Schema parsedStringSchema = new org.apache.avro.Schema.Parser().parse(s.toString());
+    Schema parsedArrayOfStringSchema =
+      new org.apache.avro.Schema.Parser().parse
+      (schemaString.substring(0, mid), schemaString.substring(mid));
+    assertNotNull(parsedStringSchema);
+    assertNotNull(parsedArrayOfStringSchema);
+    assertEquals(parsedStringSchema.toString(), parsedArrayOfStringSchema.toString());
+  }
+}

Propchange: avro/trunk/lang/java/avro/src/test/java/org/apache/avro/TestSchema.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: avro/trunk/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java?rev=1480808&r1=1480807&r2=1480808&view=diff
==============================================================================
--- avro/trunk/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java
(original)
+++ avro/trunk/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java
Thu May  9 22:15:16 2013
@@ -548,6 +548,25 @@ public class SpecificCompiler {
     return new String[0];
   }
 
+  // maximum size for string constants, to avoid javac limits
+  int maxStringChars = 8192;
+
+  /** Utility for template use. Takes a (potentially overly long) string and
+   *  splits it into a quoted, comma-separted sequence of escaped strings.
+   *  @param s The string to split
+   *  @return A sequence of quoted, comma-separated, escaped strings
+   */
+  public String javaSplit(String s) throws IOException {
+    StringBuilder b = new StringBuilder("\"");    // initial quote
+    for (int i = 0; i < s.length(); i += maxStringChars) {
+      if (i != 0) b.append("\",\"");              // insert quote-comma-quote
+      String chunk = s.substring(i, Math.min(s.length(), i + maxStringChars));
+      b.append(javaEscape(chunk));                // escape chunks
+    }
+    b.append("\"");                               // final quote
+    return b.toString();
+  }
+  
   /** Utility for template use.  Escapes quotes and backslashes. */
   public static String javaEscape(Object o) {
       return o.toString().replace("\\","\\\\").replace("\"", "\\\"");

Modified: avro/trunk/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/protocol.vm
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/protocol.vm?rev=1480808&r1=1480807&r2=1480808&view=diff
==============================================================================
--- avro/trunk/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/protocol.vm
(original)
+++ avro/trunk/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/protocol.vm
Thu May  9 22:15:16 2013
@@ -28,7 +28,7 @@ package $protocol.getNamespace();
 #end
 @org.apache.avro.specific.AvroGenerated
 public interface $this.mangle($protocol.getName()) {
-  public static final org.apache.avro.Protocol PROTOCOL = org.apache.avro.Protocol.parse("${this.javaEscape($protocol.toString())}");
+  public static final org.apache.avro.Protocol PROTOCOL = org.apache.avro.Protocol.parse(${this.javaSplit($protocol.toString())});
 #foreach ($e in $protocol.getMessages().entrySet())
 #set ($name = $e.getKey())
 #set ($message = $e.getValue())

Modified: avro/trunk/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm?rev=1480808&r1=1480807&r2=1480808&view=diff
==============================================================================
--- avro/trunk/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm
(original)
+++ avro/trunk/lang/java/compiler/src/main/velocity/org/apache/avro/compiler/specific/templates/java/classic/record.vm
Thu May  9 22:15:16 2013
@@ -27,7 +27,7 @@ package $schema.getNamespace();  
 #end
 @org.apache.avro.specific.AvroGenerated
 public class ${this.mangle($schema.getName())}#if ($schema.isError()) extends org.apache.avro.specific.SpecificExceptionBase#else
extends org.apache.avro.specific.SpecificRecordBase#end implements org.apache.avro.specific.SpecificRecord
{
-  public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("${this.javaEscape($schema.toString())}");
+  public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse(${this.javaSplit($schema.toString())});
   public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; }
 #foreach ($field in $schema.getFields())
 #if ($field.doc())

Modified: avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java?rev=1480808&r1=1480807&r2=1480808&view=diff
==============================================================================
--- avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java
(original)
+++ avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/compiler/specific/TestSpecificCompiler.java
Thu May  9 22:15:16 2013
@@ -123,9 +123,7 @@ public class TestSpecificCompiler {
 
   }
 
-  @Test
-  public void testManglingForRecords() throws IOException {
-    String schema = "" +
+  private static String SCHEMA =
       "{ \"name\": \"volatile\", \"type\": \"record\", " +
       "  \"fields\": [ {\"name\": \"package\", \"type\": \"string\" }," +
       "                {\"name\": \"data\", \"type\": \"int\" }," +
@@ -133,8 +131,12 @@ public class TestSpecificCompiler {
       "                {\"name\": \"defaultValue\", \"type\": \"int\" }," +
       "                {\"name\": \"other\", \"type\": \"int\" }," +
       "                {\"name\": \"short\", \"type\": \"volatile\" } ] }";
+
+
+  @Test
+  public void testManglingForRecords() throws IOException {
     Collection<OutputFile> c =
-      new SpecificCompiler(Schema.parse(schema)).compile();
+      new SpecificCompiler(Schema.parse(SCHEMA)).compile();
     assertEquals(1, c.size());
     String contents = c.iterator().next().contents;
 
@@ -161,6 +163,22 @@ public class TestSpecificCompiler {
   }
 
   @Test
+  public void testSchemaSplit() throws IOException {
+    SpecificCompiler compiler = new SpecificCompiler(Schema.parse(SCHEMA));
+    compiler.maxStringChars = 10;
+    Collection<OutputFile> files = compiler.compile();
+    assertCompilesWithJavaCompiler(files);
+  }
+
+  @Test
+  public void testProtocolSplit() throws IOException {
+    SpecificCompiler compiler = new SpecificCompiler(Protocol.parse(PROTOCOL));
+    compiler.maxStringChars = 10;
+    Collection<OutputFile> files = compiler.compile();
+    assertCompilesWithJavaCompiler(files);
+  }
+
+  @Test
   public void testSchemaWithDocs() {
     Collection<OutputFile> outputs = new SpecificCompiler(
         Schema.parse(TestSchema.SCHEMA_WITH_DOC_TAGS)).compile();

Modified: avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestSpecificCompilerTool.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestSpecificCompilerTool.java?rev=1480808&r1=1480807&r2=1480808&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestSpecificCompilerTool.java
(original)
+++ avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestSpecificCompilerTool.java
Thu May  9 22:15:16 2013
@@ -48,6 +48,15 @@ public class TestSpecificCompilerTool {
   private static final File TEST_EXPECTED_PLAYER =
     new File(TEST_EXPECTED_OUTPUT_DIR, "Player.java");
 
+  private static final File TEST_EXPECTED_STRING_OUTPUT_DIR =
+    new File(TEST_DIR, "output-string");
+  private static final File TEST_EXPECTED_STRING_POSITION =
+    new File(TEST_EXPECTED_STRING_OUTPUT_DIR,
+             "avro/examples/baseball/Position.java");
+  private static final File TEST_EXPECTED_STRING_PLAYER =
+    new File(TEST_EXPECTED_STRING_OUTPUT_DIR,
+             "avro/examples/baseball/Player.java");
+
   // where test output goes
   private static final File TEST_OUTPUT_DIR =
     new File("target/compiler/output");
@@ -59,9 +68,9 @@ public class TestSpecificCompilerTool {
   private static final File TEST_OUTPUT_STRING_DIR =
     new File("target/compiler/output-string");
   private static final File TEST_OUTPUT_STRING_PLAYER =
-    new File(TEST_OUTPUT_DIR, "avro/examples/baseball/Player.java");
+    new File(TEST_OUTPUT_STRING_DIR, "avro/examples/baseball/Player.java");
   private static final File TEST_OUTPUT_STRING_POSITION =
-    new File(TEST_OUTPUT_DIR, "avro/examples/baseball/Position.java");
+    new File(TEST_OUTPUT_STRING_DIR, "avro/examples/baseball/Position.java");
 
   @Before
   public void setUp() {
@@ -105,9 +114,9 @@ public class TestSpecificCompilerTool {
     doCompile(new String[]{"-string", "schema",
       TEST_INPUT_DIR.toString() + "/position.avsc",
       TEST_INPUT_DIR.toString() + "/player.avsc",
-      TEST_OUTPUT_DIR.getPath()});
-    assertFileMatch(TEST_EXPECTED_POSITION, TEST_OUTPUT_STRING_POSITION);
-    assertFileMatch(TEST_EXPECTED_PLAYER,   TEST_OUTPUT_STRING_PLAYER);
+      TEST_OUTPUT_STRING_DIR.getPath()});
+    assertFileMatch(TEST_EXPECTED_STRING_POSITION, TEST_OUTPUT_STRING_POSITION);
+    assertFileMatch(TEST_EXPECTED_STRING_PLAYER,   TEST_OUTPUT_STRING_PLAYER);
   }
 
   // Runs the actual compiler tool with the given input args



Mime
View raw message