avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r1564571 - in /avro/trunk: ./ lang/java/avro/src/main/java/org/apache/avro/ lang/java/compiler/src/test/idl/input/ lang/java/compiler/src/test/idl/output/ lang/java/ipc/src/test/java/org/apache/avro/ share/test/data/
Date Wed, 05 Feb 2014 00:09:58 GMT
Author: cutting
Date: Wed Feb  5 00:09:57 2014
New Revision: 1564571

URL: http://svn.apache.org/r1564571
Log:
AVRO-1449. Java: Optionally validate default values while reading schemas.

Modified:
    avro/trunk/CHANGES.txt
    avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Schema.java
    avro/trunk/lang/java/compiler/src/test/idl/input/interop.avdl
    avro/trunk/lang/java/compiler/src/test/idl/input/simple.avdl
    avro/trunk/lang/java/compiler/src/test/idl/output/interop.avpr
    avro/trunk/lang/java/compiler/src/test/idl/output/simple.avpr
    avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/TestSchema.java
    avro/trunk/share/test/data/schema-tests.txt

Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1564571&r1=1564570&r2=1564571&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Wed Feb  5 00:09:57 2014
@@ -18,6 +18,9 @@ Trunk (not yet released)
     AVRO-1447. Java: Remove dead code from example in documentation.
     (Jesse Anderson via cutting)
 
+    AVRO-1449. Java: Optionally validate default values while reading schemas.
+    (cutting)    
+
   BUG FIXES
 
     AVRO-1446. C#: Correctly handle system errors in RPC.

Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Schema.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Schema.java?rev=1564571&r1=1564570&r2=1564571&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Schema.java (original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/Schema.java Wed Feb  5 00:09:57
2014
@@ -372,7 +372,7 @@ public abstract class Schema extends Jso
       this.name = validateName(name);
       this.schema = schema;
       this.doc = doc;
-      this.defaultValue = defaultValue;
+      this.defaultValue = validateDefault(name, schema, defaultValue);
       this.order = order;
     }
     public String name() { return name; };
@@ -890,6 +890,7 @@ public abstract class Schema extends Jso
   public static class Parser {
     private Names names = new Names();
     private boolean validate = true;
+    private boolean validateDefaults = false;
 
     /** Adds the provided types to the set of defined, named types known to
      * this parser. */
@@ -916,6 +917,15 @@ public abstract class Schema extends Jso
     /** True iff names are validated.  True by default. */
     public boolean getValidate() { return this.validate; }
 
+    /** Enable or disable default value validation. */
+    public Parser setValidateDefaults(boolean validateDefaults) {
+      this.validateDefaults = validateDefaults;
+      return this;
+    }
+
+    /** True iff default values are validated.  False by default. */
+    public boolean getValidateDefaults() { return this.validateDefaults; }
+
     /** Parse a schema from the provided file.
      * If named, the schema is added to the names known to this parser. */
     public Schema parse(File file) throws IOException {
@@ -948,13 +958,16 @@ public abstract class Schema extends Jso
 
     private Schema parse(JsonParser parser) throws IOException {
       boolean saved = validateNames.get();
+      boolean savedValidateDefaults = VALIDATE_DEFAULTS.get();
       try {
         validateNames.set(validate);
+        VALIDATE_DEFAULTS.set(validateDefaults);
         return Schema.parse(MAPPER.readTree(parser), names);
       } catch (JsonParseException e) {
         throw new SchemaParseException(e);
       } finally {
         validateNames.set(saved);
+        VALIDATE_DEFAULTS.set(savedValidateDefaults);
       }
     }
   }
@@ -1070,6 +1083,75 @@ public abstract class Schema extends Jso
     return name;
   }
 
+  private static final ThreadLocal<Boolean> VALIDATE_DEFAULTS
+    = new ThreadLocal<Boolean>() {
+    @Override protected Boolean initialValue() {
+      return false;
+    }
+  };
+    
+  private static JsonNode validateDefault(String fieldName, Schema schema,
+                                          JsonNode defaultValue) {
+    if ((defaultValue != null)
+        && !isValidDefault(schema, defaultValue)) { // invalid default
+      String message = "Invalid default for field "+fieldName
+        +": "+defaultValue+" not a "+schema;
+      if (VALIDATE_DEFAULTS.get())
+        throw new AvroTypeException(message);     // throw exception
+      System.err.println("[WARNING] Avro: "+message); // or log warning
+    }
+    return defaultValue;
+  }
+
+  private static boolean isValidDefault(Schema schema, JsonNode defaultValue) {
+    if (defaultValue == null)
+      return false;
+    switch (schema.getType()) {
+    case STRING:  
+    case BYTES:
+    case ENUM:
+    case FIXED:
+      return defaultValue.isTextual();
+    case INT:
+    case LONG:
+    case FLOAT:
+    case DOUBLE:
+      return defaultValue.isNumber();
+    case BOOLEAN:
+      return defaultValue.isBoolean();
+    case NULL:
+      return defaultValue.isNull();
+    case ARRAY:
+      if (!defaultValue.isArray())
+        return false;
+      for (JsonNode element : defaultValue)
+        if (!isValidDefault(schema.getElementType(), element))
+          return false;
+      return true;
+    case MAP:
+      if (!defaultValue.isObject())
+        return false;
+      for (JsonNode value : defaultValue)
+        if (!isValidDefault(schema.getValueType(), value))
+          return false;
+      return true;
+    case UNION:                                   // union default: first branch
+      return isValidDefault(schema.getTypes().get(0), defaultValue);
+    case RECORD:
+      if (!defaultValue.isObject())
+        return false;
+      for (Field field : schema.getFields())
+        if (!isValidDefault(field.schema(),
+                            defaultValue.has(field.name())
+                            ? defaultValue.get(field.name())
+                            : field.defaultValue()))
+          return false;
+      return true;
+    default:
+      return false;
+    }
+  }
+
   /** @see #parse(String) */
   static Schema parse(JsonNode schema, Names names) {
     if (schema.isTextual()) {                     // name

Modified: avro/trunk/lang/java/compiler/src/test/idl/input/interop.avdl
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/compiler/src/test/idl/input/interop.avdl?rev=1564571&r1=1564570&r2=1564571&view=diff
==============================================================================
--- avro/trunk/lang/java/compiler/src/test/idl/input/interop.avdl (original)
+++ avro/trunk/lang/java/compiler/src/test/idl/input/interop.avdl Wed Feb  5 00:09:57 2014
@@ -39,7 +39,7 @@ protocol InteropProtocol {
     float floatField = 0.0;
     double doubleField = -1.0e12;
     null nullField;
-    array<double> arrayField = [{"label":"foo", "children":[]}];
+    array<double> arrayField = [];
     map<Foo> mapField;
     union { boolean, double, array<bytes> } unionFIeld;
     Kind enumField;

Modified: avro/trunk/lang/java/compiler/src/test/idl/input/simple.avdl
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/compiler/src/test/idl/input/simple.avdl?rev=1564571&r1=1564570&r2=1564571&view=diff
==============================================================================
--- avro/trunk/lang/java/compiler/src/test/idl/input/simple.avdl (original)
+++ avro/trunk/lang/java/compiler/src/test/idl/input/simple.avdl Wed Feb  5 00:09:57 2014
@@ -41,9 +41,9 @@ protocol Simple {
     /** The kind of record. */
     Kind @order("descending") kind;
 
-    @foo("bar") MD5 hash;
+    @foo("bar") MD5 hash = "0000000000000000";
 
-    union { MD5, null} @aliases(["hash", "hsh"]) nullableHash;
+    union {null, MD5} @aliases(["hash", "hsh"]) nullableHash = null;
 
     double value = NaN;
     float average = -Infinity;
@@ -55,7 +55,7 @@ protocol Simple {
 
   /** method 'hello' takes @parameter 'greeting' */
   string hello(string greeting);
-  TestRecord echo(TestRecord `record` = {"name": "bar"});
+  TestRecord echo(TestRecord `record` = {"name":"bar","kind":"BAR"});
   /** method 'add' takes @parameter 'arg1' @parameter 'arg2' */
   @specialProp("test")
   int add(int arg1, int arg2 = 0);

Modified: avro/trunk/lang/java/compiler/src/test/idl/output/interop.avpr
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/compiler/src/test/idl/output/interop.avpr?rev=1564571&r1=1564570&r2=1564571&view=diff
==============================================================================
--- avro/trunk/lang/java/compiler/src/test/idl/output/interop.avpr (original)
+++ avro/trunk/lang/java/compiler/src/test/idl/output/interop.avpr Wed Feb  5 00:09:57 2014
@@ -66,10 +66,7 @@
         "type" : "array",
         "items" : "double"
       },
-      "default" : [ {
-        "label" : "foo",
-        "children" : [ ]
-      } ]
+      "default" : [ ]
     }, {
       "name" : "mapField",
       "type" : {
@@ -93,6 +90,5 @@
       "type" : "Node"
     } ]
   } ],
-  "messages" : {
-  }
-}
+  "messages" : { }
+}
\ No newline at end of file

Modified: avro/trunk/lang/java/compiler/src/test/idl/output/simple.avpr
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/compiler/src/test/idl/output/simple.avpr?rev=1564571&r1=1564570&r2=1564571&view=diff
==============================================================================
--- avro/trunk/lang/java/compiler/src/test/idl/output/simple.avpr (original)
+++ avro/trunk/lang/java/compiler/src/test/idl/output/simple.avpr Wed Feb  5 00:09:57 2014
@@ -31,10 +31,12 @@
       "order" : "descending"
     }, {
       "name" : "hash",
-      "type" : "MD5"
+      "type" : "MD5",
+      "default" : "0000000000000000"
     }, {
       "name" : "nullableHash",
-      "type" : [ "MD5", "null" ],
+      "type" : [ "null", "MD5" ],
+      "default" : null,
       "aliases" : [ "hash", "hsh" ]
     }, {
       "name" : "value",
@@ -70,7 +72,8 @@
         "name" : "record",
         "type" : "TestRecord",
         "default" : {
-          "name" : "bar"
+          "name" : "bar",
+          "kind" : "BAR"
         }
       } ],
       "response" : "TestRecord"

Modified: avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/TestSchema.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/TestSchema.java?rev=1564571&r1=1564570&r2=1564571&view=diff
==============================================================================
--- avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/TestSchema.java (original)
+++ avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/TestSchema.java Wed Feb  5 00:09:57
2014
@@ -324,16 +324,18 @@ public class TestSchema {
     // test that erroneous default values cause errors
     for (String type : new String[]
           {"int", "long", "float", "double", "string", "bytes", "boolean"}) {
+      checkValidateDefaults("[\""+type+"\", \"null\"]", "null"); // schema parse time
       boolean error = false;
       try {
-        checkDefault("[\""+type+"\", \"null\"]", "null", 0);
+        checkDefault("[\""+type+"\", \"null\"]", "null", 0); // read time
       } catch (AvroTypeException e) {
         error = true;
       }
       assertTrue(error);
+      checkValidateDefaults("[\"null\", \""+type+"\"]", "0");  // schema parse time
       error = false;
       try {
-        checkDefault("[\"null\", \""+type+"\"]", "0", null);
+        checkDefault("[\"null\", \""+type+"\"]", "0", null); // read time
       } catch (AvroTypeException e) {
         error = true;
       }
@@ -820,6 +822,21 @@ public class TestSchema {
     assertEquals("Wrong toString", expected, Schema.parse(expected.toString()));
   }
 
+  private static void checkValidateDefaults(String schemaJson, String defaultJson) {
+    try {
+      Schema.Parser parser = new Schema.Parser();
+      parser.setValidateDefaults(true);
+      String recordJson =
+          "{\"type\":\"record\", \"name\":\"Foo\", \"fields\":[{\"name\":\"f\", "
+              +"\"type\":"+schemaJson+", "
+              +"\"default\":"+defaultJson+"}]}";
+      parser.parse(recordJson);
+      fail("Schema of type " + schemaJson + " should not have default " + defaultJson);
+    } catch (AvroTypeException e) {
+      return;
+    }
+  }
+
   @Test(expected=AvroTypeException.class)
   public void testNoDefaultField() throws Exception {
     Schema expected =

Modified: avro/trunk/share/test/data/schema-tests.txt
URL: http://svn.apache.org/viewvc/avro/trunk/share/test/data/schema-tests.txt?rev=1564571&r1=1564570&r2=1564571&view=diff
==============================================================================
--- avro/trunk/share/test/data/schema-tests.txt (original)
+++ avro/trunk/share/test/data/schema-tests.txt Wed Feb  5 00:09:57 2014
@@ -144,7 +144,7 @@
 
 // 026
 <<INPUT
-{ "fields":[{"type":"boolean", "aliases":[], "name":"f1", "default":"true"},
+{ "fields":[{"type":"boolean", "aliases":[], "name":"f1", "default":true},
             {"order":"descending","name":"f2","doc":"Hello","type":"int"}],
   "type":"record", "name":"foo"
 }



Mime
View raw message