avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tomwh...@apache.org
Subject svn commit: r1584605 - in /avro/trunk: ./ doc/src/content/xdocs/ lang/java/avro/src/main/java/org/apache/avro/generic/ lang/java/ipc/src/test/java/org/apache/avro/
Date Fri, 04 Apr 2014 10:33:03 GMT
Author: tomwhite
Date: Fri Apr  4 10:33:02 2014
New Revision: 1584605

URL: http://svn.apache.org/r1584605
Log:
AVRO-1402. Support for DECIMAL type (as a record mapping).

Added:
    avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/DecimalRecordMapping.java
    avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/RecordMapping.java
Modified:
    avro/trunk/CHANGES.txt
    avro/trunk/doc/src/content/xdocs/spec.xml
    avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java
    avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java
    avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumWriter.java
    avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/TestSchema.java

Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1584605&r1=1584604&r2=1584605&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Fri Apr  4 10:33:02 2014
@@ -11,6 +11,8 @@ Trunk (not yet released)
     AVRO-1471. Java: Permit writing generated code in different
     character encodings. (Eugene Mustaphin via cutting)
 
+    AVRO-1402. Support for DECIMAL type (as a record mapping). (tomwhite)
+
   OPTIMIZATIONS
 
     AVRO-1455. Deep copy does not need to create new instances for primitives.

Modified: avro/trunk/doc/src/content/xdocs/spec.xml
URL: http://svn.apache.org/viewvc/avro/trunk/doc/src/content/xdocs/spec.xml?rev=1584605&r1=1584604&r2=1584605&view=diff
==============================================================================
--- avro/trunk/doc/src/content/xdocs/spec.xml (original)
+++ avro/trunk/doc/src/content/xdocs/spec.xml Fri Apr  4 10:33:02 2014
@@ -1336,6 +1336,34 @@ void initFPTable() {
       </section>
     </section>
 
+    <section>
+      <title>Record Mappings</title>
+
+      <p>Implementations may optionally map the following record schemas to an appropriate
+        native type.</p>
+
+      <section>
+        <title>Decimal</title>
+        <p>The <code>Decimal</code> type represents arbitrary-precision
signed decimal
+          numbers. A <code>Decimal</code> is encoded as an <code>int</code>
<em>scale</em>
+          followed by a <code>bytes</code> <em>value</em> field containing
the
+          two's-complement representation of the unscaled integer value in big-endian
+          byte order.</p>
+        <p>The value of the number represented by this <code>Decimal</code>
type is
+          <em>value &#215; 10<sup>-scale</sup></em>.</p>
+        <source>
+{
+  "type": "record",
+  "name": "org.apache.avro.Decimal",
+  "fields": [
+    {"name": "scale", "type": "int"},
+    {"name": "value", "type": "bytes"}
+  ]
+}
+        </source>
+      </section>
+    </section>
+
   <p><em>Apache Avro, Avro, Apache, and the Avro and Apache logos are
    trademarks of The Apache Software Foundation.</em></p>
 

Added: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/DecimalRecordMapping.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/DecimalRecordMapping.java?rev=1584605&view=auto
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/DecimalRecordMapping.java
(added)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/DecimalRecordMapping.java
Fri Apr  4 10:33:02 2014
@@ -0,0 +1,65 @@
+package org.apache.avro.generic;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import org.apache.avro.Schema;
+import org.apache.avro.SchemaBuilder;
+import org.apache.avro.SchemaBuilder.RecordBuilder;
+import org.apache.avro.io.Decoder;
+import org.apache.avro.io.Encoder;
+
+/**
+ * <p>This {@link RecordMapping} writes a {@code BigDecimal}
+ * object as an Avro record with the following schema:</p>
+ * <pre><code>
+ * {
+ *   "type": "record",
+ *   "name": "org.apache.avro.Decimal",
+ *   "fields": [
+ *     {"name": "scale", "type": "int"},
+ *     {"name": "value", "type": "bytes"}
+ *   ]
+ * }
+ * </code></pre>
+ */
+public class DecimalRecordMapping extends RecordMapping<BigDecimal> {
+  public DecimalRecordMapping() {
+    this(null, null);
+  }
+
+  public DecimalRecordMapping(Integer maxPrecision, Integer maxScale) {
+    super(schema(maxPrecision, maxScale), BigDecimal.class);
+  }
+
+  private static Schema schema(Integer maxPrecision, Integer maxScale) {
+    RecordBuilder<Schema> builder = SchemaBuilder.record("org.apache.avro.Decimal");
+    if (maxPrecision != null && maxScale != null) {
+      builder.prop("maxPrecision", Integer.toString(maxPrecision))
+             .prop("maxScale", Integer.toString(maxScale));
+    }
+    return builder.fields()
+        .requiredInt("scale")
+        .requiredBytes("value")
+        .endRecord();
+  }
+
+  @Override
+  public void write(Object datum, Encoder out) throws IOException {
+    BigDecimal decimal = (BigDecimal) datum;
+    out.writeInt(decimal.scale());
+    out.writeBytes(decimal.unscaledValue().toByteArray());
+  }
+
+  @Override
+  public BigDecimal read(Object reuse, Decoder in) throws IOException {
+    // BigDecimal instances are immutable so can't reuse
+    int scale = in.readInt();
+    ByteBuffer byteBuffer = in.readBytes(null);
+    byte[] value = new byte[byteBuffer.remaining()];
+    byteBuffer.get(value);
+    return new BigDecimal(new BigInteger(value), scale);
+  }
+
+}

Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java?rev=1584605&r1=1584604&r2=1584605&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java (original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericData.java Fri Apr
 4 10:33:02 2014
@@ -64,6 +64,12 @@ public class GenericData {
 
   private final ClassLoader classLoader;
 
+  private final Map<String, RecordMapping<?>> recordMappings =
+      new HashMap<String, RecordMapping<?>>();
+
+  private final Map<Class<?>, RecordMapping<?>> recordMappingClasses =
+      new HashMap<Class<?>, RecordMapping<?>>();
+
   /** Set the Java type to be used when reading this schema.  Meaningful only
    * only string schemas and map schemas (for the keys). */
   public static void setStringType(Schema s, StringType stringType) {
@@ -91,6 +97,22 @@ public class GenericData {
   /** Return the class loader that's used (by subclasses). */
   public ClassLoader getClassLoader() { return classLoader; }
 
+  /** Register a {@link org.apache.avro.generic.RecordMapping} to be used when reading or
+   * writing records using {@link org.apache.avro.generic.GenericDatumReader} or
+   * {@link org.apache.avro.generic.GenericDatumWriter}.
+   */
+  public <T> void addRecordMapping(RecordMapping<T> recordMapping) {
+    recordMappings.put(recordMapping.getSchema().getFullName(), recordMapping);
+    recordMappingClasses.put(recordMapping.getRecordClass(), recordMapping);
+  }
+
+  /** Return the  {@link org.apache.avro.generic.RecordMapping} for the given record
+   * schema, or <code>null</code> if there is no mapping registered.
+   */
+  public RecordMapping<?> getRecordMapping(Schema schema) {
+    return recordMappings.get(schema.getFullName());
+  }
+
   /** Default implementation of {@link GenericRecord}. Note that this implementation
    * does not fill in default values for fields if they are not specified; use {@link
    * GenericRecordBuilder} in that case.
@@ -677,14 +699,25 @@ public class GenericData {
 
   /** Called by the default implementation of {@link #instanceOf}.*/
   protected boolean isRecord(Object datum) {
-    return datum instanceof IndexedRecord;
+    return datum instanceof IndexedRecord || isRecordMapping(datum);
+  }
+
+  /** Return true if the given <code>datum</code> has a registered record mapping
class.
+   * Called by the default implementation of {@link #isRecord(Object)}.
+   */
+  protected boolean isRecordMapping(Object datum) {
+    return datum != null && recordMappingClasses.containsKey(datum.getClass());
   }
 
   /** Called to obtain the schema of a record.  By default calls
-   * {GenericContainer#getSchema().  May be overridden for alternate record
-   * representations. */
+   * {@link GenericContainer#getSchema()}, or uses a registered record mapper if not a
+   * {@link GenericContainer}. May be overridden for alternate
+   * record representations. */
   protected Schema getRecordSchema(Object record) {
-    return ((GenericContainer)record).getSchema();
+    if (record instanceof GenericContainer) {
+      return ((GenericContainer) record).getSchema();
+    }
+    return recordMappingClasses.get(record.getClass()).getSchema();
   }
 
   /** Called by the default implementation of {@link #instanceOf}.*/

Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java?rev=1584605&r1=1584604&r2=1584605&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java
(original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumReader.java
Fri Apr  4 10:33:02 2014
@@ -170,6 +170,10 @@ public class GenericDatumReader<D> imple
    * representations.*/
   protected Object readRecord(Object old, Schema expected, 
       ResolvingDecoder in) throws IOException {
+    RecordMapping<?> recordMapping = data.getRecordMapping(expected);
+    if (recordMapping != null) {
+      return recordMapping.read(old, in);
+    }
     Object r = data.newRecord(old, expected);
     Object state = data.getRecordState(r, expected);
     

Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumWriter.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumWriter.java?rev=1584605&r1=1584604&r2=1584605&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumWriter.java
(original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/GenericDatumWriter.java
Fri Apr  4 10:33:02 2014
@@ -99,6 +99,11 @@ public class GenericDatumWriter<D> imple
    * representations.*/
   protected void writeRecord(Schema schema, Object datum, Encoder out)
     throws IOException {
+    RecordMapping<?> recordMapping = data.getRecordMapping(schema);
+    if (recordMapping != null) {
+      recordMapping.write(datum, out);
+      return;
+    }
     Object state = data.getRecordState(datum, schema);
     for (Field f : schema.getFields()) {
       writeField(datum, f, out, state);

Added: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/RecordMapping.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/RecordMapping.java?rev=1584605&view=auto
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/RecordMapping.java (added)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/generic/RecordMapping.java Fri
Apr  4 10:33:02 2014
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.avro.generic;
+
+import java.io.IOException;
+import org.apache.avro.Schema;
+import org.apache.avro.io.Decoder;
+import org.apache.avro.io.Encoder;
+
+/**
+ * Expert: a custom mapping that writes an object directly as an Avro record.
+ * No validation is performed to check that the encoding conforms to the schema.
+ * Invalid implementations may result in an unreadable file.
+ * The use of {@link org.apache.avro.io.ValidatingEncoder} is recommended.
+ *
+ * @param <T> The class of objects that can be serialized as Avro records using this
+ *           record mapping.
+ * @see org.apache.avro.generic.GenericData#addRecordMapping(RecordMapping)
+ */
+public abstract class RecordMapping<T> {
+
+  private final Schema schema;
+  private final Class<T> recordClass;
+
+  /**
+   * Create a record mapping for the given record {@link org.apache.avro.Schema}
+   * for serializing objects of the given {@link java.lang.Class}.
+   */
+  public RecordMapping(Schema schema, Class<T> recordClass) {
+    this.schema = schema;
+    this.recordClass = recordClass;
+  }
+
+  /**
+   * @return the schema describing the records that are serialized using this record
+   * mapping.
+   */
+  public Schema getSchema() {
+    return schema;
+  }
+
+  /**
+   * @return the class of objects that can be serialized as Avro records using this
+   * record mapping.
+   */
+  public Class<T> getRecordClass() {
+    return recordClass;
+  }
+
+  public abstract void write(Object datum, Encoder out) throws IOException;
+
+  public abstract T read(Object reuse, Decoder in) throws IOException;
+
+}

Modified: avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/TestSchema.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/TestSchema.java?rev=1584605&r1=1584604&r2=1584605&view=diff
==============================================================================
--- avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/TestSchema.java (original)
+++ avro/trunk/lang/java/ipc/src/test/java/org/apache/avro/TestSchema.java Fri Apr  4 10:33:02
2014
@@ -26,6 +26,7 @@ import static org.junit.Assert.fail;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.math.BigDecimal;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -42,6 +43,7 @@ import org.apache.avro.data.Json;
 import org.apache.avro.generic.GenericData;
 import org.apache.avro.generic.GenericDatumReader;
 import org.apache.avro.generic.GenericDatumWriter;
+import org.apache.avro.generic.DecimalRecordMapping;
 import org.apache.avro.io.DatumReader;
 import org.apache.avro.io.DatumWriter;
 import org.apache.avro.io.Decoder;
@@ -540,6 +542,40 @@ public class TestSchema {
     }
   }
 
+  @Test
+  public void testDecimalRecordMapping() throws Exception {
+    String recordJson = "{\"type\":\"record\"," +
+        "\"name\":\"org.apache.avro.Decimal\"," +
+        "\"fields\":[\n" +
+        "  {\"name\":\"scale\",\"type\":\"int\"},\n" +
+        "  {\"name\":\"value\",\"type\":\"bytes\"}\n" +
+        "]}";
+    Schema schema = Schema.parse(recordJson);
+    BigDecimal decimal = new BigDecimal("12.45");
+    GenericData data = new GenericData();
+    data.addRecordMapping(new DecimalRecordMapping());
+    checkBinary(schema, decimal,
+        new GenericDatumWriter<Object>(schema, data),
+        new GenericDatumReader<Object>(schema, schema, data));
+  }
+
+  @Test
+  public void testDecimalRecordMappingUnion() throws Exception {
+    String recordJson = "{\"type\":\"record\"," +
+        "\"name\":\"org.apache.avro.Decimal\"," +
+        "\"fields\":[\n" +
+        "  {\"name\":\"scale\",\"type\":\"int\"},\n" +
+        "  {\"name\":\"value\",\"type\":\"bytes\"}\n" +
+        "]}";
+    Schema schema = Schema.parse("[\"null\",\"string\"," + recordJson + "]");
+    BigDecimal decimal = new BigDecimal("12.45");
+    GenericData data = new GenericData();
+    data.addRecordMapping(new DecimalRecordMapping());
+    checkBinary(schema, decimal,
+        new GenericDatumWriter<Object>(schema, data),
+        new GenericDatumReader<Object>(schema, schema, data));
+  }
+
   private static void checkParseError(String json) {
     try {
       Schema.parse(json);



Mime
View raw message