avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dk...@apache.org
Subject [avro] 02/04: Add some tests to validate the include namespace functionality
Date Fri, 16 Aug 2019 18:37:55 GMT
This is an automated email from the ASF dual-hosted git repository.

dkulp pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/avro.git

commit 1267c4c8ce898143d58b18469aa9ea11726fcdd4
Author: Juan Sebastián Urrego Escobar <juancho088@gmail.com>
AuthorDate: Thu Aug 1 20:57:06 2019 +0200

    Add some tests to validate the include namespace functionality
---
 .../test/java/org/apache/avro/io/TestEncoders.java | 154 +++++++++++++++------
 1 file changed, 108 insertions(+), 46 deletions(-)

diff --git a/lang/java/avro/src/test/java/org/apache/avro/io/TestEncoders.java b/lang/java/avro/src/test/java/org/apache/avro/io/TestEncoders.java
index dbe2d6a..9f0a585 100644
--- a/lang/java/avro/src/test/java/org/apache/avro/io/TestEncoders.java
+++ b/lang/java/avro/src/test/java/org/apache/avro/io/TestEncoders.java
@@ -17,25 +17,11 @@
  */
 package org.apache.avro.io;
 
-import static java.util.Arrays.asList;
-import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.is;
-import static org.junit.Assert.assertThat;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.nio.MappedByteBuffer;
-import java.nio.channels.FileChannel;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.StandardOpenOption;
-
 import com.fasterxml.jackson.core.JsonEncoding;
 import com.fasterxml.jackson.core.JsonFactory;
 import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
 import org.apache.avro.AvroTypeException;
 import org.apache.avro.Schema;
 import org.apache.avro.Schema.Type;
@@ -46,6 +32,24 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+
+import static java.util.Arrays.asList;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+
 public class TestEncoders {
   private static final int ENCODER_BUFFER_SIZE = 32;
   private static final int EXAMPLE_DATA_SIZE = 17;
@@ -126,6 +130,32 @@ public class TestEncoders {
   }
 
   @Test
+  public void testJsonEncoderWhenIncludeNamespaceOptionIsFalse() throws IOException {
+    String value = "{\"b\": {\"string\":\"myVal\"}, \"a\": 1}";
+    String schemaStr = "{\"type\": \"record\", \"name\": \"ab\", \"fields\": ["
+      + "{\"name\": \"a\", \"type\": \"int\"}, {\"name\": \"b\", \"type\": [\"null\", \"string\"]}"
+ "]}";
+    Schema schema = new Schema.Parser().parse(schemaStr);
+    byte[] avroBytes = fromJsonToAvro(value, schema);
+    ObjectMapper mapper = new ObjectMapper();
+
+    assertEquals(mapper.readTree("{\"b\":\"myVal\",\"a\":1}"),
+      mapper.readTree(fromAvroToJson(avroBytes, schema, false)));
+  }
+
+  @Test
+  public void testJsonEncoderWhenIncludeNamespaceOptionIsTrue() throws IOException {
+    String value = "{\"b\": {\"string\":\"myVal\"}, \"a\": 1}";
+    String schemaStr = "{\"type\": \"record\", \"name\": \"ab\", \"fields\": ["
+      + "{\"name\": \"a\", \"type\": \"int\"}, {\"name\": \"b\", \"type\": [\"null\", \"string\"]}"
+ "]}";
+    Schema schema = new Schema.Parser().parse(schemaStr);
+    byte[] avroBytes = fromJsonToAvro(value, schema);
+    ObjectMapper mapper = new ObjectMapper();
+
+    assertEquals(mapper.readTree("{\"b\":{\"string\":\"myVal\"},\"a\":1}"),
+      mapper.readTree(fromAvroToJson(avroBytes, schema, true)));
+  }
+
+  @Test
   public void testValidatingEncoderInit() throws IOException {
     Schema s = new Schema.Parser().parse("\"int\"");
     OutputStream out = new ByteArrayOutputStream();
@@ -137,7 +167,7 @@ public class TestEncoders {
   public void testJsonRecordOrdering() throws IOException {
     String value = "{\"b\": 2, \"a\": 1}";
     Schema schema = new Schema.Parser().parse("{\"type\": \"record\", \"name\": \"ab\", \"fields\":
["
-        + "{\"name\": \"a\", \"type\": \"int\"}, {\"name\": \"b\", \"type\": \"int\"}" +
"]}");
+      + "{\"name\": \"a\", \"type\": \"int\"}, {\"name\": \"b\", \"type\": \"int\"}" + "]}");
     GenericDatumReader<Object> reader = new GenericDatumReader<>(schema);
     Decoder decoder = DecoderFactory.get().jsonDecoder(schema, value);
     Object o = reader.read(null, decoder);
@@ -148,11 +178,11 @@ public class TestEncoders {
   public void testJsonExcessFields() throws IOException {
     String value = "{\"b\": { \"b3\": 1.4, \"b2\": 3.14, \"b1\": \"h\"}, \"a\": {\"a0\":
45, \"a2\":true, \"a1\": null}}";
     Schema schema = new Schema.Parser().parse("{\"type\": \"record\", \"name\": \"ab\", \"fields\":
[\n"
-        + "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n"
-        + "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}},\n"
-        + "{\"name\": \"b\", \"type\": {\"type\":\"record\",\"name\":\"B\",\"fields\":\n"
-        + "[{\"name\":\"b1\", \"type\":\"string\"}, {\"name\":\"b2\", \"type\":\"float\"},
{\"name\":\"b3\", \"type\":\"double\"}]}}\n"
-        + "]}");
+      + "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n"
+      + "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}},\n"
+      + "{\"name\": \"b\", \"type\": {\"type\":\"record\",\"name\":\"B\",\"fields\":\n"
+      + "[{\"name\":\"b1\", \"type\":\"string\"}, {\"name\":\"b2\", \"type\":\"float\"},
{\"name\":\"b3\", \"type\":\"double\"}]}}\n"
+      + "]}");
     GenericDatumReader<Object> reader = new GenericDatumReader<>(schema);
     Decoder decoder = DecoderFactory.get().jsonDecoder(schema, value);
     reader.read(null, decoder);
@@ -162,30 +192,30 @@ public class TestEncoders {
   public void testJsonRecordOrdering2() throws IOException {
     String value = "{\"b\": { \"b3\": 1.4, \"b2\": 3.14, \"b1\": \"h\"}, \"a\": {\"a2\":true,
\"a1\": null}}";
     Schema schema = new Schema.Parser().parse("{\"type\": \"record\", \"name\": \"ab\", \"fields\":
[\n"
-        + "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n"
-        + "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}},\n"
-        + "{\"name\": \"b\", \"type\": {\"type\":\"record\",\"name\":\"B\",\"fields\":\n"
-        + "[{\"name\":\"b1\", \"type\":\"string\"}, {\"name\":\"b2\", \"type\":\"float\"},
{\"name\":\"b3\", \"type\":\"double\"}]}}\n"
-        + "]}");
+      + "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n"
+      + "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}},\n"
+      + "{\"name\": \"b\", \"type\": {\"type\":\"record\",\"name\":\"B\",\"fields\":\n"
+      + "[{\"name\":\"b1\", \"type\":\"string\"}, {\"name\":\"b2\", \"type\":\"float\"},
{\"name\":\"b3\", \"type\":\"double\"}]}}\n"
+      + "]}");
     GenericDatumReader<Object> reader = new GenericDatumReader<>(schema);
     Decoder decoder = DecoderFactory.get().jsonDecoder(schema, value);
     Object o = reader.read(null, decoder);
     Assert.assertEquals("{\"a\": {\"a1\": null, \"a2\": true}, \"b\": {\"b1\": \"h\", \"b2\":
3.14, \"b3\": 1.4}}",
-        o.toString());
+      o.toString());
   }
 
   @Test
   public void testJsonRecordOrderingWithProjection() throws IOException {
     String value = "{\"b\": { \"b3\": 1.4, \"b2\": 3.14, \"b1\": \"h\"}, \"a\": {\"a2\":true,
\"a1\": null}}";
     Schema writerSchema = new Schema.Parser().parse("{\"type\": \"record\", \"name\": \"ab\",
\"fields\": [\n"
-        + "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n"
-        + "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}},\n"
-        + "{\"name\": \"b\", \"type\": {\"type\":\"record\",\"name\":\"B\",\"fields\":\n"
-        + "[{\"name\":\"b1\", \"type\":\"string\"}, {\"name\":\"b2\", \"type\":\"float\"},
{\"name\":\"b3\", \"type\":\"double\"}]}}\n"
-        + "]}");
+      + "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n"
+      + "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}},\n"
+      + "{\"name\": \"b\", \"type\": {\"type\":\"record\",\"name\":\"B\",\"fields\":\n"
+      + "[{\"name\":\"b1\", \"type\":\"string\"}, {\"name\":\"b2\", \"type\":\"float\"},
{\"name\":\"b3\", \"type\":\"double\"}]}}\n"
+      + "]}");
     Schema readerSchema = new Schema.Parser().parse("{\"type\": \"record\", \"name\": \"ab\",
\"fields\": [\n"
-        + "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n"
-        + "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}}\n"
+ "]}");
+      + "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n"
+      + "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}}\n"
+ "]}");
     GenericDatumReader<Object> reader = new GenericDatumReader<>(writerSchema,
readerSchema);
     Decoder decoder = DecoderFactory.get().jsonDecoder(writerSchema, value);
     Object o = reader.read(null, decoder);
@@ -196,14 +226,14 @@ public class TestEncoders {
   public void testJsonRecordOrderingWithProjection2() throws IOException {
     String value = "{\"b\": { \"b1\": \"h\", \"b2\": [3.14, 3.56], \"b3\": 1.4}, \"a\": {\"a2\":true,
\"a1\": null}}";
     Schema writerSchema = new Schema.Parser().parse("{\"type\": \"record\", \"name\": \"ab\",
\"fields\": [\n"
-        + "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n"
-        + "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}},\n"
-        + "{\"name\": \"b\", \"type\": {\"type\":\"record\",\"name\":\"B\",\"fields\":\n"
-        + "[{\"name\":\"b1\", \"type\":\"string\"}, {\"name\":\"b2\", \"type\":{\"type\":\"array\",
\"items\":\"float\"}}, {\"name\":\"b3\", \"type\":\"double\"}]}}\n"
-        + "]}");
+      + "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n"
+      + "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}},\n"
+      + "{\"name\": \"b\", \"type\": {\"type\":\"record\",\"name\":\"B\",\"fields\":\n"
+      + "[{\"name\":\"b1\", \"type\":\"string\"}, {\"name\":\"b2\", \"type\":{\"type\":\"array\",
\"items\":\"float\"}}, {\"name\":\"b3\", \"type\":\"double\"}]}}\n"
+      + "]}");
     Schema readerSchema = new Schema.Parser().parse("{\"type\": \"record\", \"name\": \"ab\",
\"fields\": [\n"
-        + "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n"
-        + "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}}\n"
+ "]}");
+      + "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n"
+      + "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}}\n"
+ "]}");
     GenericDatumReader<Object> reader = new GenericDatumReader<>(writerSchema,
readerSchema);
     Decoder decoder = DecoderFactory.get().jsonDecoder(writerSchema, value);
     Object o = reader.read(null, decoder);
@@ -222,7 +252,7 @@ public class TestEncoders {
     Path file = Paths.get(DIR.getRoot().getPath() + "testMappedByteBuffer.avro");
     Files.write(file, someBytes(EXAMPLE_DATA_SIZE));
     MappedByteBuffer buffer = FileChannel.open(file, StandardOpenOption.READ).map(FileChannel.MapMode.READ_ONLY,
0,
-        EXAMPLE_DATA_SIZE);
+      EXAMPLE_DATA_SIZE);
 
     testWithBuffer(buffer);
   }
@@ -240,9 +270,9 @@ public class TestEncoders {
 
     assertThat(output.toByteArray(), equalTo(avroEncoded(someBytes(EXAMPLE_DATA_SIZE))));
     assertThat(asList(buffer.position(), buffer.remaining()), is(asList(0, EXAMPLE_DATA_SIZE)));
// fails if buffer is
-                                                                                        
        // not array-backed and
-                                                                                        
        // buffer overflow
-                                                                                        
        // occurs
+    // not array-backed and
+    // buffer overflow
+    // occurs
   }
 
   private byte[] someBytes(int size) {
@@ -260,4 +290,36 @@ public class TestEncoders {
     System.arraycopy(bytes, 0, result, 1, bytes.length);
     return result;
   }
+
+  private byte[] fromJsonToAvro(String json, Schema schema) throws IOException {
+    DatumReader<Object> reader = new GenericDatumReader<>(schema);
+    GenericDatumWriter<Object> writer = new GenericDatumWriter<>(schema);
+    ByteArrayOutputStream output = new ByteArrayOutputStream();
+
+    Decoder decoder = DecoderFactory.get().jsonDecoder(schema, json);
+    Encoder encoder = EncoderFactory.get().binaryEncoder(output, null);
+
+    Object datum = reader.read(null, decoder);
+
+    writer.write(datum, encoder);
+    encoder.flush();
+
+    return output.toByteArray();
+  }
+
+  private String fromAvroToJson(byte[] avroBytes, Schema schema, boolean includeNamespace)
throws IOException {
+    GenericDatumReader<Object> reader = new GenericDatumReader<>(schema);
+    DatumWriter<Object> writer = new GenericDatumWriter<>(schema);
+    ByteArrayOutputStream output = new ByteArrayOutputStream();
+
+    JsonEncoder encoder = factory.jsonEncoder(schema, output);
+    encoder.setIncludeNamespace(includeNamespace);
+    Decoder decoder = DecoderFactory.get().binaryDecoder(avroBytes, null);
+    Object datum = reader.read(null, decoder);
+    writer.write(datum, encoder);
+    encoder.flush();
+    output.flush();
+
+    return new String(output.toByteArray(), StandardCharsets.UTF_8.name());
+  }
 }


Mime
View raw message