This is an automated email from the ASF dual-hosted git repository. dkulp pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/avro.git commit 1267c4c8ce898143d58b18469aa9ea11726fcdd4 Author: Juan Sebastián Urrego Escobar AuthorDate: Thu Aug 1 20:57:06 2019 +0200 Add some tests to validate the include namespace functionality --- .../test/java/org/apache/avro/io/TestEncoders.java | 154 +++++++++++++++------ 1 file changed, 108 insertions(+), 46 deletions(-) diff --git a/lang/java/avro/src/test/java/org/apache/avro/io/TestEncoders.java b/lang/java/avro/src/test/java/org/apache/avro/io/TestEncoders.java index dbe2d6a..9f0a585 100644 --- a/lang/java/avro/src/test/java/org/apache/avro/io/TestEncoders.java +++ b/lang/java/avro/src/test/java/org/apache/avro/io/TestEncoders.java @@ -17,25 +17,11 @@ */ package org.apache.avro.io; -import static java.util.Arrays.asList; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.nio.ByteBuffer; -import java.nio.MappedByteBuffer; -import java.nio.channels.FileChannel; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; - import com.fasterxml.jackson.core.JsonEncoding; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.ObjectMapper; + import org.apache.avro.AvroTypeException; import org.apache.avro.Schema; import org.apache.avro.Schema.Type; @@ -46,6 +32,24 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; + +import static java.util.Arrays.asList; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; + public class TestEncoders { private static final int ENCODER_BUFFER_SIZE = 32; private static final int EXAMPLE_DATA_SIZE = 17; @@ -126,6 +130,32 @@ public class TestEncoders { } @Test + public void testJsonEncoderWhenIncludeNamespaceOptionIsFalse() throws IOException { + String value = "{\"b\": {\"string\":\"myVal\"}, \"a\": 1}"; + String schemaStr = "{\"type\": \"record\", \"name\": \"ab\", \"fields\": [" + + "{\"name\": \"a\", \"type\": \"int\"}, {\"name\": \"b\", \"type\": [\"null\", \"string\"]}" + "]}"; + Schema schema = new Schema.Parser().parse(schemaStr); + byte[] avroBytes = fromJsonToAvro(value, schema); + ObjectMapper mapper = new ObjectMapper(); + + assertEquals(mapper.readTree("{\"b\":\"myVal\",\"a\":1}"), + mapper.readTree(fromAvroToJson(avroBytes, schema, false))); + } + + @Test + public void testJsonEncoderWhenIncludeNamespaceOptionIsTrue() throws IOException { + String value = "{\"b\": {\"string\":\"myVal\"}, \"a\": 1}"; + String schemaStr = "{\"type\": \"record\", \"name\": \"ab\", \"fields\": [" + + "{\"name\": \"a\", \"type\": \"int\"}, {\"name\": \"b\", \"type\": [\"null\", \"string\"]}" + "]}"; + Schema schema = new Schema.Parser().parse(schemaStr); + byte[] avroBytes = fromJsonToAvro(value, schema); + ObjectMapper mapper = new ObjectMapper(); + + assertEquals(mapper.readTree("{\"b\":{\"string\":\"myVal\"},\"a\":1}"), + mapper.readTree(fromAvroToJson(avroBytes, schema, true))); + } + + @Test public void testValidatingEncoderInit() throws IOException { Schema s = new Schema.Parser().parse("\"int\""); OutputStream out = new ByteArrayOutputStream(); @@ -137,7 +167,7 @@ public class TestEncoders { public void testJsonRecordOrdering() throws IOException { String value = "{\"b\": 2, \"a\": 1}"; Schema schema = new Schema.Parser().parse("{\"type\": \"record\", \"name\": \"ab\", \"fields\": [" - + "{\"name\": \"a\", \"type\": \"int\"}, {\"name\": \"b\", \"type\": \"int\"}" + "]}"); + + "{\"name\": \"a\", \"type\": \"int\"}, {\"name\": \"b\", \"type\": \"int\"}" + "]}"); GenericDatumReader reader = new GenericDatumReader<>(schema); Decoder decoder = DecoderFactory.get().jsonDecoder(schema, value); Object o = reader.read(null, decoder); @@ -148,11 +178,11 @@ public class TestEncoders { public void testJsonExcessFields() throws IOException { String value = "{\"b\": { \"b3\": 1.4, \"b2\": 3.14, \"b1\": \"h\"}, \"a\": {\"a0\": 45, \"a2\":true, \"a1\": null}}"; Schema schema = new Schema.Parser().parse("{\"type\": \"record\", \"name\": \"ab\", \"fields\": [\n" - + "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n" - + "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}},\n" - + "{\"name\": \"b\", \"type\": {\"type\":\"record\",\"name\":\"B\",\"fields\":\n" - + "[{\"name\":\"b1\", \"type\":\"string\"}, {\"name\":\"b2\", \"type\":\"float\"}, {\"name\":\"b3\", \"type\":\"double\"}]}}\n" - + "]}"); + + "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n" + + "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}},\n" + + "{\"name\": \"b\", \"type\": {\"type\":\"record\",\"name\":\"B\",\"fields\":\n" + + "[{\"name\":\"b1\", \"type\":\"string\"}, {\"name\":\"b2\", \"type\":\"float\"}, {\"name\":\"b3\", \"type\":\"double\"}]}}\n" + + "]}"); GenericDatumReader reader = new GenericDatumReader<>(schema); Decoder decoder = DecoderFactory.get().jsonDecoder(schema, value); reader.read(null, decoder); @@ -162,30 +192,30 @@ public class TestEncoders { public void testJsonRecordOrdering2() throws IOException { String value = "{\"b\": { \"b3\": 1.4, \"b2\": 3.14, \"b1\": \"h\"}, \"a\": {\"a2\":true, \"a1\": null}}"; Schema schema = new Schema.Parser().parse("{\"type\": \"record\", \"name\": \"ab\", \"fields\": [\n" - + "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n" - + "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}},\n" - + "{\"name\": \"b\", \"type\": {\"type\":\"record\",\"name\":\"B\",\"fields\":\n" - + "[{\"name\":\"b1\", \"type\":\"string\"}, {\"name\":\"b2\", \"type\":\"float\"}, {\"name\":\"b3\", \"type\":\"double\"}]}}\n" - + "]}"); + + "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n" + + "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}},\n" + + "{\"name\": \"b\", \"type\": {\"type\":\"record\",\"name\":\"B\",\"fields\":\n" + + "[{\"name\":\"b1\", \"type\":\"string\"}, {\"name\":\"b2\", \"type\":\"float\"}, {\"name\":\"b3\", \"type\":\"double\"}]}}\n" + + "]}"); GenericDatumReader reader = new GenericDatumReader<>(schema); Decoder decoder = DecoderFactory.get().jsonDecoder(schema, value); Object o = reader.read(null, decoder); Assert.assertEquals("{\"a\": {\"a1\": null, \"a2\": true}, \"b\": {\"b1\": \"h\", \"b2\": 3.14, \"b3\": 1.4}}", - o.toString()); + o.toString()); } @Test public void testJsonRecordOrderingWithProjection() throws IOException { String value = "{\"b\": { \"b3\": 1.4, \"b2\": 3.14, \"b1\": \"h\"}, \"a\": {\"a2\":true, \"a1\": null}}"; Schema writerSchema = new Schema.Parser().parse("{\"type\": \"record\", \"name\": \"ab\", \"fields\": [\n" - + "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n" - + "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}},\n" - + "{\"name\": \"b\", \"type\": {\"type\":\"record\",\"name\":\"B\",\"fields\":\n" - + "[{\"name\":\"b1\", \"type\":\"string\"}, {\"name\":\"b2\", \"type\":\"float\"}, {\"name\":\"b3\", \"type\":\"double\"}]}}\n" - + "]}"); + + "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n" + + "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}},\n" + + "{\"name\": \"b\", \"type\": {\"type\":\"record\",\"name\":\"B\",\"fields\":\n" + + "[{\"name\":\"b1\", \"type\":\"string\"}, {\"name\":\"b2\", \"type\":\"float\"}, {\"name\":\"b3\", \"type\":\"double\"}]}}\n" + + "]}"); Schema readerSchema = new Schema.Parser().parse("{\"type\": \"record\", \"name\": \"ab\", \"fields\": [\n" - + "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n" - + "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}}\n" + "]}"); + + "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n" + + "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}}\n" + "]}"); GenericDatumReader reader = new GenericDatumReader<>(writerSchema, readerSchema); Decoder decoder = DecoderFactory.get().jsonDecoder(writerSchema, value); Object o = reader.read(null, decoder); @@ -196,14 +226,14 @@ public class TestEncoders { public void testJsonRecordOrderingWithProjection2() throws IOException { String value = "{\"b\": { \"b1\": \"h\", \"b2\": [3.14, 3.56], \"b3\": 1.4}, \"a\": {\"a2\":true, \"a1\": null}}"; Schema writerSchema = new Schema.Parser().parse("{\"type\": \"record\", \"name\": \"ab\", \"fields\": [\n" - + "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n" - + "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}},\n" - + "{\"name\": \"b\", \"type\": {\"type\":\"record\",\"name\":\"B\",\"fields\":\n" - + "[{\"name\":\"b1\", \"type\":\"string\"}, {\"name\":\"b2\", \"type\":{\"type\":\"array\", \"items\":\"float\"}}, {\"name\":\"b3\", \"type\":\"double\"}]}}\n" - + "]}"); + + "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n" + + "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}},\n" + + "{\"name\": \"b\", \"type\": {\"type\":\"record\",\"name\":\"B\",\"fields\":\n" + + "[{\"name\":\"b1\", \"type\":\"string\"}, {\"name\":\"b2\", \"type\":{\"type\":\"array\", \"items\":\"float\"}}, {\"name\":\"b3\", \"type\":\"double\"}]}}\n" + + "]}"); Schema readerSchema = new Schema.Parser().parse("{\"type\": \"record\", \"name\": \"ab\", \"fields\": [\n" - + "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n" - + "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}}\n" + "]}"); + + "{\"name\": \"a\", \"type\": {\"type\":\"record\",\"name\":\"A\",\"fields\":\n" + + "[{\"name\":\"a1\", \"type\":\"null\"}, {\"name\":\"a2\", \"type\":\"boolean\"}]}}\n" + "]}"); GenericDatumReader reader = new GenericDatumReader<>(writerSchema, readerSchema); Decoder decoder = DecoderFactory.get().jsonDecoder(writerSchema, value); Object o = reader.read(null, decoder); @@ -222,7 +252,7 @@ public class TestEncoders { Path file = Paths.get(DIR.getRoot().getPath() + "testMappedByteBuffer.avro"); Files.write(file, someBytes(EXAMPLE_DATA_SIZE)); MappedByteBuffer buffer = FileChannel.open(file, StandardOpenOption.READ).map(FileChannel.MapMode.READ_ONLY, 0, - EXAMPLE_DATA_SIZE); + EXAMPLE_DATA_SIZE); testWithBuffer(buffer); } @@ -240,9 +270,9 @@ public class TestEncoders { assertThat(output.toByteArray(), equalTo(avroEncoded(someBytes(EXAMPLE_DATA_SIZE)))); assertThat(asList(buffer.position(), buffer.remaining()), is(asList(0, EXAMPLE_DATA_SIZE))); // fails if buffer is - // not array-backed and - // buffer overflow - // occurs + // not array-backed and + // buffer overflow + // occurs } private byte[] someBytes(int size) { @@ -260,4 +290,36 @@ public class TestEncoders { System.arraycopy(bytes, 0, result, 1, bytes.length); return result; } + + private byte[] fromJsonToAvro(String json, Schema schema) throws IOException { + DatumReader reader = new GenericDatumReader<>(schema); + GenericDatumWriter writer = new GenericDatumWriter<>(schema); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + + Decoder decoder = DecoderFactory.get().jsonDecoder(schema, json); + Encoder encoder = EncoderFactory.get().binaryEncoder(output, null); + + Object datum = reader.read(null, decoder); + + writer.write(datum, encoder); + encoder.flush(); + + return output.toByteArray(); + } + + private String fromAvroToJson(byte[] avroBytes, Schema schema, boolean includeNamespace) throws IOException { + GenericDatumReader reader = new GenericDatumReader<>(schema); + DatumWriter writer = new GenericDatumWriter<>(schema); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + + JsonEncoder encoder = factory.jsonEncoder(schema, output); + encoder.setIncludeNamespace(includeNamespace); + Decoder decoder = DecoderFactory.get().binaryDecoder(avroBytes, null); + Object datum = reader.read(null, decoder); + writer.write(datum, encoder); + encoder.flush(); + output.flush(); + + return new String(output.toByteArray(), StandardCharsets.UTF_8.name()); + } }