avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From blachn...@apache.org
Subject [avro] branch master updated: AVRO-2459: Add data interop test for the C# bindings (#574)
Date Sun, 07 Jul 2019 19:29:38 GMT
This is an automated email from the ASF dual-hosted git repository.

blachniet pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/avro.git


The following commit(s) were added to refs/heads/master by this push:
     new 95f6c12  AVRO-2459: Add data interop test for the C# bindings (#574)
95f6c12 is described below

commit 95f6c12d3ec57ccc79d6a3ff4977947138b679a7
Author: Kengo Seki <sekikn@apache.org>
AuthorDate: Mon Jul 8 04:29:33 2019 +0900

    AVRO-2459: Add data interop test for the C# bindings (#574)
    
    * AVRO-2459: Add data interop test for the C# bindings
    
    * Reflect @blachniet's comments. With these fixes,
    
    * `./build.sh test` runs tests without the "Interop" category
    * `./build.sh interop-data-test` runs only the "Interop" category
      tests, always resolves the input path correctly, and fails if
      the input directory doesn't exist yet
---
 build.sh                                           |  2 +
 lang/csharp/build.sh                               | 13 ++-
 lang/csharp/src/apache/test/Avro.test.csproj       |  1 +
 .../apache/test/Interop/InteropDataConstants.cs    | 31 +++++++
 .../apache/test/Interop/InteropDataGenerator.cs    | 99 ++++++++++++++++++++++
 .../src/apache/test/Interop/InteropDataTests.cs    | 59 +++++++++++++
 6 files changed, 203 insertions(+), 2 deletions(-)

diff --git a/build.sh b/build.sh
index fec877b..e608946 100755
--- a/build.sh
+++ b/build.sh
@@ -63,6 +63,7 @@ do
       (cd lang/py; ant interop-data-generate)
       (cd lang/c; ./build.sh interop-data-generate)
       #(cd lang/c++; make interop-data-generate)
+      (cd lang/csharp; ./build.sh interop-data-generate)
       (cd lang/ruby; rake generate_interop)
       (cd lang/php; ./build.sh interop-data-generate)
 
@@ -71,6 +72,7 @@ do
       (cd lang/py; ant interop-data-test)
       (cd lang/c; ./build.sh interop-data-test)
       #(cd lang/c++; make interop-data-test)
+      (cd lang/csharp; ./build.sh interop-data-test)
       (cd lang/ruby; rake interop)
       (cd lang/php; ./build.sh test-interop)
 
diff --git a/lang/csharp/build.sh b/lang/csharp/build.sh
index d8ca0f5..db344e2 100755
--- a/lang/csharp/build.sh
+++ b/lang/csharp/build.sh
@@ -29,7 +29,8 @@ case "$1" in
     dotnet build --configuration Release Avro.sln
 
     # AVRO-2442: Explictly set LANG to work around ICU bug in `dotnet test`
-    LANG=en_US.UTF-8 dotnet test  --configuration Release --no-build Avro.sln
+    LANG=en_US.UTF-8 dotnet test  --configuration Release --no-build \
+        --filter "TestCategory!=Interop" Avro.sln
     ;;
 
   perf)
@@ -61,6 +62,14 @@ case "$1" in
     cp -pr build/doc/* ${ROOT}/build/avro-doc-${VERSION}/api/csharp
     ;;
 
+  interop-data-generate)
+    dotnet run --project src/apache/test/Avro.test.csproj --framework netcoreapp2.2 ../../share/test/schemas/interop.avsc
../../build/interop/data
+    ;;
+
+  interop-data-test)
+    LANG=en_US.UTF-8 dotnet test --filter "TestCategory=Interop"
+    ;;
+
   clean)
     rm -rf src/apache/{main,test,codegen,ipc,msbuild,perf}/{obj,bin}
     rm -rf build
@@ -68,7 +77,7 @@ case "$1" in
     ;;
 
   *)
-    echo "Usage: $0 {test|clean|dist|perf}"
+    echo "Usage: $0 {test|clean|dist|perf|interop-data-generate|interop-data-test}"
     exit 1
 esac
 
diff --git a/lang/csharp/src/apache/test/Avro.test.csproj b/lang/csharp/src/apache/test/Avro.test.csproj
index e73878c..6f776fa 100644
--- a/lang/csharp/src/apache/test/Avro.test.csproj
+++ b/lang/csharp/src/apache/test/Avro.test.csproj
@@ -22,6 +22,7 @@
     <RootNamespace>Avro.test</RootNamespace>
     <AssemblyName>Avro.test</AssemblyName>
     <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
+    <GenerateProgramFile>false</GenerateProgramFile>
   </PropertyGroup>
 
   <ItemGroup>
diff --git a/lang/csharp/src/apache/test/Interop/InteropDataConstants.cs b/lang/csharp/src/apache/test/Interop/InteropDataConstants.cs
new file mode 100644
index 0000000..219dbf0
--- /dev/null
+++ b/lang/csharp/src/apache/test/Interop/InteropDataConstants.cs
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+using System.Collections.Generic;
+using Avro.File;
+
+namespace Avro.Test.Interop
+{
+    public class InteropDataConstants
+    {
+        public static readonly HashSet<string> SupportedCodecNames = new HashSet<string>
+        {
+            DataFileConstants.NullCodec,
+            DataFileConstants.DeflateCodec
+        };
+    }
+}
\ No newline at end of file
diff --git a/lang/csharp/src/apache/test/Interop/InteropDataGenerator.cs b/lang/csharp/src/apache/test/Interop/InteropDataGenerator.cs
new file mode 100644
index 0000000..5562512
--- /dev/null
+++ b/lang/csharp/src/apache/test/Interop/InteropDataGenerator.cs
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+using Avro.File;
+using Avro.Generic;
+
+namespace Avro.Test.Interop
+{
+    public class InteropDataGenerator
+    {
+        static void GenerateInteropData(string schemaPath, string outputDir)
+        {
+            RecordSchema schema = null;
+            using (var reader = new StreamReader(schemaPath))
+            {
+                schema = Schema.Parse(reader.ReadToEnd()) as RecordSchema;
+            }
+
+            var mapFieldSchema = (schema.Fields.Find(x => x.Name == "mapField").Schema
as MapSchema).ValueSchema as RecordSchema;
+            var mapFieldRecord0 = new GenericRecord(mapFieldSchema);
+            var mapFieldRecord1 = new GenericRecord(mapFieldSchema);
+            mapFieldRecord0.Add("label", "a");
+            mapFieldRecord1.Add("label", "cee");
+            var mapFieldValue = new Dictionary<string, GenericRecord>
+            {
+                { "a", mapFieldRecord0 },
+                { "bee", mapFieldRecord1 }
+            };
+
+            var enumFieldValue = new GenericEnum(schema.Fields.Find(x => x.Name == "enumField").Schema
as EnumSchema, "C");
+
+            var fixedFieldValue = new GenericFixed(
+                schema.Fields.Find(x => x.Name == "fixedField").Schema as FixedSchema,
+                Encoding.ASCII.GetBytes("1019181716151413"));
+
+            var nodeSchema = schema.Fields.Find(x => x.Name == "recordField").Schema as
RecordSchema;
+            var recordFieldValue = new GenericRecord(nodeSchema);
+            var innerRecordFieldValue = new GenericRecord(nodeSchema);
+            innerRecordFieldValue.Add("label", "inner");
+            innerRecordFieldValue.Add("children", new GenericRecord[] { });
+            recordFieldValue.Add("label", "blah");
+            recordFieldValue.Add("children", new GenericRecord[] { innerRecordFieldValue
});
+
+            GenericRecord record = new GenericRecord(schema);
+            record.Add("intField", 12);
+            record.Add("longField", 15234324L);
+            record.Add("stringField", "hey");
+            record.Add("boolField", true);
+            record.Add("floatField", 1234.0f);
+            record.Add("doubleField", -1234.0);
+            record.Add("bytesField", Encoding.UTF8.GetBytes("12312adf"));
+            record.Add("nullField", null);
+            record.Add("arrayField", new double[] { 5.0, 0.0, 12.0 });
+            record.Add("mapField", mapFieldValue);
+            record.Add("unionField", 12.0);
+            record.Add("enumField", enumFieldValue);
+            record.Add("fixedField", fixedFieldValue);
+            record.Add("recordField", recordFieldValue);
+
+            var datumWriter = new GenericDatumWriter<GenericRecord>(schema);
+            foreach (var codecName in InteropDataConstants.SupportedCodecNames)
+            {
+                var outputFile = "csharp.avro";
+                if (codecName != DataFileConstants.NullCodec)
+                {
+                    outputFile = string.Format("csharp_{0}.avro", codecName);
+                }
+                var outputPath = Path.Combine(outputDir, outputFile);
+                var codec = Codec.CreateCodecFromString(codecName);
+                using (var dataFileWriter = DataFileWriter<GenericRecord>.OpenWriter(datumWriter,
outputPath, codec))
+                {
+                    dataFileWriter.Append(record);
+                }
+            }
+        }
+
+        static void Main(string[] args)
+        {
+            GenerateInteropData(args[0], args[1]);
+        }
+    }
+}
\ No newline at end of file
diff --git a/lang/csharp/src/apache/test/Interop/InteropDataTests.cs b/lang/csharp/src/apache/test/Interop/InteropDataTests.cs
new file mode 100644
index 0000000..4d8823a
--- /dev/null
+++ b/lang/csharp/src/apache/test/Interop/InteropDataTests.cs
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+using System.IO;
+using NUnit.Framework;
+using Avro.File;
+using Avro.Generic;
+
+namespace Avro.Test.Interop
+{
+    [TestFixture]
+    [Category("Interop")]
+    public class InteropDataTests
+    {
+        [TestCase("../../../../../../../../build/interop/data")]
+        public void TestInterop(string inputDir)
+        {
+            // Resolve inputDir relative to the TestDirectory
+            inputDir = Path.Combine(TestContext.CurrentContext.TestDirectory, inputDir);
+
+            Assert.True(Directory.Exists(inputDir),
+                "Input directory does not exist. Run `build.sh interop-data-generate` first.");
+
+            foreach (var avroFile in Directory.EnumerateFiles(inputDir, "*.avro"))
+            {
+                var codec = Path.GetFileNameWithoutExtension(avroFile).Split('_');
+                if (1 < codec.Length && !InteropDataConstants.SupportedCodecNames.Contains(codec[1]))
+                {
+                    continue;
+                }
+
+                using(var reader = DataFileReader<GenericRecord>.OpenReader(avroFile))
+                {
+                    int i = 0;
+                    foreach (var record in reader.NextEntries)
+                    {
+                        i++;
+                        Assert.IsNotNull(record);
+                    }
+                    Assert.AreNotEqual(0, i);
+                }
+            }
+        }
+    }
+}
\ No newline at end of file


Mime
View raw message