avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From fo...@apache.org
Subject [avro] branch master updated: AVRO-2469: Add data interop test to the Python3 bindings (#581)
Date Tue, 16 Jul 2019 13:50:05 GMT
This is an automated email from the ASF dual-hosted git repository.

fokko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/avro.git


The following commit(s) were added to refs/heads/master by this push:
     new fcb4764  AVRO-2469: Add data interop test to the Python3 bindings (#581)
fcb4764 is described below

commit fcb4764468cd1d70b3341c1488a394bb8f20929b
Author: Kengo Seki <sekikn@apache.org>
AuthorDate: Tue Jul 16 22:49:58 2019 +0900

    AVRO-2469: Add data interop test to the Python3 bindings (#581)
    
    * AVRO-2469: Add data interop test to the Python3 bindings
    
    * Introduce with statement and pathlib to make the syntax clean
    
    * Use DataFileReader and DataFileWriter with "with" statements
---
 build.sh                                     |  3 ++
 lang/py3/avro/tests/gen_interop_data.py      | 21 ++++++----
 lang/py3/avro/tests/test_datafile_interop.py | 61 +++++-----------------------
 lang/py3/setup.py                            | 28 ++++++++++++-
 4 files changed, 54 insertions(+), 59 deletions(-)

diff --git a/build.sh b/build.sh
index e608946..d5cf479 100755
--- a/build.sh
+++ b/build.sh
@@ -61,6 +61,8 @@ do
       (cd lang/perl; ./build.sh test)
 
       (cd lang/py; ant interop-data-generate)
+      (cd lang/py3; python3 setup.py generate_interop_data \
+        --schema-file=../../share/test/schemas/interop.avsc --output-path=../../build/interop/data)
       (cd lang/c; ./build.sh interop-data-generate)
       #(cd lang/c++; make interop-data-generate)
       (cd lang/csharp; ./build.sh interop-data-generate)
@@ -70,6 +72,7 @@ do
       # run interop data tests
       (cd lang/java; mvn -B test -P interop-data-test)
       (cd lang/py; ant interop-data-test)
+      (cd lang/py3; python3 setup.py test --test-suite avro.tests.test_datafile_interop.TestDataFileInterop)
       (cd lang/c; ./build.sh interop-data-test)
       #(cd lang/c++; make interop-data-test)
       (cd lang/csharp; ./build.sh interop-data-test)
diff --git a/lang/py3/avro/tests/gen_interop_data.py b/lang/py3/avro/tests/gen_interop_data.py
index 2180b2d..723dca5 100644
--- a/lang/py3/avro/tests/gen_interop_data.py
+++ b/lang/py3/avro/tests/gen_interop_data.py
@@ -19,6 +19,7 @@
 # limitations under the License.
 
 import sys
+from pathlib import Path
 
 from avro import datafile
 from avro import io
@@ -46,11 +47,17 @@ DATUM = {
 }
 
 
-if __name__ == "__main__":
-  interop_schema = schema.Parse(open(sys.argv[1], 'r').read())
-  writer = open(sys.argv[2], 'wb')
+def generate(schema_file, output_path):
+  interop_schema = schema.Parse(open(schema_file, 'r').read())
   datum_writer = io.DatumWriter()
-  # NB: not using compression
-  dfw = datafile.DataFileWriter(writer, datum_writer, interop_schema)
-  dfw.append(DATUM)
-  dfw.close()
+  for codec in datafile.VALID_CODECS:
+    filename = 'py3'
+    if codec != 'null':
+      filename += '_' + codec
+    with Path(output_path, filename).with_suffix('.avro').open('wb') as writer, \
+      datafile.DataFileWriter(writer, datum_writer, interop_schema, codec) as dfw:
+      dfw.append(DATUM)
+
+
+if __name__ == "__main__":
+    generate(sys.argv[1], sys.argv[2])
diff --git a/lang/py3/avro/tests/test_datafile_interop.py b/lang/py3/avro/tests/test_datafile_interop.py
index df2ce68..12d5605 100644
--- a/lang/py3/avro/tests/test_datafile_interop.py
+++ b/lang/py3/avro/tests/test_datafile_interop.py
@@ -18,65 +18,26 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import logging
+import glob
 import os
-import tempfile
 import unittest
 
 from avro import datafile
 from avro import io
-from avro import schema
-
-
-def GetInteropSchema():
-  test_dir = os.path.dirname(os.path.abspath(__file__))
-  schema_json_path = os.path.join(test_dir, 'interop.avsc')
-  with open(schema_json_path, 'r') as f:
-    schema_json = f.read()
-  return schema.Parse(schema_json)
-
-
-INTEROP_SCHEMA = GetInteropSchema()
-INTEROP_DATUM = {
-    'intField': 12,
-    'longField': 15234324,
-    'stringField': 'hey',
-    'boolField': True,
-    'floatField': 1234.0,
-    'doubleField': -1234.0,
-    'bytesField': b'12312adf',
-    'nullField': None,
-    'arrayField': [5.0, 0.0, 12.0],
-    'mapField': {'a': {'label': 'a'}, 'bee': {'label': 'cee'}},
-    'unionField': 12.0,
-    'enumField': 'C',
-    'fixedField': b'1019181716151413',
-    'recordField': {
-        'label': 'blah',
-        'children': [{'label': 'inner', 'children': []}],
-    },
-}
-
-
-def WriteDataFile(path, datum, schema):
-  datum_writer = io.DatumWriter()
-  with open(path, 'wb') as writer:
-    # NB: not using compression
-    with datafile.DataFileWriter(writer, datum_writer, schema) as dfw:
-      dfw.append(datum)
 
 
 class TestDataFileInterop(unittest.TestCase):
   def testInterop(self):
-    with tempfile.NamedTemporaryFile() as temp_path:
-      WriteDataFile(temp_path.name, INTEROP_DATUM, INTEROP_SCHEMA)
-
-      # read data in binary from file
-      datum_reader = io.DatumReader()
-      with open(temp_path.name, 'rb') as reader:
-        dfr = datafile.DataFileReader(reader, datum_reader)
-        for datum in dfr:
-          self.assertEqual(INTEROP_DATUM, datum)
+    datum_reader = io.DatumReader()
+    for avro_file in glob.glob('../../build/interop/data/*.avro'):
+      base_ext = os.path.splitext(os.path.basename(avro_file))[0].split('_', 1)
+      if len(base_ext) < 2 or base_ext[1] in datafile.VALID_CODECS:
+        with open(avro_file, 'rb') as reader, \
+          datafile.DataFileReader(reader, datum_reader) as dfr:
+          i = 0
+          for i, datum in enumerate(dfr, 1):
+            self.assertIsNotNone(datum)
+          self.assertGreater(i, 0)
 
 
 if __name__ == '__main__':
diff --git a/lang/py3/setup.py b/lang/py3/setup.py
index a752779..22b6472 100755
--- a/lang/py3/setup.py
+++ b/lang/py3/setup.py
@@ -26,9 +26,10 @@ The avro-python3 software is designed for Python 3, but this file and the
packag
 https://pypi.org/project/avro-python3/
 """
 
+import distutils.cmd
 import distutils.command.clean
-import distutils.file_util
 import distutils.dir_util
+import distutils.file_util
 import distutils.log
 import fnmatch
 import os
@@ -110,11 +111,34 @@ class CleanCommand(distutils.command.clean.clean):
                 os.remove(name)
 
 
+class GenerateInteropDataCommand(distutils.cmd.Command):
+    """A command to generate Avro files for data interop test."""
+
+    user_options = [
+      ('schema-file=', None, 'path to input Avro schema file'),
+      ('output-path=', None, 'path to output Avro data files'),
+    ]
+
+    def initialize_options(self):
+        self.schema_file = os.path.join(os.getcwd(), 'interop.avsc')
+        self.output_path = os.getcwd()
+
+    def finalize_options(self):
+        pass
+
+    def run(self):
+        from avro.tests import gen_interop_data
+        gen_interop_data.generate(self.schema_file, self.output_path)
+
+
 def main():
     if not _is_distribution():
         _generate_package_data()
 
-    setup(cmdclass={"clean": CleanCommand})
+    setup(cmdclass={
+        "clean": CleanCommand,
+        "generate_interop_data": GenerateInteropDataCommand,
+    })
 
 
 if __name__ == '__main__':


Mime
View raw message