avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From fo...@apache.org
Subject [avro] branch master updated: AVRO-2468: Fix broken data interoperability on the Perl bindings (#582)
Date Tue, 20 Aug 2019 09:37:00 GMT
This is an automated email from the ASF dual-hosted git repository.

fokko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/avro.git


The following commit(s) were added to refs/heads/master by this push:
     new 02dbe25  AVRO-2468: Fix broken data interoperability on the Perl bindings (#582)
02dbe25 is described below

commit 02dbe25e35d5ffc644f049cf602f88c1b137ea29
Author: Kengo Seki <sekikn@apache.org>
AuthorDate: Tue Aug 20 18:36:53 2019 +0900

    AVRO-2468: Fix broken data interoperability on the Perl bindings (#582)
    
    * Fix Avro::Schema to take namespace into consideration
      when parsing named types in arrays/maps/unions.
    
    * Fix Avro::Schema to encode fixed type's size value
      into numeric when it outputs Avro schema.
    
    * To ensure the above fixes work, add data interop test
      for the Perl bindings.
---
 build.sh                              |  2 +
 lang/perl/build.sh                    | 13 ++++++-
 lang/perl/lib/Avro/Schema.pm          |  9 +++--
 lang/perl/share/interop-data-generate | 72 +++++++++++++++++++++++++++++++++++
 lang/perl/xt/interop.t                | 40 +++++++++++++++++++
 lang/perl/xt/schema.t                 | 40 +++++++++++++++++++
 6 files changed, 170 insertions(+), 6 deletions(-)

diff --git a/build.sh b/build.sh
index 2660afa..73935a3 100755
--- a/build.sh
+++ b/build.sh
@@ -74,6 +74,7 @@ do
       (cd lang/csharp; ./build.sh interop-data-generate)
       (cd lang/ruby; rake generate_interop)
       (cd lang/php; ./build.sh interop-data-generate)
+      (cd lang/perl; ./build.sh interop-data-generate)
 
       # run interop data tests
       (cd lang/java; mvn -B test -P interop-data-test)
@@ -84,6 +85,7 @@ do
       (cd lang/csharp; ./build.sh interop-data-test)
       (cd lang/ruby; rake interop)
       (cd lang/php; ./build.sh test-interop)
+      (cd lang/perl; ./build.sh interop-data-test)
 
       # java needs to package the jars for the interop rpc tests
       (cd lang/java; mvn -B package -DskipTests)
diff --git a/lang/perl/build.sh b/lang/perl/build.sh
index fee0ca6..9606a24 100755
--- a/lang/perl/build.sh
+++ b/lang/perl/build.sh
@@ -18,7 +18,7 @@
 set -e # exit on error
 
 function usage {
-  echo "Usage: $0 {lint|test|dist|clean}"
+  echo "Usage: $0 {lint|test|dist|clean|interop-data-generate|interop-data-test}"
   exit 1
 }
 
@@ -47,18 +47,27 @@ case "$target" in
   lint)
     echo 'This is a stub where someone can provide linting.'
     ;;
+
   test)
     perl ./Makefile.PL && make test
     ;;
 
   dist)
-     perl ./Makefile.PL && make dist
+    perl ./Makefile.PL && make dist
     ;;
 
   clean)
     do_clean
     ;;
 
+  interop-data-generate)
+    perl -Ilib share/interop-data-generate
+    ;;
+
+  interop-data-test)
+    prove -Ilib xt/interop.t
+    ;;
+
   *)
     usage
 esac
diff --git a/lang/perl/lib/Avro/Schema.pm b/lang/perl/lib/Avro/Schema.pm
index 56eeaf9..1ecf5ee 100644
--- a/lang/perl/lib/Avro/Schema.pm
+++ b/lang/perl/lib/Avro/Schema.pm
@@ -631,7 +631,7 @@ sub new {
     my $items = $struct->{items}
         or throw Avro::Schema::Error::Parse("Array must declare 'items'");
 
-    $items = Avro::Schema->parse_struct($items, $param{names});
+    $items = Avro::Schema->parse_struct($items, $param{names}, $param{namespace});
     $schema->{items} = $items;
     return $schema;
 }
@@ -673,7 +673,7 @@ sub new {
     unless (defined $values && length $values) {
         throw Avro::Schema::Error::Parse("Map must declare 'values'");
     }
-    $values = Avro::Schema->parse_struct($values, $param{names});
+    $values = Avro::Schema->parse_struct($values, $param{names}, $param{namespace});
     $schema->{values} = $values;
 
     return $schema;
@@ -716,7 +716,7 @@ sub new {
     my @schemas;
     my %seen_types;
     for my $struct (@$union) {
-        my $sch = Avro::Schema->parse_struct($struct, $names);
+        my $sch = Avro::Schema->parse_struct($struct, $names, $param{namespace});
         my $type = $sch->type;
 
         ## 1.3.2 Unions may not contain more than one schema with the same
@@ -792,7 +792,8 @@ sub new {
             "Fixed.size should be a positive integer"
         );
     }
-    $schema->{size} = $size;
+    # Cast into numeric so that it will be encoded as a JSON number
+    $schema->{size} = $size + 0;
 
     return $schema;
 }
diff --git a/lang/perl/share/interop-data-generate b/lang/perl/share/interop-data-generate
new file mode 100644
index 0000000..c659605
--- /dev/null
+++ b/lang/perl/share/interop-data-generate
@@ -0,0 +1,72 @@
+#!/usr/bin/env perl
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+use strict;
+use warnings;
+
+use File::Path qw(make_path);
+use IO::File;
+use Avro::DataFile;
+use Avro::DataFileWriter;
+
+my $schema = do {
+    my $fh = IO::File->new('../../share/test/schemas/interop.avsc');
+    local $/ = undef;
+    <$fh>
+};
+
+my $datum = {
+    intField    => 12,
+    longField   => 15234324,
+    stringField => 'hey',
+    boolField   => 1,
+    floatField  => 1234.0,
+    doubleField => -1234.0,
+    bytesField  => '12312adf',
+    nullField   => undef,
+    arrayField  => [5.0, 0.0, 12.0],
+    mapField    => {
+        a   => { label => 'a'   },
+        bee => { label => 'cee' },
+    },
+    unionField  => 12.0,
+    enumField   => 'C',
+    fixedField  => '1019181716151413',
+    recordField => {
+        label    => 'blah',
+        children => [ { label => 'inner', children => [] } ]
+    },
+};
+
+while (my ($codec, $enabled) = each(%Avro::DataFile::ValidCodec)) {
+    next unless $enabled;
+    my $outdir = '../../build/interop/data';
+    make_path($outdir);
+    my $path = sprintf('>%s/perl%s.avro', $outdir,
+                       $codec eq 'null' ? '' : '_'.$codec);
+    my $fh = IO::File->new($path);
+    my $writer_schema = Avro::Schema->parse($schema);
+    my $writer = Avro::DataFileWriter->new(
+        fh => $fh,
+        codec => $codec,
+        writer_schema => $writer_schema
+    );
+    $writer->print($datum);
+    $writer->close;
+}
diff --git a/lang/perl/xt/interop.t b/lang/perl/xt/interop.t
new file mode 100644
index 0000000..0d6d0b1
--- /dev/null
+++ b/lang/perl/xt/interop.t
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+use strict;
+use warnings;
+
+use Test::More;
+use File::Basename qw(basename);
+use IO::File;
+use_ok 'Avro::DataFile';
+use_ok 'Avro::DataFileReader';
+
+for my $path (glob '../../build/interop/data/*.avro') {
+    my $fn = basename($path);
+    substr($fn, rindex $fn, '.') = '';
+    my $idx = rindex $fn, '_';
+    if (-1 < $idx) {
+        my $codec = substr $fn, $idx + 1;
+        next unless $Avro::DataFile::ValidCodec{$codec};
+    }
+    my $fh = IO::File->new($path);
+    Avro::DataFileReader->new(fh => $fh);
+    note("Succeeded to read ${path}");
+}
+
+done_testing;
diff --git a/lang/perl/xt/schema.t b/lang/perl/xt/schema.t
new file mode 100644
index 0000000..69c66ed
--- /dev/null
+++ b/lang/perl/xt/schema.t
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+use strict;
+use warnings;
+
+use Test::More;
+use File::Find;
+use_ok 'Avro::Schema';
+
+sub parse {
+    next unless /\.avsc$/;
+    open(my $fh, $_);
+    local $/ = undef;
+    my $schema = <$fh>;
+    close $fh;
+    Avro::Schema->parse($schema);
+    note("Successfully parsed: $_");
+}
+
+# Ensure that all schema files under the "share" directory can be parsed
+{
+    find(\&parse, '../../share');
+}
+
+done_testing;


Mime
View raw message