avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From fo...@apache.org
Subject [avro] branch master updated: AVRO-2456: Add interop test for the snappy and zstd codec (#573)
Date Fri, 12 Jul 2019 17:55:48 GMT
This is an automated email from the ASF dual-hosted git repository.

fokko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/avro.git


The following commit(s) were added to refs/heads/master by this push:
     new fbfb135  AVRO-2456: Add interop test for the snappy and zstd codec (#573)
fbfb135 is described below

commit fbfb135dd50fc39835c5323057561144eff2bc41
Author: Kengo Seki <sekikn@apache.org>
AuthorDate: Sat Jul 13 02:55:42 2019 +0900

    AVRO-2456: Add interop test for the snappy and zstd codec (#573)
---
 lang/java/avro/pom.xml            | 32 ++++++++++++++++++++++++++++++--
 lang/java/ipc/pom.xml             |  8 ++++++++
 lang/java/pom.xml                 |  1 -
 lang/py/test/gen_interop_data.py  | 32 +++++++++++++++++++++++++-------
 lang/ruby/interop/test_interop.rb | 10 +++++++++-
 share/docker/Dockerfile           |  7 ++++++-
 6 files changed, 78 insertions(+), 12 deletions(-)

diff --git a/lang/java/avro/pom.xml b/lang/java/avro/pom.xml
index 6151b62..b1c313e 100644
--- a/lang/java/avro/pom.xml
+++ b/lang/java/avro/pom.xml
@@ -126,6 +126,36 @@
                 </configuration>
                 <goals><goal>java</goal></goals>
               </execution>
+              <!-- Generate random data for interop tests, using snappy codec -->
+              <execution>
+                <id>interop-generate-snappy-codec</id>
+                <phase>generate-resources</phase>
+                <configuration>
+                  <mainClass>org.apache.avro.util.RandomData</mainClass>
+                  <arguments>
+                    <argument>../../../share/test/schemas/interop.avsc</argument>
+                    <argument>../../../build/interop/data/java_snappy.avro</argument>
+                    <argument>100</argument>
+                    <argument>snappy</argument>
+                  </arguments>
+                </configuration>
+                <goals><goal>java</goal></goals>
+              </execution>
+              <!-- Generate random data for interop tests, using zstandard codec -->
+              <execution>
+                <id>interop-generate-zstandard-codec</id>
+                <phase>generate-resources</phase>
+                <configuration>
+                  <mainClass>org.apache.avro.util.RandomData</mainClass>
+                  <arguments>
+                    <argument>../../../share/test/schemas/interop.avsc</argument>
+                    <argument>../../../build/interop/data/java_zstandard.avro</argument>
+                    <argument>100</argument>
+                    <argument>zstandard</argument>
+                  </arguments>
+                </configuration>
+                <goals><goal>java</goal></goals>
+              </execution>
             </executions>
           </plugin>
         </plugins>
@@ -149,7 +179,6 @@
     <dependency>
       <groupId>org.xerial.snappy</groupId>
       <artifactId>snappy-java</artifactId>
-      <scope>provided</scope>
       <optional>true</optional>
     </dependency>
     <dependency>
@@ -162,7 +191,6 @@
     <dependency>
       <groupId>com.github.luben</groupId>
       <artifactId>zstd-jni</artifactId>
-      <scope>provided</scope>
       <optional>true</optional>
     </dependency>
     <dependency>
diff --git a/lang/java/ipc/pom.xml b/lang/java/ipc/pom.xml
index 4265a09..c5c7c7f 100644
--- a/lang/java/ipc/pom.xml
+++ b/lang/java/ipc/pom.xml
@@ -142,6 +142,14 @@
       <artifactId>javax.annotation-api</artifactId>
       <version>1.3.2</version>
     </dependency>
+    <dependency>
+      <groupId>org.xerial.snappy</groupId>
+      <artifactId>snappy-java</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.github.luben</groupId>
+      <artifactId>zstd-jni</artifactId>
+    </dependency>
 
   </dependencies>
 
diff --git a/lang/java/pom.xml b/lang/java/pom.xml
index bf187f6..6887cfe 100644
--- a/lang/java/pom.xml
+++ b/lang/java/pom.xml
@@ -598,7 +598,6 @@
         <groupId>com.github.luben</groupId>
         <artifactId>zstd-jni</artifactId>
         <version>${zstd-jni.version}</version>
-        <scope>test</scope>
       </dependency>
     </dependencies>
   </dependencyManagement>
diff --git a/lang/py/test/gen_interop_data.py b/lang/py/test/gen_interop_data.py
index d8cd857..7db929a 100644
--- a/lang/py/test/gen_interop_data.py
+++ b/lang/py/test/gen_interop_data.py
@@ -15,6 +15,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import os
 import sys
 from avro import schema
 from avro import io
@@ -37,11 +38,28 @@ DATUM = {
   'recordField': {'label': 'blah', 'children': [{'label': 'inner', 'children': []}]},
 }
 
+CODECS_TO_VALIDATE = ('null', 'deflate')
+try:
+  import snappy
+  CODECS_TO_VALIDATE += ('snappy',)
+except ImportError:
+  print 'Snappy not present, will skip generating it.'
+try:
+  import zstandard
+  CODECS_TO_VALIDATE += ('zstandard',)
+except ImportError:
+  print 'Zstandard not present, will skip generating it.'
+
 if __name__ == "__main__":
-  interop_schema = schema.parse(open(sys.argv[1], 'r').read())
-  writer = open(sys.argv[2], 'wb')
-  datum_writer = io.DatumWriter()
-  # NB: not using compression
-  dfw = datafile.DataFileWriter(writer, datum_writer, interop_schema)
-  dfw.append(DATUM)
-  dfw.close()
+  for codec in CODECS_TO_VALIDATE:
+    interop_schema = schema.parse(open(sys.argv[1], 'r').read())
+    filename = sys.argv[2]
+    if codec != 'null':
+      base, ext = os.path.splitext(filename)
+      filename = base + "_" + codec + ext
+    writer = open(filename, 'wb')
+    datum_writer = io.DatumWriter()
+    # NB: not using compression
+    dfw = datafile.DataFileWriter(writer, datum_writer, interop_schema, codec=codec)
+    dfw.append(DATUM)
+    dfw.close()
diff --git a/lang/ruby/interop/test_interop.rb b/lang/ruby/interop/test_interop.rb
index 1a4aa74..a63c158 100644
--- a/lang/ruby/interop/test_interop.rb
+++ b/lang/ruby/interop/test_interop.rb
@@ -19,11 +19,19 @@ require 'rubygems'
 require 'test/unit'
 require 'avro'
 
+CODECS_TO_VALIDATE = ['deflate']  # The 'null' codec is implicitly included
+
 class TestInterop < Test::Unit::TestCase
   HERE = File.expand_path(File.dirname(__FILE__))
   SHARE = HERE + '/../../../share'
   SCHEMAS = SHARE + '/test/schemas'
-  Dir[HERE + '/../../../build/interop/data/*'].each do |fn|
+
+  files = Dir[HERE + '/../../../build/interop/data/*.avro'].select do |fn|
+    sep, codec = File.basename(fn, 'avro').rpartition('_')[1, 2]
+    sep.empty? || CODECS_TO_VALIDATE.include?(codec)
+  end
+
+  files.each do |fn|
     define_method("test_read_#{File.basename(fn, 'avro')}") do
       projection = Avro::Schema.parse(File.read(SCHEMAS+'/interop.avsc'))
 
diff --git a/share/docker/Dockerfile b/share/docker/Dockerfile
index f0e9f18..82f6686 100644
--- a/share/docker/Dockerfile
+++ b/share/docker/Dockerfile
@@ -67,8 +67,10 @@ RUN apt-get -qq update && \
     php5.6 \
     php5.6-gmp \
     python \
+    python-pip \
     python-setuptools \
     python-snappy \
+    python-wheel \
     python3-pip \
     python3-setuptools \
     python3-snappy \
@@ -93,7 +95,10 @@ RUN curl -L https://cpanmin.us | perl - --mirror https://www.cpan.org/
--self-up
 # Install PHPUnit
 RUN wget -O /usr/local/bin/phpunit https://phar.phpunit.de/phpunit-5.6.phar && chmod
+x /usr/local/bin/phpunit
 
-# Install Python packages
+# Install Python2 packages
+RUN pip install zstandard
+
+# Install Python3 packages
 RUN pip3 install zstandard
 
 # Install Ruby modules


Mime
View raw message