beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pabl...@apache.org
Subject [beam] branch master updated: Add Postcommit test suite for Python 3.7
Date Thu, 16 May 2019 17:31:38 GMT
This is an automated email from the ASF dual-hosted git repository.

pabloem pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new 62f431f  Add Postcommit test suite for Python 3.7
     new adb6d0c  Merge pull request #8564 from fredo838/add-suites
62f431f is described below

commit 62f431fbeeceef11b1ed3ddf61fd16255236573c
Author: frederik2 <frederik.bode@ml6.eu>
AuthorDate: Tue May 14 18:40:10 2019 +0200

    Add Postcommit test suite for Python 3.7
---
 build.gradle                                       |  2 +
 sdks/python/test-suites/dataflow/py37/build.gradle | 71 ++++++++++++++++++++++
 sdks/python/test-suites/direct/py36/build.gradle   |  1 +
 .../test-suites/direct/{py36 => py37}/build.gradle |  3 +-
 settings.gradle                                    |  2 +
 5 files changed, 78 insertions(+), 1 deletion(-)

diff --git a/build.gradle b/build.gradle
index a6cd8a7..9e9fba1 100644
--- a/build.gradle
+++ b/build.gradle
@@ -203,9 +203,11 @@ task pythonPostCommit() {
 task python3PostCommit() {
   dependsOn ":sdks:python:test-suites:dataflow:py35:postCommitIT"
   dependsOn ":sdks:python:test-suites:dataflow:py36:postCommitIT"
+  dependsOn ":sdks:python:test-suites:dataflow:py37:postCommitIT"
   dependsOn ":sdks:python:test-suites:dataflow:py35:validatesRunnerBatchTests"
   dependsOn ":sdks:python:test-suites:direct:py35:postCommitIT"
   dependsOn ":sdks:python:test-suites:direct:py36:postCommitIT"
+  dependsOn ":sdks:python:test-suites:direct:py37:postCommitIT"
 }
 
 task portablePythonPreCommit() {
diff --git a/sdks/python/test-suites/dataflow/py37/build.gradle b/sdks/python/test-suites/dataflow/py37/build.gradle
new file mode 100644
index 0000000..4d7aefc
--- /dev/null
+++ b/sdks/python/test-suites/dataflow/py37/build.gradle
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+apply plugin: org.apache.beam.gradle.BeamModulePlugin
+applyPythonNature()
+
+// Required to setup a Python 3 virtualenv.
+pythonVersion = '3.7'
+
+def runScriptsDir = "${project.rootDir}/sdks/python/scripts"
+
+// Basic test options for ITs running on Jenkins.
+def basicTestOpts = [
+    "--nocapture",  // print stdout instantly
+    "--processes=8",  // run tests in parallel
+    "--process-timeout=4500", // timeout of whole command execution
+]
+
+task postCommitIT(dependsOn: ['sdist', 'installGcpTest']) {
+  dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"
+
+  def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath
+
+  doLast {
+    def testOpts = basicTestOpts + ["--attr=IT"]
+
+     def cmdArgs = project.mapToArgString([
+        "test_opts": testOpts,
+        "sdk_location": "${project.buildDir}/apache-beam.tar.gz",
+        "worker_jar": dataflowWorkerJar
+    ])
+    exec {
+      executable 'sh'
+      args '-c', ". ${project.ext.envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh
$cmdArgs"
+    }
+  }
+}
+
+task validatesRunnerBatchTests(dependsOn: ['installGcpTest', 'sdist']) {
+  dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"
+
+  def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath
+
+  doLast {
+    def testOpts = basicTestOpts + ["--attr=ValidatesRunner"]
+    def cmdArgs = project.mapToArgString([
+        "test_opts": testOpts,
+        "sdk_location": "${project.buildDir}/apache-beam.tar.gz",
+        "worker_jar": dataflowWorkerJar
+    ])
+    exec {
+      executable 'sh'
+      args '-c', ". ${project.ext.envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh
$cmdArgs"
+    }
+  }
+}
diff --git a/sdks/python/test-suites/direct/py36/build.gradle b/sdks/python/test-suites/direct/py36/build.gradle
index cc55c5a..a2c4311 100644
--- a/sdks/python/test-suites/direct/py36/build.gradle
+++ b/sdks/python/test-suites/direct/py36/build.gradle
@@ -32,6 +32,7 @@ task postCommitIT(dependsOn: 'installGcpTest') {
         "apache_beam.io.gcp.pubsub_integration_test:PubSubIntegrationTest",
         "apache_beam.io.gcp.big_query_query_to_table_it_test:BigQueryQueryToTableIT",
         "apache_beam.io.gcp.bigquery_io_read_it_test",
+        "apache_beam.io.gcp.datastore.v1new.datastore_write_it_test",
     ]
     def testOpts = [
         "--tests=${batchTests.join(',')}",
diff --git a/sdks/python/test-suites/direct/py36/build.gradle b/sdks/python/test-suites/direct/py37/build.gradle
similarity index 95%
copy from sdks/python/test-suites/direct/py36/build.gradle
copy to sdks/python/test-suites/direct/py37/build.gradle
index cc55c5a..8d78627 100644
--- a/sdks/python/test-suites/direct/py36/build.gradle
+++ b/sdks/python/test-suites/direct/py37/build.gradle
@@ -20,7 +20,7 @@ plugins { id 'org.apache.beam.module' }
 applyPythonNature()
 
 // Required to setup a Python 3 virtualenv.
-pythonVersion = '3.6'
+pythonVersion = '3.7'
 
 def runScriptsDir = "${project.rootDir}/sdks/python/scripts"
 
@@ -32,6 +32,7 @@ task postCommitIT(dependsOn: 'installGcpTest') {
         "apache_beam.io.gcp.pubsub_integration_test:PubSubIntegrationTest",
         "apache_beam.io.gcp.big_query_query_to_table_it_test:BigQueryQueryToTableIT",
         "apache_beam.io.gcp.bigquery_io_read_it_test",
+        "apache_beam.io.gcp.datastore.v1new.datastore_write_it_test",
     ]
     def testOpts = [
         "--tests=${batchTests.join(',')}",
diff --git a/settings.gradle b/settings.gradle
index fd1201b..d433f4c 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -126,8 +126,10 @@ include ":sdks:python:container:py3"
 include ":sdks:python:test-suites:dataflow"
 include ":sdks:python:test-suites:dataflow:py35"
 include ":sdks:python:test-suites:dataflow:py36"
+include ":sdks:python:test-suites:dataflow:py37"
 include ":sdks:python:test-suites:direct:py35"
 include ":sdks:python:test-suites:direct:py36"
+include ":sdks:python:test-suites:direct:py36"
 include ":sdks:python:test-suites:tox:py35"
 include ":sdks:python:test-suites:tox:py36"
 include ":sdks:python:test-suites:tox:py37"


Mime
View raw message