beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From al...@apache.org
Subject [beam] branch master updated: [BEAM-5091] Run wordcount_it in Python 3 in ValidatesContainer test
Date Tue, 05 Feb 2019 21:24:52 GMT
This is an automated email from the ASF dual-hosted git repository.

altay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new b05ccd2  [BEAM-5091] Run wordcount_it in Python 3 in ValidatesContainer test
     new f85d953  Merge pull request #7706 from markflyhigh/py3-wordcount-it
b05ccd2 is described below

commit b05ccd2c63c1af66ceac833fd3e6dc43f9a26329
Author: Mark Liu <markliu@google.com>
AuthorDate: Fri Feb 1 15:56:56 2019 -0800

    [BEAM-5091] Run wordcount_it in Python 3 in ValidatesContainer test
---
 .../python/container/run_validatescontainer_py3.sh | 40 ++++++++++++++++++++--
 1 file changed, 38 insertions(+), 2 deletions(-)

diff --git a/sdks/python/container/run_validatescontainer_py3.sh b/sdks/python/container/run_validatescontainer_py3.sh
index 35f3d6c..f852a41 100755
--- a/sdks/python/container/run_validatescontainer_py3.sh
+++ b/sdks/python/container/run_validatescontainer_py3.sh
@@ -47,13 +47,16 @@ gcloud -v
 
 # Build the container image
 TAG=$(date +%Y%m%d-%H%M%S)
-CONTAINER=us.gcr.io/$PROJECT/$USER/python
+CONTAINER=us.gcr.io/$PROJECT/$USER/python3
 echo "Building Python 3 container $CONTAINER"
 ./gradlew :beam-sdks-python-container-py3:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER
-Pdocker-tag=$TAG -Ppython3 --info
 
 # Verify it exists
 docker images | grep $TAG
 
+# Push the container
+gcloud docker -- push $CONTAINER
+
 function cleanup_container {
   # Delete the container locally and remotely
   docker rmi $CONTAINER:$TAG || echo "Failed to remove container"
@@ -61,4 +64,37 @@ function cleanup_container {
 }
 trap cleanup_container EXIT
 
-echo "Successfully built Python 3 container $CONTAINER"
+echo ">>> Successfully built Python 3 container $CONTAINER"
+
+# Python 3 virtualenv for the rest of the script to run setup & e2e test
+virtualenv sdks/python/container/py3/venv -p python3.5
+. sdks/python/container/py3/venv/bin/activate
+cd sdks/python
+pip install -e .[gcp,test]
+
+# Install test dependencies for ValidatesContainer tests.
+echo "pyhamcrest" > postcommit_requirements.txt
+
+# Create a tarball
+python setup.py sdist
+SDK_LOCATION=$(find dist/apache-beam-*.tar.gz)
+
+# Run ValidatesRunner tests on Google Cloud Dataflow service
+echo ">>> RUNNING DATAFLOW RUNNER VALIDATESCONTAINER TEST"
+python setup.py nosetests \
+  --attr Py3IT \
+  --nologcapture \
+  --processes=1 \
+  --process-timeout=900 \
+  --test-pipeline-options=" \
+    --runner=TestDataflowRunner \
+    --project=$PROJECT \
+    --worker_harness_container_image=$CONTAINER:$TAG \
+    --staging_location=$GCS_LOCATION/staging-validatesrunner-test \
+    --temp_location=$GCS_LOCATION/temp-validatesrunner-test \
+    --output=$GCS_LOCATION/output \
+    --sdk_location=$SDK_LOCATION \
+    --requirements_file=postcommit_requirements.txt \
+    --num_workers=1"
+
+echo ">>> SUCCESS DATAFLOW RUNNER VALIDATESCONTAINER TEST"


Mime
View raw message