beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From c..@apache.org
Subject [beam] branch master updated: Update ValidatesContainer test run scripts.
Date Thu, 07 Mar 2019 23:08:49 GMT
This is an automated email from the ASF dual-hosted git repository.

ccy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new 058dfc7  Update ValidatesContainer test run scripts.
     new e835a05  Merge pull request #7905 from markflyhigh/py3-merge-valcon-script
058dfc7 is described below

commit 058dfc765a25ceb3afa3a0da65a9690a614bbd07
Author: Mark Liu <markliu@google.com>
AuthorDate: Wed Feb 20 12:09:17 2019 -0800

    Update ValidatesContainer test run scripts.
---
 ...ommit_Python_ValidatesContainer_Dataflow.groovy |   4 +-
 sdks/python/container/run_validatescontainer.sh    |  38 +++++++-
 .../python/container/run_validatescontainer_py3.sh | 101 ---------------------
 3 files changed, 35 insertions(+), 108 deletions(-)

diff --git a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy
b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy
index 0b21c2d..da5094e 100644
--- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy
+++ b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy
@@ -31,7 +31,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_ValCont',
   // Execute shell command to test Python SDK.
   // TODO: Parallel the script run with Jenkins DSL or Gradle.
   steps {
-    shell('cd ' + commonJobProperties.checkoutDir + ' && bash sdks/python/container/run_validatescontainer.sh')
-    shell('cd ' + commonJobProperties.checkoutDir + ' && bash sdks/python/container/run_validatescontainer_py3.sh')
+    shell('cd ' + commonJobProperties.checkoutDir + ' && bash sdks/python/container/run_validatescontainer.sh
python2')
+    shell('cd ' + commonJobProperties.checkoutDir + ' && bash sdks/python/container/run_validatescontainer.sh
python3')
   }
 }
diff --git a/sdks/python/container/run_validatescontainer.sh b/sdks/python/container/run_validatescontainer.sh
index 87ee7bb..47f5b8d 100755
--- a/sdks/python/container/run_validatescontainer.sh
+++ b/sdks/python/container/run_validatescontainer.sh
@@ -23,10 +23,19 @@
 # GCS_LOCATION -> Temporary location to use for service tests.
 # PROJECT      -> Project name to use for dataflow and docker images.
 #
-# Execute from the root of the repository: sdks/python/container/run_validatescontainer.sh
+# Execute from the root of the repository:
+#     test Python2 container: ./sdks/python/container/run_validatescontainer.sh python2
+#     test Python3 container: ./sdks/python/container/run_validatescontainer.sh python3
 
 echo "This script must be executed in the root of beam project. Please set LOCAL_PATH, GCS_LOCATION,
and PROJECT as desired."
 
+if [[ $# != 1 ]]; then
+  printf "Usage: \n$> ./sdks/python/container/run_validatescontainer.sh <python_version>"
+  printf "\n\tpython_version: [required] Python version used for container build and run
tests."
+  printf " Use 'python2' for Python2, 'python3' for Python3."
+  exit 1
+fi
+
 set -e
 set -v
 
@@ -36,6 +45,22 @@ GCS_LOCATION=${GCS_LOCATION:-gs://temp-storage-for-end-to-end-tests}
 # Project for the container and integration test
 PROJECT=${PROJECT:-apache-beam-testing}
 
+# Other variables branched by Python version.
+if [[ $1 == "python2" ]]; then
+  IMAGE_NAME="python"       # Use this to create CONTAINER_IMAGE variable.
+  CONTAINER_PROJECT="beam-sdks-python-container"  # Use this to build container by Gradle.
+  GRADLE_PY3_FLAG=""        # Use this in Gradle command.
+  PY_INTERPRETER="python"   # Use this in virtualenv command.
+elif [[ $1 == "python3" ]]; then
+  IMAGE_NAME="python3"          # Use this to create CONTAINER_IMAGE variable.
+  CONTAINER_PROJECT="beam-sdks-python-container-py3"  # Use this to build container by Gradle.
+  GRADLE_PY3_FLAG="-Ppython3"   # Use this in Gradle command.
+  PY_INTERPRETER="python3.5"    # Use this in virtualenv command.
+else
+  echo "Must set Python version with 'python2' or 'python3' from commandline."
+  exit 1
+fi
+
 # Verify in the root of the repository
 test -d sdks/python/container
 
@@ -47,9 +72,9 @@ gcloud -v
 
 # Build the container
 TAG=$(date +%Y%m%d-%H%M%S)
-CONTAINER=us.gcr.io/$PROJECT/$USER/python
+CONTAINER=us.gcr.io/$PROJECT/$USER/$IMAGE_NAME
 echo "Using container $CONTAINER"
-./gradlew :beam-sdks-python-container:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER
-Pdocker-tag=$TAG --info
+./gradlew :$CONTAINER_PROJECT:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$TAG
$GRADLE_PY3_FLAG --info
 
 # Verify it exists
 docker images | grep $TAG
@@ -65,9 +90,12 @@ function cleanup_container {
 }
 trap cleanup_container EXIT
 
+echo ">>> Successfully built and push container $CONTAINER"
+
 # Virtualenv for the rest of the script to run setup & e2e test
-virtualenv sdks/python/container
-. sdks/python/container/bin/activate
+VENV_PATH=sdks/python/container/venv/$PY_INTERPRETER
+virtualenv $VENV_PATH -p $PY_INTERPRETER
+. $VENV_PATH/bin/activate
 cd sdks/python
 pip install -e .[gcp,test]
 
diff --git a/sdks/python/container/run_validatescontainer_py3.sh b/sdks/python/container/run_validatescontainer_py3.sh
deleted file mode 100755
index bff133c..0000000
--- a/sdks/python/container/run_validatescontainer_py3.sh
+++ /dev/null
@@ -1,101 +0,0 @@
-#!/bin/bash
-#
-#    Licensed to the Apache Software Foundation (ASF) under one or more
-#    contributor license agreements.  See the NOTICE file distributed with
-#    this work for additional information regarding copyright ownership.
-#    The ASF licenses this file to You under the Apache License, Version 2.0
-#    (the "License"); you may not use this file except in compliance with
-#    the License.  You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
-
-# This script validates python 3 container. It will be run by Jenkins as a post commit test.
-# In order to run locally make the following changes:
-#
-# LOCAL_PATH   -> Path of tox and virtualenv if you have them already installed.
-# GCS_LOCATION -> Temporary location to use for service tests.
-# PROJECT      -> Project name to use for dataflow and docker images.
-#
-# Execute from the root of the repository: sdks/python/container/run_validatescontainer.sh
-
-echo "This script must be executed in the root of beam project. Please set LOCAL_PATH, GCS_LOCATION,
and PROJECT as desired."
-
-set -e
-set -v
-
-# Where to store integration test outputs.
-GCS_LOCATION=${GCS_LOCATION:-gs://temp-storage-for-end-to-end-tests}
-
-# Project for the container and integration test
-PROJECT=${PROJECT:-apache-beam-testing}
-
-# Verify in the root of the repository
-test -d sdks/python/container
-
-# Verify docker and gcloud commands exist
-command -v docker
-command -v gcloud
-docker -v
-gcloud -v
-
-# Build the container image
-TAG=$(date +%Y%m%d-%H%M%S)
-CONTAINER=us.gcr.io/$PROJECT/$USER/python3
-echo "Building Python 3 container $CONTAINER"
-./gradlew :beam-sdks-python-container-py3:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER
-Pdocker-tag=$TAG -Ppython3 --info
-
-# Verify it exists
-docker images | grep $TAG
-
-# Push the container
-gcloud docker -- push $CONTAINER
-
-function cleanup_container {
-  # Delete the container locally and remotely
-  docker rmi $CONTAINER:$TAG || echo "Failed to remove container"
-  gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to delete container"
-  echo "Removed the container"
-}
-trap cleanup_container EXIT
-
-echo ">>> Successfully built Python 3 container $CONTAINER"
-
-# Python 3 virtualenv for the rest of the script to run setup & e2e test
-virtualenv sdks/python/container/py3/venv -p python3.5
-. sdks/python/container/py3/venv/bin/activate
-cd sdks/python
-pip install -e .[gcp,test]
-
-# Install test dependencies for ValidatesContainer tests.
-echo "pyhamcrest" > postcommit_requirements.txt
-
-# Create a tarball
-python setup.py sdist
-SDK_LOCATION=$(find dist/apache-beam-*.tar.gz)
-
-# Run ValidatesRunner tests on Google Cloud Dataflow service
-echo ">>> RUNNING DATAFLOW RUNNER VALIDATESCONTAINER TEST"
-python setup.py nosetests \
-  --attr ValidatesContainer \
-  --nologcapture \
-  --processes=1 \
-  --process-timeout=900 \
-  --test-pipeline-options=" \
-    --runner=TestDataflowRunner \
-    --project=$PROJECT \
-    --worker_harness_container_image=$CONTAINER:$TAG \
-    --staging_location=$GCS_LOCATION/staging-validatesrunner-test \
-    --temp_location=$GCS_LOCATION/temp-validatesrunner-test \
-    --output=$GCS_LOCATION/output \
-    --sdk_location=$SDK_LOCATION \
-    --requirements_file=postcommit_requirements.txt \
-    --num_workers=1"
-
-echo ">>> SUCCESS DATAFLOW RUNNER VALIDATESCONTAINER TEST"


Mime
View raw message