beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From u...@apache.org
Subject [beam] branch master updated: [BEAM-7060] Introduce Python3-only test modules (#9223)
Date Fri, 02 Aug 2019 23:01:59 GMT
This is an automated email from the ASF dual-hosted git repository.

udim pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new 149153b  [BEAM-7060] Introduce Python3-only test modules (#9223)
149153b is described below

commit 149153b525236327badb138b09235ff735045adf
Author: Udi Meiri <udim@users.noreply.github.com>
AuthorDate: Fri Aug 2 16:01:31 2019 -0700

    [BEAM-7060] Introduce Python3-only test modules (#9223)
    
    This is required for testing type annotations support (PEP 484).
---
 .../apache_beam/testing/load_tests/build.gradle    |  2 +-
 .../typehints/trivial_inference_test.py            | 21 ---------
 .../typehints/trivial_inference_test_py3.py        | 50 ++++++++++++++++++++++
 sdks/python/container/run_validatescontainer.sh    |  1 +
 sdks/python/scripts/generate_pydoc.sh              |  2 +
 sdks/python/scripts/run_integration_test.sh        |  1 +
 sdks/python/scripts/run_pylint.sh                  | 27 ++++++++----
 sdks/python/tox.ini                                |  6 +--
 8 files changed, 77 insertions(+), 33 deletions(-)

diff --git a/sdks/python/apache_beam/testing/load_tests/build.gradle b/sdks/python/apache_beam/testing/load_tests/build.gradle
index cc3742a..80e976b 100644
--- a/sdks/python/apache_beam/testing/load_tests/build.gradle
+++ b/sdks/python/apache_beam/testing/load_tests/build.gradle
@@ -41,7 +41,7 @@ task run(type: Exec, dependsOn: installGcpTest) {
     environment "LOAD_TEST_ENABLED", 'true'
     setWorkingDir "${project.rootDir}/sdks/python"
 
-    commandLine 'sh', '-c', "${project.ext.envdir}/bin/python setup.py nosetests --test-pipeline-options=\"${parseOptions(loadTestArgs)}\"
--tests ${mainClass}"
+    commandLine 'sh', '-c', "${project.ext.envdir}/bin/python setup.py nosetests --test-pipeline-options=\"${parseOptions(loadTestArgs)}\"
--tests ${mainClass}" --ignore-files '.*py3.py$'
     ignoreExitValue true
 
     doLast {
diff --git a/sdks/python/apache_beam/typehints/trivial_inference_test.py b/sdks/python/apache_beam/typehints/trivial_inference_test.py
index 9734993..e32a375 100644
--- a/sdks/python/apache_beam/typehints/trivial_inference_test.py
+++ b/sdks/python/apache_beam/typehints/trivial_inference_test.py
@@ -235,27 +235,6 @@ class TrivialInferenceTest(unittest.TestCase):
     self.assertReturnType(typehints.Any, lambda: A.m(A(), 3.0), depth=0)
     self.assertReturnType(float, lambda: A.m(A(), 3.0), depth=1)
 
-  # pylint: disable=eval-used
-  # TODO(udim): Remove eval() call once Python 2 support is removed from Beam.
-  @unittest.skipIf(sys.version_info < (3,), 'Python 3 only')
-  def testBuildListUnpack(self):
-    # Lambda uses BUILD_LIST_UNPACK opcode in Python 3.
-    # Uses eval() to hide Python 3 syntax from Python 2.
-    eval('''self.assertReturnType(typehints.List[int],
-                                  lambda _list: [*_list, *_list, *_list],
-                                  [typehints.List[int]])''')
-
-  # TODO(udim): Remove eval() call once Python 2 support is removed from Beam.
-  @unittest.skipIf(sys.version_info < (3,), 'Python 3 only')
-  def testBuildTupleUnpack(self):
-    # Lambda uses BUILD_TUPLE_UNPACK opcode in Python 3.
-    # Uses eval() to hide Python 3 syntax from Python 2.
-    eval('''self.assertReturnType(
-                typehints.Tuple[int, str, str],
-                lambda _list1, _list2: (*_list1, *_list2, *_list2),
-                [typehints.List[int], typehints.List[str]])''')
-  # pylint: enable=eval-used
-
   def testBuildTupleUnpackWithCall(self):
     # Lambda uses BUILD_TUPLE_UNPACK_WITH_CALL opcode in Python 3.6, 3.7.
     def fn(x1, x2, *unused_args):
diff --git a/sdks/python/apache_beam/typehints/trivial_inference_test_py3.py b/sdks/python/apache_beam/typehints/trivial_inference_test_py3.py
new file mode 100644
index 0000000..291e52e
--- /dev/null
+++ b/sdks/python/apache_beam/typehints/trivial_inference_test_py3.py
@@ -0,0 +1,50 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Tests for apache_beam.typehints.trivial_inference that use Python 3 syntax.
+"""
+
+from __future__ import absolute_import
+
+import unittest
+
+from apache_beam.typehints import trivial_inference
+from apache_beam.typehints import typehints
+
+
+class TrivialInferenceTest(unittest.TestCase):
+
+  def assertReturnType(self, expected, f, inputs=(), depth=5):
+    self.assertEqual(
+        expected,
+        trivial_inference.infer_return_type(f, inputs, debug=True, depth=depth))
+
+  def testBuildListUnpack(self):
+    # Lambda uses BUILD_LIST_UNPACK opcode in Python 3.
+    self.assertReturnType(typehints.List[int],
+                          lambda _list: [*_list, *_list, *_list],
+                          [typehints.List[int]])
+
+  def testBuildTupleUnpack(self):
+    # Lambda uses BUILD_TUPLE_UNPACK opcode in Python 3.
+    self.assertReturnType(typehints.Tuple[int, str, str],
+                          lambda _list1, _list2: (*_list1, *_list2, *_list2),
+                          [typehints.List[int], typehints.List[str]])
+
+
+if __name__ == '__main__':
+  unittest.main()
diff --git a/sdks/python/container/run_validatescontainer.sh b/sdks/python/container/run_validatescontainer.sh
index 790b08b..005f925 100755
--- a/sdks/python/container/run_validatescontainer.sh
+++ b/sdks/python/container/run_validatescontainer.sh
@@ -113,6 +113,7 @@ python setup.py nosetests \
   --process-timeout=900 \
   --with-xunitmp \
   --xunitmp-file=$XUNIT_FILE \
+  --ignore-files '.*py3.py$' \
   --test-pipeline-options=" \
     --runner=TestDataflowRunner \
     --project=$PROJECT \
diff --git a/sdks/python/scripts/generate_pydoc.sh b/sdks/python/scripts/generate_pydoc.sh
index 7309f90..d74dc62 100755
--- a/sdks/python/scripts/generate_pydoc.sh
+++ b/sdks/python/scripts/generate_pydoc.sh
@@ -76,6 +76,8 @@ excluded_patterns=(
     *_pb2.py
     *_test.py
     *_test_common.py
+    # TODO(BEAM-7847): Remove this once doc generation can parse Py3 syntax.
+    *py3.py
 )
 
 python $(type -p sphinx-apidoc) -fMeT -o target/docs/source apache_beam \
diff --git a/sdks/python/scripts/run_integration_test.sh b/sdks/python/scripts/run_integration_test.sh
index b5f0c60..586d338 100755
--- a/sdks/python/scripts/run_integration_test.sh
+++ b/sdks/python/scripts/run_integration_test.sh
@@ -246,4 +246,5 @@ echo ">>>   test options: $TEST_OPTS"
 python setup.py nosetests \
   --test-pipeline-options="$PIPELINE_OPTS" \
   --with-xunitmp --xunitmp-file=$XUNIT_FILE \
+  --ignore-files '.*py3.py$' \
   $TEST_OPTS
diff --git a/sdks/python/scripts/run_pylint.sh b/sdks/python/scripts/run_pylint.sh
index 751d9c6..8a4b5e1 100755
--- a/sdks/python/scripts/run_pylint.sh
+++ b/sdks/python/scripts/run_pylint.sh
@@ -60,23 +60,34 @@ EXCLUDED_GENERATED_FILES=(
 apache_beam/portability/api/*pb2*.py
 )
 
+PYTHON_MAJOR=$(python -c 'import sys; print(sys.version_info[0])')
+if [[ "${PYTHON_MAJOR}" == 2 ]]; then
+  EXCLUDED_PY3_FILES=$(find ${MODULE} | grep 'py3\.py$')
+  echo -e "Excluding Py3 files:\n${EXCLUDED_PY3_FILES}"
+else
+  EXCLUDED_PY3_FILES=""
+fi
+
 FILES_TO_IGNORE=""
-for file in "${EXCLUDED_GENERATED_FILES[@]}"; do
+for file in "${EXCLUDED_GENERATED_FILES[@]}" ${EXCLUDED_PY3_FILES}; do
   if test -z "$FILES_TO_IGNORE"
     then FILES_TO_IGNORE="$(basename $file)"
     else FILES_TO_IGNORE="$FILES_TO_IGNORE, $(basename $file)"
   fi
 done
-echo "Skipping lint for generated files: $FILES_TO_IGNORE"
 
-echo "Running pylint for module $MODULE:"
+echo -e "Skipping lint for files:\n${FILES_TO_IGNORE}"
+echo -e "Linting modules:\n${MODULE}"
+
+echo "Running pylint..."
 pylint -j8 ${MODULE} --ignore-patterns="$FILES_TO_IGNORE"
-echo "Running pycodestyle for module $MODULE:"
+echo "Running pycodestyle..."
 pycodestyle ${MODULE} --exclude="$FILES_TO_IGNORE"
-echo "Running flake8 for module $MODULE:"
-flake8 ${MODULE} --count --select=E9,F821,F822,F823 --show-source --statistics
+echo "Running flake8..."
+flake8 ${MODULE} --count --select=E9,F821,F822,F823 --show-source --statistics \
+  --exclude="${FILES_TO_IGNORE}"
 
-echo "Running isort for module $MODULE:"
+echo "Running isort..."
 # Skip files where isort is behaving weirdly
 ISORT_EXCLUDED=(
   "apiclient.py"
@@ -104,7 +115,7 @@ done
 isort ${MODULE} -p apache_beam --line-width 120 --check-only --order-by-type \
     --combine-star --force-single-line-imports --diff --recursive ${SKIP_PARAM}
 
-echo "Checking unittest.main for module ${MODULE}:"
+echo "Checking unittest.main..."
 TESTS_MISSING_MAIN=$(find ${MODULE} | grep '\.py$' | xargs grep -l '^import unittest$' |
xargs grep -L unittest.main)
 if [ -n "${TESTS_MISSING_MAIN}" ]; then
   echo -e "\nThe following files are missing a call to unittest.main():"
diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini
index 2a221b0..f6f8430 100644
--- a/sdks/python/tox.ini
+++ b/sdks/python/tox.ini
@@ -54,7 +54,7 @@ commands_post =
 [testenv:py27]
 commands =
   python apache_beam/examples/complete/autocomplete_test.py
-  python setup.py nosetests
+  python setup.py nosetests --ignore-files '.*py3.py$'
 
 [testenv:py35]
 setenv =
@@ -85,7 +85,7 @@ commands =
 platform = linux2
 commands =
   python apache_beam/examples/complete/autocomplete_test.py
-  python setup.py nosetests
+  python setup.py nosetests --ignore-files '.*py3.py$'
 
 [testenv:py35-cython]
 # cython tests are only expected to work in linux (2.x and 3.x)
@@ -127,7 +127,7 @@ commands =
 extras = test,gcp
 commands =
   python apache_beam/examples/complete/autocomplete_test.py
-  python setup.py nosetests
+  python setup.py nosetests --ignore-files '.*py3.py$'
   # Old and new Datastore client unit tests cannot be run in the same process
   # due to conflicting protobuf modules.
   # TODO(BEAM-4543): Remove these separate nosetests invocations once the


Mime
View raw message