beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From al...@apache.org
Subject [beam] branch master updated: [BEAM-6583] Update minimal Python 3 requirements. (#7839)
Date Thu, 14 Feb 2019 05:57:27 GMT
This is an automated email from the ASF dual-hosted git repository.

altay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new dbe86f8  [BEAM-6583] Update minimal Python 3 requirements. (#7839)
dbe86f8 is described below

commit dbe86f8590ae494314c8e96308aaf46d1f8123b3
Author: tvalentyn <tvalentyn@users.noreply.github.com>
AuthorDate: Wed Feb 13 21:57:17 2019 -0800

    [BEAM-6583] Update minimal Python 3 requirements. (#7839)
    
    * Refine python_requires stanza.
    * Update Dataflow Runner requirements on Python 3
---
 .../runners/dataflow/internal/apiclient.py         | 26 +++++++++++++
 .../runners/dataflow/internal/apiclient_test.py    | 45 ++++++++++++++++++++--
 sdks/python/setup.py                               |  5 +--
 3 files changed, 70 insertions(+), 6 deletions(-)

diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
index 6d5e140..4f0da29 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient.py
@@ -31,6 +31,7 @@ import re
 import sys
 import tempfile
 import time
+import warnings
 from datetime import datetime
 
 from builtins import object
@@ -159,6 +160,7 @@ class Environment(object):
             key='version', value=to_json_value(beam_version.__version__))])
     # Version information.
     self.proto.version = dataflow.Environment.VersionValue()
+    _verify_interpreter_version_is_supported(options)
     if self.standard_options.streaming:
       job_type = 'FNAPI_STREAMING'
     else:
@@ -938,6 +940,30 @@ def get_response_encoding():
   return None if sys.version_info[0] < 3 else 'utf8'
 
 
+def _verify_interpreter_version_is_supported(pipeline_options):
+  if sys.version_info[0] == 2:
+    return
+
+  if sys.version_info[0] == 3 and sys.version_info[1] == 5:
+    if sys.version_info[2] < 3:
+      warnings.warn(
+          'You are using an early release of Python 3.5. It is recommended '
+          'to use Python 3.5.3 or higher with Dataflow '
+          'runner.')
+    return
+
+  debug_options = pipeline_options.view_as(DebugOptions)
+  if (debug_options.experiments and 'ignore_py3_minor_version' in
+      debug_options.experiments):
+    return
+
+  raise Exception(
+      'Dataflow runner currently supports Python versions '
+      '2.7 and 3.5. To ignore this requirement and start a job using a '
+      'different version of Python 3 interpreter, pass '
+      '--experiment ignore_py3_minor_version pipeline option.')
+
+
 # To enable a counter on the service, add it to this dictionary.
 structured_counter_translations = {
     cy_combiners.CountCombineFn: (
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
index 0e37a0a..054b62f 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py
@@ -396,19 +396,58 @@ class UtilTest(unittest.TestCase):
     self.assertNotIn(
         "use_multiple_sdk_containers", environment.proto.experiments)
 
-  @mock.patch('apache_beam.runners.dataflow.internal.apiclient.sys')
-  def test_get_python_sdk_name(self, mock_sys):
+  @mock.patch(
+      'apache_beam.runners.dataflow.internal.apiclient.sys.version_info',
+      [2, 333])
+  def test_get_python_sdk_name(self):
     pipeline_options = PipelineOptions(
         ['--project', 'test_project', '--job_name', 'test_job_name',
          '--temp_location', 'gs://test-location/temp',
          '--experiments', 'beam_fn_api',
          '--experiments', 'use_multiple_sdk_containers'])
-    mock_sys.version_info = [2, 333]
     environment = apiclient.Environment(
         [], pipeline_options, 1, FAKE_PIPELINE_URL)
     self.assertEqual('Apache Beam Python 2.333 SDK',
                      environment._get_python_sdk_name())
 
+  @mock.patch(
+      'apache_beam.runners.dataflow.internal.apiclient.sys.version_info',
+      [2, 7])
+  def test_interpreter_version_check_passes_py27(self):
+    pipeline_options = PipelineOptions([])
+    apiclient._verify_interpreter_version_is_supported(pipeline_options)
+
+  @mock.patch(
+      'apache_beam.runners.dataflow.internal.apiclient.sys.version_info',
+      [3, 5, 2])
+  def test_interpreter_version_check_passes_py352(self):
+    pipeline_options = PipelineOptions([])
+    apiclient._verify_interpreter_version_is_supported(pipeline_options)
+
+  @mock.patch(
+      'apache_beam.runners.dataflow.internal.apiclient.sys.version_info',
+      [3, 5, 6])
+  def test_interpreter_version_check_passes_py356(self):
+    pipeline_options = PipelineOptions([])
+    apiclient._verify_interpreter_version_is_supported(pipeline_options)
+
+  @mock.patch(
+      'apache_beam.runners.dataflow.internal.apiclient.sys.version_info',
+      [3, 9, 0])
+  def test_interpreter_version_check_passes_with_experiment(self):
+    pipeline_options = PipelineOptions(
+        ["--experiment=ignore_py3_minor_version"])
+    apiclient._verify_interpreter_version_is_supported(pipeline_options)
+
+  @mock.patch(
+      'apache_beam.runners.dataflow.internal.apiclient.sys.version_info',
+      [3, 9, 0])
+  def test_interpreter_version_check_fails_py39(self):
+    pipeline_options = PipelineOptions([])
+    self.assertRaises(
+        Exception,
+        apiclient._verify_interpreter_version_is_supported, pipeline_options)
+
 
 if __name__ == '__main__':
   unittest.main()
diff --git a/sdks/python/setup.py b/sdks/python/setup.py
index 8b23605..020ab43 100644
--- a/sdks/python/setup.py
+++ b/sdks/python/setup.py
@@ -167,9 +167,7 @@ def generate_protos_first(original_cmd):
     return original_cmd
 
 
-# TODO(BEAM-6583): audit Python 3.x version compatibility and refine this
-# requirement range if necessary.
-python_requires = '>=2.7<=3.7'
+python_requires = '>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*'
 
 if sys.version_info[0] == 3:
   warnings.warn(
@@ -216,6 +214,7 @@ setuptools.setup(
         'License :: OSI Approved :: Apache Software License',
         'Operating System :: POSIX :: Linux',
         'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3.5',
         'Topic :: Software Development :: Libraries',
         'Topic :: Software Development :: Libraries :: Python Modules',
     ],


Mime
View raw message