beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From al...@apache.org
Subject [beam] branch master updated: [BEAM-5315] python 3 datastore fail gracefully
Date Wed, 13 Feb 2019 18:42:43 GMT
This is an automated email from the ASF dual-hosted git repository.

altay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new c6c12cc  [BEAM-5315] python 3 datastore fail gracefully
     new 5406b98  Merge pull request #7804 from Juta/io-gcp
c6c12cc is described below

commit c6c12ccbcea10cde99dfbc39c1d58585977a4adc
Author: Juta <juta_staes@hotmail.com>
AuthorDate: Mon Feb 11 15:13:49 2019 +0100

    [BEAM-5315] python 3 datastore fail gracefully
---
 .../examples/cookbook/datastore_wordcount_it_test.py           |  6 ++++++
 sdks/python/apache_beam/examples/snippets/snippets_test.py     |  4 ++++
 .../apache_beam/io/gcp/datastore/v1/adaptive_throttler_test.py |  6 ++++++
 sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py     |  7 ++++++-
 .../python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py | 10 ++++++++++
 sdks/python/apache_beam/io/gcp/datastore/v1/util_test.py       |  6 ++++++
 sdks/python/apache_beam/io/gcp/datastore_write_it_test.py      |  6 ++++++
 sdks/python/tox.ini                                            |  2 +-
 8 files changed, 45 insertions(+), 2 deletions(-)

diff --git a/sdks/python/apache_beam/examples/cookbook/datastore_wordcount_it_test.py b/sdks/python/apache_beam/examples/cookbook/datastore_wordcount_it_test.py
index d3d63cc..5a4fea9 100644
--- a/sdks/python/apache_beam/examples/cookbook/datastore_wordcount_it_test.py
+++ b/sdks/python/apache_beam/examples/cookbook/datastore_wordcount_it_test.py
@@ -20,6 +20,8 @@
 from __future__ import absolute_import
 
 import logging
+import os
+import sys
 import time
 import unittest
 
@@ -32,6 +34,10 @@ from apache_beam.testing.pipeline_verifiers import PipelineStateMatcher
 from apache_beam.testing.test_pipeline import TestPipeline
 
 
+@unittest.skipIf(sys.version_info[0] == 3 and
+                 os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+                 'This test still needs to be fixed on Python 3'
+                 'TODO: BEAM-4543')
 class DatastoreWordCountIT(unittest.TestCase):
 
   DATASTORE_WORDCOUNT_KIND = "DatastoreWordCount"
diff --git a/sdks/python/apache_beam/examples/snippets/snippets_test.py b/sdks/python/apache_beam/examples/snippets/snippets_test.py
index d6ac1f5..64ed9fe 100644
--- a/sdks/python/apache_beam/examples/snippets/snippets_test.py
+++ b/sdks/python/apache_beam/examples/snippets/snippets_test.py
@@ -631,6 +631,10 @@ class SnippetsTest(unittest.TestCase):
     snippets.model_textio_compressed(
         {'read': gzip_file_name}, ['aa', 'bb', 'cc'])
 
+  @unittest.skipIf(sys.version_info[0] == 3 and
+                   os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+                   'This test still needs to be fixed on Python 3'
+                   'TODO: BEAM-4543')
   @unittest.skipIf(datastore_pb2 is None, 'GCP dependencies are not installed')
   def test_model_datastoreio(self):
     # We cannot test DatastoreIO functionality in unit tests, therefore we limit
diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/adaptive_throttler_test.py b/sdks/python/apache_beam/io/gcp/datastore/v1/adaptive_throttler_test.py
index e3ccb92..1d9c46e 100644
--- a/sdks/python/apache_beam/io/gcp/datastore/v1/adaptive_throttler_test.py
+++ b/sdks/python/apache_beam/io/gcp/datastore/v1/adaptive_throttler_test.py
@@ -18,6 +18,8 @@
 from __future__ import absolute_import
 from __future__ import division
 
+import os
+import sys
 import unittest
 from builtins import range
 
@@ -26,6 +28,10 @@ from mock import patch
 from apache_beam.io.gcp.datastore.v1.adaptive_throttler import AdaptiveThrottler
 
 
+@unittest.skipIf(sys.version_info[0] == 3 and
+                 os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+                 'This test still needs to be fixed on Python 3'
+                 'TODO: BEAM-4543')
 class AdaptiveThrottlerTest(unittest.TestCase):
 
   START_TIME = 1500000000000
diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py
index 123d063..d12be78 100644
--- a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py
+++ b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio.py
@@ -20,7 +20,9 @@ from __future__ import absolute_import
 from __future__ import division
 
 import logging
+import sys
 import time
+import warnings
 from builtins import object
 from builtins import round
 
@@ -44,7 +46,10 @@ try:
   from google.cloud.proto.datastore.v1 import datastore_pb2
   from googledatastore import helper as datastore_helper
 except ImportError:
-  pass
+  if sys.version_info[0] == 3:
+    warnings.warn('Datastore IO will support Python 3 after replacing '
+                  'googledatastore by google-cloud-datastore, '
+                  'see: BEAM-4543.')
 # pylint: enable=wrong-import-order, wrong-import-position
 
 
diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py
index b7bc22a..d345fef 100644
--- a/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py
+++ b/sdks/python/apache_beam/io/gcp/datastore/v1/datastoreio_test.py
@@ -19,6 +19,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import os
+import sys
 import unittest
 from builtins import map
 from builtins import range
@@ -48,6 +50,10 @@ except ImportError:
 # pylint: enable=wrong-import-order, wrong-import-position, ungrouped-imports
 
 
+@unittest.skipIf(sys.version_info[0] == 3 and
+                 os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+                 'This test still needs to be fixed on Python 3'
+                 'TODO: BEAM-4543')
 @unittest.skipIf(datastore_pb2 is None, 'GCP dependencies are not installed')
 class DatastoreioTest(unittest.TestCase):
   _PROJECT = 'project'
@@ -272,6 +278,10 @@ class DatastoreioTest(unittest.TestCase):
     return split_queries
 
 
+@unittest.skipIf(sys.version_info[0] == 3 and
+                 os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+                 'This test still needs to be fixed on Python 3'
+                 'TODO: BEAM-4543')
 @unittest.skipIf(datastore_pb2 is None, 'GCP dependencies are not installed')
 class DynamicWriteBatcherTest(unittest.TestCase):
 
diff --git a/sdks/python/apache_beam/io/gcp/datastore/v1/util_test.py b/sdks/python/apache_beam/io/gcp/datastore/v1/util_test.py
index 8b05ebe..0f04727 100644
--- a/sdks/python/apache_beam/io/gcp/datastore/v1/util_test.py
+++ b/sdks/python/apache_beam/io/gcp/datastore/v1/util_test.py
@@ -18,11 +18,17 @@
 """Tests for util.py."""
 from __future__ import absolute_import
 
+import os
+import sys
 import unittest
 
 from apache_beam.io.gcp.datastore.v1 import util
 
 
+@unittest.skipIf(sys.version_info[0] == 3 and
+                 os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+                 'This test still needs to be fixed on Python 3'
+                 'TODO: BEAM-4543')
 class MovingSumTest(unittest.TestCase):
 
   TIMESTAMP = 1500000000
diff --git a/sdks/python/apache_beam/io/gcp/datastore_write_it_test.py b/sdks/python/apache_beam/io/gcp/datastore_write_it_test.py
index a7947a2..44af815 100644
--- a/sdks/python/apache_beam/io/gcp/datastore_write_it_test.py
+++ b/sdks/python/apache_beam/io/gcp/datastore_write_it_test.py
@@ -28,7 +28,9 @@ results in the pipeline.
 from __future__ import absolute_import
 
 import logging
+import os
 import random
+import sys
 import unittest
 from datetime import datetime
 
@@ -40,6 +42,10 @@ from apache_beam.testing.pipeline_verifiers import PipelineStateMatcher
 from apache_beam.testing.test_pipeline import TestPipeline
 
 
+@unittest.skipIf(sys.version_info[0] == 3 and
+                 os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
+                 'This test still needs to be fixed on Python 3'
+                 'TODO: BEAM-4543')
 class DatastoreWriteIT(unittest.TestCase):
 
   NUM_ENTITIES = 1001
diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini
index 743e0bd..3661c9d 100644
--- a/sdks/python/tox.ini
+++ b/sdks/python/tox.ini
@@ -93,7 +93,7 @@ setenv =
   RUN_SKIPPED_PY3_TESTS=0
 extras = test,gcp
 modules =
-  apache_beam.typehints,apache_beam.coders,apache_beam.options,apache_beam.tools,apache_beam.utils,apache_beam.internal,apache_beam.metrics,apache_beam.portability,apache_beam.pipeline_test,apache_beam.pvalue_test,apache_beam.runners,apache_beam.io.hadoopfilesystem_test,apache_beam.io.gcp.tests.utils_test,apache_beam.io.gcp.big_query_query_to_table_it_test,apache_beam.io.gcp.bigquery_io_read_it_test,apache_beam.io.gcp.bigquery_test,apache_beam.io.gcp.pubsub_integration_test,apache_beam.i
[...]
+  apache_beam.typehints,apache_beam.coders,apache_beam.options,apache_beam.tools,apache_beam.utils,apache_beam.internal,apache_beam.metrics,apache_beam.portability,apache_beam.pipeline_test,apache_beam.pvalue_test,apache_beam.runners,apache_beam.io.hadoopfilesystem_test,apache_beam.io.gcp.tests.utils_test,apache_beam.io.gcp.big_query_query_to_table_it_test,apache_beam.io.gcp.bigquery_io_read_it_test,apache_beam.io.gcp.bigquery_test,apache_beam.io.gcp.pubsub_integration_test,apache_beam.i
[...]
 commands =
   python --version
   pip --version


Mime
View raw message