beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From al...@apache.org
Subject [beam] branch master updated: [BEAM-6619] [BEAM-6593] [BEAM-6711] Add bigquery integration tests to postcommit
Date Tue, 02 Apr 2019 17:09:38 GMT
This is an automated email from the ASF dual-hosted git repository.

altay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new 9dfd935  [BEAM-6619] [BEAM-6593] [BEAM-6711] Add bigquery integration tests to postcommit
     new b6800ce  Merge pull request #8076 from Juta/bq-io-tests
9dfd935 is described below

commit 9dfd9358bd89ef963e30487c15dbf8f7f40d8189
Author: Juta <juta.staes@gmail.com>
AuthorDate: Mon Mar 18 09:46:36 2019 +0100

    [BEAM-6619] [BEAM-6593] [BEAM-6711] Add bigquery integration tests to postcommit
---
 sdks/python/apache_beam/examples/complete/game/game_stats.py     | 1 +
 .../apache_beam/examples/complete/game/game_stats_it_test.py     | 9 ++-------
 .../examples/complete/game/hourly_team_score_it_test.py          | 6 ------
 sdks/python/apache_beam/examples/complete/game/leader_board.py   | 1 +
 .../apache_beam/examples/complete/game/leader_board_it_test.py   | 9 ++-------
 5 files changed, 6 insertions(+), 20 deletions(-)

diff --git a/sdks/python/apache_beam/examples/complete/game/game_stats.py b/sdks/python/apache_beam/examples/complete/game/game_stats.py
index 5f7a0ed..9b5cc32 100644
--- a/sdks/python/apache_beam/examples/complete/game/game_stats.py
+++ b/sdks/python/apache_beam/examples/complete/game/game_stats.py
@@ -306,6 +306,7 @@ def run(argv=None):
           topic=args.topic)
     raw_events = (
         scores
+        | 'DecodeString' >> beam.Map(lambda b: b.decode('utf-8'))
         | 'ParseGameEventFn' >> beam.ParDo(ParseGameEventFn())
         | 'AddEventTimestamps' >> beam.Map(
             lambda elem: beam.window.TimestampedValue(elem, elem['timestamp'])))
diff --git a/sdks/python/apache_beam/examples/complete/game/game_stats_it_test.py b/sdks/python/apache_beam/examples/complete/game/game_stats_it_test.py
index e7b89aa..cba4b00 100644
--- a/sdks/python/apache_beam/examples/complete/game/game_stats_it_test.py
+++ b/sdks/python/apache_beam/examples/complete/game/game_stats_it_test.py
@@ -33,8 +33,6 @@ Usage:
 from __future__ import absolute_import
 
 import logging
-import os
-import sys
 import time
 import unittest
 import uuid
@@ -51,10 +49,6 @@ from apache_beam.testing.pipeline_verifiers import PipelineStateMatcher
 from apache_beam.testing.test_pipeline import TestPipeline
 
 
-@unittest.skipIf(sys.version_info[0] == 3 and
-                 os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
-                 'This test still needs to be fixed on Python 3'
-                 'TODO: BEAM-6711')
 class GameStatsIT(unittest.TestCase):
 
   # Input events containing user, team, score, processing time, window start.
@@ -102,7 +96,8 @@ class GameStatsIT(unittest.TestCase):
 
     for _ in range(message_count):
       self.pub_client.publish(topic.name,
-                              self.INPUT_EVENT % self._test_timestamp)
+                              (self.INPUT_EVENT % self._test_timestamp
+                              ).encode('utf-8'))
 
   def _cleanup_pubsub(self):
     test_utils.cleanup_subscriptions(self.sub_client, [self.input_sub])
diff --git a/sdks/python/apache_beam/examples/complete/game/hourly_team_score_it_test.py b/sdks/python/apache_beam/examples/complete/game/hourly_team_score_it_test.py
index 2fce1fc..5685132 100644
--- a/sdks/python/apache_beam/examples/complete/game/hourly_team_score_it_test.py
+++ b/sdks/python/apache_beam/examples/complete/game/hourly_team_score_it_test.py
@@ -33,8 +33,6 @@ Usage:
 from __future__ import absolute_import
 
 import logging
-import os
-import sys
 import unittest
 
 from hamcrest.core.core.allof import all_of
@@ -48,10 +46,6 @@ from apache_beam.testing.pipeline_verifiers import PipelineStateMatcher
 from apache_beam.testing.test_pipeline import TestPipeline
 
 
-@unittest.skipIf(sys.version_info[0] == 3 and
-                 os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
-                 'This test still needs to be fixed on Python 3'
-                 'TODO: BEAM-6870')
 class HourlyTeamScoreIT(unittest.TestCase):
 
   DEFAULT_INPUT_FILE = 'gs://dataflow-samples/game/gaming_data*'
diff --git a/sdks/python/apache_beam/examples/complete/game/leader_board.py b/sdks/python/apache_beam/examples/complete/game/leader_board.py
index cde1544..43a599b 100644
--- a/sdks/python/apache_beam/examples/complete/game/leader_board.py
+++ b/sdks/python/apache_beam/examples/complete/game/leader_board.py
@@ -315,6 +315,7 @@ def run(argv=None):
 
     events = (
         scores
+        | 'DecodeString' >> beam.Map(lambda b: b.decode('utf-8'))
         | 'ParseGameEventFn' >> beam.ParDo(ParseGameEventFn())
         | 'AddEventTimestamps' >> beam.Map(
             lambda elem: beam.window.TimestampedValue(elem, elem['timestamp'])))
diff --git a/sdks/python/apache_beam/examples/complete/game/leader_board_it_test.py b/sdks/python/apache_beam/examples/complete/game/leader_board_it_test.py
index b86e49e..9f057fd 100644
--- a/sdks/python/apache_beam/examples/complete/game/leader_board_it_test.py
+++ b/sdks/python/apache_beam/examples/complete/game/leader_board_it_test.py
@@ -33,8 +33,6 @@ Usage:
 from __future__ import absolute_import
 
 import logging
-import os
-import sys
 import time
 import unittest
 import uuid
@@ -52,10 +50,6 @@ from apache_beam.testing.pipeline_verifiers import PipelineStateMatcher
 from apache_beam.testing.test_pipeline import TestPipeline
 
 
-@unittest.skipIf(sys.version_info[0] == 3 and
-                 os.environ.get('RUN_SKIPPED_PY3_TESTS') != '1',
-                 'This test still needs to be fixed on Python 3'
-                 'TODO: BEAM-6711')
 class LeaderBoardIT(unittest.TestCase):
 
   # Input event containing user, team, score, processing time, window start.
@@ -104,7 +98,8 @@ class LeaderBoardIT(unittest.TestCase):
 
     for _ in range(message_count):
       self.pub_client.publish(topic.name,
-                              self.INPUT_EVENT % self._test_timestamp)
+                              (self.INPUT_EVENT % self._test_timestamp
+                              ).encode('utf-8'))
 
   def _cleanup_pubsub(self):
     test_utils.cleanup_subscriptions(self.sub_client, [self.input_sub])


Mime
View raw message