beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From al...@apache.org
Subject [beam] branch master updated: [BEAM-7389] Add code examples for MapTuple and FlatMapTuple
Date Wed, 07 Aug 2019 18:02:22 GMT
This is an automated email from the ASF dual-hosted git repository.

altay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new 1a3eed5  [BEAM-7389] Add code examples for MapTuple and FlatMapTuple
     new 5e6ec5d5 Merge pull request #9276 from davidcavazos/map-flatmap-tuple
1a3eed5 is described below

commit 1a3eed5969986aa178655e6bc0442ae586f99519
Author: David Cavazos <dcavazos@google.com>
AuthorDate: Tue Aug 6 11:07:49 2019 -0700

    [BEAM-7389] Add code examples for MapTuple and FlatMapTuple
---
 .../snippets/transforms/element_wise/flat_map.py   | 27 ++++++++
 .../transforms/element_wise/flat_map_test.py       | 71 ++++++++++++----------
 .../snippets/transforms/element_wise/map.py        | 23 +++++++
 .../snippets/transforms/element_wise/map_test.py   | 71 ++++++++++++----------
 4 files changed, 126 insertions(+), 66 deletions(-)

diff --git a/sdks/python/apache_beam/examples/snippets/transforms/element_wise/flat_map.py
b/sdks/python/apache_beam/examples/snippets/transforms/element_wise/flat_map.py
index c227a71..e6da218 100644
--- a/sdks/python/apache_beam/examples/snippets/transforms/element_wise/flat_map.py
+++ b/sdks/python/apache_beam/examples/snippets/transforms/element_wise/flat_map.py
@@ -125,6 +125,33 @@ def flat_map_multiple_arguments(test=None):
       test(plants)
 
 
+def flat_map_tuple(test=None):
+  # [START flat_map_tuple]
+  import apache_beam as beam
+
+  def format_plant(icon, plant):
+    if icon:
+      yield '{}{}'.format(icon, plant)
+
+  with beam.Pipeline() as pipeline:
+    plants = (
+        pipeline
+        | 'Gardening plants' >> beam.Create([
+            ('🍓', 'Strawberry'),
+            ('🥕', 'Carrot'),
+            ('🍆', 'Eggplant'),
+            ('🍅', 'Tomato'),
+            ('🥔', 'Potato'),
+            (None, 'Invalid'),
+        ])
+        | 'Format' >> beam.FlatMapTuple(format_plant)
+        | beam.Map(print)
+    )
+    # [END flat_map_tuple]
+    if test:
+      test(plants)
+
+
 def flat_map_side_inputs_singleton(test=None):
   # [START flat_map_side_inputs_singleton]
   import apache_beam as beam
diff --git a/sdks/python/apache_beam/examples/snippets/transforms/element_wise/flat_map_test.py
b/sdks/python/apache_beam/examples/snippets/transforms/element_wise/flat_map_test.py
index 43b3b49..9825118 100644
--- a/sdks/python/apache_beam/examples/snippets/transforms/element_wise/flat_map_test.py
+++ b/sdks/python/apache_beam/examples/snippets/transforms/element_wise/flat_map_test.py
@@ -23,64 +23,69 @@ import unittest
 
 import mock
 
-from apache_beam.examples.snippets.transforms.element_wise.flat_map import *
 from apache_beam.testing.test_pipeline import TestPipeline
 from apache_beam.testing.util import assert_that
 from apache_beam.testing.util import equal_to
 
+from . import flat_map
+
+
+def check_plants(actual):
+  # [START plants]
+  plants = [
+      '🍓Strawberry',
+      '🥕Carrot',
+      '🍆Eggplant',
+      '🍅Tomato',
+      '🥔Potato',
+  ]
+  # [END plants]
+  assert_that(actual, equal_to(plants))
+
+
+def check_valid_plants(actual):
+  # [START valid_plants]
+  valid_plants = [
+      {'icon': '🍓', 'name': 'Strawberry', 'duration': 'perennial'},
+      {'icon': '🥕', 'name': 'Carrot', 'duration': 'biennial'},
+      {'icon': '🍆', 'name': 'Eggplant', 'duration': 'perennial'},
+      {'icon': '🍅', 'name': 'Tomato', 'duration': 'annual'},
+  ]
+  # [END valid_plants]
+  assert_that(actual, equal_to(valid_plants))
+
 
 @mock.patch('apache_beam.Pipeline', TestPipeline)
 # pylint: disable=line-too-long
 @mock.patch('apache_beam.examples.snippets.transforms.element_wise.flat_map.print', lambda
elem: elem)
 # pylint: enable=line-too-long
 class FlatMapTest(unittest.TestCase):
-  def __init__(self, methodName):
-    super(FlatMapTest, self).__init__(methodName)
-    # [START plants]
-    plants = [
-        '🍓Strawberry',
-        '🥕Carrot',
-        '🍆Eggplant',
-        '🍅Tomato',
-        '🥔Potato',
-    ]
-    # [END plants]
-    self.plants_test = lambda actual: assert_that(actual, equal_to(plants))
-
-    # [START valid_plants]
-    valid_plants = [
-        {'icon': '🍓', 'name': 'Strawberry', 'duration': 'perennial'},
-        {'icon': '🥕', 'name': 'Carrot', 'duration': 'biennial'},
-        {'icon': '🍆', 'name': 'Eggplant', 'duration': 'perennial'},
-        {'icon': '🍅', 'name': 'Tomato', 'duration': 'annual'},
-    ]
-    # [END valid_plants]
-    self.valid_plants_test = lambda actual: \
-        assert_that(actual, equal_to(valid_plants))
-
   def test_flat_map_simple(self):
-    flat_map_simple(self.plants_test)
+    flat_map.flat_map_simple(check_plants)
 
   def test_flat_map_function(self):
-    flat_map_function(self.plants_test)
+    flat_map.flat_map_function(check_plants)
 
   def test_flat_map_lambda(self):
-    flat_map_lambda(self.plants_test)
+    flat_map.flat_map_lambda(check_plants)
 
   def test_flat_map_generator(self):
-    flat_map_generator(self.plants_test)
+    flat_map.flat_map_generator(check_plants)
 
   def test_flat_map_multiple_arguments(self):
-    flat_map_multiple_arguments(self.plants_test)
+    flat_map.flat_map_multiple_arguments(check_plants)
+
+  def test_flat_map_tuple(self):
+    flat_map.flat_map_tuple(check_plants)
 
   def test_flat_map_side_inputs_singleton(self):
-    flat_map_side_inputs_singleton(self.plants_test)
+    flat_map.flat_map_side_inputs_singleton(check_plants)
 
   def test_flat_map_side_inputs_iter(self):
-    flat_map_side_inputs_iter(self.valid_plants_test)
+    flat_map.flat_map_side_inputs_iter(check_valid_plants)
 
   def test_flat_map_side_inputs_dict(self):
-    flat_map_side_inputs_dict(self.valid_plants_test)
+    flat_map.flat_map_side_inputs_dict(check_valid_plants)
 
 
 if __name__ == '__main__':
diff --git a/sdks/python/apache_beam/examples/snippets/transforms/element_wise/map.py b/sdks/python/apache_beam/examples/snippets/transforms/element_wise/map.py
index 147cee4..9defd47 100644
--- a/sdks/python/apache_beam/examples/snippets/transforms/element_wise/map.py
+++ b/sdks/python/apache_beam/examples/snippets/transforms/element_wise/map.py
@@ -114,6 +114,29 @@ def map_multiple_arguments(test=None):
       test(plants)
 
 
+def map_tuple(test=None):
+  # [START map_tuple]
+  import apache_beam as beam
+
+  with beam.Pipeline() as pipeline:
+    plants = (
+        pipeline
+        | 'Gardening plants' >> beam.Create([
+            ('🍓', 'Strawberry'),
+            ('🥕', 'Carrot'),
+            ('🍆', 'Eggplant'),
+            ('🍅', 'Tomato'),
+            ('🥔', 'Potato'),
+        ])
+        | 'Format' >> beam.MapTuple(
+            lambda icon, plant: '{}{}'.format(icon, plant))
+        | beam.Map(print)
+    )
+    # [END map_tuple]
+    if test:
+      test(plants)
+
+
 def map_side_inputs_singleton(test=None):
   # [START map_side_inputs_singleton]
   import apache_beam as beam
diff --git a/sdks/python/apache_beam/examples/snippets/transforms/element_wise/map_test.py
b/sdks/python/apache_beam/examples/snippets/transforms/element_wise/map_test.py
index 9d34fb8..5fcee8a 100644
--- a/sdks/python/apache_beam/examples/snippets/transforms/element_wise/map_test.py
+++ b/sdks/python/apache_beam/examples/snippets/transforms/element_wise/map_test.py
@@ -23,62 +23,67 @@ import unittest
 
 import mock
 
-from apache_beam.examples.snippets.transforms.element_wise.map import *
 from apache_beam.testing.test_pipeline import TestPipeline
 from apache_beam.testing.util import assert_that
 from apache_beam.testing.util import equal_to
 
+from . import map
+
+
+def check_plants(actual):
+  # [START plants]
+  plants = [
+      '🍓Strawberry',
+      '🥕Carrot',
+      '🍆Eggplant',
+      '🍅Tomato',
+      '🥔Potato',
+  ]
+  # [END plants]
+  assert_that(actual, equal_to(plants))
+
+
+def check_plant_details(actual):
+  # [START plant_details]
+  plant_details = [
+      {'icon': '🍓', 'name': 'Strawberry', 'duration': 'perennial'},
+      {'icon': '🥕', 'name': 'Carrot', 'duration': 'biennial'},
+      {'icon': '🍆', 'name': 'Eggplant', 'duration': 'perennial'},
+      {'icon': '🍅', 'name': 'Tomato', 'duration': 'annual'},
+      {'icon': '🥔', 'name': 'Potato', 'duration': 'perennial'},
+  ]
+  # [END plant_details]
+  assert_that(actual, equal_to(plant_details))
+
 
 @mock.patch('apache_beam.Pipeline', TestPipeline)
 # pylint: disable=line-too-long
 @mock.patch('apache_beam.examples.snippets.transforms.element_wise.map.print', lambda elem:
elem)
 # pylint: enable=line-too-long
 class MapTest(unittest.TestCase):
-  def __init__(self, methodName):
-    super(MapTest, self).__init__(methodName)
-    # [START plants]
-    plants = [
-        '🍓Strawberry',
-        '🥕Carrot',
-        '🍆Eggplant',
-        '🍅Tomato',
-        '🥔Potato',
-    ]
-    # [END plants]
-    self.plants_test = lambda actual: assert_that(actual, equal_to(plants))
-
-    # [START plant_details]
-    plant_details = [
-        {'icon': '🍓', 'name': 'Strawberry', 'duration': 'perennial'},
-        {'icon': '🥕', 'name': 'Carrot', 'duration': 'biennial'},
-        {'icon': '🍆', 'name': 'Eggplant', 'duration': 'perennial'},
-        {'icon': '🍅', 'name': 'Tomato', 'duration': 'annual'},
-        {'icon': '🥔', 'name': 'Potato', 'duration': 'perennial'},
-    ]
-    # [END plant_details]
-    self.plant_details_test = lambda actual: \
-        assert_that(actual, equal_to(plant_details))
-
   def test_map_simple(self):
-    map_simple(self.plants_test)
+    map.map_simple(check_plants)
 
   def test_map_function(self):
-    map_function(self.plants_test)
+    map.map_function(check_plants)
 
   def test_map_lambda(self):
-    map_lambda(self.plants_test)
+    map.map_lambda(check_plants)
 
   def test_map_multiple_arguments(self):
-    map_multiple_arguments(self.plants_test)
+    map.map_multiple_arguments(check_plants)
+
+  def test_map_tuple(self):
+    map.map_tuple(check_plants)
 
   def test_map_side_inputs_singleton(self):
-    map_side_inputs_singleton(self.plants_test)
+    map.map_side_inputs_singleton(check_plants)
 
   def test_map_side_inputs_iter(self):
-    map_side_inputs_iter(self.plants_test)
+    map.map_side_inputs_iter(check_plants)
 
   def test_map_side_inputs_dict(self):
-    map_side_inputs_dict(self.plant_details_test)
+    map.map_side_inputs_dict(check_plant_details)
 
 
 if __name__ == '__main__':


Mime
View raw message