beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pabl...@apache.org
Subject [beam] branch master updated: Fixing fileio tests for windows
Date Tue, 07 May 2019 18:07:41 GMT
This is an automated email from the ASF dual-hosted git repository.

pabloem pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new bbf40b0  Fixing fileio tests for windows
     new 1382505  Merge pull request #8516 from pabloem/fix-win
bbf40b0 is described below

commit bbf40b0029fbbe4f8324c8b99a0409e066642ad0
Author: pabloem <pabloem@apache.org>
AuthorDate: Tue May 7 09:06:02 2019 -0700

    Fixing fileio tests for windows
---
 sdks/python/apache_beam/io/fileio.py      |  5 +++--
 sdks/python/apache_beam/io/fileio_test.py | 12 ++++++------
 sdks/python/apache_beam/io/filesystem.py  |  4 ++--
 3 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/sdks/python/apache_beam/io/fileio.py b/sdks/python/apache_beam/io/fileio.py
index 65ce76b..10890ca 100644
--- a/sdks/python/apache_beam/io/fileio.py
+++ b/sdks/python/apache_beam/io/fileio.py
@@ -129,9 +129,10 @@ class _ReadMatchesFn(beam.DoFn):
                 if isinstance(file_metadata, (str, unicode))
                 else file_metadata)
 
-    if metadata.path.endswith('/') and self._skip_directories:
+    if ((metadata.path.endswith('/') or metadata.path.endswith('\\'))
+        and self._skip_directories):
       return
-    elif metadata.path.endswith('/'):
+    elif metadata.path.endswith('/') or metadata.path.endswith('\\'):
       raise BeamIOError(
           'Directories are not allowed in ReadMatches transform.'
           'Found %s.' % metadata.path)
diff --git a/sdks/python/apache_beam/io/fileio_test.py b/sdks/python/apache_beam/io/fileio_test.py
index 07b2fcd..096149b 100644
--- a/sdks/python/apache_beam/io/fileio_test.py
+++ b/sdks/python/apache_beam/io/fileio_test.py
@@ -37,7 +37,6 @@ from apache_beam.testing.util import assert_that
 from apache_beam.testing.util import equal_to
 
 
-@unittest.skipIf(sys.platform.startswith('win'), "skipping for windows")
 class MatchTest(_TestCaseWithTempDirCleanUp):
 
   def test_basic_two_files(self):
@@ -111,7 +110,6 @@ class MatchTest(_TestCaseWithTempDirCleanUp):
       assert_that(files_pc, equal_to(files))
 
 
-@unittest.skipIf(sys.platform.startswith('win'), "skipping for windows")
 class ReadTest(_TestCaseWithTempDirCleanUp):
 
   def test_basic_file_name_provided(self):
@@ -124,9 +122,10 @@ class ReadTest(_TestCaseWithTempDirCleanUp):
                     | beam.Create([dir])
                     | fileio.MatchAll()
                     | fileio.ReadMatches()
-                    | beam.Map(lambda f: f.read().decode('utf-8')))
+                    | beam.FlatMap(
+                        lambda f: f.read().decode('utf-8').splitlines()))
 
-      assert_that(content_pc, equal_to([content]))
+      assert_that(content_pc, equal_to(content.splitlines()))
 
   def test_csv_file_source(self):
     content = 'name,year,place\ngoogle,1999,CA\nspotify,2006,sweden'
@@ -163,9 +162,10 @@ class ReadTest(_TestCaseWithTempDirCleanUp):
       contents_pc = (p
                      | beam.Create(files + [tempdir])
                      | fileio.ReadMatches()
-                     | beam.Map(lambda x: x.read().decode('utf-8')))
+                     | beam.FlatMap(
+                         lambda x: x.read().decode('utf-8').splitlines()))
 
-      assert_that(contents_pc, equal_to([content]*2))
+      assert_that(contents_pc, equal_to(content.splitlines()*2))
 
   def test_fail_on_directories(self):
     content = 'thecontent\n'
diff --git a/sdks/python/apache_beam/io/filesystem.py b/sdks/python/apache_beam/io/filesystem.py
index 015af8b..cfdf472 100644
--- a/sdks/python/apache_beam/io/filesystem.py
+++ b/sdks/python/apache_beam/io/filesystem.py
@@ -660,7 +660,7 @@ class FileSystem(with_metaclass(abc.ABCMeta, BeamPlugin)):
     See Also:
       :meth:`translate_pattern`
 
-    Patterns ending with '/' will be appended with '*'.
+    Patterns ending with '/' or '\\' will be appended with '*'.
 
     Args:
       patterns: list of string for the file path pattern to match against
@@ -679,7 +679,7 @@ class FileSystem(with_metaclass(abc.ABCMeta, BeamPlugin)):
 
     def _match(pattern, limit):
       """Find all matching paths to the pattern provided."""
-      if pattern.endswith('/'):
+      if pattern.endswith('/') or pattern.endswith('\\'):
         pattern += '*'
       # Get the part of the pattern before the first globbing character.
       # For example scheme://path/foo* will become scheme://path/foo for


Mime
View raw message