beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dhalp...@apache.org
Subject [1/2] incubator-beam git commit: [BEAM-363] Return false for non-AUTO compression mode
Date Mon, 20 Jun 2016 23:21:58 GMT
Repository: incubator-beam
Updated Branches:
  refs/heads/master 682a19c4e -> 2b9906e8d


[BEAM-363] Return false for non-AUTO compression mode


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/278a76e0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/278a76e0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/278a76e0

Branch: refs/heads/master
Commit: 278a76e096ced0ba51b5ed0ae8205bbd32491c77
Parents: 682a19c
Author: Dan Halperin <dhalperi@google.com>
Authored: Mon Jun 20 15:44:54 2016 -0700
Committer: Dan Halperin <dhalperi@google.com>
Committed: Mon Jun 20 15:44:54 2016 -0700

----------------------------------------------------------------------
 .../apache/beam/sdk/io/CompressedSource.java    |  2 +-
 .../beam/sdk/io/CompressedSourceTest.java       | 50 ++++++++++++++++++++
 2 files changed, 51 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/278a76e0/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CompressedSource.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CompressedSource.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CompressedSource.java
index 8bccf5f..a5c54b3 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CompressedSource.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/CompressedSource.java
@@ -292,7 +292,7 @@ public class CompressedSource<T> extends FileBasedSource<T>
{
           (FileNameBasedDecompressingChannelFactory) channelFactory;
       return !fileNameBasedChannelFactory.isCompressed(getFileOrPatternSpec());
     }
-    return true;
+    return false;
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/278a76e0/sdks/java/core/src/test/java/org/apache/beam/sdk/io/CompressedSourceTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/CompressedSourceTest.java
b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/CompressedSourceTest.java
index c34d2dd..abf1de3 100644
--- a/sdks/java/core/src/test/java/org/apache/beam/sdk/io/CompressedSourceTest.java
+++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/io/CompressedSourceTest.java
@@ -97,6 +97,56 @@ public class CompressedSourceTest {
   }
 
   /**
+   * Test splittability of files in AUTO mode.
+   */
+  @Test
+  public void testAutoSplittable() throws Exception {
+    CompressedSource<Byte> source;
+
+    // GZip files are not splittable
+    source = CompressedSource.from(new ByteSource("input.gz", 1));
+    assertFalse(source.isSplittable());
+    source = CompressedSource.from(new ByteSource("input.GZ", 1));
+    assertFalse(source.isSplittable());
+
+    // BZ2 files are not splittable
+    source = CompressedSource.from(new ByteSource("input.bz2", 1));
+    assertFalse(source.isSplittable());
+    source = CompressedSource.from(new ByteSource("input.BZ2", 1));
+    assertFalse(source.isSplittable());
+
+    // Other extensions are assumed to be splittable.
+    source = CompressedSource.from(new ByteSource("input.txt", 1));
+    assertTrue(source.isSplittable());
+    source = CompressedSource.from(new ByteSource("input.csv", 1));
+    assertTrue(source.isSplittable());
+  }
+
+  /**
+   * Test splittability of files in GZIP mode -- none should be splittable.
+   */
+  @Test
+  public void testGzipSplittable() throws Exception {
+    CompressedSource<Byte> source;
+
+    // GZip files are not splittable
+    source = CompressedSource.from(new ByteSource("input.gz", 1))
+        .withDecompression(CompressionMode.GZIP);
+    assertFalse(source.isSplittable());
+    source = CompressedSource.from(new ByteSource("input.GZ", 1))
+        .withDecompression(CompressionMode.GZIP);
+    assertFalse(source.isSplittable());
+
+    // Other extensions are also not splittable.
+    source = CompressedSource.from(new ByteSource("input.txt", 1))
+        .withDecompression(CompressionMode.GZIP);
+    assertFalse(source.isSplittable());
+    source = CompressedSource.from(new ByteSource("input.csv", 1))
+        .withDecompression(CompressionMode.GZIP);
+    assertFalse(source.isSplittable());
+  }
+
+  /**
    * Test reading nonempty input with bzip2.
    */
   @Test


Mime
View raw message