beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jbono...@apache.org
Subject [1/2] incubator-beam git commit: [BEAM-1034] Clean up tmp area in tests
Date Thu, 24 Nov 2016 07:43:27 GMT
Repository: incubator-beam
Updated Branches:
  refs/heads/master 26a30a22d -> 6d0c205a3


[BEAM-1034] Clean up tmp area in tests


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/ef74e192
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/ef74e192
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/ef74e192

Branch: refs/heads/master
Commit: ef74e192eaee79e4cb8c7c901a296dd76559d76d
Parents: 26a30a2
Author: Daniel Kulp <dkulp@apache.org>
Authored: Tue Nov 22 13:31:19 2016 -0500
Committer: Jean-Baptiste Onofré <jbonofre@apache.org>
Committed: Thu Nov 24 07:55:58 2016 +0100

----------------------------------------------------------------------
 .../sorter/BufferedExternalSorter.java          |  6 +-
 .../sdk/extensions/sorter/ExternalSorter.java   |  6 +-
 .../sorter/BufferedExternalSorterTest.java      | 58 +++++++++++++++++---
 .../extensions/sorter/ExternalSorterTest.java   | 53 +++++++++++++++---
 4 files changed, 103 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/ef74e192/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/BufferedExternalSorter.java
----------------------------------------------------------------------
diff --git a/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/BufferedExternalSorter.java
b/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/BufferedExternalSorter.java
index 0f89e30..1dfd339 100644
--- a/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/BufferedExternalSorter.java
+++ b/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/BufferedExternalSorter.java
@@ -35,12 +35,13 @@ public class BufferedExternalSorter implements Sorter {
     private int memoryMB = 100;
 
     /** Sets the path to a temporary location where the sorter writes intermediate files.
*/
-    public void setTempLocation(String tempLocation) {
+    public Options setTempLocation(String tempLocation) {
       checkArgument(
           !tempLocation.startsWith("gs://"),
           "BufferedExternalSorter does not support GCS temporary location");
 
       this.tempLocation = tempLocation;
+      return this;
     }
 
     /** Returns the configured temporary location. */
@@ -52,9 +53,10 @@ public class BufferedExternalSorter implements Sorter {
      * Sets the size of the memory buffer in megabytes. This controls both the buffer for
initial in
      * memory sorting and the buffer used when external sorting. Must be greater than zero.
      */
-    public void setMemoryMB(int memoryMB) {
+    public Options setMemoryMB(int memoryMB) {
       checkArgument(memoryMB > 0, "memoryMB must be greater than zero");
       this.memoryMB = memoryMB;
+      return this;
     }
 
     /** Returns the configured size of the memory buffer. */

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/ef74e192/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/ExternalSorter.java
----------------------------------------------------------------------
diff --git a/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/ExternalSorter.java
b/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/ExternalSorter.java
index 3cf0cc0..beef1ee 100644
--- a/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/ExternalSorter.java
+++ b/sdks/java/extensions/sorter/src/main/java/org/apache/beam/sdk/extensions/sorter/ExternalSorter.java
@@ -67,12 +67,13 @@ class ExternalSorter implements Sorter {
     private int memoryMB = 100;
 
     /** Sets the path to a temporary location where the sorter writes intermediate files.
*/
-    public void setTempLocation(String tempLocation) {
+    public Options setTempLocation(String tempLocation) {
       if (tempLocation.startsWith("gs://")) {
         throw new IllegalArgumentException("Sorter doesn't support GCS temporary location.");
       }
 
       this.tempLocation = tempLocation;
+      return this;
     }
 
     /** Returns the configured temporary location. */
@@ -81,9 +82,10 @@ class ExternalSorter implements Sorter {
     }
 
     /** Sets the size of the memory buffer in megabytes. */
-    public void setMemoryMB(int memoryMB) {
+    public Options setMemoryMB(int memoryMB) {
       checkArgument(memoryMB > 0, "memoryMB must be greater than zero");
       this.memoryMB = memoryMB;
+      return this;
     }
 
     /** Returns the configured size of the memory buffer. */

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/ef74e192/sdks/java/extensions/sorter/src/test/java/org/apache/beam/sdk/extensions/sorter/BufferedExternalSorterTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/extensions/sorter/src/test/java/org/apache/beam/sdk/extensions/sorter/BufferedExternalSorterTest.java
b/sdks/java/extensions/sorter/src/test/java/org/apache/beam/sdk/extensions/sorter/BufferedExternalSorterTest.java
index 63dbedf..8c108eb 100644
--- a/sdks/java/extensions/sorter/src/test/java/org/apache/beam/sdk/extensions/sorter/BufferedExternalSorterTest.java
+++ b/sdks/java/extensions/sorter/src/test/java/org/apache/beam/sdk/extensions/sorter/BufferedExternalSorterTest.java
@@ -27,9 +27,17 @@ import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
 import java.util.Arrays;
 import org.apache.beam.sdk.extensions.sorter.SorterTestUtils.SorterGenerator;
 import org.apache.beam.sdk.values.KV;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
@@ -40,6 +48,29 @@ import org.junit.runners.JUnit4;
 @RunWith(JUnit4.class)
 public class BufferedExternalSorterTest {
   @Rule public ExpectedException thrown = ExpectedException.none();
+  static Path tmpLocation;
+
+  @BeforeClass
+  public static void setupTempDir() throws IOException {
+    tmpLocation = Files.createTempDirectory("tmp");
+  }
+
+  @AfterClass
+  public static void cleanupTempDir() throws IOException {
+    Files.walkFileTree(tmpLocation, new SimpleFileVisitor<Path>() {
+      @Override
+      public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException
{
+        Files.delete(file);
+        return FileVisitResult.CONTINUE;
+      }
+
+      @Override
+      public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException
{
+        Files.delete(dir);
+        return FileVisitResult.CONTINUE;
+      }
+    });
+  }
 
   @SuppressWarnings("unchecked")
   @Test
@@ -106,25 +137,29 @@ public class BufferedExternalSorterTest {
 
   @Test
   public void testEmpty() throws Exception {
-    SorterTestUtils.testEmpty(BufferedExternalSorter.create(new BufferedExternalSorter.Options()));
+    SorterTestUtils.testEmpty(BufferedExternalSorter.create(new BufferedExternalSorter.Options()
+        .setTempLocation(tmpLocation.toString())));
   }
 
   @Test
   public void testSingleElement() throws Exception {
     SorterTestUtils.testSingleElement(
-        BufferedExternalSorter.create(new BufferedExternalSorter.Options()));
+        BufferedExternalSorter.create(new BufferedExternalSorter.Options()
+            .setTempLocation(tmpLocation.toString())));
   }
 
   @Test
   public void testEmptyKeyValueElement() throws Exception {
     SorterTestUtils.testEmptyKeyValueElement(
-        BufferedExternalSorter.create(new BufferedExternalSorter.Options()));
+        BufferedExternalSorter.create(new BufferedExternalSorter.Options()
+            .setTempLocation(tmpLocation.toString())));
   }
 
   @Test
   public void testMultipleIterations() throws Exception {
     SorterTestUtils.testMultipleIterations(
-        BufferedExternalSorter.create(new BufferedExternalSorter.Options()));
+        BufferedExternalSorter.create(new BufferedExternalSorter.Options()
+            .setTempLocation(tmpLocation.toString())));
   }
 
   @Test
@@ -133,7 +168,8 @@ public class BufferedExternalSorterTest {
         new SorterGenerator() {
           @Override
           public Sorter generateSorter() throws Exception {
-            return BufferedExternalSorter.create(new BufferedExternalSorter.Options());
+            return BufferedExternalSorter.create(new BufferedExternalSorter.Options()
+                .setTempLocation(tmpLocation.toString()));
           }
         },
         1000000,
@@ -146,7 +182,8 @@ public class BufferedExternalSorterTest {
         new SorterGenerator() {
           @Override
           public Sorter generateSorter() throws Exception {
-            return BufferedExternalSorter.create(new BufferedExternalSorter.Options());
+            return BufferedExternalSorter.create(new BufferedExternalSorter.Options()
+                .setTempLocation(tmpLocation.toString()));
           }
         },
         1,
@@ -156,14 +193,16 @@ public class BufferedExternalSorterTest {
   @Test
   public void testAddAfterSort() throws Exception {
     SorterTestUtils.testAddAfterSort(
-        BufferedExternalSorter.create(new BufferedExternalSorter.Options()), thrown);
+        BufferedExternalSorter.create(new BufferedExternalSorter.Options()
+              .setTempLocation(tmpLocation.toString())), thrown);
     fail();
   }
 
   @Test
   public void testSortTwice() throws Exception {
     SorterTestUtils.testSortTwice(
-        BufferedExternalSorter.create(new BufferedExternalSorter.Options()), thrown);
+        BufferedExternalSorter.create(new BufferedExternalSorter.Options()
+            .setTempLocation(tmpLocation.toString())), thrown);
     fail();
   }
 
@@ -171,7 +210,8 @@ public class BufferedExternalSorterTest {
   public void testNegativeMemory() throws Exception {
     thrown.expect(IllegalArgumentException.class);
     thrown.expectMessage("memoryMB must be greater than zero");
-    BufferedExternalSorter.Options options = new BufferedExternalSorter.Options();
+    BufferedExternalSorter.Options options = new BufferedExternalSorter.Options()
+        .setTempLocation(tmpLocation.toString());
     options.setMemoryMB(-1);
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/ef74e192/sdks/java/extensions/sorter/src/test/java/org/apache/beam/sdk/extensions/sorter/ExternalSorterTest.java
----------------------------------------------------------------------
diff --git a/sdks/java/extensions/sorter/src/test/java/org/apache/beam/sdk/extensions/sorter/ExternalSorterTest.java
b/sdks/java/extensions/sorter/src/test/java/org/apache/beam/sdk/extensions/sorter/ExternalSorterTest.java
index 9232b62..bcfbdad 100644
--- a/sdks/java/extensions/sorter/src/test/java/org/apache/beam/sdk/extensions/sorter/ExternalSorterTest.java
+++ b/sdks/java/extensions/sorter/src/test/java/org/apache/beam/sdk/extensions/sorter/ExternalSorterTest.java
@@ -20,7 +20,16 @@ package org.apache.beam.sdk.extensions.sorter;
 
 import static org.junit.Assert.fail;
 
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+
 import org.apache.beam.sdk.extensions.sorter.SorterTestUtils.SorterGenerator;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
@@ -31,25 +40,52 @@ import org.junit.runners.JUnit4;
 @RunWith(JUnit4.class)
 public class ExternalSorterTest {
   @Rule public ExpectedException thrown = ExpectedException.none();
+  static Path tmpLocation;
+
+  @BeforeClass
+  public static void setupTempDir() throws IOException {
+    tmpLocation = Files.createTempDirectory("tmp");
+  }
+
+  @AfterClass
+  public static void cleanupTempDir() throws IOException {
+    Files.walkFileTree(tmpLocation, new SimpleFileVisitor<Path>() {
+      @Override
+      public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException
{
+        Files.delete(file);
+        return FileVisitResult.CONTINUE;
+      }
+
+      @Override
+      public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException
{
+        Files.delete(dir);
+        return FileVisitResult.CONTINUE;
+      }
+    });
+  }
 
   @Test
   public void testEmpty() throws Exception {
-    SorterTestUtils.testEmpty(ExternalSorter.create(new ExternalSorter.Options()));
+    SorterTestUtils.testEmpty(ExternalSorter.create(new ExternalSorter.Options()
+        .setTempLocation(tmpLocation.toString())));
   }
 
   @Test
   public void testSingleElement() throws Exception {
-    SorterTestUtils.testSingleElement(ExternalSorter.create(new ExternalSorter.Options()));
+    SorterTestUtils.testSingleElement(ExternalSorter.create(new ExternalSorter.Options()
+        .setTempLocation(tmpLocation.toString())));
   }
 
   @Test
   public void testEmptyKeyValueElement() throws Exception {
-    SorterTestUtils.testEmptyKeyValueElement(ExternalSorter.create(new ExternalSorter.Options()));
+    SorterTestUtils.testEmptyKeyValueElement(ExternalSorter.create(new ExternalSorter.Options()
+        .setTempLocation(tmpLocation.toString())));
   }
 
   @Test
   public void testMultipleIterations() throws Exception {
-    SorterTestUtils.testMultipleIterations(ExternalSorter.create(new ExternalSorter.Options()));
+    SorterTestUtils.testMultipleIterations(ExternalSorter.create(new ExternalSorter.Options()
+        .setTempLocation(tmpLocation.toString())));
   }
 
   @Test
@@ -58,7 +94,8 @@ public class ExternalSorterTest {
         new SorterGenerator() {
           @Override
           public Sorter generateSorter() throws Exception {
-            return ExternalSorter.create(new ExternalSorter.Options());
+            return ExternalSorter.create(new ExternalSorter.Options()
+                .setTempLocation(tmpLocation.toString()));
           }
         },
         1,
@@ -67,13 +104,15 @@ public class ExternalSorterTest {
 
   @Test
   public void testAddAfterSort() throws Exception {
-    SorterTestUtils.testAddAfterSort(ExternalSorter.create(new ExternalSorter.Options()),
thrown);
+    SorterTestUtils.testAddAfterSort(ExternalSorter.create(new ExternalSorter.Options()
+        .setTempLocation(tmpLocation.toString())), thrown);
     fail();
   }
 
   @Test
   public void testSortTwice() throws Exception {
-    SorterTestUtils.testSortTwice(ExternalSorter.create(new ExternalSorter.Options()), thrown);
+    SorterTestUtils.testSortTwice(ExternalSorter.create(new ExternalSorter.Options()
+        .setTempLocation(tmpLocation.toString())), thrown);
     fail();
   }
 


Mime
View raw message