beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From c..@apache.org
Subject [beam] 01/01: Revert "Adding Autocomplete IT for Java"
Date Fri, 21 Sep 2018 19:02:17 GMT
This is an automated email from the ASF dual-hosted git repository.

ccy pushed a commit to branch revert-6372-autocomplete-it
in repository https://gitbox.apache.org/repos/asf/beam.git

commit e3ddecdda6587233b333da4aa8a0e0dba66c7337
Author: Charles Chen <charlesccychen@users.noreply.github.com>
AuthorDate: Fri Sep 21 12:02:06 2018 -0700

    Revert "Adding Autocomplete IT for Java"
---
 .../beam/examples/complete/AutoComplete.java       | 41 +--------------
 .../beam/examples/complete/AutoCompleteIT.java     | 59 ----------------------
 2 files changed, 2 insertions(+), 98 deletions(-)

diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java
b/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java
index 34e4247..0f9b8fd 100644
--- a/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java
+++ b/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java
@@ -51,7 +51,6 @@ import org.apache.beam.sdk.options.Description;
 import org.apache.beam.sdk.options.PipelineOptionsFactory;
 import org.apache.beam.sdk.options.StreamingOptions;
 import org.apache.beam.sdk.options.Validation;
-import org.apache.beam.sdk.testing.PAssert;
 import org.apache.beam.sdk.transforms.Count;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.Filter;
@@ -61,7 +60,6 @@ import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.Partition;
 import org.apache.beam.sdk.transforms.Partition.PartitionFn;
 import org.apache.beam.sdk.transforms.SerializableFunction;
-import org.apache.beam.sdk.transforms.Sum;
 import org.apache.beam.sdk.transforms.Top;
 import org.apache.beam.sdk.transforms.windowing.GlobalWindows;
 import org.apache.beam.sdk.transforms.windowing.SlidingWindows;
@@ -433,17 +431,6 @@ public class AutoComplete {
 
     void setOutputToBigQuery(Boolean value);
 
-    @Description("Whether to send output to checksum Transform.")
-    @Default.Boolean(true)
-    Boolean getOutputToChecksum();
-
-    void setOutputToChecksum(Boolean value);
-
-    @Description("Expected result of the checksum transform.")
-    Long getExpectedChecksum();
-
-    void setExpectedChecksum(Long value);
-
     @Description("Whether output to Cloud Datastore")
     @Default.Boolean(false)
     Boolean getOutputToDatastore();
@@ -462,7 +449,8 @@ public class AutoComplete {
     void setOutputProject(String value);
   }
 
-  public static void runAutocompletePipeline(Options options) throws IOException {
+  public static void main(String[] args) throws IOException {
+    Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class);
 
     options.setBigQuerySchema(FormatForBigquery.getSchema());
     ExampleUtils exampleUtils = new ExampleUtils(options);
@@ -518,35 +506,10 @@ public class AutoComplete {
                           : BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE));
     }
 
-    if (options.getOutputToChecksum()) {
-      PCollection<Long> checksum =
-          toWrite
-              .apply(
-                  ParDo.of(
-                      new DoFn<KV<String, List<CompletionCandidate>>, Long>()
{
-                        @ProcessElement
-                        public void process(ProcessContext c) {
-                          KV<String, List<CompletionCandidate>> elm = c.element();
-                          Long listHash =
-                              c.element().getValue().stream().mapToLong(cc -> cc.hashCode()).sum();
-                          c.output(Long.valueOf(elm.getKey().hashCode()) + listHash);
-                        }
-                      }))
-              .apply(Sum.longsGlobally());
-
-      PAssert.that(checksum).containsInAnyOrder(options.getExpectedChecksum());
-    }
-
     // Run the pipeline.
     PipelineResult result = p.run();
 
     // ExampleUtils will try to cancel the pipeline and the injector before the program exists.
     exampleUtils.waitToFinish(result);
   }
-
-  public static void main(String[] args) throws IOException {
-    Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class);
-
-    runAutocompletePipeline(options);
-  }
 }
diff --git a/examples/java/src/test/java/org/apache/beam/examples/complete/AutoCompleteIT.java
b/examples/java/src/test/java/org/apache/beam/examples/complete/AutoCompleteIT.java
deleted file mode 100644
index 6b01d1a..0000000
--- a/examples/java/src/test/java/org/apache/beam/examples/complete/AutoCompleteIT.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.beam.examples.complete;
-
-import org.apache.beam.examples.complete.AutoComplete.Options;
-import org.apache.beam.sdk.options.PipelineOptionsFactory;
-import org.apache.beam.sdk.testing.TestPipeline;
-import org.apache.beam.sdk.testing.TestPipelineOptions;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.JUnit4;
-
-/** Run integration tests for a basic AutoComplete pipeline. */
-@RunWith(JUnit4.class)
-public class AutoCompleteIT {
-
-  public static final String DEFAULT_INPUT =
-      "gs://apache-beam-samples/shakespeare/kinglear-hashtag.txt";
-
-  public static final Long DEFAULT_INPUT_CHECKSUM = -25447108232L;
-
-  /** Options for the Autocomplete Integration test. */
-  public interface AutoCompleteITOptions extends TestPipelineOptions, Options {}
-
-  @BeforeClass
-  public static void setUp() {
-    PipelineOptionsFactory.register(TestPipelineOptions.class);
-  }
-
-  @Test
-  public void testE2EAutoComplete() throws Exception {
-    AutoCompleteITOptions options =
-        TestPipeline.testingPipelineOptions().as(AutoCompleteITOptions.class);
-
-    options.setInputFile(DEFAULT_INPUT);
-    options.setOutputToBigQuery(false);
-    options.setOutputToDatastore(false);
-    options.setOutputToChecksum(true);
-    options.setExpectedChecksum(DEFAULT_INPUT_CHECKSUM);
-
-    AutoComplete.runAutocompletePipeline(options);
-  }
-}


Mime
View raw message