beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "ASF GitHub Bot (JIRA)" <j...@apache.org>
Subject [jira] [Work logged] (BEAM-3194) Support annotating that a DoFn requires stable / deterministic input for replay/retry
Date Tue, 18 Sep 2018 18:37:00 GMT

     [ https://issues.apache.org/jira/browse/BEAM-3194?focusedWorklogId=145423&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-145423
]

ASF GitHub Bot logged work on BEAM-3194:
----------------------------------------

                Author: ASF GitHub Bot
            Created on: 18/Sep/18 18:36
            Start Date: 18/Sep/18 18:36
    Worklog Time Spent: 10m 
      Work Description: lukecwik closed pull request #6388: [BEAM-3194] Fail if @RequiresStableInput
is used on runners that don't supoort it
URL: https://github.com/apache/beam/pull/6388
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
index 79489b43dca..3601741f82d 100644
--- a/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
+++ b/runners/apex/src/main/java/org/apache/beam/runners/apex/ApexRunner.java
@@ -46,6 +46,7 @@
 import org.apache.beam.runners.core.construction.SingleInputOutputOverrideFactory;
 import org.apache.beam.runners.core.construction.SplittableParDo;
 import org.apache.beam.runners.core.construction.SplittableParDoNaiveBounded;
+import org.apache.beam.runners.core.construction.UnsupportedOverrideFactory;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.PipelineRunner;
 import org.apache.beam.sdk.coders.Coder;
@@ -132,6 +133,12 @@ public static ApexRunner fromOptions(PipelineOptions options) {
             PTransformOverride.of(
                 PTransformMatchers.splittableProcessKeyedUnbounded(),
                 new SplittableParDoViaKeyedWorkItems.OverrideFactory<>()))
+        // TODO: [BEAM-5360] Support @RequiresStableInput on Apex runner
+        .add(
+            PTransformOverride.of(
+                PTransformMatchers.requiresStableInputParDoMulti(),
+                UnsupportedOverrideFactory.withMessage(
+                    "Apex runner currently doesn't support @RequiresStableInput annotation.")))
         .build();
   }
 
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkTransformOverrides.java
b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkTransformOverrides.java
index dc8461b96f8..f0fd698637c 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkTransformOverrides.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkTransformOverrides.java
@@ -24,6 +24,7 @@
 import org.apache.beam.runners.core.construction.PTransformTranslation;
 import org.apache.beam.runners.core.construction.SplittableParDo;
 import org.apache.beam.runners.core.construction.SplittableParDoNaiveBounded;
+import org.apache.beam.runners.core.construction.UnsupportedOverrideFactory;
 import org.apache.beam.sdk.runners.PTransformOverride;
 import org.apache.beam.sdk.transforms.PTransform;
 
@@ -32,6 +33,12 @@
   public static List<PTransformOverride> getDefaultOverrides(boolean streaming) {
     ImmutableList.Builder<PTransformOverride> builder = ImmutableList.builder();
     builder
+        // TODO: [BEAM-5359] Support @RequiresStableInput on Flink runner
+        .add(
+            PTransformOverride.of(
+                PTransformMatchers.requiresStableInputParDoMulti(),
+                UnsupportedOverrideFactory.withMessage(
+                    "Flink runner currently doesn't support @RequiresStableInput annotation.")))
         .add(
             PTransformOverride.of(
                 PTransformMatchers.splittableParDo(), new SplittableParDo.OverrideFactory()))
diff --git a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GearpumpPipelineTranslator.java
b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GearpumpPipelineTranslator.java
index 445fe4bfda0..3745eb3df45 100644
--- a/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GearpumpPipelineTranslator.java
+++ b/runners/gearpump/src/main/java/org/apache/beam/runners/gearpump/translators/GearpumpPipelineTranslator.java
@@ -25,6 +25,7 @@
 import org.apache.beam.runners.core.construction.PTransformMatchers;
 import org.apache.beam.runners.core.construction.SplittableParDo;
 import org.apache.beam.runners.core.construction.SplittableParDoNaiveBounded;
+import org.apache.beam.runners.core.construction.UnsupportedOverrideFactory;
 import org.apache.beam.sdk.Pipeline;
 import org.apache.beam.sdk.io.Read;
 import org.apache.beam.sdk.runners.PTransformOverride;
@@ -89,6 +90,12 @@ public void translate(Pipeline pipeline) {
                 PTransformOverride.of(
                     PTransformMatchers.splittableProcessKeyedBounded(),
                     new SplittableParDoNaiveBounded.OverrideFactory()))
+            // TODO: [BEAM-5361] Support @RequiresStableInput on Gearpump runner
+            .add(
+                PTransformOverride.of(
+                    PTransformMatchers.requiresStableInputParDoMulti(),
+                    UnsupportedOverrideFactory.withMessage(
+                        "Gearpump runner currently doesn't support @RequiresStableInput annotation.")))
             .build();
 
     pipeline.replaceAll(overrides);
diff --git a/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/SamzaTransformOverrides.java
b/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/SamzaTransformOverrides.java
index 473e43a855e..4b9fe84d5de 100644
--- a/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/SamzaTransformOverrides.java
+++ b/runners/samza/src/main/java/org/apache/beam/runners/samza/translation/SamzaTransformOverrides.java
@@ -24,6 +24,7 @@
 import org.apache.beam.runners.core.construction.PTransformTranslation;
 import org.apache.beam.runners.core.construction.SplittableParDo;
 import org.apache.beam.runners.core.construction.SplittableParDoNaiveBounded;
+import org.apache.beam.runners.core.construction.UnsupportedOverrideFactory;
 import org.apache.beam.sdk.runners.PTransformOverride;
 
 /** {@link org.apache.beam.sdk.transforms.PTransform} overrides for Samza runner. */
@@ -41,6 +42,12 @@
             PTransformOverride.of(
                 PTransformMatchers.splittableProcessKeyedBounded(),
                 new SplittableParDoNaiveBounded.OverrideFactory()))
+        // TODO: [BEAM-5362] Support @RequiresStableInput on Samza runner
+        .add(
+            PTransformOverride.of(
+                PTransformMatchers.requiresStableInputParDoMulti(),
+                UnsupportedOverrideFactory.withMessage(
+                    "Samza runner currently doesn't support @RequiresStableInput annotation.")))
         .build();
   }
 }
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkTransformOverrides.java
b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkTransformOverrides.java
index 1718f3323f5..897ac017d56 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkTransformOverrides.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkTransformOverrides.java
@@ -23,6 +23,7 @@
 import org.apache.beam.runners.core.construction.PTransformTranslation;
 import org.apache.beam.runners.core.construction.SplittableParDo;
 import org.apache.beam.runners.core.construction.SplittableParDoNaiveBounded;
+import org.apache.beam.runners.core.construction.UnsupportedOverrideFactory;
 import org.apache.beam.sdk.runners.PTransformOverride;
 import org.apache.beam.sdk.transforms.PTransform;
 
@@ -30,6 +31,12 @@
 public class SparkTransformOverrides {
   public static List<PTransformOverride> getDefaultOverrides(boolean streaming) {
     ImmutableList.Builder<PTransformOverride> builder = ImmutableList.builder();
+    // TODO: [BEAM-5358] Support @RequiresStableInput on Spark runner
+    builder.add(
+        PTransformOverride.of(
+            PTransformMatchers.requiresStableInputParDoMulti(),
+            UnsupportedOverrideFactory.withMessage(
+                "Spark runner currently doesn't support @RequiresStableInput annotation.")));
     if (!streaming) {
       builder
           .add(


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


Issue Time Tracking
-------------------

    Worklog Id:     (was: 145423)
    Time Spent: 2h 40m  (was: 2.5h)

> Support annotating that a DoFn requires stable / deterministic input for replay/retry
> -------------------------------------------------------------------------------------
>
>                 Key: BEAM-3194
>                 URL: https://issues.apache.org/jira/browse/BEAM-3194
>             Project: Beam
>          Issue Type: New Feature
>          Components: beam-model, sdk-java-core
>            Reporter: Kenneth Knowles
>            Assignee: Yueyang Qiu
>            Priority: Major
>          Time Spent: 2h 40m
>  Remaining Estimate: 0h
>
> See the thread: https://lists.apache.org/thread.html/5fd81ce371aeaf642665348f8e6940e308e04275dd7072f380f9f945@%3Cdev.beam.apache.org%3E
> We need this in order to have truly cross-runner end-to-end exactly once via replay +
idempotence.



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

Mime
View raw message