beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From echauc...@apache.org
Subject [beam] 20/50: Add precise TODO for multiple TransformTranslator per transform URN
Date Fri, 04 Jan 2019 10:38:42 GMT
This is an automated email from the ASF dual-hosted git repository.

echauchot pushed a commit to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git

commit b37da3e80a8f678463fb5e3cf2c991ff93de5b23
Author: Etienne Chauchot <echauchot@apache.org>
AuthorDate: Mon Nov 26 11:31:36 2018 +0100

    Add precise TODO for multiple TransformTranslator per transform URN
---
 .../translation/batch/PipelineTranslatorBatch.java                  | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/PipelineTranslatorBatch.java
b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/PipelineTranslatorBatch.java
index e883131..318d74c 100644
--- a/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/PipelineTranslatorBatch.java
+++ b/runners/spark-structured-streaming/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/PipelineTranslatorBatch.java
@@ -42,6 +42,12 @@ public class PipelineTranslatorBatch extends PipelineTranslator {
   @SuppressWarnings("rawtypes")
   private static final Map<String, TransformTranslator> TRANSFORM_TRANSLATORS = new
HashMap<>();
 
+  //TODO the ability to have more than one TransformTranslator per URN
+  // that could be dynamically chosen by a predicated that evaluates based on PCollection
+  // obtainable though node.getInputs.getValue()
+  // See https://github.com/seznam/euphoria/blob/master/euphoria-spark/src/main/java/cz/seznam/euphoria/spark/SparkFlowTranslator.java#L83
+  // And https://github.com/seznam/euphoria/blob/master/euphoria-spark/src/main/java/cz/seznam/euphoria/spark/SparkFlowTranslator.java#L106
+
   static {
     TRANSFORM_TRANSLATORS.put(
         PTransformTranslation.COMBINE_PER_KEY_TRANSFORM_URN, new CombinePerKeyTranslatorBatch());


Mime
View raw message