beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From echauc...@apache.org
Subject [beam] 08/37: Add a simple spark native test to test Beam coders wrapping into Spark Encoders
Date Thu, 24 Oct 2019 10:15:17 GMT
This is an automated email from the ASF dual-hosted git repository.

echauchot pushed a commit to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git

commit e4478ffc2a9fd35d76080ff8f33cc8d3340cba1c
Author: Etienne Chauchot <echauchot@apache.org>
AuthorDate: Fri Aug 30 17:34:13 2019 +0200

    Add a simple spark native test to test Beam coders wrapping into Spark Encoders
---
 .../structuredstreaming/utils/EncodersTest.java    | 29 ++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/structuredstreaming/utils/EncodersTest.java
b/runners/spark/src/test/java/org/apache/beam/runners/spark/structuredstreaming/utils/EncodersTest.java
new file mode 100644
index 0000000..490e3dc
--- /dev/null
+++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/structuredstreaming/utils/EncodersTest.java
@@ -0,0 +1,29 @@
+package org.apache.beam.runners.spark.structuredstreaming.utils;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.runners.spark.structuredstreaming.translation.helpers.EncoderHelpers;
+import org.apache.beam.sdk.coders.VarIntCoder;
+import org.apache.spark.sql.SparkSession;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+@RunWith(JUnit4.class)
+/**
+ * Test of the wrapping of Beam Coders as Spark ExpressionEncoders.
+ */
+public class EncodersTest {
+
+  @Test
+  public void beamCoderToSparkEncoderTest() {
+    SparkSession sparkSession = SparkSession.builder().appName("beamCoderToSparkEncoderTest")
+        .master("local[4]").getOrCreate();
+    List<Integer> data = new ArrayList<>();
+    data.add(1);
+    data.add(2);
+    data.add(3);
+//    sparkSession.createDataset(data, EncoderHelpers.fromBeamCoder(VarIntCoder.of()));
+    sparkSession.createDataset(data, EncoderHelpers.genericEncoder());
+  }
+}


Mime
View raw message