beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ieme...@apache.org
Subject [beam] branch spark-runner_structured-streaming updated: Add Batch Validates Runner tests for Structured Streaming Runner
Date Fri, 03 May 2019 15:32:34 GMT
This is an automated email from the ASF dual-hosted git repository.

iemejia pushed a commit to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/spark-runner_structured-streaming by this
push:
     new cc31b85  Add Batch Validates Runner tests for Structured Streaming Runner
cc31b85 is described below

commit cc31b8570e4e3a6c65633dd5ec9ec7c0898280ba
Author: Ismaël Mejía <iemejia@gmail.com>
AuthorDate: Fri May 3 17:26:26 2019 +0200

    Add Batch Validates Runner tests for Structured Streaming Runner
---
 runners/spark/build.gradle | 49 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)

diff --git a/runners/spark/build.gradle b/runners/spark/build.gradle
index 3414a34..8ea30ec 100644
--- a/runners/spark/build.gradle
+++ b/runners/spark/build.gradle
@@ -158,11 +158,60 @@ task validatesRunnerStreaming(type: Test) {
   }
 }
 
+task validatesStructuredStreamingRunnerBatch(type: Test) {
+  group = "Verification"
+  def pipelineOptions = JsonOutput.toJson([
+          "--runner=SparkStructuredStreamingRunner",
+          "--streaming=false",
+  ])
+  systemProperty "beamTestPipelineOptions", pipelineOptions
+  systemProperty "beam.spark.test.reuseSparkContext", "true"
+  systemProperty "spark.ui.enabled", "false"
+  systemProperty "spark.ui.showConsoleProgress", "false"
+
+  classpath = configurations.validatesRunner
+  testClassesDirs = files(project(":beam-sdks-java-core").sourceSets.test.output.classesDirs)
+ files(project(":beam-sdks-java-io-hadoop-format").sourceSets.test.output.classesDirs) +
files(project.sourceSets.test.output.classesDirs)
+  // Only one SparkContext may be running in a JVM (SPARK-2243)
+  forkEvery 1
+  maxParallelForks 4
+  useJUnit {
+    includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner'
+    excludeCategories 'org.apache.beam.sdk.testing.UsesCustomWindowMerging'
+    excludeCategories 'org.apache.beam.sdk.testing.UsesFailureMessage'
+    excludeCategories 'org.apache.beam.sdk.testing.UsesParDoLifecycle'
+    // Unbounded
+    excludeCategories 'org.apache.beam.sdk.testing.UsesUnboundedPCollections'
+    excludeCategories 'org.apache.beam.sdk.testing.UsesTestStream'
+    // State and Timers
+    excludeCategories 'org.apache.beam.sdk.testing.UsesStatefulParDo'
+    excludeCategories 'org.apache.beam.sdk.testing.UsesMapState'
+    excludeCategories 'org.apache.beam.sdk.testing.UsesSetState'
+    excludeCategories 'org.apache.beam.sdk.testing.UsesTimersInParDo'
+    // Metrics
+    excludeCategories 'org.apache.beam.sdk.testing.UsesAttemptedMetrics'
+    excludeCategories 'org.apache.beam.sdk.testing.UsesCommittedMetrics'
+    excludeCategories 'org.apache.beam.sdk.testing.UsesCounterMetrics'
+    excludeCategories 'org.apache.beam.sdk.testing.UsesDistributionMetrics'
+    excludeCategories 'org.apache.beam.sdk.testing.UsesGaugeMetrics'
+    excludeCategories 'org.apache.beam.sdk.testing.UsesMetricsPusher'
+    // SDF
+    excludeCategories 'org.apache.beam.sdk.testing.UsesBoundedSplittableParDo'
+    excludeCategories 'org.apache.beam.sdk.testing.UsesSplittableParDoWithWindowedSideInputs'
+    excludeCategories 'org.apache.beam.sdk.testing.UsesUnboundedSplittableParDo'
+    // Portability
+    excludeCategories 'org.apache.beam.sdk.testing.UsesImpulse'
+    excludeCategories 'org.apache.beam.sdk.testing.UsesCrossLanguageTransforms'
+    // Schema
+    excludeCategories 'org.apache.beam.sdk.testing.UsesSchema'
+  }
+}
+
 task validatesRunner {
   group = "Verification"
   description "Validates Spark runner"
   dependsOn validatesRunnerBatch
   dependsOn validatesRunnerStreaming
+  dependsOn validatesStructuredStreamingRunnerBatch
 }
 
 // Generates :beam-runners-spark:runQuickstartJavaSpark


Mime
View raw message