beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mark...@apache.org
Subject [beam] branch master updated: [BEAM-7339] Enable 1Gb input for Python wordcount benchmark
Date Fri, 17 May 2019 22:36:22 GMT
This is an automated email from the ASF dual-hosted git repository.

markliu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new ddbdcd1  [BEAM-7339] Enable 1Gb input for Python wordcount benchmark
     new 21fece9  Merge pull request #8608 from markflyhigh/py-large-input
ddbdcd1 is described below

commit ddbdcd11ba37d2dd1767a7c1f303393947b7dc86
Author: Mark Liu <markliu@google.com>
AuthorDate: Fri May 17 13:00:10 2019 -0700

    [BEAM-7339] Enable 1Gb input for Python wordcount benchmark
---
 .test-infra/jenkins/job_PerformanceTests_Python.groovy | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/.test-infra/jenkins/job_PerformanceTests_Python.groovy b/.test-infra/jenkins/job_PerformanceTests_Python.groovy
index 732272f..0ccd4ea 100644
--- a/.test-infra/jenkins/job_PerformanceTests_Python.groovy
+++ b/.test-infra/jenkins/job_PerformanceTests_Python.groovy
@@ -62,24 +62,32 @@ def dataflowPipelineArgs = [
 def testConfigurations = [
     new PerformanceTestConfigurations(
         jobName           : 'beam_PerformanceTests_WordCountIT_Py27',
-        jobDescription    : 'Python SDK Performance Test - Run WordCountIT in Py27',
+        jobDescription    : 'Python SDK Performance Test - Run WordCountIT in Py27 with 1Gb
files',
         jobTriggerPhrase  : 'Run Python27 WordCountIT Performance Test',
         resultTable       : 'beam_performance.wordcount_py27_pkb_results',
         itClass           : 'apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_it',
         itModule          : 'sdks/python',
         extraPipelineArgs : dataflowPipelineArgs + [
-            output: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output'
+            input: 'gs://apache-beam-samples/input_small_files/ascii_sort_1MB_input.0000*',
// 1Gb
+            output: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output',
+            expect_checksum: 'ea0ca2e5ee4ea5f218790f28d0b9fe7d09d8d710',
+            num_workers: '10',
+            autoscaling_algorithm: 'NONE',  // Disable autoscale the worker pool.
         ],
     ),
     new PerformanceTestConfigurations(
         jobName           : 'beam_PerformanceTests_WordCountIT_Py35',
-        jobDescription    : 'Python SDK Performance Test - Run WordCountIT in Py35',
+        jobDescription    : 'Python SDK Performance Test - Run WordCountIT in Py35 with 1Gb
files',
         jobTriggerPhrase  : 'Run Python35 WordCountIT Performance Test',
         resultTable       : 'beam_performance.wordcount_py35_pkb_results',
         itClass           : 'apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_it',
         itModule          : 'sdks/python/test-suites/dataflow/py35',
         extraPipelineArgs : dataflowPipelineArgs + [
-            output: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output'
+            input: 'gs://apache-beam-samples/input_small_files/ascii_sort_1MB_input.0000*',
// 1Gb
+            output: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output',
+            expect_checksum: 'ea0ca2e5ee4ea5f218790f28d0b9fe7d09d8d710',
+            num_workers: '10',
+            autoscaling_algorithm: 'NONE',  // Disable autoscale the worker pool.
         ],
     )
 ]


Mime
View raw message