beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bhule...@apache.org
Subject [beam] 01/01: Merge remote-tracking branch 'origin/master' into website-revamp
Date Tue, 08 Dec 2020 02:10:22 GMT
This is an automated email from the ASF dual-hosted git repository.

bhulette pushed a commit to branch website-revamp
in repository https://gitbox.apache.org/repos/asf/beam.git

commit 8b22fb4b9aa8607313f1e39374a226bc226a349c
Merge: 140a913 31716b0
Author: Brian Hulette <bhulette@google.com>
AuthorDate: Mon Dec 7 18:09:14 2020 -0800

    Merge remote-tracking branch 'origin/master' into website-revamp

 .test-infra/jenkins/PythonTestProperties.groovy    |  17 +-
 .../jenkins/job_LoadTests_Combine_Flink_Go.groovy  | 136 ++++++++++
 .../jenkins/job_LoadTests_GBK_Flink_Go.groovy      | 220 ++++++++++++++++
 .../jenkins/job_LoadTests_coGBK_Flink_Go.groovy    | 182 +++++++++++++
 .../job_PostCommit_PortableJar_Flink.groovy        |   3 +-
 .../job_PostCommit_PortableJar_Spark.groovy        |   2 +-
 ...Commit_Python_Chicago_Taxi_Example_Flink.groovy |   8 +-
 ...ommit_Python_ValidatesContainer_Dataflow.groovy |   9 +-
 .../dashboards/Post-Commits_status_dashboard.json  |   2 +-
 CHANGES.md                                         |   4 +
 build.gradle                                       |  13 +-
 buildSrc/build.gradle                              |   4 +-
 .../org/apache/beam/gradle/BeamModulePlugin.groovy |  23 +-
 dev-support/docker/Dockerfile                      | 117 +++++++++
 dev-support/docker/bashcolors.sh                   |  91 +++++++
 dev-support/docker/beam_env_checks.sh              | 121 +++++++++
 model/pipeline/src/main/proto/metrics.proto        |  13 +
 .../DefaultExpansionServiceClientFactory.java      |   7 +-
 .../core/construction/ExpansionServiceClient.java  |   2 +-
 .../ExpansionServiceClientFactory.java             |   2 +-
 .../beam/runners/core/construction/External.java   |   2 +-
 .../core/construction/PTransformTranslation.java   |   1 +
 .../beam/runners/core/SideInputHandlerTest.java    | 118 +++++----
 runners/direct-java/build.gradle                   |  22 +-
 runners/flink/job-server/flink_job_server.gradle   |  66 -----
 .../worker/windmill/GrpcWindmillServer.java        |  14 +-
 .../test_flink_uber_jar.sh                         |   0
 runners/portability/test_pipeline_jar.sh           |   3 +-
 runners/spark/job-server/build.gradle              |  32 ---
 sdks/go/test/load/build.gradle                     |   3 +
 sdks/go/test/load/cogbk/cogbk.go                   | 106 ++++++++
 sdks/go/test/load/combine/combine.go               |  87 +++++++
 sdks/go/test/load/group_by_key/group_by_key.go     |  89 +++++++
 sdks/java/build-tools/beam-linkage-check.sh        |  80 +++---
 .../apache/beam/sdk/annotations/Experimental.java  |   7 -
 .../main/java/org/apache/beam/sdk/io/AvroIO.java   |   7 +-
 .../main/java/org/apache/beam/sdk/io/FileIO.java   |   5 +-
 .../main/java/org/apache/beam/sdk/io/TextIO.java   |   5 +-
 .../java/org/apache/beam/sdk/state/Timers.java     |   2 +-
 .../apache/beam/sdk/transforms/Deduplicate.java    |   3 -
 .../java/org/apache/beam/sdk/transforms/DoFn.java  |  13 -
 .../beam/sdk/transforms/PeriodicSequence.java      |   3 -
 .../java/org/apache/beam/sdk/transforms/Watch.java |   3 -
 .../splittabledofn/ByteKeyRangeTracker.java        |   3 -
 .../splittabledofn/GrowableOffsetRangeTracker.java |   3 -
 .../splittabledofn/HasDefaultTracker.java          |   4 -
 .../HasDefaultWatermarkEstimator.java              |   3 -
 .../splittabledofn/ManualWatermarkEstimator.java   |   3 -
 .../splittabledofn/OffsetRangeTracker.java         |   3 -
 .../splittabledofn/RestrictionTracker.java         |   3 -
 .../sdk/transforms/splittabledofn/SplitResult.java |   3 -
 .../TimestampObservingWatermarkEstimator.java      |   3 -
 .../splittabledofn/WatermarkEstimator.java         |   3 -
 .../splittabledofn/WatermarkEstimators.java        |   3 -
 .../transforms/splittabledofn/package-info.java    |   3 -
 sdks/java/extensions/ml/build.gradle               |  33 ++-
 sdks/java/extensions/sql/build.gradle              |   1 +
 .../sql/meta/provider/bigtable/BigtableTable.java  | 211 +++++++++++++++
 .../provider/bigtable/BigtableTableProvider.java   |  85 ++++++
 .../sql/meta/provider/bigtable/package-info.java   |   9 +-
 .../BigtableTableCreationFailuresTest.java         | 145 +++++++++++
 .../provider/bigtable/BigtableTableFlatTest.java   | 163 ++++++++++++
 .../meta/provider/bigtable/BigtableTableTest.java  |  94 +++++++
 .../bigtable/BigtableTableWithRowsTest.java        | 158 +++++++++++
 .../apache/beam/fn/harness/ToStringFnRunner.java   |  66 +++++
 .../beam/fn/harness/ToStringFnRunnerTest.java      |  72 +++++
 sdks/java/io/amazon-web-services2/build.gradle     |   2 +-
 .../beam/sdk/io/aws2/options/AwsOptions.java       |   4 +-
 .../beam/sdk/io/gcp/bigquery/BigQueryIO.java       |   5 +
 .../io/gcp/bigtable/BeamRowToBigtableMutation.java | 123 +++++++++
 .../beam/sdk/io/gcp/bigtable/BigtableConfig.java   |  15 ++
 .../beam/sdk/io/gcp/bigtable/BigtableIO.java       |  22 ++
 .../sdk/io/gcp/bigtable/BigtableRowToBeamRow.java  | 176 +++++++++++++
 .../io/gcp/bigtable/BigtableRowToBeamRowFlat.java  | 108 ++++++++
 .../io/gcp/bigtable/BigtableRowToBeamRowFn.java    |  46 ++++
 .../beam/sdk/io/gcp/bigtable/CellValueParser.java  | 106 ++++++++
 .../apache/beam/sdk/io/gcp/bigtable/RowUtils.java  |  13 +-
 .../apache/beam/sdk/io/gcp/healthcare/DicomIO.java | 199 ++++++++++++++
 .../sdk/io/gcp/healthcare/HealthcareApiClient.java |  12 +
 .../io/gcp/healthcare/HttpHealthcareApiClient.java |  79 ++++++
 .../beam/sdk/io/gcp/healthcare/WebPathParser.java  |  63 +++++
 .../io/gcp/testing/BigtableEmulatorWrapper.java    |  75 ++++++
 .../beam/sdk/io/gcp/testing/BigtableTestUtils.java | 154 +++++++++++
 .../sdk/io/gcp/bigquery/BigQueryIOWriteTest.java   |   9 +
 .../bigtable/BeamRowToBigtableMutationTest.java    |  89 +++++++
 .../gcp/bigtable/BigtableRowToBeamRowFlatTest.java |  65 +++++
 .../io/gcp/bigtable/BigtableRowToBeamRowTest.java  |  93 +++++++
 .../sdk/io/gcp/bigtable/CellValueParserTest.java   | 277 ++++++++++++++++++++
 .../apache/beam/sdk/io/gcp/bigtable/TestUtils.java | 109 ++++++++
 .../beam/sdk/io/gcp/healthcare/DicomIOReadIT.java  |  93 +++++++
 .../beam/sdk/io/gcp/healthcare/DicomIOTest.java    |  60 +++++
 .../sdk/io/gcp/healthcare/WebPathParserTest.java   |  48 ++++
 .../src/test/resources/DICOM/testDicomFile.dcm     | Bin 0 -> 10366 bytes
 .../java/org/apache/beam/sdk/io/hbase/HBaseIO.java |   1 -
 sdks/java/io/kafka/build.gradle                    |  90 +++++--
 .../org/apache/beam/sdk/io/kafka/KafkaIOIT.java    |  69 ++++-
 sdks/python/apache_beam/coders/coders.py           |  16 ++
 .../apache_beam/coders/coders_test_common.py       |  17 ++
 sdks/python/apache_beam/dataframe/io.py            | 143 +++++++++-
 sdks/python/apache_beam/dataframe/io_test.py       |  41 +++
 .../apache_beam/examples/kafkataxi/README.md       |   4 +-
 .../apache_beam/examples/snippets/snippets.py      |   4 +
 sdks/python/apache_beam/io/gcp/bigquery.py         | 273 +++++++++++++------
 .../apache_beam/io/gcp/bigquery_read_internal.py   | 289 +++++++++++++++++++++
 .../apache_beam/io/gcp/bigquery_read_it_test.py    | 103 ++++++++
 sdks/python/apache_beam/io/gcp/bigquery_test.py    |   2 +-
 sdks/python/apache_beam/io/gcp/bigquery_tools.py   |  26 +-
 sdks/python/apache_beam/io/gcp/bigtableio.py       |   7 +-
 sdks/python/apache_beam/io/gcp/pubsub.py           | 114 ++++++++
 sdks/python/apache_beam/io/gcp/pubsub_test.py      | 123 +++++++++
 sdks/python/apache_beam/io/iobase.py               |  20 +-
 sdks/python/apache_beam/io/restriction_trackers.py |  17 ++
 sdks/python/apache_beam/metrics/cells.pxd          |   2 +
 sdks/python/apache_beam/metrics/cells.py           |  15 +-
 sdks/python/apache_beam/metrics/cells_test.py      |  25 ++
 .../python/apache_beam/options/pipeline_options.py |   6 +
 sdks/python/apache_beam/options/value_provider.py  |  71 ++++-
 .../apache_beam/options/value_provider_test.py     |  40 +++
 sdks/python/apache_beam/runners/common.py          |   5 +-
 .../Linux/7a35f487b2a5f3a9b9852a8659eeb4bd.png     | Bin 700548 -> 703677 bytes
 .../runners/portability/flink_runner_test.py       |   2 +-
 .../portability/fn_api_runner/fn_runner_test.py    | 115 +++++++-
 .../portability/fn_api_runner/translations.py      |   8 +-
 .../portability/fn_api_runner/translations_test.py |  31 +++
 .../portability/fn_api_runner/worker_handlers.py   |   2 +-
 .../apache_beam/runners/portability/job_server.py  |   3 +-
 .../runners/portability/sdk_container_builder.py   |  30 ++-
 .../runners/portability/spark_runner_test.py       |   6 +-
 .../apache_beam/runners/worker/bundle_processor.py |  17 ++
 .../apache_beam/runners/worker/sdk_worker_test.py  |   2 +
 sdks/python/apache_beam/transforms/core.py         |  32 ++-
 .../apache_beam/typehints/sharded_key_type.py      |  75 ++++++
 .../apache_beam/typehints/sharded_key_type_test.py |  80 ++++++
 sdks/python/apache_beam/utils/shared.py            |   2 +-
 sdks/python/build-requirements.txt                 |   3 +
 sdks/python/setup.py                               |   2 +-
 sdks/python/test-suites/portable/common.gradle     |  69 ++++-
 start-build-env.sh                                 | 132 ++++++++++
 .../www/site/content/en/community/powered-by.md    |   2 +
 website/www/site/content/en/contribute/_index.md   |  16 +-
 .../dsls/sql/extensions/create-external-table.md   | 132 ++++++++++
 .../en/documentation/io/developing-io-java.md      |   3 +
 .../en/documentation/io/developing-io-overview.md  |  80 +++---
 .../en/documentation/io/developing-io-python.md    |   3 +
 .../content/en/documentation/programming-guide.md  |   2 +-
 .../en/documentation/sdks/python-dependencies.md   |  12 +-
 .../site/content/en/get-started/quickstart-py.md   |   4 +-
 .../content/en/get-started/wordcount-example.md    |   2 +-
 148 files changed, 6607 insertions(+), 549 deletions(-)



Mime
View raw message