beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From github-...@apache.org
Subject [beam] tag nightly-master updated (f137246 -> b8c001c)
Date Thu, 17 Dec 2020 03:03:35 GMT
This is an automated email from the ASF dual-hosted git repository.

github-bot pushed a change to tag nightly-master
in repository https://gitbox.apache.org/repos/asf/beam.git.


*** WARNING: tag nightly-master was modified! ***

    from f137246  (commit)
      to b8c001c  (commit)
    from f137246  Merge pull request #13556 [BEAM-11472] Support SDF on non-splitting runners
such as legacy Dataflow.
     add 31dcd53  [BEAM-10114] Convert PubsubLiteIO read to use SplittableDoFn.
     add ec3efe2  Fix test.
     add d3336fe  fix: Testing issues
     add 8892c1c  [BEAM-10114] Convert PubsubLiteIO to use an SDF for reads.
     add f5c6108  [BEAM-10114] Add byte and time limiting
     add 6d8c0a9  [BEAM-10114] Add byte and time limiting
     add 762b7ba  [BEAM-10114] Add byte and time limiting
     add 65b2c42  [BEAM-10114] Add byte and time limiting
     add 418b3ac  [BEAM-10114] Add byte and time limiting
     add 843fc4a  [BEAM-10114] Add byte and time limiting
     add fa3fc2d  fix: Linter issues
     add ea33f9e  fix: linters
     add 88a3021  fix: linters
     add b731da8  fix: Remove unnecessary shuffle.
     add a432e8f  fix: Address comments
     add f87c984  Merge pull request #13470 from [BEAM-10114] Convert PubsubLiteIO read to
use SplittableDoFn.
     add dee8d49  [BEAM-11443] Implement byte-based GBK for Flink Streaming Runner.
     add 56d4ac9  Merge pull request #13538: [BEAM-11443] Implement byte-based GBK for Flink
Streaming Runner.
     add ded0325  [BEAM-11075] Go SDK SideInput load tests (#13436)
     add f1c1514  [BEAM-11267] Remove unecessary reshuffle for stateful ParDo after keyed
operation.
     add 76c6129  Merge pull request #13353: [BEAM-11267] Remove unnecessary reshuffle for
stateful ParDo after key…
     add 12b6015  Update DoFn docstring (#13559)
     add cf20794  Fix concat source fraction_consumed() in degenerate case.
     add 8f79065  Merge pull request #13563 from robertwb/concat-isdone
     add b2f4784  [BEAM-11361] Dynamic splitting of dataframe csv reads. (#13488)
     add 97ad770  fix PerPartitionSubscriptionSdf methods with incorrect visibility
     add 92209d1  Merge pull request #13566 from fix PerPartitionSubscriptionSdf methods with
incorrect visibility
     add 1ccdee8  [BEAM-11432] put latest tag when publishing SDK head containers to gcr (#13517)
     add 8de3288  [BEAM-10925] Add interface for SQL Java scalar UDF. (#13305)
     add 554d254  Track transform processing thread in Java SDK harness and set log entry
transform id field.
     add a6ba301  Add javadoc
     add 23b2007  Add tracking to other ProcessElement methods
     add 945e794  Use LoadingCache instead of ConcurrentHashMap to limit the size for thread
tracker
     add 857d41f  Address comment
     add c8d9d1d  Fix checkstyle
     add 6869dfa  Merge pull request #13533 from [BEAM-11474] Track transform processing thread
in Java SDK harness and set log entry field
     add 3de4f88  [BEAM-10925] Add SQL AggregateFn interface.
     add a7457c0  Merge pull request #13306 from ibzib/aggregate-udf
     add 3cacd4a  [BEAM-11407] Add IT test to Bigtable for BeamSQL
     add 1a5505b  Merge pull request #13512 from piotr-szuberski/bigtable-it
     add 8fa3e87  Thread safe addition of new metrics
     add c7efdc1  Thread safe addition of new metrics
     add 5f253b6  Merge pull request #13526 from [BEAM-11459] Thread safe addition of new
metrics
     add b8c001c  Moving to 2.28.0-SNAPSHOT on master branch.

No new revisions were added by this update.

Summary of changes:
 .../job_LoadTests_SideInput_Flink_Go.groovy        | 103 +++++++
 .../jenkins/job_LoadTests_SideInput_Go.groovy      |  94 ++++++
 .../jenkins/job_Publish_SDK_Image_Snapshots.groovy |   8 +-
 .../perftests_metrics/SideInput_Load_Tests.json    |   2 +-
 .../org/apache/beam/gradle/BeamModulePlugin.groovy |  15 +-
 gradle.properties                                  |   4 +-
 .../FlinkStreamingPortablePipelineTranslator.java  |  27 +-
 .../flink/FlinkStreamingTransformTranslators.java  | 215 +++++++++----
 .../flink/FlinkStreamingTranslationContext.java    |  24 +-
 .../wrappers/streaming/DoFnOperator.java           |   5 +-
 .../streaming/SingletonKeyedWorkItemCoder.java     |  13 +-
 .../wrappers/streaming/WorkItemKeySelector.java    |  13 +-
 .../FlinkStreamingPipelineTranslatorTest.java      | 109 +++++++
 .../wrappers/streaming/WindowDoFnOperatorTest.java |   5 +-
 sdks/go/container/build.gradle                     |   2 +
 sdks/go/test/load/build.gradle                     |   1 +
 sdks/go/test/load/sideinput/sideinput.go           | 100 +++++++
 sdks/java/container/common.gradle                  |   2 +
 .../provider/bigtable/BigtableClientWrapper.java   | 115 +++++++
 .../BigtableTableCreationFailuresTest.java         |   2 +-
 .../provider/bigtable/BigtableTableFlatTest.java   | 110 +++----
 .../meta/provider/bigtable/BigtableTableIT.java    | 200 +++++++++++++
 .../meta/provider/bigtable/BigtableTableTest.java  |  94 ------
 .../provider/bigtable/BigtableTableTestUtils.java  | 237 +++++++++++++++
 .../bigtable/BigtableTableWithRowsTest.java        | 120 ++++----
 .../sql/udf/build.gradle}                          |  22 +-
 .../beam/sdk/extensions/sql/udf/AggregateFn.java   |  70 +++++
 .../beam/sdk/extensions/sql/udf/ScalarFn.java      |  53 ++++
 .../beam/sdk/extensions/sql/udf/package-info.java} |  15 +-
 .../apache/beam/fn/harness/FnApiDoFnRunner.java    |  27 ++
 .../harness/TransformProcessingThreadTracker.java  |  49 +++
 .../fn/harness/logging/BeamFnLoggingClient.java    |   8 +
 ...fsetFinalizer.java => InitialOffsetReader.java} |  12 +-
 .../io/gcp/pubsublite/InitialOffsetReaderImpl.java |  54 ++++
 .../gcp/pubsublite/LimitingTopicBacklogReader.java |  81 +++++
 ...ffsetFinalizer.java => OffsetByteProgress.java} |  17 +-
 .../io/gcp/pubsublite/OffsetByteRangeTracker.java  | 172 +++++++++++
 .../io/gcp/pubsublite/OffsetCheckpointMark.java    |  74 -----
 .../pubsublite/PerSubscriptionPartitionSdf.java    | 110 +++++++
 .../beam/sdk/io/gcp/pubsublite/PublisherCache.java |   2 +-
 .../beam/sdk/io/gcp/pubsublite/PubsubLiteIO.java   |  14 +-
 .../gcp/pubsublite/PubsubLiteUnboundedReader.java  | 333 ---------------------
 .../gcp/pubsublite/PubsubLiteUnboundedSource.java  | 126 --------
 ...zer.java => SerializableSubscriberFactory.java} |  17 +-
 .../io/gcp/pubsublite/SerializableSupplier.java    |   2 +-
 .../sdk/io/gcp/pubsublite/SubscribeTransform.java  | 124 ++++++++
 .../sdk/io/gcp/pubsublite/SubscriberOptions.java   |  42 ++-
 ...etFinalizer.java => SubscriptionPartition.java} |  19 +-
 ...dCoder.java => SubscriptionPartitionCoder.java} |  36 ++-
 ...er.java => SubscriptionPartitionProcessor.java} |  14 +-
 ... => SubscriptionPartitionProcessorFactory.java} |  14 +-
 .../SubscriptionPartitionProcessorImpl.java        | 152 ++++++++++
 .../sdk/io/gcp/pubsublite/TopicBacklogReader.java  |  33 +-
 .../io/gcp/pubsublite/TopicBacklogReaderImpl.java  |  80 ++---
 .../gcp/pubsublite/TopicBacklogReaderSettings.java |  31 +-
 .../gcp/pubsublite/TranslatingPullSubscriber.java  |  55 ----
 .../beam/sdk/io/gcp/pubsublite/UuidCoder.java      |   7 +-
 .../io/gcp/testing/BigtableEmulatorWrapper.java    |  75 -----
 .../beam/sdk/io/gcp/testing/BigtableTestUtils.java | 153 ----------
 .../beam/sdk/io/gcp/testing/BigtableUtils.java     |  49 +++
 .../bigtable/BeamRowToBigtableMutationTest.java    |  14 +-
 .../gcp/bigtable/BigtableRowToBeamRowFlatTest.java |  14 +-
 .../io/gcp/bigtable/BigtableRowToBeamRowTest.java  |  17 +-
 .../{TestUtils.java => BigtableTestUtils.java}     |  50 +++-
 .../sdk/io/gcp/bigtable/CellValueParserTest.java   |  13 +-
 .../gcp/pubsublite/OffsetByteRangeTrackerTest.java | 158 ++++++++++
 .../gcp/pubsublite/OffsetCheckpointMarkTest.java   |  81 -----
 .../PerSubscriptionPartitionSdfTest.java           | 189 ++++++++++++
 .../pubsublite/PubsubLiteUnboundedReaderTest.java  | 330 --------------------
 .../SubscriptionPartitionProcessorImplTest.java    | 227 ++++++++++++++
 .../gcp/pubsublite/TopicBacklogReaderImplTest.java | 129 ++------
 sdks/python/apache_beam/dataframe/io.py            | 258 ++++++++++++----
 sdks/python/apache_beam/dataframe/io_test.py       |  88 +++++-
 sdks/python/apache_beam/io/concat_source.py        |   9 +-
 sdks/python/apache_beam/io/concat_source_test.py   |   8 +
 sdks/python/apache_beam/metrics/execution.pxd      |   1 +
 sdks/python/apache_beam/metrics/execution.py       |   9 +-
 sdks/python/apache_beam/transforms/core.py         |  41 +--
 sdks/python/apache_beam/version.py                 |   2 +-
 sdks/python/container/common.gradle                |   2 +
 settings.gradle                                    |   1 +
 81 files changed, 3479 insertions(+), 1924 deletions(-)
 create mode 100644 .test-infra/jenkins/job_LoadTests_SideInput_Flink_Go.groovy
 create mode 100644 .test-infra/jenkins/job_LoadTests_SideInput_Go.groovy
 create mode 100644 sdks/go/test/load/sideinput/sideinput.go
 create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableClientWrapper.java
 create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableIT.java
 delete mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableTest.java
 create mode 100644 sdks/java/extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider/bigtable/BigtableTableTestUtils.java
 copy sdks/java/{io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/OffsetFinalizer.java
=> extensions/sql/udf/build.gradle} (50%)
 create mode 100644 sdks/java/extensions/sql/udf/src/main/java/org/apache/beam/sdk/extensions/sql/udf/AggregateFn.java
 create mode 100644 sdks/java/extensions/sql/udf/src/main/java/org/apache/beam/sdk/extensions/sql/udf/ScalarFn.java
 copy sdks/java/{io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/SerializableSupplier.java
=> extensions/sql/udf/src/main/java/org/apache/beam/sdk/extensions/sql/udf/package-info.java}
(68%)
 create mode 100644 sdks/java/harness/src/main/java/org/apache/beam/fn/harness/TransformProcessingThreadTracker.java
 copy sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{OffsetFinalizer.java
=> InitialOffsetReader.java} (75%)
 create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/InitialOffsetReaderImpl.java
 create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/LimitingTopicBacklogReader.java
 copy sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{OffsetFinalizer.java
=> OffsetByteProgress.java} (68%)
 create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/OffsetByteRangeTracker.java
 delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/OffsetCheckpointMark.java
 create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/PerSubscriptionPartitionSdf.java
 delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/PubsubLiteUnboundedReader.java
 delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/PubsubLiteUnboundedSource.java
 copy sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{OffsetFinalizer.java
=> SerializableSubscriberFactory.java} (65%)
 create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/SubscribeTransform.java
 copy sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{OffsetFinalizer.java
=> SubscriptionPartition.java} (64%)
 copy sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{UuidCoder.java
=> SubscriptionPartitionCoder.java} (50%)
 rename sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{OffsetFinalizer.java
=> SubscriptionPartitionProcessor.java} (74%)
 copy sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/{SerializableSupplier.java
=> SubscriptionPartitionProcessorFactory.java} (60%)
 create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/SubscriptionPartitionProcessorImpl.java
 delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsublite/TranslatingPullSubscriber.java
 delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/BigtableEmulatorWrapper.java
 delete mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/BigtableTestUtils.java
 create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/testing/BigtableUtils.java
 rename sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigtable/{TestUtils.java
=> BigtableTestUtils.java} (71%)
 create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/OffsetByteRangeTrackerTest.java
 delete mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/OffsetCheckpointMarkTest.java
 create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/PerSubscriptionPartitionSdfTest.java
 delete mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/PubsubLiteUnboundedReaderTest.java
 create mode 100644 sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsublite/SubscriptionPartitionProcessorImplTest.java


Mime
View raw message