This is an automated email from the ASF dual-hosted git repository.
bhulette pushed a commit to branch website-revamp
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 7d0d220278b65376ab84832421be613e901b1177
Merge: 8b22fb4 53615fe
Author: Brian Hulette <bhulette@google.com>
AuthorDate: Tue Jan 12 17:53:24 2021 -0800
Merge remote-tracking branch 'origin/master' into website-revamp
.github/workflows/build_wheels.yml | 2 +-
.github/workflows/cancel.yml | 2 +-
.gitignore | 10 +
.test-infra/jenkins/README.md | 9 +-
.test-infra/jenkins/job_Inventory.groovy | 15 +-
.../jenkins/job_LoadTests_Combine_Flink_Go.groovy | 2 +-
...k_Go.groovy => job_LoadTests_Combine_Go.groovy} | 85 +-
.../jenkins/job_LoadTests_GBK_Flink_Go.groovy | 19 +-
.test-infra/jenkins/job_LoadTests_GBK_Go.groovy | 211 +++++
.test-infra/jenkins/job_LoadTests_ParDo_Go.groovy | 145 ++++
.../job_LoadTests_SideInput_Flink_Go.groovy | 103 +++
.../jenkins/job_LoadTests_SideInput_Go.groovy | 94 +++
.test-infra/jenkins/job_LoadTests_coGBK_Go.groovy | 169 ++++
...ython36.groovy => job_PostCommit_Python.groovy} | 39 +-
.test-infra/jenkins/job_PostCommit_Python37.groovy | 44 -
.test-infra/jenkins/job_PostCommit_Python38.groovy | 44 -
...stCommit_Python_ValidatesRunner_Dataflow.groovy | 8 +-
...ommit_Python_ValidatesRunner_Dataflow_V2.groovy | 9 +-
.../jenkins/job_Publish_SDK_Image_Snapshots.groovy | 8 +-
.../perftests_metrics/SideInput_Load_Tests.json | 2 +-
CHANGES.md | 51 +-
build.gradle | 9 -
buildSrc/build.gradle | 4 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 31 +-
examples/java/build.gradle | 11 +-
.../org/apache/beam/examples/complete/README.md | 5 +
.../kafkatopubsub/KafkaPubsubConstants.java | 15 +-
.../complete/kafkatopubsub/KafkaToPubsub.java | 235 ++++++
.../beam/examples/complete/kafkatopubsub/README.md | 200 +++++
.../complete/kafkatopubsub/avro/AvroDataClass.java | 63 ++
.../avro/AvroDataClassKafkaAvroDeserializer.java | 27 +-
.../complete/kafkatopubsub/avro/package-info.java | 10 +-
.../kafka/consumer/SslConsumerFactoryFn.java | 135 ++++
.../kafkatopubsub/kafka/consumer/Utils.java | 165 ++++
.../kafkatopubsub/kafka/consumer/package-info.java | 10 +-
.../options/KafkaToPubsubOptions.java | 90 +++
.../kafkatopubsub/options/package-info.java | 10 +-
.../complete/kafkatopubsub/package-info.java | 10 +-
.../kafkatopubsub/transforms/FormatTransform.java | 128 +++
.../kafkatopubsub/transforms/package-info.java | 10 +-
.../examples/complete/game/LeaderBoardTest.java | 2 +-
.../complete/kafkatopubsub/KafkaToPubsubTest.java | 90 +++
.../cookbook/CombinePerKeyExamplesTest.java | 13 +-
examples/kotlin/build.gradle | 2 +-
.../beam/examples/kotlin/MinimalWordCount.kt | 2 +-
gradle.properties | 7 +-
gradle/wrapper/gradle-wrapper.properties | 2 +-
.../beam/learning/katas/util/ContainsKvs.java | 2 +-
.../pipeline/src/main/proto/beam_runner_api.proto | 18 +
.../src/main/scripts/build_release_candidate.sh | 6 +-
.../src/main/scripts/preparation_before_release.sh | 2 +
release/src/main/scripts/publish_docker_images.sh | 4 +-
.../core/construction/CoderTranslationTest.java | 2 +-
.../runners/core/construction/CommonCoderTest.java | 2 +-
.../CreatePCollectionViewTranslationTest.java | 2 +-
.../DeduplicatedFlattenFactoryTest.java | 2 +-
.../construction/DefaultArtifactResolverTest.java | 2 +-
.../EmptyFlattenAsCreateFactoryTest.java | 2 +-
.../core/construction/EnvironmentsTest.java | 2 +-
.../ExecutableStageTranslationTest.java | 2 +-
.../construction/ForwardingPTransformTest.java | 2 +-
.../construction/GroupByKeyTranslationTest.java | 2 +-
.../runners/core/construction/ModelCodersTest.java | 2 +-
.../core/construction/MorePipelineTest.java | 2 +-
.../construction/PCollectionTranslationTest.java | 2 +-
.../core/construction/PTransformMatchersTest.java | 2 +-
.../construction/PTransformReplacementsTest.java | 2 +-
.../construction/PTransformTranslationTest.java | 2 +-
.../core/construction/ParDoTranslationTest.java | 2 +-
.../PipelineOptionsTranslationTest.java | 2 +-
.../core/construction/PipelineTranslationTest.java | 2 +-
.../core/construction/ReadTranslationTest.java | 2 +-
.../construction/RehydratedComponentsTest.java | 2 +-
.../core/construction/ReplacementOutputsTest.java | 2 +-
.../construction/ReshuffleTranslationTest.java | 2 +-
.../core/construction/SchemaTranslationTest.java | 2 +-
.../core/construction/SdkComponentsTest.java | 2 +-
.../SingleInputOutputOverrideFactoryTest.java | 2 +-
.../construction/TestStreamTranslationTest.java | 2 +-
.../core/construction/TransformInputsTest.java | 2 +-
.../core/construction/TriggerTranslationTest.java | 2 +-
.../core/construction/UnconsumedReadsTest.java | 2 +-
.../core/construction/ValidateRunnerXlangTest.java | 2 +-
.../WindowingStrategyTranslationTest.java | 161 +++-
.../construction/WriteFilesTranslationTest.java | 2 +-
.../construction/graph/ExecutableStageTest.java | 2 +-
.../core/construction/graph/FusedPipelineTest.java | 2 +-
.../graph/GreedyPipelineFuserTest.java | 2 +-
.../construction/graph/GreedyStageFuserTest.java | 2 +-
.../graph/ImmutableExecutableStageTest.java | 2 +-
.../core/construction/graph/NetworksTest.java | 2 +-
.../construction/graph/OutputDeduplicatorTest.java | 2 +-
.../construction/graph/ProtoOverridesTest.java | 2 +-
.../construction/graph/QueryablePipelineTest.java | 2 +-
.../core/metrics/ExecutionStateTracker.java | 20 +-
.../runners/core/InMemoryStateInternalsTest.java | 2 +-
.../runners/core/InMemoryTimerInternalsTest.java | 2 +-
.../core/LateDataDroppingDoFnRunnerTest.java | 2 +-
.../beam/runners/core/LateDataUtilsTest.java | 2 +-
...BoundedSplittableProcessElementInvokerTest.java | 2 +-
.../beam/runners/core/ReduceFnRunnerTest.java | 2 +-
.../beam/runners/core/SideInputHandlerTest.java | 2 +-
.../beam/runners/core/SimpleDoFnRunnerTest.java | 2 +-
.../SimplePushbackSideInputDoFnRunnerTest.java | 2 +-
.../runners/core/SplittableParDoProcessFnTest.java | 2 +-
.../beam/runners/core/StateNamespacesTest.java | 2 +-
.../beam/runners/core/TimerInternalsTest.java | 2 +-
.../beam/runners/core/WindowMatchersTest.java | 2 +-
.../beam/runners/core/metrics/CounterCellTest.java | 2 +-
.../beam/runners/core/metrics/DirtyStateTest.java | 2 +-
.../runners/core/metrics/DistributionCellTest.java | 2 +-
.../core/metrics/ExecutionStateSamplerTest.java | 2 +-
.../core/metrics/ExecutionStateTrackerTest.java | 2 +-
.../beam/runners/core/metrics/GaugeCellTest.java | 2 +-
.../core/metrics/MetricsContainerImplTest.java | 2 +-
.../core/metrics/MetricsContainerStepMapTest.java | 2 +-
.../beam/runners/core/metrics/MetricsMapTest.java | 2 +-
.../runners/core/metrics/MetricsPusherTest.java | 2 +-
.../core/metrics/MonitoringInfoConstantsTest.java | 2 +-
.../core/metrics/SimpleStateRegistryTest.java | 2 +-
.../triggers/AfterWatermarkStateMachineTest.java | 2 +-
.../core/triggers/FinishedTriggersBitSetTest.java | 2 +-
.../core/triggers/FinishedTriggersSetTest.java | 2 +-
.../core/triggers/NeverStateMachineTest.java | 2 +-
.../core/triggers/TriggerStateMachinesTest.java | 2 +-
.../direct/BoundedReadEvaluatorFactoryTest.java | 2 +-
.../runners/direct/CloningBundleFactoryTest.java | 2 +-
.../beam/runners/direct/CommittedResultTest.java | 2 +-
.../CopyOnAccessInMemoryStateInternalsTest.java | 2 +-
.../runners/direct/DirectGraphVisitorTest.java | 2 +-
.../DirectGroupByKeyOverrideFactoryTest.java | 2 +-
.../beam/runners/direct/DirectMetricsTest.java | 2 +-
.../runners/direct/DirectRunnerApiSurfaceTest.java | 2 +-
.../beam/runners/direct/DirectRunnerTest.java | 2 +-
.../runners/direct/DirectTimerInternalsTest.java | 2 +-
.../direct/DirectTransformExecutorTest.java | 2 +-
...cycleManagerRemovingTransformEvaluatorTest.java | 2 +-
.../runners/direct/DoFnLifecycleManagerTest.java | 2 +-
.../beam/runners/direct/EvaluationContextTest.java | 2 +-
.../direct/FlattenEvaluatorFactoryTest.java | 2 +-
.../direct/GroupByKeyEvaluatorFactoryTest.java | 2 +-
.../direct/GroupByKeyOnlyEvaluatorFactoryTest.java | 2 +-
.../ImmutabilityCheckingBundleFactoryTest.java | 2 +-
.../direct/ImmutableListBundleFactoryTest.java | 2 +-
.../direct/ImpulseEvaluatorFactoryTest.java | 2 +-
.../direct/KeyedPValueTrackingVisitorTest.java | 2 +-
.../beam/runners/direct/MultiStepCombineTest.java | 2 +-
.../beam/runners/direct/ParDoEvaluatorTest.java | 2 +-
.../runners/direct/SideInputContainerTest.java | 2 +-
.../direct/StatefulParDoEvaluatorFactoryTest.java | 2 +-
.../runners/direct/StepTransformResultTest.java | 2 +-
.../direct/TestStreamEvaluatorFactoryTest.java | 2 +-
.../direct/UnboundedReadDeduplicatorTest.java | 2 +-
.../direct/UnboundedReadEvaluatorFactoryTest.java | 2 +-
.../runners/direct/ViewEvaluatorFactoryTest.java | 2 +-
.../direct/WatermarkCallbackExecutorTest.java | 2 +-
.../beam/runners/direct/WatermarkManagerTest.java | 2 +-
.../runners/direct/WindowEvaluatorFactoryTest.java | 2 +-
.../direct/WriteWithShardingFactoryTest.java | 2 +-
.../runners/flink/streaming/StreamSources.java | 13 +
{sdks/go => runners/flink/1.12}/build.gradle | 38 +-
.../flink/1.12/job-server-container/build.gradle | 16 +-
.../flink/1.12/job-server}/build.gradle | 34 +-
.../streaming/AbstractStreamOperatorCompat.java | 53 ++
.../runners/flink/streaming/StreamSources.java | 14 +
.../runners/flink/streaming/StreamSources.java | 13 +
.../runners/flink/streaming/StreamSources.java | 13 +
runners/flink/flink_runner.gradle | 19 +-
.../runners/flink/FlinkExecutionEnvironments.java | 12 +
.../flink/FlinkPortableClientEntryPoint.java | 4 +-
.../FlinkStreamingPortablePipelineTranslator.java | 27 +-
.../flink/FlinkStreamingTransformTranslators.java | 215 +++--
.../flink/FlinkStreamingTranslationContext.java | 24 +-
.../wrappers/streaming/DoFnOperator.java | 77 +-
.../streaming/KeyedPushedBackElementsHandler.java | 5 +-
.../streaming/SingletonKeyedWorkItemCoder.java | 13 +-
.../wrappers/streaming/WorkItemKeySelector.java | 13 +-
.../streaming/state/FlinkStateInternals.java | 4 +-
.../FlinkStreamingPipelineTranslatorTest.java | 109 +++
.../FlinkStreamingTransformTranslatorsTest.java | 7 +-
.../runners/flink/FlinkTransformOverridesTest.java | 2 +-
.../wrappers/streaming/DedupingOperatorTest.java | 2 +-
.../wrappers/streaming/DoFnOperatorTest.java | 315 ++++++--
.../wrappers/streaming/WindowDoFnOperatorTest.java | 5 +-
runners/google-cloud-dataflow-java/build.gradle | 2 +-
.../beam/runners/dataflow/DataflowRunner.java | 53 +-
.../options/DataflowPipelineDebugOptions.java | 10 +
.../dataflow/BatchStatefulParDoOverridesTest.java | 2 +-
.../runners/dataflow/BatchViewOverridesTest.java | 2 +-
.../dataflow/DataflowPTransformMatchersTest.java | 2 +-
.../dataflow/DataflowPipelineTranslatorTest.java | 2 +-
.../runners/dataflow/DataflowRunnerInfoTest.java | 2 +-
.../beam/runners/dataflow/DataflowRunnerTest.java | 56 ++
.../dataflow/PrimitiveParDoSingleFactoryTest.java | 2 +-
.../runners/dataflow/TestDataflowRunnerTest.java | 2 +-
.../options/DataflowPipelineDebugOptionsTest.java | 2 +-
.../options/DataflowProfilingOptionsTest.java | 2 +-
.../runners/dataflow/util/PackageUtilTest.java | 2 +-
.../beam/runners/dataflow/util/StructsTest.java | 25 +-
.../beam/runners/dataflow/worker/ReaderCache.java | 38 +-
.../dataflow/worker/StreamingDataflowWorker.java | 61 +-
.../worker/fn/data/BeamFnDataGrpcService.java | 14 +-
.../logging/DataflowWorkerLoggingInitializer.java | 14 +-
.../JulHandlerPrintStreamAdapterFactory.java | 15 +-
.../dataflow/harness/test/TestStreamsTest.java | 2 +-
.../dataflow/worker/AvroByteReaderFactoryTest.java | 5 +-
.../dataflow/worker/AvroByteSinkFactoryTest.java | 3 +-
.../dataflow/worker/BatchDataflowWorkerTest.java | 2 +-
.../worker/BatchModeExecutionContextTest.java | 2 +-
.../dataflow/worker/ConcatReaderFactoryTest.java | 2 +-
.../runners/dataflow/worker/ConcatReaderTest.java | 2 +-
.../ContextActivationObserverRegistryTest.java | 2 +-
...CreateIsmShardKeyAndSortKeyDoFnFactoryTest.java | 2 +-
.../DataflowElementExecutionTrackerTest.java | 2 +-
.../worker/DataflowExecutionStateTrackerTest.java | 2 +-
.../worker/DataflowSideInputReadCounterTest.java | 2 +-
.../worker/DataflowWorkerHarnessHelperTest.java | 2 +-
.../dataflow/worker/DefaultParDoFnFactoryTest.java | 2 +-
.../dataflow/worker/DeltaCounterCellTest.java | 2 +-
.../dataflow/worker/DoFnInstanceManagersTest.java | 2 +-
.../dataflow/worker/FakeWindmillServer.java | 14 +-
.../runners/dataflow/worker/FilepatternsTest.java | 2 +-
.../GroupAlsoByWindowParDoFnFactoryTest.java | 2 +-
.../dataflow/worker/InMemoryReaderFactoryTest.java | 3 +-
.../IntrinsicMapTaskExecutorFactoryTest.java | 2 +-
.../worker/IntrinsicMapTaskExecutorTest.java | 2 +-
.../runners/dataflow/worker/IsmFormatTest.java | 2 +-
.../dataflow/worker/IsmSideInputReaderTest.java | 2 +-
.../dataflow/worker/LogRecordMatcherTest.java | 2 +-
.../beam/runners/dataflow/worker/LogSaverTest.java | 2 +-
...tContextActivationObserverRegistrationTest.java | 2 +-
.../worker/PairWithConstantKeyDoFnFactoryTest.java | 2 +-
.../worker/PartialGroupByKeyParDoFnsTest.java | 2 +-
.../runners/dataflow/worker/ReaderCacheTest.java | 6 +-
.../runners/dataflow/worker/ReaderFactoryTest.java | 6 +-
...ReifyTimestampAndWindowsParDoFnFactoryTest.java | 2 +-
...essCoderCloudObjectTranslatorRegistrarTest.java | 2 +-
.../dataflow/worker/ShuffleReaderFactoryTest.java | 3 +-
.../dataflow/worker/ShuffleSinkFactoryTest.java | 3 +-
.../runners/dataflow/worker/SimpleParDoFnTest.java | 2 +-
.../runners/dataflow/worker/SinkRegistryTest.java | 5 +-
.../worker/SourceOperationExecutorFactoryTest.java | 2 +-
.../runners/dataflow/worker/StateFetcherTest.java | 2 +-
.../worker/StreamingDataflowWorkerTest.java | 10 +-
.../worker/StreamingGroupAlsoByWindowFnsTest.java | 2 +-
...reamingGroupAlsoByWindowsReshuffleDoFnTest.java | 2 +-
.../worker/StreamingModeExecutionContextTest.java | 5 +-
...eamingPCollectionViewWriterDoFnFactoryTest.java | 2 +-
.../worker/StreamingSideInputDoFnRunnerTest.java | 2 +-
.../worker/StreamingSideInputFetcherTest.java | 2 +-
.../worker/StreamingStepMetricsContainerTest.java | 2 +-
.../ToIsmRecordForMultimapDoFnFactoryTest.java | 2 +-
.../dataflow/worker/UserParDoFnFactoryTest.java | 2 +-
.../dataflow/worker/ValuesDoFnFactoryTest.java | 2 +-
.../dataflow/worker/WindmillKeyedWorkItemTest.java | 2 +-
.../worker/WindmillStateInternalsTest.java | 2 +-
.../dataflow/worker/WindmillStateReaderTest.java | 2 +-
.../worker/WindmillTimerInternalsTest.java | 2 +-
.../dataflow/worker/WorkItemStatusClientTest.java | 2 +-
.../dataflow/worker/WorkerCustomSourcesTest.java | 2 +-
.../dataflow/worker/counters/CounterSetTest.java | 2 +-
.../fn/control/BeamFnMapTaskExecutorTest.java | 2 +-
.../RegisterAndProcessBundleOperationTest.java | 2 +-
.../worker/fn/data/BeamFnDataGrpcServiceTest.java | 2 +-
.../fn/data/RemoteGrpcPortReadOperationTest.java | 2 +-
.../fn/data/RemoteGrpcPortWriteOperationTest.java | 2 +-
.../fn/logging/BeamFnLoggingServiceTest.java | 2 +-
.../fn/stream/ServerStreamObserverFactoryTest.java | 2 +-
.../graph/CloneAmbiguousFlattensFunctionTest.java | 2 +-
.../CreateRegisterFnOperationFunctionTest.java | 2 +-
.../graph/DeduceNodeLocationsFunctionTest.java | 2 +-
...tFetchAndFilterStreamingSideInputNodesTest.java | 2 +-
.../worker/graph/MapTaskToNetworkFunctionTest.java | 2 +-
.../dataflow/worker/graph/NetworksTest.java | 2 +-
.../RemoveFlattenInstructionsFunctionTest.java | 2 +-
.../ReplacePgbkWithPrecombineFunctionTest.java | 2 +-
.../DataflowWorkerLoggingInitializerTest.java | 2 +-
.../JulHandlerPrintStreamAdapterFactoryTest.java | 16 +-
.../worker/profiler/ScopedProfilerTest.java | 2 +-
.../dataflow/worker/status/ThreadzServletTest.java | 2 +-
.../worker/status/WorkerStatusPagesTest.java | 2 +-
.../worker/util/BatchGroupAlsoByWindowFnsTest.java | 2 +-
.../BatchGroupAlsoByWindowReshuffleDoFnTest.java | 2 +-
.../worker/util/GroupAlsoByWindowProperties.java | 2 +-
.../dataflow/worker/util/MemoryMonitorTest.java | 2 +-
.../worker/util/ScalableBloomFilterTest.java | 2 +-
.../dataflow/worker/util/TimerOrElementTest.java | 2 +-
.../worker/BatchingShuffleEntryReaderTest.java | 2 +-
.../worker/CachingShuffleBatchReaderTest.java | 2 +-
.../util/common/worker/FlattenOperationTest.java | 5 +-
.../util/common/worker/GroupingTablesTest.java | 2 +-
.../util/common/worker/MapTaskExecutorTest.java | 2 +-
.../util/common/worker/OutputReceiverTest.java | 5 +-
.../util/common/worker/ParDoOperationTest.java | 2 +-
.../util/common/worker/ReadOperationTest.java | 2 +-
.../util/common/worker/ShuffleEntryTest.java | 2 +-
.../util/common/worker/WriteOperationTest.java | 2 +-
.../runners/fnexecution/data/GrpcDataService.java | 6 +-
.../runners/fnexecution/ServerFactoryTest.java | 2 +-
.../control/DefaultJobBundleFactoryTest.java | 62 +-
.../control/FnApiControlClientPoolServiceTest.java | 2 +-
.../control/FnApiControlClientTest.java | 2 +-
...gleEnvironmentInstanceJobBundleFactoryTest.java | 2 +-
.../fnexecution/data/GrpcDataServiceTest.java | 2 +-
.../environment/ProcessEnvironmentFactoryTest.java | 2 +-
.../environment/ProcessManagerTest.java | 8 +-
.../environment/RemoteEnvironmentTest.java | 2 +-
.../logging/GrpcLoggingServiceTest.java | 2 +-
.../StaticGrpcProvisionServiceTest.java | 2 +-
.../fnexecution/state/GrpcStateServiceTest.java | 2 +-
.../translation/PipelineTranslatorUtilsTest.java | 2 +-
.../beam/runners/local/StructuralKeyTest.java | 18 +-
runners/portability/java/build.gradle | 24 -
runners/samza/build.gradle | 17 +-
.../beam/runners/samza/SamzaJobInvocation.java | 114 +++
.../beam/runners/samza/SamzaJobServerDriver.java | 60 +-
.../beam/runners/samza/SamzaPipelineOptions.java | 22 +-
.../samza/SamzaPipelineOptionsValidator.java | 38 +-
.../samza/SamzaPortablePipelineOptions.java | 13 +
.../runners/samza/SamzaRunnerOverrideConfigs.java | 15 +
.../samza/adapter/UnboundedSourceSystem.java | 39 +-
.../samza/container/BeamContainerRunner.java | 6 +-
.../samza/container/BeamJobCoordinatorRunner.java | 78 ++
...inerCfgFactory.java => ContainerCfgLoader.java} | 18 +-
.../ContainerCfgLoaderFactory.java} | 16 +-
.../beam/runners/samza/runtime/BundleManager.java | 349 ++++++++
.../apache/beam/runners/samza/runtime/DoFnOp.java | 277 ++++---
.../runners/samza/runtime/FutureCollector.java | 60 ++
.../beam/runners/samza/runtime/GroupByKeyOp.java | 11 +-
.../beam/runners/samza/runtime/KeyedTimerData.java | 1 +
.../beam/runners/samza/runtime/OpAdapter.java | 48 +-
.../beam/runners/samza/runtime/OpEmitter.java | 5 +
.../samza/runtime/OutputManagerFactory.java | 5 +
.../samza/runtime/SamzaStoreStateInternals.java | 230 ++++--
.../samza/runtime/SamzaTimerInternalsFactory.java | 217 ++++-
.../runners/samza/translation/ConfigBuilder.java | 77 +-
.../runners/samza/translation/ConfigContext.java | 8 +
.../translation/FlattenPCollectionsTranslator.java | 2 +-
.../samza/translation/GroupByKeyTranslator.java | 24 +-
.../translation/ParDoBoundMultiTranslator.java | 56 +-
.../samza/translation/SamzaPipelineTranslator.java | 21 +-
.../translation/SamzaTestStreamSystemFactory.java | 179 +++++
.../translation/SamzaTestStreamTranslator.java | 100 +++
.../translation/SplittableParDoTranslators.java | 6 +-
.../samza/translation/TranslationContext.java | 59 +-
.../samza/translation/WindowAssignTranslator.java | 4 +-
.../beam/runners/samza/util/FutureUtils.java | 50 ++
.../samza/SamzaPipelineOptionsValidatorTest.java | 60 ++
.../samza/adapter/UnboundedSourceSystemTest.java | 28 +
.../runners/samza/runtime/BundleManagerTest.java | 474 +++++++++++
.../samza/runtime/FutureCollectorImplTest.java | 92 +++
.../runners/samza/runtime/KeyedTimerDataTest.java | 14 +-
.../runtime/SamzaStoreStateInternalsTest.java | 38 +-
.../runtime/SamzaTimerInternalsFactoryTest.java | 416 +++++++++-
.../samza/translation/ConfigGeneratorTest.java | 86 +-
.../samza/translation/TranslationContextTest.java | 94 +++
.../beam/runners/samza/util/FutureUtilsTest.java | 107 +++
.../runners/spark/GlobalWatermarkHolderTest.java | 2 +-
.../beam/runners/spark/SparkPipelineStateTest.java | 2 +-
.../runners/spark/SparkRunnerDebuggerTest.java | 2 +-
.../metrics/sink/SparkMetricsSinkTest.java | 2 +-
.../spark/io/ReaderToIteratorAdapterTest.java | 2 +-
.../runners/spark/metrics/SparkBeamMetricTest.java | 2 +-
.../spark/metrics/SparkMetricsPusherTest.java | 2 +-
.../metrics/sink/SparkMetricsSinkTest.java | 2 +-
.../metrics/BeamMetricTest.java | 2 +-
.../translation/batch/GroupByKeyTest.java | 2 +-
.../translation/streaming/CreateStreamTest.java | 2 +-
.../ResumeFromCheckpointStreamingTest.java | 2 +-
.../streaming/SparkCoGroupByKeyStreamingTest.java | 2 +-
.../streaming/StreamingSourceMetricsTest.java | 2 +-
.../streaming/TrackStreamingSourcesTest.java | 2 +-
sdks/go/build.gradle | 2 +-
sdks/go/container/build.gradle | 2 +
sdks/go/pkg/beam/core/graph/coder/double_test.go | 53 ++
sdks/go/pkg/beam/core/graph/coder/map_test.go | 11 +-
sdks/go/pkg/beam/core/graph/coder/row.go | 337 ++------
sdks/go/pkg/beam/core/graph/coder/row_decoder.go | 308 +++++++
sdks/go/pkg/beam/core/graph/coder/row_encoder.go | 271 +++++++
sdks/go/pkg/beam/core/graph/coder/row_test.go | 488 +++++++++---
.../pkg/beam/core/graph/coder/testutil/testutil.go | 154 ++++
.../core/graph/coder/testutil/testutil_test.go | 201 +++++
sdks/go/pkg/beam/core/metrics/metrics.go | 39 +
sdks/go/pkg/beam/core/runtime/graphx/xlang.go | 9 +-
sdks/go/pkg/beam/core/runtime/metricsx/metricsx.go | 38 +-
sdks/go/pkg/beam/pipeline.go | 1 +
sdks/go/pkg/beam/runners/dataflow/dataflow.go | 5 +-
.../beam/runners/dataflow/dataflowlib/execute.go | 57 +-
.../pkg/beam/runners/dataflow/dataflowlib/job.go | 8 +
.../beam/runners/dataflow/dataflowlib/metrics.go | 124 +++
.../runners/dataflow/dataflowlib/metrics_test.go | 133 ++++
.../beam/runners/universal/runnerlib/compile.go | 13 +-
.../beam/runners/universal/runnerlib/execute.go | 21 +-
sdks/go/pkg/beam/runners/universal/universal.go | 1 +
sdks/go/pkg/beam/testing/ptest/ptest.go | 21 +
sdks/go/test/build.gradle | 94 ++-
sdks/go/test/integration/{ => driver}/driver.go | 0
sdks/go/test/integration/integration.go | 147 ++++
sdks/go/test/integration/primitives/cogbk_test.go | 16 +-
.../go/test/integration/primitives/flatten_test.go | 11 +-
sdks/go/test/integration/primitives/pardo_test.go | 16 +-
.../test/integration/synthetic/synthetic_test.go | 75 ++
.../test/integration/wordcount/wordcount_test.go | 6 +
sdks/go/test/integration/xlang/xlang_test.go | 260 ++++++
sdks/go/test/load/build.gradle | 1 +
sdks/go/test/load/sideinput/sideinput.go | 100 +++
sdks/go/test/run_integration_tests.sh | 2 +-
...ation_tests.sh => run_validatesrunner_tests.sh} | 220 +++--
.../src/main/resources/beam/checkstyle.xml | 17 +
.../src/main/resources/beam/suppressions.xml | 4 +
sdks/java/container/build.gradle | 4 +-
sdks/java/container/common.gradle | 12 +-
sdks/java/core/build.gradle | 5 +-
.../main/java/org/apache/beam/sdk/io/AvroIO.java | 38 +-
.../apache/beam/sdk/io/AvroSchemaIOProvider.java | 4 +-
.../beam/sdk/io/GenericRecordWriteConverter.java | 67 --
.../src/main/java/org/apache/beam/sdk/io/Read.java | 97 ++-
.../sdk/schemas/FieldValueTypeInformation.java | 53 +-
.../org/apache/beam/sdk/schemas/RowMessages.java | 132 +++
.../beam/sdk/schemas/transforms/Convert.java | 5 +
.../apache/beam/sdk/schemas/utils/AvroUtils.java | 4 +-
.../beam/sdk/schemas/utils/ConvertHelpers.java | 14 +-
.../apache/beam/sdk/testing/CoderProperties.java | 2 +-
.../apache/beam/sdk/testing/CombineFnTester.java | 2 +-
.../java/org/apache/beam/sdk/testing/PAssert.java | 2 +-
.../apache/beam/sdk/testing/SourceTestUtils.java | 2 +-
.../org/apache/beam/sdk/testing/TestPipeline.java | 45 ++
.../apache/beam/sdk/testing/WindowFnTestUtils.java | 2 +-
.../beam/sdk/transforms/ApproximateUnique.java | 20 +-
.../beam/sdk/transforms/windowing/Trigger.java | 3 +-
.../resources/org/apache/beam/sdk/sdk.properties | 5 +-
.../java/org/apache/beam/sdk/PipelineTest.java | 2 +-
.../org/apache/beam/sdk/coders/AvroCoderTest.java | 2 +-
.../beam/sdk/coders/BigDecimalCoderTest.java | 2 +-
.../beam/sdk/coders/BigEndianIntegerCoderTest.java | 2 +-
.../beam/sdk/coders/BigEndianLongCoderTest.java | 2 +-
.../beam/sdk/coders/BigIntegerCoderTest.java | 2 +-
.../apache/beam/sdk/coders/BitSetCoderTest.java | 2 +-
.../apache/beam/sdk/coders/ByteArrayCoderTest.java | 2 +-
.../org/apache/beam/sdk/coders/ByteCoderTest.java | 2 +-
.../java/org/apache/beam/sdk/coders/CoderTest.java | 2 +-
.../beam/sdk/coders/CollectionCoderTest.java | 2 +-
.../apache/beam/sdk/coders/DefaultCoderTest.java | 2 +-
.../apache/beam/sdk/coders/DelegateCoderTest.java | 2 +-
.../apache/beam/sdk/coders/DoubleCoderTest.java | 2 +-
.../apache/beam/sdk/coders/DurationCoderTest.java | 2 +-
.../org/apache/beam/sdk/coders/FloatCoderTest.java | 2 +-
.../apache/beam/sdk/coders/InstantCoderTest.java | 2 +-
.../apache/beam/sdk/coders/IterableCoderTest.java | 2 +-
.../org/apache/beam/sdk/coders/KvCoderTest.java | 2 +-
.../org/apache/beam/sdk/coders/ListCoderTest.java | 2 +-
.../org/apache/beam/sdk/coders/MapCoderTest.java | 2 +-
.../apache/beam/sdk/coders/NullableCoderTest.java | 2 +-
.../sdk/coders/PCollectionCustomCoderTest.java | 2 +-
.../beam/sdk/coders/SerializableCoderTest.java | 2 +-
.../org/apache/beam/sdk/coders/SetCoderTest.java | 2 +-
.../beam/sdk/coders/StringDelegateCoderTest.java | 2 +-
.../beam/sdk/coders/StringUtf8CoderTest.java | 2 +-
.../beam/sdk/coders/StructuredCoderTest.java | 11 +-
.../beam/sdk/coders/TextualIntegerCoderTest.java | 2 +-
.../apache/beam/sdk/coders/VarIntCoderTest.java | 2 +-
.../apache/beam/sdk/coders/VarLongCoderTest.java | 2 +-
.../org/apache/beam/sdk/coders/VoidCoderTest.java | 2 +-
.../java/org/apache/beam/sdk/io/AvroIOTest.java | 2 +-
.../org/apache/beam/sdk/io/AvroSourceTest.java | 2 +-
.../sdk/io/BoundedReadFromUnboundedSourceTest.java | 2 +-
.../apache/beam/sdk/io/CompressedSourceTest.java | 2 +-
.../org/apache/beam/sdk/io/CountingSourceTest.java | 2 +-
.../org/apache/beam/sdk/io/FileBasedSinkTest.java | 2 +-
.../apache/beam/sdk/io/FileBasedSourceTest.java | 2 +-
.../org/apache/beam/sdk/io/FileSystemsTest.java | 2 +-
.../beam/sdk/io/LocalFileSystemRegistrarTest.java | 2 +-
.../apache/beam/sdk/io/LocalFileSystemTest.java | 2 +-
.../test/java/org/apache/beam/sdk/io/ReadTest.java | 130 +++
.../org/apache/beam/sdk/io/TextIOReadTest.java | 2 +-
.../org/apache/beam/sdk/io/TextIOWriteTest.java | 2 +-
.../org/apache/beam/sdk/io/WriteFilesTest.java | 2 +-
.../io/range/ByteKeyRangeEstimateFractionTest.java | 2 +-
.../io/range/ByteKeyRangeInterpolateKeyTest.java | 2 +-
.../apache/beam/sdk/io/range/ByteKeyRangeTest.java | 2 +-
.../org/apache/beam/sdk/io/range/ByteKeyTest.java | 2 +-
.../org/apache/beam/sdk/metrics/MetricsTest.java | 2 +-
.../sdk/options/PipelineOptionsFactoryTest.java | 2 +-
.../sdk/options/ProxyInvocationHandlerTest.java | 2 +-
.../beam/sdk/runners/PipelineRunnerTest.java | 2 +-
.../beam/sdk/runners/TransformHierarchyTest.java | 2 +-
.../apache/beam/sdk/runners/TransformTreeTest.java | 2 +-
.../beam/sdk/schemas/AutoValueSchemaTest.java | 4 +-
.../beam/sdk/schemas/JavaBeanSchemaTest.java | 38 +-
.../beam/sdk/schemas/JavaFieldSchemaTest.java | 6 +-
.../apache/beam/sdk/schemas/SchemaCoderTest.java | 2 +-
.../sdk/schemas/transforms/CastValidatorTest.java | 2 +-
.../beam/sdk/schemas/transforms/CoGroupTest.java | 2 +-
.../beam/sdk/schemas/transforms/ConvertTest.java | 15 +
.../beam/sdk/schemas/transforms/GroupTest.java | 2 +-
.../beam/sdk/schemas/utils/JavaBeanUtilsTest.java | 26 +-
.../beam/sdk/schemas/utils/POJOUtilsTest.java | 6 +-
.../beam/sdk/schemas/utils/SchemaZipFoldTest.java | 2 +-
.../beam/sdk/schemas/utils/TestJavaBeans.java | 17 +
.../apache/beam/sdk/state/StateContextsTest.java | 2 +-
.../beam/sdk/testing/CoderPropertiesTest.java | 2 +-
.../beam/sdk/testing/CombineFnTesterTest.java | 2 +-
.../org/apache/beam/sdk/testing/PAssertTest.java | 2 +-
.../beam/sdk/testing/PaneExtractorsTest.java | 2 +-
.../beam/sdk/testing/SerializableMatchersTest.java | 2 +-
.../apache/beam/sdk/testing/StaticWindowsTest.java | 2 +-
.../apache/beam/sdk/testing/TestPipelineTest.java | 2 +-
.../beam/sdk/testing/WindowSupplierTest.java | 2 +-
.../apache/beam/sdk/transforms/CombineFnsTest.java | 2 +-
.../apache/beam/sdk/transforms/CombineTest.java | 2 +-
.../org/apache/beam/sdk/transforms/CreateTest.java | 2 +-
.../apache/beam/sdk/transforms/DistinctTest.java | 2 +-
.../org/apache/beam/sdk/transforms/DoFnTest.java | 2 +-
.../apache/beam/sdk/transforms/DoFnTesterTest.java | 2 +-
.../apache/beam/sdk/transforms/FlattenTest.java | 2 +-
.../apache/beam/sdk/transforms/GroupByKeyTest.java | 2 +-
.../apache/beam/sdk/transforms/PTransformTest.java | 2 +-
.../org/apache/beam/sdk/transforms/ParDoTest.java | 2 +-
.../org/apache/beam/sdk/transforms/ReifyTest.java | 2 +-
.../beam/sdk/transforms/ReifyTimestampsTest.java | 2 +-
.../apache/beam/sdk/transforms/ReshuffleTest.java | 2 +-
.../beam/sdk/transforms/SimpleFunctionTest.java | 2 +-
.../beam/sdk/transforms/WithFailuresTest.java | 2 +-
.../display/DisplayDataEvaluatorTest.java | 2 +-
.../display/DisplayDataMatchersTest.java | 2 +-
.../sdk/transforms/display/DisplayDataTest.java | 2 +-
.../beam/sdk/transforms/join/CoGbkResultTest.java | 2 +-
.../beam/sdk/transforms/join/CoGroupByKeyTest.java | 2 +-
.../beam/sdk/transforms/join/UnionCoderTest.java | 2 +-
.../sdk/transforms/reflect/DoFnInvokersTest.java | 2 +-
.../sdk/transforms/reflect/DoFnSignaturesTest.java | 2 +-
.../transforms/reflect/OnTimerInvokersTest.java | 2 +-
.../sdk/transforms/windowing/FixedWindowsTest.java | 2 +-
.../transforms/windowing/IntervalWindowTest.java | 2 +-
.../sdk/transforms/windowing/RepeatedlyTest.java | 2 +-
.../sdk/transforms/windowing/SessionsTest.java | 2 +-
.../transforms/windowing/SlidingWindowsTest.java | 2 +-
.../beam/sdk/transforms/windowing/WindowTest.java | 2 +-
.../org/apache/beam/sdk/util/ApiSurfaceTest.java | 2 +-
.../BufferedElementCountingOutputStreamTest.java | 2 +-
.../apache/beam/sdk/util/FluentBackoffTest.java | 2 +-
.../org/apache/beam/sdk/util/HistogramTest.java | 2 +-
.../org/apache/beam/sdk/util/MoreFuturesTest.java | 2 +-
.../org/apache/beam/sdk/util/ReleaseInfoTest.java | 2 +-
.../beam/sdk/util/UserCodeExceptionTest.java | 2 +-
.../java/org/apache/beam/sdk/util/VarIntTest.java | 2 +-
.../apache/beam/sdk/util/WindowedValueTest.java | 2 +-
.../org/apache/beam/sdk/util/ZipFilesTest.java | 2 +-
.../beam/sdk/util/common/ReflectHelpersTest.java | 2 +-
.../java/org/apache/beam/sdk/values/KVTest.java | 2 +-
.../beam/sdk/values/PCollectionListTest.java | 2 +-
.../beam/sdk/values/PCollectionTupleTest.java | 2 +-
.../org/apache/beam/sdk/values/TupleTagTest.java | 2 +-
.../beam/sdk/values/TypeDescriptorsTest.java | 2 +-
.../apache/beam/sdk/values/TypedPValueTest.java | 2 +-
.../beam/sdk/expansion/service/ExternalTest.java | 2 +-
.../sdk/extensions/gcp/util/GceMetadataUtil.java | 4 +-
.../gcp/storage/GcsFileSystemRegistrarTest.java | 2 +-
.../extensions/gcp/storage/GcsFileSystemTest.java | 2 +-
.../beam/sdk/extensions/gcp/util/GcsUtilTest.java | 2 +-
...LatencyRecordingHttpRequestInitializerTest.java | 2 +-
.../gcp/util/RetryHttpRequestInitializerTest.java | 2 +-
.../sdk/extensions/gcp/util/gcsfs/GcsPathTest.java | 2 +-
.../extensions/protobuf/ProtoMessageSchema.java | 68 +-
.../extensions/protobuf/ByteStringCoderTest.java | 2 +-
.../sdk/extensions/protobuf/ProtobufUtilTest.java | 2 +-
.../ExternalSchemaIOTransformRegistrarTest.java | 4 +-
.../extensions/sketching/TDigestQuantilesTest.java | 2 +-
.../extensions/sorter/ExternalSorterBenchmark.java | 6 +-
.../beam/sdk/extensions/sorter/SortValuesTest.java | 2 +-
sdks/java/extensions/sql/build.gradle | 15 +-
.../datacatalog/DataCatalogTableProvider.java | 61 +-
.../sql/meta/provider/datacatalog/SchemaUtils.java | 93 +++
.../meta/provider/datacatalog/SchemaUtilsTest.java | 157 ++++
.../sdk/extensions/sql/jdbc/BeamSqlLineIT.java | 2 +-
.../sdk/extensions/sql/jdbc/BeamSqlLineTest.java | 2 +-
.../sdk/extensions/sql/impl/ScalarFnReflector.java | 73 ++
.../extensions/sql/impl/ScalarFunctionImpl.java | 47 +-
.../sql/meta/provider/bigtable/BigtableFilter.java | 136 ++++
.../sql/meta/provider/bigtable/BigtableTable.java | 47 +-
.../meta/provider/kafka/BeamKafkaThriftTable.java | 138 ++++
.../meta/provider/kafka/KafkaTableProvider.java | 56 +-
.../sql/BeamSqlDslSqlStdOperatorsTest.java | 2 +-
.../sdk/extensions/sql/impl/JdbcDriverTest.java | 2 +-
.../extensions/sql/impl/ScalarFnReflectorTest.java | 141 ++++
.../provider/bigtable/BigtableClientWrapper.java | 115 +++
.../meta/provider/bigtable/BigtableFilterTest.java | 114 +++
.../BigtableTableCreationFailuresTest.java | 2 +-
.../provider/bigtable/BigtableTableFlatTest.java | 132 +--
.../meta/provider/bigtable/BigtableTableIT.java | 200 +++++
.../meta/provider/bigtable/BigtableTableTest.java | 94 ---
.../provider/bigtable/BigtableTableTestUtils.java | 237 ++++++
.../bigtable/BigtableTableWithRowsTest.java | 120 ++-
.../provider/kafka/BeamKafkaTableProtoTest.java | 2 +-
...rotoTest.java => BeamKafkaTableThriftTest.java} | 53 +-
.../meta/provider/kafka/KafkaTableProviderIT.java | 214 ++++-
.../provider/kafka/KafkaTableProviderProtoIT.java | 53 --
.../provider/kafka/KafkaTableProviderTest.java | 55 +-
.../provider/kafka/thrift/ItThriftMessage.java | 612 ++++++++++++++
.../provider/kafka/thrift/SimpleThriftMessage.java | 509 ++++++++++++
.../provider/kafka/thrift/TestThriftMessage.java | 881 +++++++++++++++++++++
.../sql/meta/provider/pubsub/PubsubAvroIT.java | 102 ---
.../sql/meta/provider/pubsub/PubsubJsonIT.java | 76 --
.../provider/pubsub/PubsubTableProviderIT.java | 273 +++++--
.../meta/provider/text/TextTableProviderTest.java | 2 +-
.../sql/meta/store/InMemoryMetaStoreTest.java | 2 +-
.../sql/src/test/thrift/kafka/messages.thrift | 36 +-
sdks/{go => java/extensions/sql/udf}/build.gradle | 27 +-
.../beam/sdk/extensions/sql/udf/AggregateFn.java | 71 ++
.../beam/sdk/extensions/sql/udf/ScalarFn.java | 53 ++
.../beam/sdk/extensions/sql/udf/UdfProvider.java} | 23 +-
.../beam/sdk/extensions/sql/udf/package-info.java | 16 +-
.../sdk/extensions/sql/zetasql/SqlAnalyzer.java | 6 +
.../extensions/sql/zetasql/ZetaSQLPlannerImpl.java | 5 +-
.../sql/zetasql/translation/ConversionContext.java | 2 +-
.../zetasql/translation/ExpressionConverter.java | 11 +-
.../sql/zetasql/translation/SqlOperators.java | 76 +-
.../translation/UserFunctionDefinitions.java | 30 +-
.../translation/ZetaSqlScalarFunctionImpl.java | 86 ++
.../zetasketch/ApproximateCountDistinct.java | 288 +++++++
.../zetasketch/ApproximateCountDistinctTest.java | 342 ++++++++
.../sdk/fn/data/BeamFnDataInboundObserver.java | 35 +-
.../beam/sdk/fn/data/DecodingFnDataReceiver.java | 46 ++
.../org/apache/beam/sdk/fn/IdGeneratorsTest.java | 2 +-
.../sdk/fn/channel/SocketAddressFactoryTest.java | 2 +-
.../sdk/fn/data/BeamFnDataGrpcMultiplexerTest.java | 2 +-
...DataSizeBasedBufferingOutboundObserverTest.java | 2 +-
.../CompletableFutureInboundDataClientTest.java | 2 +-
.../beam/sdk/fn/data/RemoteGrpcPortReadTest.java | 2 +-
.../beam/sdk/fn/data/RemoteGrpcPortWriteTest.java | 2 +-
.../fn/splittabledofn/RestrictionTrackersTest.java | 2 +-
.../beam/sdk/fn/stream/AdvancingPhaserTest.java | 2 +-
.../apache/beam/sdk/fn/stream/DataStreamsTest.java | 2 +-
.../sdk/fn/stream/OutboundObserverFactoryTest.java | 2 +-
.../apache/beam/sdk/fn/test/TestStreamsTest.java | 2 +-
.../fn/harness/control/ProcessBundleHandler.java | 87 +-
.../beam/fn/harness/data/BeamFnDataClient.java | 14 +-
.../beam/fn/harness/data/BeamFnDataGrpcClient.java | 10 +-
.../fn/harness/data/QueueingBeamFnDataClient.java | 11 +-
.../fn/harness/logging/BeamFnLoggingClient.java | 15 +
.../beam/fn/harness/AssignWindowsRunnerTest.java | 2 +-
.../beam/fn/harness/BeamFnDataReadRunnerTest.java | 2 +-
.../beam/fn/harness/BeamFnDataWriteRunnerTest.java | 2 +-
.../apache/beam/fn/harness/CombineRunnersTest.java | 2 +-
.../apache/beam/fn/harness/FlattenRunnerTest.java | 2 +-
.../beam/fn/harness/FnApiDoFnRunnerTest.java | 2 +-
.../apache/beam/fn/harness/MapFnRunnersTest.java | 2 +-
.../fn/harness/PrecombineGroupingTableTest.java | 2 +-
.../beam/fn/harness/WindowMergingFnRunnerTest.java | 2 +-
.../harness/control/BeamFnControlClientTest.java | 2 +-
.../harness/control/BundleSplitListenerTest.java | 2 +-
.../harness/control/ProcessBundleHandlerTest.java | 2 +-
.../fn/harness/data/BeamFnDataGrpcClientTest.java | 2 +-
.../data/BeamFnDataInboundObserverTest.java | 13 +-
.../data/PCollectionConsumerRegistryTest.java | 2 +-
.../harness/data/QueueingBeamFnDataClientTest.java | 2 +-
.../harness/logging/BeamFnLoggingClientTest.java | 2 +-
.../state/BeamFnStateGrpcClientCacheTest.java | 2 +-
.../stream/HarnessStreamObserverFactoriesTest.java | 2 +-
.../beam/sdk/io/aws/dynamodb/DynamoDBIO.java | 40 +-
.../beam/sdk/io/aws/coders/AwsCodersTest.java | 2 +-
.../beam/sdk/io/aws/dynamodb/DynamoDBIOTest.java | 68 +-
.../sdk/io/aws/dynamodb/DynamoDBIOTestHelper.java | 2 +-
.../beam/sdk/io/aws/options/AwsModuleTest.java | 2 +-
.../beam/sdk/io/aws/s3/S3FileSystemTest.java | 2 +-
.../sdk/io/aws/sns/PublishResultCodersTest.java | 2 +-
.../aws2/dynamodb/BasicDynamoDbClientProvider.java | 32 +-
.../beam/sdk/io/aws2/dynamodb/DynamoDBIO.java | 40 +-
.../sdk/io/aws2/options/AwsSerializableUtils.java | 50 ++
.../io/aws2/sns/BasicSnsAsyncClientProvider.java | 32 +-
.../sdk/io/aws2/sns/BasicSnsClientProvider.java | 34 +-
.../sdk/io/aws2/sqs/BasicSqsClientProvider.java | 34 +-
.../dynamodb/BasicDynamoDbClientProviderTest.java | 51 ++
.../beam/sdk/io/aws2/dynamodb/DynamoDBIOTest.java | 85 +-
.../beam/sdk/io/aws2/options/AwsModuleTest.java | 2 +-
.../io/aws2/options/AwsSerializableUtilsTest.java | 70 ++
.../aws2/sns/BasicSnsAsyncClientProviderTest.java | 51 ++
.../io/aws2/sns/BasicSnsClientProviderTest.java | 51 ++
.../io/aws2/sqs/BasicSqsClientProviderTest.java | 51 ++
.../blobstore/AzureBlobStoreFileSystemTest.java | 2 +-
.../beam/sdk/io/cassandra/CassandraIOTest.java | 2 +-
.../io/contextualtextio/ContextualTextIOTest.java | 2 +-
.../elasticsearch/ElasticsearchIOTestCommon.java | 2 +-
.../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 7 +-
.../gcp/bigquery/BigQueryStorageStreamSource.java | 6 +
.../apache/beam/sdk/io/gcp/bigtable/RowUtils.java | 10 +
.../apache/beam/sdk/io/gcp/healthcare/FhirIO.java | 232 ++++++
.../sdk/io/gcp/healthcare/HealthcareApiClient.java | 17 +
.../io/gcp/healthcare/HttpHealthcareApiClient.java | 174 ++++
.../sdk/io/gcp/healthcare/JsonArrayCoder.java} | 34 +-
.../beam/sdk/io/gcp/pubsub/RowToPubsubMessage.java | 28 +-
.../apache/beam/sdk/io/gcp/pubsub/TestPubsub.java | 17 +-
.../beam/sdk/io/gcp/pubsub/TestPubsubSignal.java | 150 ++--
.../sdk/io/gcp/pubsublite/CloudPubsubChecks.java | 21 +-
...fsetFinalizer.java => InitialOffsetReader.java} | 12 +-
.../io/gcp/pubsublite/InitialOffsetReaderImpl.java | 54 ++
.../gcp/pubsublite/LimitingTopicBacklogReader.java | 81 ++
...ffsetFinalizer.java => OffsetByteProgress.java} | 17 +-
.../io/gcp/pubsublite/OffsetByteRangeTracker.java | 172 ++++
.../io/gcp/pubsublite/OffsetCheckpointMark.java | 78 --
.../pubsublite/PerSubscriptionPartitionSdf.java | 110 +++
.../beam/sdk/io/gcp/pubsublite/PublisherCache.java | 10 +-
.../sdk/io/gcp/pubsublite/PublisherOptions.java | 47 +-
.../sdk/io/gcp/pubsublite/PublisherOrError.java | 6 +-
.../beam/sdk/io/gcp/pubsublite/Publishers.java | 56 ++
.../beam/sdk/io/gcp/pubsublite/PubsubLiteIO.java | 14 +-
.../beam/sdk/io/gcp/pubsublite/PubsubLiteSink.java | 18 +-
.../gcp/pubsublite/PubsubLiteUnboundedReader.java | 333 --------
.../gcp/pubsublite/PubsubLiteUnboundedSource.java | 140 ----
...zer.java => SerializableSubscriberFactory.java} | 17 +-
.../io/gcp/pubsublite/SerializableSupplier.java | 5 +-
.../sdk/io/gcp/pubsublite/SubscribeTransform.java | 124 +++
.../sdk/io/gcp/pubsublite/SubscriberOptions.java | 157 ++--
...etFinalizer.java => SubscriptionPartition.java} | 19 +-
...dCoder.java => SubscriptionPartitionCoder.java} | 36 +-
...er.java => SubscriptionPartitionProcessor.java} | 16 +-
... => SubscriptionPartitionProcessorFactory.java} | 15 +-
.../SubscriptionPartitionProcessorImpl.java | 152 ++++
.../sdk/io/gcp/pubsublite/TopicBacklogReader.java | 34 +-
.../io/gcp/pubsublite/TopicBacklogReaderImpl.java | 80 +-
.../gcp/pubsublite/TopicBacklogReaderSettings.java | 50 +-
.../gcp/pubsublite/TranslatingPullSubscriber.java | 48 --
.../beam/sdk/io/gcp/pubsublite/UuidCoder.java | 7 +-
.../gcp/pubsublite/UuidDeduplicationOptions.java | 15 +-
.../beam/sdk/io/gcp/spanner/SpannerAccessor.java | 22 +-
.../io/gcp/testing/BigtableEmulatorWrapper.java | 75 --
.../beam/sdk/io/gcp/testing/BigtableTestUtils.java | 154 ----
.../beam/sdk/io/gcp/testing/BigtableUtils.java | 52 +-
.../cloud/spanner/FakeBatchTransactionId.java | 1 +
.../apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java | 3 +-
.../sdk/io/gcp/bigquery/BigQueryIOReadTest.java | 2 +-
.../io/gcp/bigquery/BigQueryIOStorageReadTest.java | 13 +-
.../sdk/io/gcp/bigquery/BigQueryIOWriteTest.java | 2 +-
.../io/gcp/bigquery/BigQueryServicesImplTest.java | 2 +-
.../sdk/io/gcp/bigquery/BigQueryUtilsTest.java | 2 +-
.../sdk/io/gcp/bigquery/TableRowJsonCoderTest.java | 2 +-
.../bigtable/BeamRowToBigtableMutationTest.java | 14 +-
.../sdk/io/gcp/bigtable/BigtableConfigTest.java | 2 +-
.../beam/sdk/io/gcp/bigtable/BigtableIOTest.java | 2 +-
.../gcp/bigtable/BigtableRowToBeamRowFlatTest.java | 14 +-
.../io/gcp/bigtable/BigtableRowToBeamRowTest.java | 17 +-
.../{TestUtils.java => BigtableTestUtils.java} | 50 +-
.../beam/sdk/io/gcp/bigtable/BigtableWriteIT.java | 2 +-
.../sdk/io/gcp/bigtable/CellValueParserTest.java | 13 +-
.../io/gcp/datastore/AdaptiveThrottlerTest.java | 2 +-
.../beam/sdk/io/gcp/datastore/DatastoreV1Test.java | 2 +-
.../gcp/datastore/EntityToRowRowToEntityTest.java | 6 +-
.../beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java | 156 ++++
.../beam/sdk/io/gcp/healthcare/FhirIOTest.java | 21 +
.../beam/sdk/io/gcp/pubsub/PubsubIOTest.java | 2 +-
.../pubsub/PubsubMessagePayloadOnlyCoderTest.java | 2 +-
...MessageWithAttributesAndMessageIdCoderTest.java | 2 +-
.../PubsubMessageWithAttributesCoderTest.java | 2 +-
.../PubsubMessageWithMessageIdCoderTest.java | 2 +-
.../io/gcp/pubsub/PubsubUnboundedSourceTest.java | 2 +-
.../gcp/pubsublite/OffsetByteRangeTrackerTest.java | 158 ++++
.../gcp/pubsublite/OffsetCheckpointMarkTest.java | 81 --
.../PerSubscriptionPartitionSdfTest.java | 189 +++++
.../sdk/io/gcp/pubsublite/PubsubLiteSinkTest.java | 47 +-
.../pubsublite/PubsubLiteUnboundedReaderTest.java | 330 --------
.../SubscriptionPartitionProcessorImplTest.java | 227 ++++++
.../gcp/pubsublite/TopicBacklogReaderImplTest.java | 140 +---
.../pubsublite/UuidDeduplicationTransformTest.java | 13 +-
.../io/gcp/spanner/MutationSizeEstimatorTest.java | 2 +-
.../sdk/io/gcp/spanner/ReadSpannerSchemaTest.java | 2 +-
.../sdk/io/gcp/spanner/SpannerIOWriteTest.java | 2 +-
.../beam/sdk/io/gcp/spanner/SpannerWriteIT.java | 2 +-
.../beam/sdk/io/gcp/spanner/StructUtilsTest.java | 2 +-
.../sdk/io/hadoop/SerializableConfiguration.java | 11 +
.../io/hadoop/SerializableConfigurationTest.java | 9 +
.../beam/sdk/io/hadoop/WritableCoderTest.java | 2 +-
.../beam/sdk/io/hdfs/HadoopFileSystemOptions.java | 9 +-
.../sdk/io/hdfs/HadoopFileSystemModuleTest.java | 2 +-
.../hdfs/HadoopFileSystemOptionsRegistrarTest.java | 2 +-
.../sdk/io/hdfs/HadoopFileSystemOptionsTest.java | 22 +-
.../beam/sdk/io/hdfs/HadoopFileSystemTest.java | 2 +-
.../beam/sdk/io/hadoop/format/HadoopFormatIO.java | 85 +-
.../hadoop/format/HadoopFormatIOCassandraTest.java | 6 +-
.../io/hadoop/format/HadoopFormatIOReadTest.java | 162 +++-
.../org/apache/beam/sdk/io/hbase/HBaseIOTest.java | 2 +-
.../beam/sdk/io/hcatalog/HCatalogIOTest.java | 2 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 2 +-
.../java/org/apache/beam/sdk/io/jms/JmsIOTest.java | 2 +-
.../CustomTimestampPolicyWithLimitedDelayTest.java | 2 +-
.../org/apache/beam/sdk/io/kafka/KafkaIOTest.java | 8 +-
.../apache/beam/sdk/io/kinesis/KinesisIOIT.java | 2 +-
.../java/org/apache/beam/sdk/io/kudu/KuduIOIT.java | 4 +-
.../sdk/io/parquet/GenericRecordReadConverter.java | 64 --
.../org/apache/beam/sdk/io/parquet/ParquetIO.java | 472 +++++++++--
.../sdk/io/parquet/ParquetSchemaIOProvider.java | 11 +-
.../sdk/io/parquet/GenericRecordToRowTest.java | 79 --
.../apache/beam/sdk/io/parquet/ParquetIOTest.java | 132 ++-
.../apache/beam/sdk/io/snowflake/KeyPairUtils.java | 4 +-
.../io/snowflake/crosslanguage/ReadBuilder.java | 4 +-
.../services/SnowflakeBatchServiceImpl.java | 4 +-
.../test/FakeSnowflakeStreamingServiceImpl.java | 4 +-
.../beam/sdk/io/snowflake/test/TestUtils.java | 6 +-
.../beam/sdk/io/solr/JavaBinCodecCoderTest.java | 2 +-
.../org/apache/beam/sdk/io/solr/SolrIOTest.java | 4 +-
sdks/java/io/thrift/build.gradle | 10 +-
.../org/apache/beam/sdk/io/thrift/ThriftCoder.java | 9 +-
.../apache/beam/sdk/io/thrift/ThriftSchema.java | 408 ++++++++++
.../apache/beam/sdk/io/thrift/TestThriftEnum.java} | 45 +-
.../beam/sdk/io/thrift/TestThriftInnerStruct.java | 526 ++++++++++++
.../beam/sdk/io/thrift/TestThriftStruct.java | 760 ++++++++++++++++--
.../apache/beam/sdk/io/thrift/TestThriftUnion.java | 401 ++++++++++
.../apache/beam/sdk/io/thrift/ThriftIOTest.java | 38 +-
.../beam/sdk/io/thrift/ThriftSchemaTest.java | 251 ++++++
.../src/test/resources/thrift/thrift_test.thrift | 35 +-
.../apache/beam/sdk/io/tika/ParseResultTest.java | 2 +-
.../org/apache/beam/sdk/io/tika/TikaIOTest.java | 2 +-
.../java/org/apache/beam/sdk/io/xml/XmlSource.java | 22 +-
.../org/apache/beam/sdk/io/xml/JAXBCoderTest.java | 2 +-
.../java/org/apache/beam/sdk/io/xml/XmlIOTest.java | 11 +-
.../org/apache/beam/sdk/io/xml/XmlSourceTest.java | 44 +-
sdks/java/maven-archetypes/examples/build.gradle | 2 +-
sdks/java/testing/jpms-tests/build.gradle | 2 +-
.../java/org/apache/beam/sdk/nexmark/Monitor.java | 13 +-
.../apache/beam/sdk/nexmark/NexmarkLauncher.java | 50 +-
.../apache/beam/sdk/nexmark/NexmarkOptions.java | 14 +
.../beam/sdk/nexmark/queries/NexmarkQuery.java | 3 -
.../sdk/nexmark/queries/NexmarkQueryModel.java | 4 +-
.../apache/beam/sdk/nexmark/queries/Query10.java | 8 +-
.../apache/beam/sdk/nexmark/queries/Query3.java | 191 +++--
.../apache/beam/sdk/nexmark/queries/Query5.java | 133 +++-
.../beam/sdk/nexmark/NexmarkConfigurationTest.java | 2 +-
.../beam/sdk/nexmark/PerfsToBigQueryTest.java | 2 +-
.../nexmark/queries/BoundedSideInputJoinTest.java | 2 +-
.../nexmark/queries/SessionSideInputJoinTest.java | 2 +-
.../queries/sql/SqlBoundedSideInputJoinTest.java | 2 +-
sdks/python/apache_beam/coders/coders.py | 71 +-
.../apache_beam/coders/coders_test_common.py | 1 -
sdks/python/apache_beam/coders/typecoders.py | 3 -
sdks/python/apache_beam/dataframe/frames.py | 26 +-
sdks/python/apache_beam/dataframe/frames_test.py | 78 +-
sdks/python/apache_beam/dataframe/io.py | 263 ++++--
sdks/python/apache_beam/dataframe/io_test.py | 88 +-
.../examples/cookbook/bigquery_side_input.py | 7 +-
.../apache_beam/examples/cookbook/filters.py | 2 +-
.../apache_beam/examples/snippets/snippets.py | 16 +-
.../apache_beam/examples/snippets/snippets_test.py | 10 +-
sdks/python/apache_beam/examples/sql_taxi.py | 3 +-
sdks/python/apache_beam/io/concat_source.py | 9 +-
sdks/python/apache_beam/io/concat_source_test.py | 8 +
.../flink/flink_streaming_impulse_source_test.py | 3 +-
sdks/python/apache_beam/io/gcp/bigquery.py | 2 +-
.../apache_beam/io/gcp/bigquery_file_loads.py | 21 +-
.../apache_beam/io/gcp/bigquery_read_internal.py | 40 +-
sdks/python/apache_beam/io/gcp/bigquery_test.py | 14 +-
sdks/python/apache_beam/io/gcp/pubsub.py | 8 +-
sdks/python/apache_beam/io/gcp/pubsub_test.py | 3 +-
sdks/python/apache_beam/io/mongodbio.py | 29 +-
sdks/python/apache_beam/io/mongodbio_test.py | 41 +
sdks/python/apache_beam/io/restriction_trackers.py | 2 +-
sdks/python/apache_beam/metrics/execution.pxd | 1 +
sdks/python/apache_beam/metrics/execution.py | 9 +-
.../python/apache_beam/options/pipeline_options.py | 29 +-
sdks/python/apache_beam/pipeline.py | 163 ++--
sdks/python/apache_beam/portability/common_urns.py | 1 +
sdks/python/apache_beam/runners/common.pxd | 1 +
sdks/python/apache_beam/runners/common.py | 36 +-
.../runners/dataflow/dataflow_runner.py | 184 ++---
.../runners/dataflow/dataflow_runner_test.py | 62 ++
.../apache_beam/runners/dataflow/internal/names.py | 9 +-
.../runners/dataflow/ptransform_overrides.py | 45 ++
.../apache_beam/runners/direct/direct_runner.py | 4 +-
.../apache_beam/runners/direct/direct_userstate.py | 35 +-
.../runners/direct/transform_evaluator.py | 8 +-
sdks/python/apache_beam/runners/direct/util.py | 18 +-
.../runners/direct/watermark_manager.py | 10 +-
.../python/apache_beam/runners/pipeline_context.py | 2 -
.../runners/portability/flink_runner_test.py | 3 +
.../portability/fn_api_runner/fn_runner_test.py | 24 +
.../portability/fn_api_runner/translations.py | 81 +-
.../portability/fn_api_runner/translations_test.py | 18 +-
.../portability/fn_api_runner/worker_handlers.py | 7 +-
.../apache_beam/runners/portability/job_server.py | 10 +-
.../runners/portability/job_server_test.py | 84 ++
.../runners/portability/portable_runner.py | 85 +-
.../runners/portability/sdk_container_builder.py | 11 +-
.../runners/portability/spark_runner_test.py | 5 +-
sdks/python/apache_beam/runners/runner.py | 24 +-
.../apache_beam/runners/worker/bundle_processor.py | 41 +-
.../apache_beam/runners/worker/operations.py | 6 +-
sdks/python/apache_beam/transforms/core.py | 145 +---
sdks/python/apache_beam/transforms/environments.py | 26 +-
sdks/python/apache_beam/transforms/external.py | 24 +-
sdks/python/apache_beam/transforms/ptransform.py | 46 +-
sdks/python/apache_beam/transforms/trigger.py | 39 +-
sdks/python/apache_beam/transforms/trigger_test.py | 10 +-
sdks/python/apache_beam/transforms/userstate.py | 55 +-
.../apache_beam/transforms/userstate_test.py | 216 ++++-
sdks/python/apache_beam/transforms/util.py | 123 ++-
sdks/python/apache_beam/transforms/util_test.py | 65 ++
.../typehints/native_type_compatibility.py | 7 -
.../apache_beam/typehints/sharded_key_type.py | 31 +-
.../apache_beam/typehints/sharded_key_type_test.py | 35 +-
sdks/python/apache_beam/version.py | 2 +-
sdks/python/container/common.gradle | 2 +
sdks/python/setup.py | 6 +-
sdks/python/test-suites/dataflow/build.gradle | 24 +
sdks/python/test-suites/gradle.properties | 5 +
sdks/python/test-suites/portable/common.gradle | 34 +-
settings.gradle | 5 +
start-build-env.sh | 16 +-
website/.gitignore | 23 -
website/www/package.json | 2 +-
website/www/site/config.toml | 2 +-
website/www/site/content/en/blog/beam-2.13.0.md | 2 +-
website/www/site/content/en/blog/beam-2.21.0.md | 2 +-
website/www/site/content/en/blog/beam-2.22.0.md | 2 +-
website/www/site/content/en/blog/beam-2.23.0.md | 4 +-
website/www/site/content/en/blog/beam-2.24.0.md | 4 +-
website/www/site/content/en/blog/beam-2.25.0.md | 4 +-
website/www/site/content/en/blog/beam-2.26.0.md | 65 ++
website/www/site/content/en/blog/beam-2.27.0.md | 69 ++
.../en/blog/dataframe-api-preview-available.md | 178 +++++
.../site/content/en/blog/pattern-match-beam-sql.md | 2 +-
.../en/blog/splittable-do-fn-is-available.md | 91 +++
website/www/site/content/en/contribute/_index.md | 2 +-
.../site/content/en/contribute/release-guide.md | 7 +-
.../en/documentation/dsls/dataframes/overview.md | 131 +++
.../dsls/sql/extensions/create-external-table.md | 15 +-
.../content/en/documentation/dsls/sql/overview.md | 2 +-
.../documentation/io/built-in/google-bigquery.md | 15 +-
.../en/documentation/io/built-in/snowflake.md | 6 +-
.../content/en/documentation/programming-guide.md | 4 +-
.../content/en/documentation/runners/direct.md | 112 +--
.../site/content/en/documentation/runners/flink.md | 42 +-
.../site/content/en/documentation/runners/samza.md | 5 +
.../en/documentation/runtime/environments.md | 305 ++++---
.../site/content/en/get-started/beam-overview.md | 5 +-
.../www/site/content/en/get-started/downloads.md | 28 +-
.../www/site/content/en/get-started/from-spark.md | 268 +++++++
website/www/site/data/authors.yml | 38 +-
website/www/site/data/capability_matrix.yaml | 714 +++++++++++++++--
.../www/site/data/capability_matrix_snapshot.yaml | 195 +++++
website/www/site/layouts/blog/single.html | 1 +
.../partials/section-menu/en/get-started.html | 7 +-
.../layouts/partials/section-menu/en/sdks.html | 17 +-
website/www/site/static/.htaccess | 2 +-
941 files changed, 26264 insertions(+), 6811 deletions(-)
|