beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From boyu...@apache.org
Subject [beam-site] branch release-docs updated: Update beam-site for release 2.19.0\n\nContent generated
Date Wed, 05 Feb 2020 20:03:25 GMT
This is an automated email from the ASF dual-hosted git repository.

boyuanz pushed a commit to branch release-docs
in repository https://gitbox.apache.org/repos/asf/beam-site.git


The following commit(s) were added to refs/heads/release-docs by this push:
     new 1548175  Update beam-site for release 2.19.0\n\nContent generated
     new 96895f5  Merge pull request #601 from boyuanzz/updates_release_2.19.0_py
1548175 is described below

commit 154817516ad2f0bad8ab84d7f90207e30dd129c7
Author: Boyuan Zhang <boyuanz@google.com>
AuthorDate: Wed Feb 5 11:30:30 2020 -0800

    Update beam-site for release 2.19.0\n\nContent generated
---
 pydoc/2.19.0/.buildinfo                            |     4 +
 pydoc/2.19.0/_modules/abc.html                     |   420 +
 .../_modules/apache_beam/coders/avro_record.html   |   275 +
 .../2.19.0/_modules/apache_beam/coders/coders.html |  1642 +++
 .../_modules/apache_beam/coders/observable.html    |   283 +
 .../_modules/apache_beam/coders/row_coder.html     |   411 +
 .../_modules/apache_beam/coders/slow_stream.html   |   452 +
 pydoc/2.19.0/_modules/apache_beam/error.html       |   285 +
 .../_modules/apache_beam/internal/gcp/auth.html    |   378 +
 .../apache_beam/internal/gcp/json_value.html       |   406 +
 .../_modules/apache_beam/internal/http_client.html |   307 +
 .../_modules/apache_beam/internal/pickler.html     |   549 +
 .../2.19.0/_modules/apache_beam/internal/util.html |   398 +
 pydoc/2.19.0/_modules/apache_beam/io/avroio.html   |   869 ++
 .../io/aws/clients/s3/boto3_client.html            |   477 +
 .../apache_beam/io/aws/clients/s3/fake_client.html |   473 +
 .../apache_beam/io/aws/clients/s3/messages.html    |   404 +
 .../_modules/apache_beam/io/aws/s3filesystem.html  |   512 +
 pydoc/2.19.0/_modules/apache_beam/io/aws/s3io.html |   843 ++
 .../_modules/apache_beam/io/concat_source.html     |   509 +
 .../apache_beam/io/external/gcp/pubsub.html        |   405 +
 .../apache_beam/io/external/generate_sequence.html |   300 +
 .../_modules/apache_beam/io/external/kafka.html    |   414 +
 .../_modules/apache_beam/io/filebasedsink.html     |   637 ++
 .../_modules/apache_beam/io/filebasedsource.html   |   663 ++
 pydoc/2.19.0/_modules/apache_beam/io/fileio.html   |   997 ++
 .../2.19.0/_modules/apache_beam/io/filesystem.html |  1088 ++
 .../_modules/apache_beam/io/filesystemio.html      |   542 +
 .../_modules/apache_beam/io/filesystems.html       |   580 ++
 .../io/flink/flink_streaming_impulse_source.html   |   316 +
 .../io/gcp/big_query_query_to_table_pipeline.html  |   320 +
 .../_modules/apache_beam/io/gcp/bigquery.html      |  1848 ++++
 .../apache_beam/io/gcp/bigquery_file_loads.html    |  1066 ++
 .../io/gcp/bigquery_io_read_pipeline.html          |   316 +
 .../apache_beam/io/gcp/bigquery_tools.html         |  1399 +++
 .../_modules/apache_beam/io/gcp/bigtableio.html    |   380 +
 .../v1new/datastore_write_it_pipeline.html         |   410 +
 .../io/gcp/datastore/v1new/datastoreio.html        |   751 ++
 .../apache_beam/io/gcp/datastore/v1new/helper.html |   314 +
 .../io/gcp/datastore/v1new/query_splitter.html     |   525 +
 .../apache_beam/io/gcp/datastore/v1new/types.html  |   497 +
 .../_modules/apache_beam/io/gcp/gcsfilesystem.html |   568 ++
 .../2.19.0/_modules/apache_beam/io/gcp/gcsio.html  |   828 ++
 .../apache_beam/io/gcp/gcsio_overrides.html        |   290 +
 .../2.19.0/_modules/apache_beam/io/gcp/pubsub.html |   676 ++
 .../apache_beam/io/gcp/pubsub_it_pipeline.html     |   325 +
 .../_modules/apache_beam/io/hadoopfilesystem.html  |   613 ++
 pydoc/2.19.0/_modules/apache_beam/io/iobase.html   |  1831 ++++
 .../_modules/apache_beam/io/localfilesystem.html   |   575 ++
 .../2.19.0/_modules/apache_beam/io/mongodbio.html  |   753 ++
 .../2.19.0/_modules/apache_beam/io/parquetio.html  |   799 ++
 .../_modules/apache_beam/io/range_trackers.html    |   657 ++
 .../apache_beam/io/restriction_trackers.html       |   394 +
 .../_modules/apache_beam/io/source_test_utils.html |   908 ++
 pydoc/2.19.0/_modules/apache_beam/io/textio.html   |   850 ++
 .../2.19.0/_modules/apache_beam/io/tfrecordio.html |   566 ++
 pydoc/2.19.0/_modules/apache_beam/io/utils.html    |   305 +
 pydoc/2.19.0/_modules/apache_beam/io/vcfio.html    |   739 ++
 .../2.19.0/_modules/apache_beam/metrics/cells.html |   794 ++
 .../_modules/apache_beam/metrics/metric.html       |   493 +
 .../_modules/apache_beam/metrics/metricbase.html   |   359 +
 .../apache_beam/metrics/monitoring_infos.html      |   635 ++
 .../apache_beam/options/pipeline_options.html      |  1347 +++
 .../options/pipeline_options_validator.html        |   477 +
 .../apache_beam/options/value_provider.html        |   377 +
 pydoc/2.19.0/_modules/apache_beam/pipeline.html    |  1342 +++
 .../api/beam_artifact_api_pb2_grpc.html            |   364 +
 .../api/beam_artifact_api_pb2_urns.html            |   245 +
 .../api/beam_expansion_api_pb2_grpc.html           |   283 +
 .../portability/api/beam_fn_api_pb2_grpc.html      |   528 +
 .../portability/api/beam_fn_api_pb2_urns.html      |   254 +
 .../portability/api/beam_job_api_pb2_grpc.html     |   437 +
 .../portability/api/beam_job_api_pb2_urns.html     |   278 +
 .../api/beam_provision_api_pb2_grpc.html           |   287 +
 .../portability/api/beam_runner_api_pb2_grpc.html  |   283 +
 .../portability/api/beam_runner_api_pb2_urns.html  |   409 +
 .../portability/api/metrics_pb2_urns.html          |   272 +
 .../api/standard_window_fns_pb2_urns.html          |   263 +
 .../_modules/apache_beam/portability/utils.html    |   269 +
 pydoc/2.19.0/_modules/apache_beam/pvalue.html      |   872 ++
 .../dataflow_exercise_metrics_pipeline.html        |   419 +
 ...taflow_exercise_streaming_metrics_pipeline.html |   339 +
 .../runners/dataflow/dataflow_metrics.html         |   530 +
 .../runners/dataflow/dataflow_runner.html          |  1723 ++++
 .../runners/dataflow/native_io/iobase.html         |   565 ++
 .../runners/dataflow/ptransform_overrides.html     |   343 +
 .../runners/dataflow/test_dataflow_runner.html     |   339 +
 .../apache_beam/runners/direct/bundle_factory.html |   461 +
 .../_modules/apache_beam/runners/direct/clock.html |   289 +
 .../direct/consumer_tracking_pipeline_visitor.html |   304 +
 .../apache_beam/runners/direct/direct_metrics.html |   364 +
 .../apache_beam/runners/direct/direct_runner.html  |   742 ++
 .../runners/direct/direct_userstate.html           |   479 +
 .../runners/direct/evaluation_context.html         |   703 ++
 .../apache_beam/runners/direct/executor.html       |   979 ++
 .../runners/direct/helper_transforms.html          |   347 +
 .../runners/direct/sdf_direct_runner.html          |   748 ++
 .../runners/direct/test_direct_runner.html         |   294 +
 .../runners/direct/transform_evaluator.html        |  1333 +++
 .../_modules/apache_beam/runners/direct/util.html  |   317 +
 .../runners/direct/watermark_manager.html          |   559 +
 .../interactive/background_caching_job.html        |   394 +
 .../runners/interactive/cache_manager.html         |   508 +
 .../interactive/caching/streaming_cache.html       |   408 +
 .../interactive/display/display_manager.html       |   415 +
 .../display/interactive_pipeline_graph.html        |   397 +
 .../interactive/display/pcoll_visualization.html   |   530 +
 .../interactive/display/pipeline_graph.html        |   524 +
 .../display/pipeline_graph_renderer.html           |   380 +
 .../runners/interactive/interactive_beam.html      |   324 +
 .../interactive/interactive_environment.html       |   515 +
 .../runners/interactive/interactive_runner.html    |   433 +
 .../runners/interactive/pipeline_analyzer.html     |   713 ++
 .../runners/interactive/pipeline_instrument.html   |   849 ++
 .../runners/interactive/testing/mock_ipython.html  |   302 +
 .../interactive/testing/pipeline_assertion.html    |   321 +
 .../_modules/apache_beam/runners/job/manager.html  |   294 +
 .../_modules/apache_beam/runners/job/utils.html    |   273 +
 .../apache_beam/runners/pipeline_context.html      |   453 +
 .../_modules/apache_beam/runners/runner.html       |   672 ++
 .../benchmarks/nexmark/models/nexmark_model.html   |   320 +
 .../benchmarks/nexmark/nexmark_launcher.html       |   504 +
 .../testing/benchmarks/nexmark/nexmark_util.html   |   342 +
 .../testing/benchmarks/nexmark/queries/query0.html |   274 +
 .../testing/benchmarks/nexmark/queries/query1.html |   284 +
 .../testing/benchmarks/nexmark/queries/query2.html |   279 +
 .../apache_beam/testing/datatype_inference.html    |   381 +
 .../apache_beam/testing/extra_assertions.html      |   301 +
 .../load_tests/load_test_metrics_utils.html        |   695 ++
 .../streaming/group_by_key_streaming_pipeline.html |   335 +
 .../testing/metric_result_matchers.html            |   422 +
 .../apache_beam/testing/pipeline_verifiers.html    |   394 +
 .../apache_beam/testing/synthetic_pipeline.html    |  1059 ++
 .../apache_beam/testing/test_pipeline.html         |   429 +
 .../_modules/apache_beam/testing/test_stream.html  |   518 +
 .../apache_beam/testing/test_stream_service.html   |   291 +
 .../_modules/apache_beam/testing/test_utils.html   |   434 +
 .../2.19.0/_modules/apache_beam/testing/util.html  |   564 ++
 .../apache_beam/tools/coders_microbenchmark.html   |   506 +
 .../tools/distribution_counter_microbenchmark.html |   307 +
 .../tools/fn_api_runner_microbenchmark.html        |   385 +
 .../tools/sideinput_microbenchmark.html            |   318 +
 pydoc/2.19.0/_modules/apache_beam/tools/utils.html |   425 +
 .../_modules/apache_beam/transforms/combiners.html |  1178 +++
 .../_modules/apache_beam/transforms/core.html      |  2881 ++++++
 .../_modules/apache_beam/transforms/display.html   |   606 ++
 .../apache_beam/transforms/environments.html       |   741 ++
 .../_modules/apache_beam/transforms/external.html  |   708 ++
 .../apache_beam/transforms/external_test_it.html   |   307 +
 .../apache_beam/transforms/ptransform.html         |  1227 +++
 .../apache_beam/transforms/sideinputs.html         |   352 +
 .../_modules/apache_beam/transforms/stats.html     |   873 ++
 .../_modules/apache_beam/transforms/timeutil.html  |   376 +
 .../_modules/apache_beam/transforms/trigger.html   |  1649 +++
 .../_modules/apache_beam/transforms/userstate.html |   561 ++
 .../_modules/apache_beam/transforms/util.html      |  1362 +++
 .../_modules/apache_beam/transforms/window.html    |   828 ++
 .../_modules/apache_beam/typehints/decorators.html |  1148 +++
 .../typehints/native_type_compatibility.html       |   609 ++
 .../_modules/apache_beam/typehints/opcodes.html    |   652 ++
 .../_modules/apache_beam/typehints/schemas.html    |   455 +
 .../apache_beam/typehints/trivial_inference.html   |   796 ++
 .../_modules/apache_beam/typehints/typecheck.html  |   494 +
 .../_modules/apache_beam/typehints/typehints.html  |  1475 +++
 .../_modules/apache_beam/utils/annotations.html    |   394 +
 .../apache_beam/utils/interactive_utils.html       |   337 +
 .../2.19.0/_modules/apache_beam/utils/plugin.html  |   283 +
 .../_modules/apache_beam/utils/processes.html      |   339 +
 .../_modules/apache_beam/utils/profiler.html       |   423 +
 .../_modules/apache_beam/utils/proto_utils.html    |   372 +
 pydoc/2.19.0/_modules/apache_beam/utils/retry.html |   493 +
 .../apache_beam/utils/subprocess_server.html       |   467 +
 .../apache_beam/utils/thread_pool_executor.html    |   407 +
 .../_modules/apache_beam/utils/timestamp.html      |   629 ++
 pydoc/2.19.0/_modules/apache_beam/utils/urns.html  |   411 +
 pydoc/2.19.0/_modules/future/types/newobject.html  |   352 +
 pydoc/2.19.0/_modules/index.html                   |   408 +
 .../apache_beam.coders.avro_record.rst.txt         |     7 +
 .../_sources/apache_beam.coders.coders.rst.txt     |     7 +
 .../_sources/apache_beam.coders.observable.rst.txt |     7 +
 .../_sources/apache_beam.coders.row_coder.rst.txt  |     7 +
 pydoc/2.19.0/_sources/apache_beam.coders.rst.txt   |    20 +
 .../apache_beam.coders.slow_stream.rst.txt         |     7 +
 .../_sources/apache_beam.coders.typecoders.rst.txt |     7 +
 pydoc/2.19.0/_sources/apache_beam.error.rst.txt    |     7 +
 .../_sources/apache_beam.internal.gcp.auth.rst.txt |     7 +
 .../apache_beam.internal.gcp.json_value.rst.txt    |     7 +
 .../_sources/apache_beam.internal.gcp.rst.txt      |    16 +
 .../apache_beam.internal.http_client.rst.txt       |     7 +
 .../_sources/apache_beam.internal.pickler.rst.txt  |     7 +
 pydoc/2.19.0/_sources/apache_beam.internal.rst.txt |    24 +
 .../_sources/apache_beam.internal.util.rst.txt     |     7 +
 .../2.19.0/_sources/apache_beam.io.avroio.rst.txt  |     7 +
 .../_sources/apache_beam.io.aws.clients.rst.txt    |    15 +
 ...che_beam.io.aws.clients.s3.boto3_client.rst.txt |     7 +
 ...ache_beam.io.aws.clients.s3.fake_client.rst.txt |     7 +
 .../apache_beam.io.aws.clients.s3.messages.rst.txt |     7 +
 .../_sources/apache_beam.io.aws.clients.s3.rst.txt |    17 +
 pydoc/2.19.0/_sources/apache_beam.io.aws.rst.txt   |    23 +
 .../apache_beam.io.aws.s3filesystem.rst.txt        |     7 +
 .../_sources/apache_beam.io.aws.s3io.rst.txt       |     7 +
 .../_sources/apache_beam.io.concat_source.rst.txt  |     7 +
 .../apache_beam.io.external.gcp.pubsub.rst.txt     |     7 +
 .../_sources/apache_beam.io.external.gcp.rst.txt   |    15 +
 ...ache_beam.io.external.generate_sequence.rst.txt |     7 +
 .../_sources/apache_beam.io.external.kafka.rst.txt |     7 +
 .../_sources/apache_beam.io.external.rst.txt       |    23 +
 .../_sources/apache_beam.io.filebasedsink.rst.txt  |     7 +
 .../apache_beam.io.filebasedsource.rst.txt         |     7 +
 .../2.19.0/_sources/apache_beam.io.fileio.rst.txt  |     7 +
 .../_sources/apache_beam.io.filesystem.rst.txt     |     7 +
 .../_sources/apache_beam.io.filesystemio.rst.txt   |     7 +
 .../_sources/apache_beam.io.filesystems.rst.txt    |     7 +
 ...io.flink.flink_streaming_impulse_source.rst.txt |     7 +
 pydoc/2.19.0/_sources/apache_beam.io.flink.rst.txt |    15 +
 ...o.gcp.big_query_query_to_table_pipeline.rst.txt |     7 +
 .../_sources/apache_beam.io.gcp.bigquery.rst.txt   |     7 +
 .../apache_beam.io.gcp.bigquery_file_loads.rst.txt |     7 +
 ...e_beam.io.gcp.bigquery_io_read_pipeline.rst.txt |     7 +
 .../apache_beam.io.gcp.bigquery_tools.rst.txt      |     7 +
 .../_sources/apache_beam.io.gcp.bigtableio.rst.txt |     7 +
 .../_sources/apache_beam.io.gcp.datastore.rst.txt  |    15 +
 ...store.v1new.datastore_write_it_pipeline.rst.txt |     7 +
 ...beam.io.gcp.datastore.v1new.datastoreio.rst.txt |     7 +
 ...ache_beam.io.gcp.datastore.v1new.helper.rst.txt |     7 +
 ...m.io.gcp.datastore.v1new.query_splitter.rst.txt |     7 +
 .../apache_beam.io.gcp.datastore.v1new.rst.txt     |    19 +
 ...pache_beam.io.gcp.datastore.v1new.types.rst.txt |     7 +
 .../apache_beam.io.gcp.gcsfilesystem.rst.txt       |     7 +
 .../_sources/apache_beam.io.gcp.gcsio.rst.txt      |     7 +
 .../apache_beam.io.gcp.gcsio_overrides.rst.txt     |     7 +
 .../_sources/apache_beam.io.gcp.pubsub.rst.txt     |     7 +
 .../apache_beam.io.gcp.pubsub_it_pipeline.rst.txt  |     7 +
 pydoc/2.19.0/_sources/apache_beam.io.gcp.rst.txt   |    32 +
 .../apache_beam.io.hadoopfilesystem.rst.txt        |     7 +
 .../2.19.0/_sources/apache_beam.io.iobase.rst.txt  |     7 +
 .../apache_beam.io.localfilesystem.rst.txt         |     7 +
 .../_sources/apache_beam.io.mongodbio.rst.txt      |     7 +
 .../_sources/apache_beam.io.parquetio.rst.txt      |     7 +
 .../_sources/apache_beam.io.range_trackers.rst.txt |     7 +
 .../apache_beam.io.restriction_trackers.rst.txt    |     7 +
 pydoc/2.19.0/_sources/apache_beam.io.rst.txt       |    44 +
 .../apache_beam.io.source_test_utils.rst.txt       |     7 +
 .../2.19.0/_sources/apache_beam.io.textio.rst.txt  |     7 +
 .../_sources/apache_beam.io.tfrecordio.rst.txt     |     7 +
 pydoc/2.19.0/_sources/apache_beam.io.utils.rst.txt |     7 +
 pydoc/2.19.0/_sources/apache_beam.io.vcfio.rst.txt |     7 +
 .../_sources/apache_beam.metrics.cells.rst.txt     |     7 +
 .../_sources/apache_beam.metrics.metric.rst.txt    |     7 +
 .../apache_beam.metrics.metricbase.rst.txt         |     7 +
 .../apache_beam.metrics.monitoring_infos.rst.txt   |     7 +
 pydoc/2.19.0/_sources/apache_beam.metrics.rst.txt  |    18 +
 .../apache_beam.options.pipeline_options.rst.txt   |     7 +
 ...beam.options.pipeline_options_validator.rst.txt |     7 +
 pydoc/2.19.0/_sources/apache_beam.options.rst.txt  |    17 +
 .../apache_beam.options.value_provider.rst.txt     |     7 +
 pydoc/2.19.0/_sources/apache_beam.pipeline.rst.txt |     7 +
 ...tability.api.beam_artifact_api_pb2_grpc.rst.txt |     7 +
 ...tability.api.beam_artifact_api_pb2_urns.rst.txt |     7 +
 ...ability.api.beam_expansion_api_pb2_grpc.rst.txt |     7 +
 ...am.portability.api.beam_fn_api_pb2_grpc.rst.txt |     7 +
 ...am.portability.api.beam_fn_api_pb2_urns.rst.txt |     7 +
 ...ility.api.beam_interactive_api_pb2_grpc.rst.txt |     7 +
 ...m.portability.api.beam_job_api_pb2_grpc.rst.txt |     7 +
 ...m.portability.api.beam_job_api_pb2_urns.rst.txt |     7 +
 ...ability.api.beam_provision_api_pb2_grpc.rst.txt |     7 +
 ...ortability.api.beam_runner_api_pb2_grpc.rst.txt |     7 +
 ...ortability.api.beam_runner_api_pb2_urns.rst.txt |     7 +
 ...beam.portability.api.endpoints_pb2_grpc.rst.txt |     7 +
 ...bility.api.external_transforms_pb2_grpc.rst.txt |     7 +
 ...e_beam.portability.api.metrics_pb2_grpc.rst.txt |     7 +
 ...e_beam.portability.api.metrics_pb2_urns.rst.txt |     7 +
 .../_sources/apache_beam.portability.api.rst.txt   |    32 +
 ...he_beam.portability.api.schema_pb2_grpc.rst.txt |     7 +
 ...bility.api.standard_window_fns_pb2_grpc.rst.txt |     7 +
 ...bility.api.standard_window_fns_pb2_urns.rst.txt |     7 +
 .../apache_beam.portability.common_urns.rst.txt    |     7 +
 .../apache_beam.portability.python_urns.rst.txt    |     7 +
 .../_sources/apache_beam.portability.rst.txt       |    24 +
 .../_sources/apache_beam.portability.utils.rst.txt |     7 +
 pydoc/2.19.0/_sources/apache_beam.pvalue.rst.txt   |     7 +
 ...flow.dataflow_exercise_metrics_pipeline.rst.txt |     7 +
 ...low_exercise_streaming_metrics_pipeline.rst.txt |     7 +
 ..._beam.runners.dataflow.dataflow_metrics.rst.txt |     7 +
 ...e_beam.runners.dataflow.dataflow_runner.rst.txt |     7 +
 ..._beam.runners.dataflow.native_io.iobase.rst.txt |     7 +
 .../apache_beam.runners.dataflow.native_io.rst.txt |    15 +
 ...m.runners.dataflow.ptransform_overrides.rst.txt |     7 +
 .../_sources/apache_beam.runners.dataflow.rst.txt  |    27 +
 ...m.runners.dataflow.test_dataflow_runner.rst.txt |     7 +
 ...ache_beam.runners.direct.bundle_factory.rst.txt |     7 +
 .../apache_beam.runners.direct.clock.rst.txt       |     7 +
 ...rect.consumer_tracking_pipeline_visitor.rst.txt |     7 +
 ...ache_beam.runners.direct.direct_metrics.rst.txt |     7 +
 ...pache_beam.runners.direct.direct_runner.rst.txt |     7 +
 ...he_beam.runners.direct.direct_userstate.rst.txt |     7 +
 ..._beam.runners.direct.evaluation_context.rst.txt |     7 +
 .../apache_beam.runners.direct.executor.rst.txt    |     7 +
 ...e_beam.runners.direct.helper_transforms.rst.txt |     7 +
 .../_sources/apache_beam.runners.direct.rst.txt    |    29 +
 ...e_beam.runners.direct.sdf_direct_runner.rst.txt |     7 +
 ..._beam.runners.direct.test_direct_runner.rst.txt |     7 +
 ...he_beam.runners.direct.test_stream_impl.rst.txt |     7 +
 ...beam.runners.direct.transform_evaluator.rst.txt |     7 +
 .../apache_beam.runners.direct.util.rst.txt        |     7 +
 ...e_beam.runners.direct.watermark_manager.rst.txt |     7 +
 ...ners.interactive.background_caching_job.rst.txt |     7 +
 ..._beam.runners.interactive.cache_manager.rst.txt |     7 +
 ...apache_beam.runners.interactive.caching.rst.txt |    15 +
 ...ers.interactive.caching.streaming_cache.rst.txt |     7 +
 ...ers.interactive.display.display_manager.rst.txt |     7 +
 ...tive.display.interactive_pipeline_graph.rst.txt |     7 +
 ...interactive.display.pcoll_visualization.rst.txt |     7 +
 ...ners.interactive.display.pipeline_graph.rst.txt |     7 +
 ...ractive.display.pipeline_graph_renderer.rst.txt |     7 +
 ...apache_beam.runners.interactive.display.rst.txt |    19 +
 ...am.runners.interactive.interactive_beam.rst.txt |     7 +
 ...ers.interactive.interactive_environment.rst.txt |     7 +
 ....runners.interactive.interactive_runner.rst.txt |     7 +
 ...m.runners.interactive.pipeline_analyzer.rst.txt |     7 +
 ...runners.interactive.pipeline_instrument.rst.txt |     7 +
 .../apache_beam.runners.interactive.rst.txt        |    30 +
 ...unners.interactive.testing.mock_ipython.rst.txt |     7 +
 ....interactive.testing.pipeline_assertion.rst.txt |     7 +
 ...apache_beam.runners.interactive.testing.rst.txt |    16 +
 .../apache_beam.runners.internal.names.rst.txt     |     7 +
 .../_sources/apache_beam.runners.internal.rst.txt  |    15 +
 .../apache_beam.runners.job.manager.rst.txt        |     7 +
 .../_sources/apache_beam.runners.job.rst.txt       |    16 +
 .../_sources/apache_beam.runners.job.utils.rst.txt |     7 +
 .../apache_beam.runners.pipeline_context.rst.txt   |     7 +
 pydoc/2.19.0/_sources/apache_beam.runners.rst.txt  |    27 +
 .../_sources/apache_beam.runners.runner.rst.txt    |     7 +
 ...benchmarks.nexmark.models.nexmark_model.rst.txt |     7 +
 ..._beam.testing.benchmarks.nexmark.models.rst.txt |    15 +
 ...ing.benchmarks.nexmark.nexmark_launcher.rst.txt |     7 +
 ...testing.benchmarks.nexmark.nexmark_util.rst.txt |     7 +
 ...sting.benchmarks.nexmark.queries.query0.rst.txt |     7 +
 ...sting.benchmarks.nexmark.queries.query1.rst.txt |     7 +
 ...sting.benchmarks.nexmark.queries.query2.rst.txt |     7 +
 ...beam.testing.benchmarks.nexmark.queries.rst.txt |    17 +
 .../apache_beam.testing.benchmarks.nexmark.rst.txt |    24 +
 .../apache_beam.testing.benchmarks.rst.txt         |    15 +
 .../apache_beam.testing.datatype_inference.rst.txt |     7 +
 .../apache_beam.testing.extra_assertions.rst.txt   |     7 +
 ...ting.load_tests.load_test_metrics_utils.rst.txt |     7 +
 .../apache_beam.testing.load_tests.rst.txt         |    22 +
 ...reaming.group_by_key_streaming_pipeline.rst.txt |     7 +
 ...pache_beam.testing.load_tests.streaming.rst.txt |    15 +
 ...che_beam.testing.metric_result_matchers.rst.txt |     7 +
 .../apache_beam.testing.pipeline_verifiers.rst.txt |     7 +
 pydoc/2.19.0/_sources/apache_beam.testing.rst.txt  |    32 +
 .../apache_beam.testing.synthetic_pipeline.rst.txt |     7 +
 .../apache_beam.testing.test_pipeline.rst.txt      |     7 +
 .../apache_beam.testing.test_stream.rst.txt        |     7 +
 ...apache_beam.testing.test_stream_service.rst.txt |     7 +
 .../apache_beam.testing.test_utils.rst.txt         |     7 +
 .../_sources/apache_beam.testing.util.rst.txt      |     7 +
 ...apache_beam.tools.coders_microbenchmark.rst.txt |     7 +
 ...ols.distribution_counter_microbenchmark.rst.txt |     7 +
 ...beam.tools.fn_api_runner_microbenchmark.rst.txt |     7 +
 pydoc/2.19.0/_sources/apache_beam.tools.rst.txt    |    19 +
 ...che_beam.tools.sideinput_microbenchmark.rst.txt |     7 +
 .../_sources/apache_beam.tools.utils.rst.txt       |     7 +
 .../apache_beam.transforms.combiners.rst.txt       |     7 +
 .../_sources/apache_beam.transforms.core.rst.txt   |     7 +
 .../apache_beam.transforms.create_source.rst.txt   |     7 +
 .../apache_beam.transforms.display.rst.txt         |     7 +
 .../apache_beam.transforms.environments.rst.txt    |     7 +
 .../apache_beam.transforms.external.rst.txt        |     7 +
 ...apache_beam.transforms.external_test_it.rst.txt |     7 +
 .../apache_beam.transforms.ptransform.rst.txt      |     7 +
 .../2.19.0/_sources/apache_beam.transforms.rst.txt |    29 +
 .../apache_beam.transforms.sideinputs.rst.txt      |     7 +
 .../_sources/apache_beam.transforms.stats.rst.txt  |     7 +
 .../apache_beam.transforms.timeutil.rst.txt        |     7 +
 .../apache_beam.transforms.trigger.rst.txt         |     7 +
 .../apache_beam.transforms.userstate.rst.txt       |     7 +
 .../_sources/apache_beam.transforms.util.rst.txt   |     7 +
 .../_sources/apache_beam.transforms.window.rst.txt |     7 +
 .../apache_beam.typehints.decorators.rst.txt       |     7 +
 ...eam.typehints.native_type_compatibility.rst.txt |     7 +
 .../_sources/apache_beam.typehints.opcodes.rst.txt |     7 +
 .../2.19.0/_sources/apache_beam.typehints.rst.txt  |    21 +
 .../_sources/apache_beam.typehints.schemas.rst.txt |     7 +
 ...apache_beam.typehints.trivial_inference.rst.txt |     7 +
 .../apache_beam.typehints.typecheck.rst.txt        |     7 +
 .../apache_beam.typehints.typehints.rst.txt        |     7 +
 .../_sources/apache_beam.utils.annotations.rst.txt |     7 +
 .../apache_beam.utils.interactive_utils.rst.txt    |     7 +
 .../_sources/apache_beam.utils.plugin.rst.txt      |     7 +
 .../_sources/apache_beam.utils.processes.rst.txt   |     7 +
 .../_sources/apache_beam.utils.profiler.rst.txt    |     7 +
 .../_sources/apache_beam.utils.proto_utils.rst.txt |     7 +
 .../_sources/apache_beam.utils.retry.rst.txt       |     7 +
 pydoc/2.19.0/_sources/apache_beam.utils.rst.txt    |    25 +
 .../apache_beam.utils.subprocess_server.rst.txt    |     7 +
 .../apache_beam.utils.thread_pool_executor.rst.txt |     7 +
 .../_sources/apache_beam.utils.timestamp.rst.txt   |     7 +
 .../2.19.0/_sources/apache_beam.utils.urns.rst.txt |     7 +
 pydoc/2.19.0/_sources/apache_beam.version.rst.txt  |     7 +
 pydoc/2.19.0/_sources/index.rst.txt                |     2 +
 pydoc/2.19.0/_static/ajax-loader.gif               |   Bin 0 -> 673 bytes
 pydoc/2.19.0/_static/basic.css                     |   643 ++
 pydoc/2.19.0/_static/comment-bright.png            |   Bin 0 -> 756 bytes
 pydoc/2.19.0/_static/comment-close.png             |   Bin 0 -> 829 bytes
 pydoc/2.19.0/_static/comment.png                   |   Bin 0 -> 641 bytes
 pydoc/2.19.0/_static/css/badge_only.css            |     2 +
 pydoc/2.19.0/_static/css/theme.css                 |     5 +
 pydoc/2.19.0/_static/doctools.js                   |   311 +
 pydoc/2.19.0/_static/down-pressed.png              |   Bin 0 -> 222 bytes
 pydoc/2.19.0/_static/down.png                      |   Bin 0 -> 202 bytes
 pydoc/2.19.0/_static/file.png                      |   Bin 0 -> 286 bytes
 pydoc/2.19.0/_static/fonts/Inconsolata-Bold.ttf    |   Bin 0 -> 109948 bytes
 pydoc/2.19.0/_static/fonts/Inconsolata-Regular.ttf |   Bin 0 -> 96964 bytes
 pydoc/2.19.0/_static/fonts/Lato-Bold.ttf           |   Bin 0 -> 656544 bytes
 pydoc/2.19.0/_static/fonts/Lato-Regular.ttf        |   Bin 0 -> 656568 bytes
 pydoc/2.19.0/_static/fonts/RobotoSlab-Bold.ttf     |   Bin 0 -> 170616 bytes
 pydoc/2.19.0/_static/fonts/RobotoSlab-Regular.ttf  |   Bin 0 -> 169064 bytes
 pydoc/2.19.0/_static/fonts/fontawesome-webfont.eot |   Bin 0 -> 76518 bytes
 pydoc/2.19.0/_static/fonts/fontawesome-webfont.svg |   685 ++
 pydoc/2.19.0/_static/fonts/fontawesome-webfont.ttf |   Bin 0 -> 152796 bytes
 .../2.19.0/_static/fonts/fontawesome-webfont.woff  |   Bin 0 -> 90412 bytes
 pydoc/2.19.0/_static/jquery-3.1.0.js               | 10074 +++++++++++++++++++
 pydoc/2.19.0/_static/jquery.js                     |     4 +
 pydoc/2.19.0/_static/js/modernizr.min.js           |     4 +
 pydoc/2.19.0/_static/js/theme.js                   |   169 +
 pydoc/2.19.0/_static/minus.png                     |   Bin 0 -> 90 bytes
 pydoc/2.19.0/_static/plus.png                      |   Bin 0 -> 90 bytes
 pydoc/2.19.0/_static/pygments.css                  |     2 +
 pydoc/2.19.0/_static/searchtools.js                |   761 ++
 pydoc/2.19.0/_static/underscore-1.3.1.js           |   999 ++
 pydoc/2.19.0/_static/underscore.js                 |    31 +
 pydoc/2.19.0/_static/up-pressed.png                |   Bin 0 -> 214 bytes
 pydoc/2.19.0/_static/up.png                        |   Bin 0 -> 203 bytes
 pydoc/2.19.0/_static/websupport.js                 |   808 ++
 pydoc/2.19.0/apache_beam.coders.avro_record.html   |   269 +
 pydoc/2.19.0/apache_beam.coders.coders.html        |   891 ++
 pydoc/2.19.0/apache_beam.coders.html               |   271 +
 pydoc/2.19.0/apache_beam.coders.observable.html    |   282 +
 pydoc/2.19.0/apache_beam.coders.row_coder.html     |   315 +
 pydoc/2.19.0/apache_beam.coders.slow_stream.html   |   409 +
 pydoc/2.19.0/apache_beam.coders.typecoders.html    |   309 +
 pydoc/2.19.0/apache_beam.error.html                |   297 +
 pydoc/2.19.0/apache_beam.internal.gcp.auth.html    |   297 +
 pydoc/2.19.0/apache_beam.internal.gcp.html         |   270 +
 .../apache_beam.internal.gcp.json_value.html       |   342 +
 pydoc/2.19.0/apache_beam.internal.html             |   285 +
 pydoc/2.19.0/apache_beam.internal.http_client.html |   293 +
 pydoc/2.19.0/apache_beam.internal.pickler.html     |   304 +
 pydoc/2.19.0/apache_beam.internal.util.html        |   348 +
 pydoc/2.19.0/apache_beam.io.avroio.html            |   464 +
 pydoc/2.19.0/apache_beam.io.aws.clients.html       |   282 +
 ...apache_beam.io.aws.clients.s3.boto3_client.html |   396 +
 .../apache_beam.io.aws.clients.s3.fake_client.html |   369 +
 pydoc/2.19.0/apache_beam.io.aws.clients.s3.html    |   278 +
 .../apache_beam.io.aws.clients.s3.messages.html    |   351 +
 pydoc/2.19.0/apache_beam.io.aws.html               |   295 +
 pydoc/2.19.0/apache_beam.io.aws.s3filesystem.html  |   542 +
 pydoc/2.19.0/apache_beam.io.aws.s3io.html          |   593 ++
 pydoc/2.19.0/apache_beam.io.concat_source.html     |   390 +
 pydoc/2.19.0/apache_beam.io.external.gcp.html      |   274 +
 .../2.19.0/apache_beam.io.external.gcp.pubsub.html |   429 +
 .../apache_beam.io.external.generate_sequence.html |   293 +
 pydoc/2.19.0/apache_beam.io.external.html          |   287 +
 pydoc/2.19.0/apache_beam.io.external.kafka.html    |   439 +
 pydoc/2.19.0/apache_beam.io.filebasedsink.html     |   361 +
 pydoc/2.19.0/apache_beam.io.filebasedsource.html   |   407 +
 pydoc/2.19.0/apache_beam.io.fileio.html            |   438 +
 pydoc/2.19.0/apache_beam.io.filesystem.html        |   816 ++
 pydoc/2.19.0/apache_beam.io.filesystemio.html      |   552 +
 pydoc/2.19.0/apache_beam.io.filesystems.html       |   604 ++
 ...am.io.flink.flink_streaming_impulse_source.html |   321 +
 pydoc/2.19.0/apache_beam.io.flink.html             |   271 +
 ...m.io.gcp.big_query_query_to_table_pipeline.html |   283 +
 pydoc/2.19.0/apache_beam.io.gcp.bigquery.html      |   901 ++
 .../apache_beam.io.gcp.bigquery_file_loads.html    |   516 +
 ...ache_beam.io.gcp.bigquery_io_read_pipeline.html |   284 +
 .../2.19.0/apache_beam.io.gcp.bigquery_tools.html  |   698 ++
 pydoc/2.19.0/apache_beam.io.gcp.bigtableio.html    |   309 +
 pydoc/2.19.0/apache_beam.io.gcp.datastore.html     |   284 +
 ...atastore.v1new.datastore_write_it_pipeline.html |   308 +
 ...he_beam.io.gcp.datastore.v1new.datastoreio.html |   382 +
 .../apache_beam.io.gcp.datastore.v1new.helper.html |   296 +
 .../2.19.0/apache_beam.io.gcp.datastore.v1new.html |   280 +
 ...beam.io.gcp.datastore.v1new.query_splitter.html |   320 +
 .../apache_beam.io.gcp.datastore.v1new.types.html  |   402 +
 pydoc/2.19.0/apache_beam.io.gcp.gcsfilesystem.html |   542 +
 pydoc/2.19.0/apache_beam.io.gcp.gcsio.html         |   503 +
 .../2.19.0/apache_beam.io.gcp.gcsio_overrides.html |   278 +
 pydoc/2.19.0/apache_beam.io.gcp.html               |   314 +
 pydoc/2.19.0/apache_beam.io.gcp.pubsub.html        |   402 +
 .../apache_beam.io.gcp.pubsub_it_pipeline.html     |   273 +
 pydoc/2.19.0/apache_beam.io.hadoopfilesystem.html  |   426 +
 pydoc/2.19.0/apache_beam.io.html                   |   421 +
 pydoc/2.19.0/apache_beam.io.iobase.html            |  1301 +++
 pydoc/2.19.0/apache_beam.io.localfilesystem.html   |   543 +
 pydoc/2.19.0/apache_beam.io.mongodbio.html         |   398 +
 pydoc/2.19.0/apache_beam.io.parquetio.html         |   525 +
 pydoc/2.19.0/apache_beam.io.range_trackers.html    |   492 +
 .../apache_beam.io.restriction_trackers.html       |   346 +
 pydoc/2.19.0/apache_beam.io.source_test_utils.html |   530 +
 pydoc/2.19.0/apache_beam.io.textio.html            |   470 +
 pydoc/2.19.0/apache_beam.io.tfrecordio.html        |   353 +
 pydoc/2.19.0/apache_beam.io.utils.html             |   305 +
 pydoc/2.19.0/apache_beam.io.vcfio.html             |   428 +
 pydoc/2.19.0/apache_beam.metrics.cells.html        |   312 +
 pydoc/2.19.0/apache_beam.metrics.html              |   267 +
 pydoc/2.19.0/apache_beam.metrics.metric.html       |   424 +
 pydoc/2.19.0/apache_beam.metrics.metricbase.html   |   368 +
 .../apache_beam.metrics.monitoring_infos.html      |   538 +
 pydoc/2.19.0/apache_beam.options.html              |   265 +
 .../apache_beam.options.pipeline_options.html      |   685 ++
 ...he_beam.options.pipeline_options_validator.html |   433 +
 .../2.19.0/apache_beam.options.value_provider.html |   334 +
 pydoc/2.19.0/apache_beam.pipeline.html             |   466 +
 ...portability.api.beam_artifact_api_pb2_grpc.html |   346 +
 ...portability.api.beam_artifact_api_pb2_urns.html |   279 +
 ...ortability.api.beam_expansion_api_pb2_grpc.html |   295 +
 ..._beam.portability.api.beam_fn_api_pb2_grpc.html |   479 +
 ..._beam.portability.api.beam_fn_api_pb2_urns.html |   320 +
 ...tability.api.beam_interactive_api_pb2_grpc.html |   262 +
 ...beam.portability.api.beam_job_api_pb2_grpc.html |   351 +
 ...beam.portability.api.beam_job_api_pb2_urns.html |   418 +
 ...ortability.api.beam_provision_api_pb2_grpc.html |   300 +
 ...m.portability.api.beam_runner_api_pb2_grpc.html |   294 +
 ...m.portability.api.beam_runner_api_pb2_urns.html |   886 ++
 ...he_beam.portability.api.endpoints_pb2_grpc.html |   262 +
 ...rtability.api.external_transforms_pb2_grpc.html |   262 +
 pydoc/2.19.0/apache_beam.portability.api.html      |   287 +
 ...ache_beam.portability.api.metrics_pb2_grpc.html |   262 +
 ...ache_beam.portability.api.metrics_pb2_urns.html |   388 +
 ...pache_beam.portability.api.schema_pb2_grpc.html |   262 +
 ...rtability.api.standard_window_fns_pb2_grpc.html |   262 +
 ...rtability.api.standard_window_fns_pb2_urns.html |   330 +
 .../apache_beam.portability.common_urns.html       |   260 +
 pydoc/2.19.0/apache_beam.portability.html          |   301 +
 .../apache_beam.portability.python_urns.html       |   261 +
 pydoc/2.19.0/apache_beam.portability.utils.html    |   291 +
 pydoc/2.19.0/apache_beam.pvalue.html               |   410 +
 ...ataflow.dataflow_exercise_metrics_pipeline.html |   303 +
 ...taflow_exercise_streaming_metrics_pipeline.html |   297 +
 ...che_beam.runners.dataflow.dataflow_metrics.html |   314 +
 ...ache_beam.runners.dataflow.dataflow_runner.html |   474 +
 pydoc/2.19.0/apache_beam.runners.dataflow.html     |   295 +
 .../apache_beam.runners.dataflow.native_io.html    |   275 +
 ...che_beam.runners.dataflow.native_io.iobase.html |   507 +
 ...beam.runners.dataflow.ptransform_overrides.html |   319 +
 ...beam.runners.dataflow.test_dataflow_runner.html |   292 +
 .../apache_beam.runners.direct.bundle_factory.html |   295 +
 pydoc/2.19.0/apache_beam.runners.direct.clock.html |   314 +
 ....direct.consumer_tracking_pipeline_visitor.html |   283 +
 .../apache_beam.runners.direct.direct_metrics.html |   329 +
 .../apache_beam.runners.direct.direct_runner.html  |   322 +
 ...pache_beam.runners.direct.direct_userstate.html |   371 +
 ...che_beam.runners.direct.evaluation_context.html |   399 +
 .../apache_beam.runners.direct.executor.html       |   311 +
 ...ache_beam.runners.direct.helper_transforms.html |   323 +
 pydoc/2.19.0/apache_beam.runners.direct.html       |   289 +
 ...ache_beam.runners.direct.sdf_direct_runner.html |   538 +
 ...che_beam.runners.direct.test_direct_runner.html |   279 +
 ...pache_beam.runners.direct.test_stream_impl.html |   271 +
 ...he_beam.runners.direct.transform_evaluator.html |   334 +
 pydoc/2.19.0/apache_beam.runners.direct.util.html  |   289 +
 ...ache_beam.runners.direct.watermark_manager.html |   316 +
 pydoc/2.19.0/apache_beam.runners.html              |   380 +
 ...runners.interactive.background_caching_job.html |   335 +
 ...che_beam.runners.interactive.cache_manager.html |   445 +
 .../apache_beam.runners.interactive.caching.html   |   275 +
 ...unners.interactive.caching.streaming_cache.html |   298 +
 ...unners.interactive.display.display_manager.html |   327 +
 .../apache_beam.runners.interactive.display.html   |   279 +
 ...ractive.display.interactive_pipeline_graph.html |   323 +
 ...rs.interactive.display.pcoll_visualization.html |   337 +
 ...runners.interactive.display.pipeline_graph.html |   310 +
 ...nteractive.display.pipeline_graph_renderer.html |   375 +
 pydoc/2.19.0/apache_beam.runners.interactive.html  |   312 +
 ..._beam.runners.interactive.interactive_beam.html |   329 +
 ...unners.interactive.interactive_environment.html |   426 +
 ...eam.runners.interactive.interactive_runner.html |   382 +
 ...beam.runners.interactive.pipeline_analyzer.html |   398 +
 ...am.runners.interactive.pipeline_instrument.html |   456 +
 .../apache_beam.runners.interactive.testing.html   |   276 +
 ...m.runners.interactive.testing.mock_ipython.html |   297 +
 ...ers.interactive.testing.pipeline_assertion.html |   302 +
 pydoc/2.19.0/apache_beam.runners.internal.html     |   272 +
 .../2.19.0/apache_beam.runners.internal.names.html |   267 +
 pydoc/2.19.0/apache_beam.runners.job.html          |   273 +
 pydoc/2.19.0/apache_beam.runners.job.manager.html  |   274 +
 pydoc/2.19.0/apache_beam.runners.job.utils.html    |   277 +
 .../apache_beam.runners.pipeline_context.html      |   293 +
 pydoc/2.19.0/apache_beam.runners.runner.html       |   516 +
 pydoc/2.19.0/apache_beam.testing.benchmarks.html   |   295 +
 .../apache_beam.testing.benchmarks.nexmark.html    |   295 +
 ...che_beam.testing.benchmarks.nexmark.models.html |   273 +
 ...ng.benchmarks.nexmark.models.nexmark_model.html |   299 +
 ...esting.benchmarks.nexmark.nexmark_launcher.html |   335 +
 ...am.testing.benchmarks.nexmark.nexmark_util.html |   319 +
 ...he_beam.testing.benchmarks.nexmark.queries.html |   275 +
 ....testing.benchmarks.nexmark.queries.query0.html |   278 +
 ....testing.benchmarks.nexmark.queries.query1.html |   277 +
 ....testing.benchmarks.nexmark.queries.query2.html |   277 +
 .../apache_beam.testing.datatype_inference.html    |   341 +
 .../apache_beam.testing.extra_assertions.html      |   286 +
 pydoc/2.19.0/apache_beam.testing.html              |   341 +
 pydoc/2.19.0/apache_beam.testing.load_tests.html   |   284 +
 ...testing.load_tests.load_test_metrics_utils.html |   610 ++
 ....streaming.group_by_key_streaming_pipeline.html |   278 +
 .../apache_beam.testing.load_tests.streaming.html  |   272 +
 ...apache_beam.testing.metric_result_matchers.html |   328 +
 .../apache_beam.testing.pipeline_verifiers.html    |   331 +
 .../apache_beam.testing.synthetic_pipeline.html    |   580 ++
 .../2.19.0/apache_beam.testing.test_pipeline.html  |   365 +
 pydoc/2.19.0/apache_beam.testing.test_stream.html  |   388 +
 .../apache_beam.testing.test_stream_service.html   |   288 +
 pydoc/2.19.0/apache_beam.testing.test_utils.html   |   385 +
 pydoc/2.19.0/apache_beam.testing.util.html         |   396 +
 .../apache_beam.tools.coders_microbenchmark.html   |   399 +
 ....tools.distribution_counter_microbenchmark.html |   278 +
 ...he_beam.tools.fn_api_runner_microbenchmark.html |   327 +
 pydoc/2.19.0/apache_beam.tools.html                |   271 +
 ...apache_beam.tools.sideinput_microbenchmark.html |   277 +
 pydoc/2.19.0/apache_beam.tools.utils.html          |   377 +
 pydoc/2.19.0/apache_beam.transforms.combiners.html |   594 ++
 pydoc/2.19.0/apache_beam.transforms.core.html      |  1741 ++++
 .../apache_beam.transforms.create_source.html      |   270 +
 pydoc/2.19.0/apache_beam.transforms.display.html   |   473 +
 .../apache_beam.transforms.environments.html       |   468 +
 pydoc/2.19.0/apache_beam.transforms.external.html  |   487 +
 .../apache_beam.transforms.external_test_it.html   |   285 +
 pydoc/2.19.0/apache_beam.transforms.html           |   290 +
 .../2.19.0/apache_beam.transforms.ptransform.html  |   482 +
 .../2.19.0/apache_beam.transforms.sideinputs.html  |   297 +
 pydoc/2.19.0/apache_beam.transforms.stats.html     |   407 +
 pydoc/2.19.0/apache_beam.transforms.timeutil.html  |   303 +
 pydoc/2.19.0/apache_beam.transforms.trigger.html   |   785 ++
 pydoc/2.19.0/apache_beam.transforms.userstate.html |   495 +
 pydoc/2.19.0/apache_beam.transforms.util.html      |   794 ++
 pydoc/2.19.0/apache_beam.transforms.window.html    |   702 ++
 pydoc/2.19.0/apache_beam.typehints.decorators.html |   496 +
 pydoc/2.19.0/apache_beam.typehints.html            |   274 +
 ...e_beam.typehints.native_type_compatibility.html |   356 +
 pydoc/2.19.0/apache_beam.typehints.opcodes.html    |   693 ++
 pydoc/2.19.0/apache_beam.typehints.schemas.html    |   324 +
 .../apache_beam.typehints.trivial_inference.html   |   436 +
 pydoc/2.19.0/apache_beam.typehints.typecheck.html  |   373 +
 pydoc/2.19.0/apache_beam.typehints.typehints.html  |   331 +
 pydoc/2.19.0/apache_beam.utils.annotations.html    |   370 +
 pydoc/2.19.0/apache_beam.utils.html                |   283 +
 .../apache_beam.utils.interactive_utils.html       |   295 +
 pydoc/2.19.0/apache_beam.utils.plugin.html         |   288 +
 pydoc/2.19.0/apache_beam.utils.processes.html      |   288 +
 pydoc/2.19.0/apache_beam.utils.profiler.html       |   341 +
 pydoc/2.19.0/apache_beam.utils.proto_utils.html    |   311 +
 pydoc/2.19.0/apache_beam.utils.retry.html          |   403 +
 .../apache_beam.utils.subprocess_server.html       |   361 +
 .../apache_beam.utils.thread_pool_executor.html    |   284 +
 pydoc/2.19.0/apache_beam.utils.timestamp.html      |   409 +
 pydoc/2.19.0/apache_beam.utils.urns.html           |   319 +
 pydoc/2.19.0/apache_beam.version.html              |   245 +
 pydoc/2.19.0/doctest/output.txt                    |    38 +
 .../apache_beam.coders.avro_record.doctree         |   Bin 0 -> 7156 bytes
 .../doctrees/apache_beam.coders.coders.doctree     |   Bin 0 -> 216403 bytes
 pydoc/2.19.0/doctrees/apache_beam.coders.doctree   |   Bin 0 -> 3958 bytes
 .../doctrees/apache_beam.coders.observable.doctree |   Bin 0 -> 11384 bytes
 .../doctrees/apache_beam.coders.row_coder.doctree  |   Bin 0 -> 22145 bytes
 .../apache_beam.coders.slow_stream.doctree         |   Bin 0 -> 54078 bytes
 .../doctrees/apache_beam.coders.typecoders.doctree |   Bin 0 -> 10891 bytes
 pydoc/2.19.0/doctrees/apache_beam.error.doctree    |   Bin 0 -> 22928 bytes
 pydoc/2.19.0/doctrees/apache_beam.internal.doctree |   Bin 0 -> 4737 bytes
 .../doctrees/apache_beam.internal.gcp.auth.doctree |   Bin 0 -> 11620 bytes
 .../doctrees/apache_beam.internal.gcp.doctree      |   Bin 0 -> 4293 bytes
 .../apache_beam.internal.gcp.json_value.doctree    |   Bin 0 -> 29125 bytes
 .../apache_beam.internal.http_client.doctree       |   Bin 0 -> 11527 bytes
 .../doctrees/apache_beam.internal.pickler.doctree  |   Bin 0 -> 19110 bytes
 .../doctrees/apache_beam.internal.util.doctree     |   Bin 0 -> 25792 bytes
 .../2.19.0/doctrees/apache_beam.io.avroio.doctree  |   Bin 0 -> 57667 bytes
 .../doctrees/apache_beam.io.aws.clients.doctree    |   Bin 0 -> 3853 bytes
 ...che_beam.io.aws.clients.s3.boto3_client.doctree |   Bin 0 -> 34619 bytes
 .../doctrees/apache_beam.io.aws.clients.s3.doctree |   Bin 0 -> 4007 bytes
 ...ache_beam.io.aws.clients.s3.fake_client.doctree |   Bin 0 -> 36045 bytes
 .../apache_beam.io.aws.clients.s3.messages.doctree |   Bin 0 -> 42783 bytes
 pydoc/2.19.0/doctrees/apache_beam.io.aws.doctree   |   Bin 0 -> 4241 bytes
 .../apache_beam.io.aws.s3filesystem.doctree        |   Bin 0 -> 64060 bytes
 .../doctrees/apache_beam.io.aws.s3io.doctree       |   Bin 0 -> 82382 bytes
 .../doctrees/apache_beam.io.concat_source.doctree  |   Bin 0 -> 41736 bytes
 pydoc/2.19.0/doctrees/apache_beam.io.doctree       |   Bin 0 -> 5541 bytes
 .../doctrees/apache_beam.io.external.doctree       |   Bin 0 -> 4318 bytes
 .../doctrees/apache_beam.io.external.gcp.doctree   |   Bin 0 -> 3866 bytes
 .../apache_beam.io.external.gcp.pubsub.doctree     |   Bin 0 -> 51873 bytes
 ...ache_beam.io.external.generate_sequence.doctree |   Bin 0 -> 12339 bytes
 .../doctrees/apache_beam.io.external.kafka.doctree |   Bin 0 -> 50573 bytes
 .../doctrees/apache_beam.io.filebasedsink.doctree  |   Bin 0 -> 37446 bytes
 .../apache_beam.io.filebasedsource.doctree         |   Bin 0 -> 48985 bytes
 .../2.19.0/doctrees/apache_beam.io.fileio.doctree  |   Bin 0 -> 57544 bytes
 .../doctrees/apache_beam.io.filesystem.doctree     |   Bin 0 -> 140522 bytes
 .../doctrees/apache_beam.io.filesystemio.doctree   |   Bin 0 -> 70057 bytes
 .../doctrees/apache_beam.io.filesystems.doctree    |   Bin 0 -> 79919 bytes
 pydoc/2.19.0/doctrees/apache_beam.io.flink.doctree |   Bin 0 -> 3797 bytes
 ...io.flink.flink_streaming_impulse_source.doctree |   Bin 0 -> 25477 bytes
 ...o.gcp.big_query_query_to_table_pipeline.doctree |   Bin 0 -> 8097 bytes
 .../doctrees/apache_beam.io.gcp.bigquery.doctree   |   Bin 0 -> 200723 bytes
 .../apache_beam.io.gcp.bigquery_file_loads.doctree |   Bin 0 -> 97968 bytes
 ...e_beam.io.gcp.bigquery_io_read_pipeline.doctree |   Bin 0 -> 11910 bytes
 .../apache_beam.io.gcp.bigquery_tools.doctree      |   Bin 0 -> 130672 bytes
 .../doctrees/apache_beam.io.gcp.bigtableio.doctree |   Bin 0 -> 14261 bytes
 .../doctrees/apache_beam.io.gcp.datastore.doctree  |   Bin 0 -> 3882 bytes
 ...store.v1new.datastore_write_it_pipeline.doctree |   Bin 0 -> 18550 bytes
 ...beam.io.gcp.datastore.v1new.datastoreio.doctree |   Bin 0 -> 40404 bytes
 .../apache_beam.io.gcp.datastore.v1new.doctree     |   Bin 0 -> 4210 bytes
 ...ache_beam.io.gcp.datastore.v1new.helper.doctree |   Bin 0 -> 14307 bytes
 ...m.io.gcp.datastore.v1new.query_splitter.doctree |   Bin 0 -> 19581 bytes
 ...pache_beam.io.gcp.datastore.v1new.types.doctree |   Bin 0 -> 45213 bytes
 pydoc/2.19.0/doctrees/apache_beam.io.gcp.doctree   |   Bin 0 -> 4714 bytes
 .../apache_beam.io.gcp.gcsfilesystem.doctree       |   Bin 0 -> 64224 bytes
 .../doctrees/apache_beam.io.gcp.gcsio.doctree      |   Bin 0 -> 60853 bytes
 .../apache_beam.io.gcp.gcsio_overrides.doctree     |   Bin 0 -> 8596 bytes
 .../doctrees/apache_beam.io.gcp.pubsub.doctree     |   Bin 0 -> 45240 bytes
 .../apache_beam.io.gcp.pubsub_it_pipeline.doctree  |   Bin 0 -> 6875 bytes
 .../apache_beam.io.hadoopfilesystem.doctree        |   Bin 0 -> 43505 bytes
 .../2.19.0/doctrees/apache_beam.io.iobase.doctree  |   Bin 0 -> 266090 bytes
 .../apache_beam.io.localfilesystem.doctree         |   Bin 0 -> 62229 bytes
 .../doctrees/apache_beam.io.mongodbio.doctree      |   Bin 0 -> 40904 bytes
 .../doctrees/apache_beam.io.parquetio.doctree      |   Bin 0 -> 86757 bytes
 .../doctrees/apache_beam.io.range_trackers.doctree |   Bin 0 -> 80325 bytes
 .../apache_beam.io.restriction_trackers.doctree    |   Bin 0 -> 29714 bytes
 .../apache_beam.io.source_test_utils.doctree       |   Bin 0 -> 79264 bytes
 .../2.19.0/doctrees/apache_beam.io.textio.doctree  |   Bin 0 -> 79433 bytes
 .../doctrees/apache_beam.io.tfrecordio.doctree     |   Bin 0 -> 25706 bytes
 pydoc/2.19.0/doctrees/apache_beam.io.utils.doctree |   Bin 0 -> 13753 bytes
 pydoc/2.19.0/doctrees/apache_beam.io.vcfio.doctree |   Bin 0 -> 60434 bytes
 .../doctrees/apache_beam.metrics.cells.doctree     |   Bin 0 -> 17704 bytes
 pydoc/2.19.0/doctrees/apache_beam.metrics.doctree  |   Bin 0 -> 3890 bytes
 .../doctrees/apache_beam.metrics.metric.doctree    |   Bin 0 -> 50994 bytes
 .../apache_beam.metrics.metricbase.doctree         |   Bin 0 -> 37676 bytes
 .../apache_beam.metrics.monitoring_infos.doctree   |   Bin 0 -> 84197 bytes
 pydoc/2.19.0/doctrees/apache_beam.options.doctree  |   Bin 0 -> 3876 bytes
 .../apache_beam.options.pipeline_options.doctree   |   Bin 0 -> 107894 bytes
 ...beam.options.pipeline_options_validator.doctree |   Bin 0 -> 61487 bytes
 .../apache_beam.options.value_provider.doctree     |   Bin 0 -> 31622 bytes
 pydoc/2.19.0/doctrees/apache_beam.pipeline.doctree |   Bin 0 -> 69593 bytes
 ...tability.api.beam_artifact_api_pb2_grpc.doctree |   Bin 0 -> 35612 bytes
 ...tability.api.beam_artifact_api_pb2_urns.doctree |   Bin 0 -> 10889 bytes
 ...ability.api.beam_expansion_api_pb2_grpc.doctree |   Bin 0 -> 16264 bytes
 ...am.portability.api.beam_fn_api_pb2_grpc.doctree |   Bin 0 -> 79497 bytes
 ...am.portability.api.beam_fn_api_pb2_urns.doctree |   Bin 0 -> 22672 bytes
 ...ility.api.beam_interactive_api_pb2_grpc.doctree |   Bin 0 -> 3523 bytes
 ...m.portability.api.beam_job_api_pb2_grpc.doctree |   Bin 0 -> 38340 bytes
 ...m.portability.api.beam_job_api_pb2_urns.doctree |   Bin 0 -> 53692 bytes
 ...ability.api.beam_provision_api_pb2_grpc.doctree |   Bin 0 -> 17572 bytes
 ...ortability.api.beam_runner_api_pb2_grpc.doctree |   Bin 0 -> 15972 bytes
 ...ortability.api.beam_runner_api_pb2_urns.doctree |   Bin 0 -> 218948 bytes
 .../doctrees/apache_beam.portability.api.doctree   |   Bin 0 -> 5685 bytes
 ...beam.portability.api.endpoints_pb2_grpc.doctree |   Bin 0 -> 3398 bytes
 ...bility.api.external_transforms_pb2_grpc.doctree |   Bin 0 -> 3510 bytes
 ...e_beam.portability.api.metrics_pb2_grpc.doctree |   Bin 0 -> 3376 bytes
 ...e_beam.portability.api.metrics_pb2_urns.doctree |   Bin 0 -> 50347 bytes
 ...he_beam.portability.api.schema_pb2_grpc.doctree |   Bin 0 -> 3365 bytes
 ...bility.api.standard_window_fns_pb2_grpc.doctree |   Bin 0 -> 3512 bytes
 ...bility.api.standard_window_fns_pb2_urns.doctree |   Bin 0 -> 30926 bytes
 .../apache_beam.portability.common_urns.doctree    |   Bin 0 -> 3672 bytes
 .../doctrees/apache_beam.portability.doctree       |   Bin 0 -> 4796 bytes
 .../apache_beam.portability.python_urns.doctree    |   Bin 0 -> 3908 bytes
 .../doctrees/apache_beam.portability.utils.doctree |   Bin 0 -> 13075 bytes
 pydoc/2.19.0/doctrees/apache_beam.pvalue.doctree   |   Bin 0 -> 48605 bytes
 ...flow.dataflow_exercise_metrics_pipeline.doctree |   Bin 0 -> 18647 bytes
 ...low_exercise_streaming_metrics_pipeline.doctree |   Bin 0 -> 17086 bytes
 ..._beam.runners.dataflow.dataflow_metrics.doctree |   Bin 0 -> 18213 bytes
 ...e_beam.runners.dataflow.dataflow_runner.doctree |   Bin 0 -> 80615 bytes
 .../doctrees/apache_beam.runners.dataflow.doctree  |   Bin 0 -> 5540 bytes
 .../apache_beam.runners.dataflow.native_io.doctree |   Bin 0 -> 4011 bytes
 ..._beam.runners.dataflow.native_io.iobase.doctree |   Bin 0 -> 76812 bytes
 ...m.runners.dataflow.ptransform_overrides.doctree |   Bin 0 -> 26619 bytes
 ...m.runners.dataflow.test_dataflow_runner.doctree |   Bin 0 -> 14866 bytes
 ...ache_beam.runners.direct.bundle_factory.doctree |   Bin 0 -> 13088 bytes
 .../apache_beam.runners.direct.clock.doctree       |   Bin 0 -> 21164 bytes
 ...rect.consumer_tracking_pipeline_visitor.doctree |   Bin 0 -> 10938 bytes
 ...ache_beam.runners.direct.direct_metrics.doctree |   Bin 0 -> 27310 bytes
 ...pache_beam.runners.direct.direct_runner.doctree |   Bin 0 -> 24278 bytes
 ...he_beam.runners.direct.direct_userstate.doctree |   Bin 0 -> 48681 bytes
 .../doctrees/apache_beam.runners.direct.doctree    |   Bin 0 -> 5451 bytes
 ..._beam.runners.direct.evaluation_context.doctree |   Bin 0 -> 48395 bytes
 .../apache_beam.runners.direct.executor.doctree    |   Bin 0 -> 22378 bytes
 ...e_beam.runners.direct.helper_transforms.doctree |   Bin 0 -> 27140 bytes
 ...e_beam.runners.direct.sdf_direct_runner.doctree |   Bin 0 -> 113956 bytes
 ..._beam.runners.direct.test_direct_runner.doctree |   Bin 0 -> 9621 bytes
 ...he_beam.runners.direct.test_stream_impl.doctree |   Bin 0 -> 4418 bytes
 ...beam.runners.direct.transform_evaluator.doctree |   Bin 0 -> 29301 bytes
 .../apache_beam.runners.direct.util.doctree        |   Bin 0 -> 16119 bytes
 ...e_beam.runners.direct.watermark_manager.doctree |   Bin 0 -> 21064 bytes
 pydoc/2.19.0/doctrees/apache_beam.runners.doctree  |   Bin 0 -> 5081 bytes
 ...ners.interactive.background_caching_job.doctree |   Bin 0 -> 26207 bytes
 ..._beam.runners.interactive.cache_manager.doctree |   Bin 0 -> 65657 bytes
 ...apache_beam.runners.interactive.caching.doctree |   Bin 0 -> 4031 bytes
 ...ers.interactive.caching.streaming_cache.doctree |   Bin 0 -> 15267 bytes
 ...ers.interactive.display.display_manager.doctree |   Bin 0 -> 19401 bytes
 ...apache_beam.runners.interactive.display.doctree |   Bin 0 -> 4327 bytes
 ...tive.display.interactive_pipeline_graph.doctree |   Bin 0 -> 21152 bytes
 ...interactive.display.pcoll_visualization.doctree |   Bin 0 -> 22018 bytes
 ...ners.interactive.display.pipeline_graph.doctree |   Bin 0 -> 16518 bytes
 ...ractive.display.pipeline_graph_renderer.doctree |   Bin 0 -> 42854 bytes
 .../apache_beam.runners.interactive.doctree        |   Bin 0 -> 4900 bytes
 ...am.runners.interactive.interactive_beam.doctree |   Bin 0 -> 15063 bytes
 ...ers.interactive.interactive_environment.doctree |   Bin 0 -> 55521 bytes
 ....runners.interactive.interactive_runner.doctree |   Bin 0 -> 40113 bytes
 ...m.runners.interactive.pipeline_analyzer.doctree |   Bin 0 -> 49639 bytes
 ...runners.interactive.pipeline_instrument.doctree |   Bin 0 -> 59465 bytes
 ...apache_beam.runners.interactive.testing.doctree |   Bin 0 -> 4101 bytes
 ...unners.interactive.testing.mock_ipython.doctree |   Bin 0 -> 8274 bytes
 ....interactive.testing.pipeline_assertion.doctree |   Bin 0 -> 14069 bytes
 .../doctrees/apache_beam.runners.internal.doctree  |   Bin 0 -> 3876 bytes
 .../apache_beam.runners.internal.names.doctree     |   Bin 0 -> 3663 bytes
 .../doctrees/apache_beam.runners.job.doctree       |   Bin 0 -> 3857 bytes
 .../apache_beam.runners.job.manager.doctree        |   Bin 0 -> 7442 bytes
 .../doctrees/apache_beam.runners.job.utils.doctree |   Bin 0 -> 7588 bytes
 .../apache_beam.runners.pipeline_context.doctree   |   Bin 0 -> 18369 bytes
 .../doctrees/apache_beam.runners.runner.doctree    |   Bin 0 -> 74326 bytes
 .../apache_beam.testing.benchmarks.doctree         |   Bin 0 -> 3908 bytes
 .../apache_beam.testing.benchmarks.nexmark.doctree |   Bin 0 -> 4615 bytes
 ..._beam.testing.benchmarks.nexmark.models.doctree |   Bin 0 -> 4109 bytes
 ...benchmarks.nexmark.models.nexmark_model.doctree |   Bin 0 -> 19186 bytes
 ...ing.benchmarks.nexmark.nexmark_launcher.doctree |   Bin 0 -> 24780 bytes
 ...testing.benchmarks.nexmark.nexmark_util.doctree |   Bin 0 -> 22055 bytes
 ...beam.testing.benchmarks.nexmark.queries.doctree |   Bin 0 -> 4249 bytes
 ...sting.benchmarks.nexmark.queries.query0.doctree |   Bin 0 -> 7256 bytes
 ...sting.benchmarks.nexmark.queries.query1.doctree |   Bin 0 -> 7133 bytes
 ...sting.benchmarks.nexmark.queries.query2.doctree |   Bin 0 -> 7127 bytes
 .../apache_beam.testing.datatype_inference.doctree |   Bin 0 -> 24426 bytes
 pydoc/2.19.0/doctrees/apache_beam.testing.doctree  |   Bin 0 -> 4732 bytes
 .../apache_beam.testing.extra_assertions.doctree   |   Bin 0 -> 11493 bytes
 .../apache_beam.testing.load_tests.doctree         |   Bin 0 -> 4386 bytes
 ...ting.load_tests.load_test_metrics_utils.doctree |   Bin 0 -> 118932 bytes
 ...pache_beam.testing.load_tests.streaming.doctree |   Bin 0 -> 4062 bytes
 ...reaming.group_by_key_streaming_pipeline.doctree |   Bin 0 -> 7533 bytes
 ...che_beam.testing.metric_result_matchers.doctree |   Bin 0 -> 25485 bytes
 .../apache_beam.testing.pipeline_verifiers.doctree |   Bin 0 -> 26448 bytes
 .../apache_beam.testing.synthetic_pipeline.doctree |   Bin 0 -> 115830 bytes
 .../apache_beam.testing.test_pipeline.doctree      |   Bin 0 -> 35616 bytes
 .../apache_beam.testing.test_stream.doctree        |   Bin 0 -> 48821 bytes
 ...apache_beam.testing.test_stream_service.doctree |   Bin 0 -> 12752 bytes
 .../apache_beam.testing.test_utils.doctree         |   Bin 0 -> 39505 bytes
 .../doctrees/apache_beam.testing.util.doctree      |   Bin 0 -> 38202 bytes
 ...apache_beam.tools.coders_microbenchmark.doctree |   Bin 0 -> 51119 bytes
 ...ols.distribution_counter_microbenchmark.doctree |   Bin 0 -> 10747 bytes
 pydoc/2.19.0/doctrees/apache_beam.tools.doctree    |   Bin 0 -> 4551 bytes
 ...beam.tools.fn_api_runner_microbenchmark.doctree |   Bin 0 -> 24402 bytes
 ...che_beam.tools.sideinput_microbenchmark.doctree |   Bin 0 -> 10006 bytes
 .../doctrees/apache_beam.tools.utils.doctree       |   Bin 0 -> 29769 bytes
 .../apache_beam.transforms.combiners.doctree       |   Bin 0 -> 115960 bytes
 .../doctrees/apache_beam.transforms.core.doctree   |   Bin 0 -> 449839 bytes
 .../apache_beam.transforms.create_source.doctree   |   Bin 0 -> 3284 bytes
 .../apache_beam.transforms.display.doctree         |   Bin 0 -> 63600 bytes
 .../2.19.0/doctrees/apache_beam.transforms.doctree |   Bin 0 -> 4776 bytes
 .../apache_beam.transforms.environments.doctree    |   Bin 0 -> 83200 bytes
 .../apache_beam.transforms.external.doctree        |   Bin 0 -> 66553 bytes
 ...apache_beam.transforms.external_test_it.doctree |   Bin 0 -> 9610 bytes
 .../apache_beam.transforms.ptransform.doctree      |   Bin 0 -> 65756 bytes
 .../apache_beam.transforms.sideinputs.doctree      |   Bin 0 -> 13910 bytes
 .../doctrees/apache_beam.transforms.stats.doctree  |   Bin 0 -> 44796 bytes
 .../apache_beam.transforms.timeutil.doctree        |   Bin 0 -> 14290 bytes
 .../apache_beam.transforms.trigger.doctree         |   Bin 0 -> 177453 bytes
 .../apache_beam.transforms.userstate.doctree       |   Bin 0 -> 84202 bytes
 .../doctrees/apache_beam.transforms.util.doctree   |   Bin 0 -> 151334 bytes
 .../doctrees/apache_beam.transforms.window.doctree |   Bin 0 -> 136765 bytes
 .../apache_beam.typehints.decorators.doctree       |   Bin 0 -> 53740 bytes
 .../2.19.0/doctrees/apache_beam.typehints.doctree  |   Bin 0 -> 4504 bytes
 ...eam.typehints.native_type_compatibility.doctree |   Bin 0 -> 27425 bytes
 .../doctrees/apache_beam.typehints.opcodes.doctree |   Bin 0 -> 155517 bytes
 .../doctrees/apache_beam.typehints.schemas.doctree |   Bin 0 -> 20793 bytes
 ...apache_beam.typehints.trivial_inference.doctree |   Bin 0 -> 57847 bytes
 .../apache_beam.typehints.typecheck.doctree        |   Bin 0 -> 49636 bytes
 .../apache_beam.typehints.typehints.doctree        |   Bin 0 -> 19924 bytes
 .../doctrees/apache_beam.utils.annotations.doctree |   Bin 0 -> 23173 bytes
 pydoc/2.19.0/doctrees/apache_beam.utils.doctree    |   Bin 0 -> 4755 bytes
 .../apache_beam.utils.interactive_utils.doctree    |   Bin 0 -> 13050 bytes
 .../doctrees/apache_beam.utils.plugin.doctree      |   Bin 0 -> 11483 bytes
 .../doctrees/apache_beam.utils.processes.doctree   |   Bin 0 -> 11747 bytes
 .../doctrees/apache_beam.utils.profiler.doctree    |   Bin 0 -> 24514 bytes
 .../doctrees/apache_beam.utils.proto_utils.doctree |   Bin 0 -> 20082 bytes
 .../doctrees/apache_beam.utils.retry.doctree       |   Bin 0 -> 46263 bytes
 .../apache_beam.utils.subprocess_server.doctree    |   Bin 0 -> 35044 bytes
 .../apache_beam.utils.thread_pool_executor.doctree |   Bin 0 -> 11417 bytes
 .../doctrees/apache_beam.utils.timestamp.doctree   |   Bin 0 -> 44584 bytes
 .../2.19.0/doctrees/apache_beam.utils.urns.doctree |   Bin 0 -> 21785 bytes
 pydoc/2.19.0/doctrees/apache_beam.version.doctree  |   Bin 0 -> 3474 bytes
 pydoc/2.19.0/doctrees/environment.pickle           |   Bin 0 -> 4780942 bytes
 pydoc/2.19.0/doctrees/index.doctree                |   Bin 0 -> 21069 bytes
 pydoc/2.19.0/genindex.html                         |  6429 ++++++++++++
 pydoc/2.19.0/index.html                            |   774 ++
 pydoc/2.19.0/objects.inv                           |   Bin 0 -> 26517 bytes
 pydoc/2.19.0/py-modindex.html                      |  1372 +++
 pydoc/2.19.0/search.html                           |   249 +
 pydoc/2.19.0/searchindex.js                        |     1 +
 893 files changed, 211887 insertions(+)

diff --git a/pydoc/2.19.0/.buildinfo b/pydoc/2.19.0/.buildinfo
new file mode 100644
index 0000000..0885fbe
--- /dev/null
+++ b/pydoc/2.19.0/.buildinfo
@@ -0,0 +1,4 @@
+# Sphinx build info version 1
+# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
+config: 
+tags: 
diff --git a/pydoc/2.19.0/_modules/abc.html b/pydoc/2.19.0/_modules/abc.html
new file mode 100644
index 0000000..19ed934
--- /dev/null
+++ b/pydoc/2.19.0/_modules/abc.html
@@ -0,0 +1,420 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>abc &mdash; Apache Beam  documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="../genindex.html"/>
+        <link rel="search" title="Search" href="../search.html"/>
+    <link rel="top" title="Apache Beam  documentation" href="../index.html"/>
+        <link rel="up" title="Module code" href="index.html"/> 
+
+  
+  <script src="../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../index.html" class="icon icon-home"> Apache Beam
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../apache_beam.coders.html">apache_beam.coders package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../apache_beam.internal.html">apache_beam.internal package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../apache_beam.io.html">apache_beam.io package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../apache_beam.metrics.html">apache_beam.metrics package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../apache_beam.options.html">apache_beam.options package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../apache_beam.portability.html">apache_beam.portability package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../apache_beam.runners.html">apache_beam.runners package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../apache_beam.testing.html">apache_beam.testing package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../apache_beam.tools.html">apache_beam.tools package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../apache_beam.transforms.html">apache_beam.transforms package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../apache_beam.typehints.html">apache_beam.typehints package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../apache_beam.utils.html">apache_beam.utils package</a></li>
+</ul>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../apache_beam.error.html">apache_beam.error module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../apache_beam.pipeline.html">apache_beam.pipeline module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../apache_beam.pvalue.html">apache_beam.pvalue module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../apache_beam.version.html">apache_beam.version module</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../index.html">Apache Beam</a>
+        
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="index.html">Module code</a> &raquo;</li>
+        
+      <li>abc</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+            
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for abc</h1><div class="highlight"><pre>
+<span></span><span class="c1"># Copyright 2007 Google, Inc. All Rights Reserved.</span>
+<span class="c1"># Licensed to PSF under a Contributor Agreement.</span>
+
+<span class="sd">&quot;&quot;&quot;Abstract Base Classes (ABCs) according to PEP 3119.&quot;&quot;&quot;</span>
+
+<span class="kn">import</span> <span class="nn">types</span>
+
+<span class="kn">from</span> <span class="nn">_weakrefset</span> <span class="kn">import</span> <span class="n">WeakSet</span>
+
+<span class="c1"># Instance of old-style class</span>
+<span class="k">class</span> <span class="nc">_C</span><span class="p">:</span> <span class="k">pass</span>
+<span class="n">_InstanceType</span> <span class="o">=</span> <span class="nb">type</span><span class="p">(</span><span class="n">_C</span><span class="p">())</span>
+
+
+<span class="k">def</span> <span class="nf">abstractmethod</span><span class="p">(</span><span class="n">funcobj</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;A decorator indicating abstract methods.</span>
+
+<span class="sd">    Requires that the metaclass is ABCMeta or derived from it.  A</span>
+<span class="sd">    class that has a metaclass derived from ABCMeta cannot be</span>
+<span class="sd">    instantiated unless all of its abstract methods are overridden.</span>
+<span class="sd">    The abstract methods can be called using any of the normal</span>
+<span class="sd">    &#39;super&#39; call mechanisms.</span>
+
+<span class="sd">    Usage:</span>
+
+<span class="sd">        class C:</span>
+<span class="sd">            __metaclass__ = ABCMeta</span>
+<span class="sd">            @abstractmethod</span>
+<span class="sd">            def my_abstract_method(self, ...):</span>
+<span class="sd">                ...</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">funcobj</span><span class="o">.</span><span class="n">__isabstractmethod__</span> <span class="o">=</span> <span class="kc">True</span>
+    <span class="k">return</span> <span class="n">funcobj</span>
+
+
+<span class="k">class</span> <span class="nc">abstractproperty</span><span class="p">(</span><span class="nb">property</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;A decorator indicating abstract properties.</span>
+
+<span class="sd">    Requires that the metaclass is ABCMeta or derived from it.  A</span>
+<span class="sd">    class that has a metaclass derived from ABCMeta cannot be</span>
+<span class="sd">    instantiated unless all of its abstract properties are overridden.</span>
+<span class="sd">    The abstract properties can be called using any of the normal</span>
+<span class="sd">    &#39;super&#39; call mechanisms.</span>
+
+<span class="sd">    Usage:</span>
+
+<span class="sd">        class C:</span>
+<span class="sd">            __metaclass__ = ABCMeta</span>
+<span class="sd">            @abstractproperty</span>
+<span class="sd">            def my_abstract_property(self):</span>
+<span class="sd">                ...</span>
+
+<span class="sd">    This defines a read-only property; you can also define a read-write</span>
+<span class="sd">    abstract property using the &#39;long&#39; form of property declaration:</span>
+
+<span class="sd">        class C:</span>
+<span class="sd">            __metaclass__ = ABCMeta</span>
+<span class="sd">            def getx(self): ...</span>
+<span class="sd">            def setx(self, value): ...</span>
+<span class="sd">            x = abstractproperty(getx, setx)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">__isabstractmethod__</span> <span class="o">=</span> <span class="kc">True</span>
+
+
+<span class="k">class</span> <span class="nc">ABCMeta</span><span class="p">(</span><span class="nb">type</span><span class="p">):</span>
+
+    <span class="sd">&quot;&quot;&quot;Metaclass for defining Abstract Base Classes (ABCs).</span>
+
+<span class="sd">    Use this metaclass to create an ABC.  An ABC can be subclassed</span>
+<span class="sd">    directly, and then acts as a mix-in class.  You can also register</span>
+<span class="sd">    unrelated concrete classes (even built-in classes) and unrelated</span>
+<span class="sd">    ABCs as &#39;virtual subclasses&#39; -- these and their descendants will</span>
+<span class="sd">    be considered subclasses of the registering ABC by the built-in</span>
+<span class="sd">    issubclass() function, but the registering ABC won&#39;t show up in</span>
+<span class="sd">    their MRO (Method Resolution Order) nor will method</span>
+<span class="sd">    implementations defined by the registering ABC be callable (not</span>
+<span class="sd">    even via super()).</span>
+
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="c1"># A global counter that is incremented each time a class is</span>
+    <span class="c1"># registered as a virtual subclass of anything.  It forces the</span>
+    <span class="c1"># negative cache to be cleared before its next use.</span>
+    <span class="n">_abc_invalidation_counter</span> <span class="o">=</span> <span class="mi">0</span>
+
+    <span class="k">def</span> <span class="fm">__new__</span><span class="p">(</span><span class="n">mcls</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">bases</span><span class="p">,</span> <span class="n">namespace</span><span class="p">):</span>
+        <span class="bp">cls</span> <span class="o">=</span> <span class="nb">super</span><span class="p">(</span><span class="n">ABCMeta</span><span class="p">,</span> <span class="n">mcls</span><span class="p">)</span><span class="o">.</span><span class="fm">__new__</span><span class="p">(</span><span class="n">mcls</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">bases</span><span class="p">,</span> <span class="n">namespace</span><spa [...]
+        <span class="c1"># Compute set of abstract method names</span>
+        <span class="n">abstracts</span> <span class="o">=</span> <span class="nb">set</span><span class="p">(</span><span class="n">name</span>
+                     <span class="k">for</span> <span class="n">name</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="n">namespace</span><span class="o">.</span><span class="n">items</span><span class="p">()</span>
+                     <span class="k">if</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="s2">&quot;__isabstractmethod__&quot;</span><span class="p">,</span> <span class="kc">False</span><span class="p">))</span>
+        <span class="k">for</span> <span class="n">base</span> <span class="ow">in</span> <span class="n">bases</span><span class="p">:</span>
+            <span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">base</span><span class="p">,</span> <span class="s2">&quot;__abstractmethods__&quot;</span><span class="p">,</span> <span class="nb">set</span><span class="p">()):</span>
+                <span class="n">value</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+                <span class="k">if</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="s2">&quot;__isabstractmethod__&quot;</span><span class="p">,</span> <span class="kc">False</span><span class="p">):</span>
+                    <span class="n">abstracts</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">name</span><span class="p">)</span>
+        <span class="bp">cls</span><span class="o">.</span><span class="n">__abstractmethods__</span> <span class="o">=</span> <span class="nb">frozenset</span><span class="p">(</span><span class="n">abstracts</span><span class="p">)</span>
+        <span class="c1"># Set up inheritance registry</span>
+        <span class="bp">cls</span><span class="o">.</span><span class="n">_abc_registry</span> <span class="o">=</span> <span class="n">WeakSet</span><span class="p">()</span>
+        <span class="bp">cls</span><span class="o">.</span><span class="n">_abc_cache</span> <span class="o">=</span> <span class="n">WeakSet</span><span class="p">()</span>
+        <span class="bp">cls</span><span class="o">.</span><span class="n">_abc_negative_cache</span> <span class="o">=</span> <span class="n">WeakSet</span><span class="p">()</span>
+        <span class="bp">cls</span><span class="o">.</span><span class="n">_abc_negative_cache_version</span> <span class="o">=</span> <span class="n">ABCMeta</span><span class="o">.</span><span class="n">_abc_invalidation_counter</span>
+        <span class="k">return</span> <span class="bp">cls</span>
+
+    <span class="k">def</span> <span class="nf">register</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">subclass</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;Register a virtual subclass of an ABC.&quot;&quot;&quot;</span>
+        <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">subclass</span><span class="p">,</span> <span class="p">(</span><span class="nb">type</span><span class="p">,</span> <span class="n">types</span><span class="o">.</span><span class="n">ClassType</span><span class="p">)):</span>
+            <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">&quot;Can only register classes&quot;</span><span class="p">)</span>
+        <span class="k">if</span> <span class="nb">issubclass</span><span class="p">(</span><span class="n">subclass</span><span class="p">,</span> <span class="bp">cls</span><span class="p">):</span>
+            <span class="k">return</span>  <span class="c1"># Already a subclass</span>
+        <span class="c1"># Subtle: test for cycles *after* testing for &quot;already a subclass&quot;;</span>
+        <span class="c1"># this means we allow X.register(X) and interpret it as a no-op.</span>
+        <span class="k">if</span> <span class="nb">issubclass</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">subclass</span><span class="p">):</span>
+            <span class="c1"># This would create a cycle, which is bad for the algorithm below</span>
+            <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">&quot;Refusing to create an inheritance cycle&quot;</span><span class="p">)</span>
+        <span class="bp">cls</span><span class="o">.</span><span class="n">_abc_registry</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">subclass</span><span class="p">)</span>
+        <span class="n">ABCMeta</span><span class="o">.</span><span class="n">_abc_invalidation_counter</span> <span class="o">+=</span> <span class="mi">1</span>  <span class="c1"># Invalidate negative cache</span>
+
+    <span class="k">def</span> <span class="nf">_dump_registry</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">file</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;Debug helper to print the ABC registry.&quot;&quot;&quot;</span>
+        <span class="nb">print</span> <span class="o">&gt;&gt;</span> <span class="n">file</span><span class="p">,</span> <span class="s2">&quot;Class: </span><span class="si">%s</span><span class="s2">.</span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="vm">__module__</span><span class="p">,</span> <span class="bp">cls</span><span class="o">.</span><span class="vm [...]
+        <span class="nb">print</span> <span class="o">&gt;&gt;</span> <span class="n">file</span><span class="p">,</span> <span class="s2">&quot;Inv.counter: </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="n">ABCMeta</span><span class="o">.</span><span class="n">_abc_invalidation_counter</span>
+        <span class="k">for</span> <span class="n">name</span> <span class="ow">in</span> <span class="nb">sorted</span><span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">keys</span><span class="p">()):</span>
+            <span class="k">if</span> <span class="n">name</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s2">&quot;_abc_&quot;</span><span class="p">):</span>
+                <span class="n">value</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">name</span><span class="p">)</span>
+                <span class="nb">print</span> <span class="o">&gt;&gt;</span> <span class="n">file</span><span class="p">,</span> <span class="s2">&quot;</span><span class="si">%s</span><span class="s2">: </span><span class="si">%r</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">name</span><span class="p">,</span> <span class="n">value</span><span class="p">)</span>
+
+    <span class="k">def</span> <span class="fm">__instancecheck__</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">instance</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;Override for isinstance(instance, cls).&quot;&quot;&quot;</span>
+        <span class="c1"># Inline the cache checking when it&#39;s simple.</span>
+        <span class="n">subclass</span> <span class="o">=</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">instance</span><span class="p">,</span> <span class="s1">&#39;__class__&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">subclass</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">subclass</span> <span class="ow">in</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_abc_cache</span><span class="p">:</span>
+            <span class="k">return</span> <span class="kc">True</span>
+        <span class="n">subtype</span> <span class="o">=</span> <span class="nb">type</span><span class="p">(</span><span class="n">instance</span><span class="p">)</span>
+        <span class="c1"># Old-style instances</span>
+        <span class="k">if</span> <span class="n">subtype</span> <span class="ow">is</span> <span class="n">_InstanceType</span><span class="p">:</span>
+            <span class="n">subtype</span> <span class="o">=</span> <span class="n">subclass</span>
+        <span class="k">if</span> <span class="n">subtype</span> <span class="ow">is</span> <span class="n">subclass</span> <span class="ow">or</span> <span class="n">subclass</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+            <span class="k">if</span> <span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="n">_abc_negative_cache_version</span> <span class="o">==</span>
+                <span class="n">ABCMeta</span><span class="o">.</span><span class="n">_abc_invalidation_counter</span> <span class="ow">and</span>
+                <span class="n">subtype</span> <span class="ow">in</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_abc_negative_cache</span><span class="p">):</span>
+                <span class="k">return</span> <span class="kc">False</span>
+            <span class="c1"># Fall back to the subclass check.</span>
+            <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="fm">__subclasscheck__</span><span class="p">(</span><span class="n">subtype</span><span class="p">)</span>
+        <span class="k">return</span> <span class="p">(</span><span class="bp">cls</span><span class="o">.</span><span class="fm">__subclasscheck__</span><span class="p">(</span><span class="n">subclass</span><span class="p">)</span> <span class="ow">or</span>
+                <span class="bp">cls</span><span class="o">.</span><span class="fm">__subclasscheck__</span><span class="p">(</span><span class="n">subtype</span><span class="p">))</span>
+
+    <span class="k">def</span> <span class="fm">__subclasscheck__</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">subclass</span><span class="p">):</span>
+        <span class="sd">&quot;&quot;&quot;Override for issubclass(subclass, cls).&quot;&quot;&quot;</span>
+        <span class="c1"># Check cache</span>
+        <span class="k">if</span> <span class="n">subclass</span> <span class="ow">in</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_abc_cache</span><span class="p">:</span>
+            <span class="k">return</span> <span class="kc">True</span>
+        <span class="c1"># Check negative cache; may have to invalidate</span>
+        <span class="k">if</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_abc_negative_cache_version</span> <span class="o">&lt;</span> <span class="n">ABCMeta</span><span class="o">.</span><span class="n">_abc_invalidation_counter</span><span class="p">:</span>
+            <span class="c1"># Invalidate the negative cache</span>
+            <span class="bp">cls</span><span class="o">.</span><span class="n">_abc_negative_cache</span> <span class="o">=</span> <span class="n">WeakSet</span><span class="p">()</span>
+            <span class="bp">cls</span><span class="o">.</span><span class="n">_abc_negative_cache_version</span> <span class="o">=</span> <span class="n">ABCMeta</span><span class="o">.</span><span class="n">_abc_invalidation_counter</span>
+        <span class="k">elif</span> <span class="n">subclass</span> <span class="ow">in</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_abc_negative_cache</span><span class="p">:</span>
+            <span class="k">return</span> <span class="kc">False</span>
+        <span class="c1"># Check the subclass hook</span>
+        <span class="n">ok</span> <span class="o">=</span> <span class="bp">cls</span><span class="o">.</span><span class="n">__subclasshook__</span><span class="p">(</span><span class="n">subclass</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">ok</span> <span class="ow">is</span> <span class="ow">not</span> <span class="bp">NotImplemented</span><span class="p">:</span>
+            <span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">ok</span><span class="p">,</span> <span class="nb">bool</span><span class="p">)</span>
+            <span class="k">if</span> <span class="n">ok</span><span class="p">:</span>
+                <span class="bp">cls</span><span class="o">.</span><span class="n">_abc_cache</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">subclass</span><span class="p">)</span>
+            <span class="k">else</span><span class="p">:</span>
+                <span class="bp">cls</span><span class="o">.</span><span class="n">_abc_negative_cache</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">subclass</span><span class="p">)</span>
+            <span class="k">return</span> <span class="n">ok</span>
+        <span class="c1"># Check if it&#39;s a direct subclass</span>
+        <span class="k">if</span> <span class="bp">cls</span> <span class="ow">in</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">subclass</span><span class="p">,</span> <span class="s1">&#39;__mro__&#39;</span><span class="p">,</span> <span class="p">()):</span>
+            <span class="bp">cls</span><span class="o">.</span><span class="n">_abc_cache</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">subclass</span><span class="p">)</span>
+            <span class="k">return</span> <span class="kc">True</span>
+        <span class="c1"># Check if it&#39;s a subclass of a registered class (recursive)</span>
+        <span class="k">for</span> <span class="n">rcls</span> <span class="ow">in</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_abc_registry</span><span class="p">:</span>
+            <span class="k">if</span> <span class="nb">issubclass</span><span class="p">(</span><span class="n">subclass</span><span class="p">,</span> <span class="n">rcls</span><span class="p">):</span>
+                <span class="bp">cls</span><span class="o">.</span><span class="n">_abc_cache</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">subclass</span><span class="p">)</span>
+                <span class="k">return</span> <span class="kc">True</span>
+        <span class="c1"># Check if it&#39;s a subclass of a subclass (recursive)</span>
+        <span class="k">for</span> <span class="n">scls</span> <span class="ow">in</span> <span class="bp">cls</span><span class="o">.</span><span class="n">__subclasses__</span><span class="p">():</span>
+            <span class="k">if</span> <span class="nb">issubclass</span><span class="p">(</span><span class="n">subclass</span><span class="p">,</span> <span class="n">scls</span><span class="p">):</span>
+                <span class="bp">cls</span><span class="o">.</span><span class="n">_abc_cache</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">subclass</span><span class="p">)</span>
+                <span class="k">return</span> <span class="kc">True</span>
+        <span class="c1"># No dice; update negative cache</span>
+        <span class="bp">cls</span><span class="o">.</span><span class="n">_abc_negative_cache</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">subclass</span><span class="p">)</span>
+        <span class="k">return</span> <span class="kc">False</span>
+</pre></div>
+
+           </div>
+           <div class="articleComments">
+            
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+        &copy; Copyright .
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="../_static/jquery.js"></script>
+      <script type="text/javascript" src="../_static/underscore.js"></script>
+      <script type="text/javascript" src="../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file
diff --git a/pydoc/2.19.0/_modules/apache_beam/coders/avro_record.html b/pydoc/2.19.0/_modules/apache_beam/coders/avro_record.html
new file mode 100644
index 0000000..1c44ed8
--- /dev/null
+++ b/pydoc/2.19.0/_modules/apache_beam/coders/avro_record.html
@@ -0,0 +1,275 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>apache_beam.coders.avro_record &mdash; Apache Beam  documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../search.html"/>
+    <link rel="top" title="Apache Beam  documentation" href="../../../index.html"/>
+        <link rel="up" title="Module code" href="../../index.html"/> 
+
+  
+  <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../index.html" class="icon icon-home"> Apache Beam
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.coders.html">apache_beam.coders package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.internal.html">apache_beam.internal package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.io.html">apache_beam.io package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.metrics.html">apache_beam.metrics package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.options.html">apache_beam.options package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.portability.html">apache_beam.portability package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.runners.html">apache_beam.runners package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.testing.html">apache_beam.testing package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.tools.html">apache_beam.tools package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.transforms.html">apache_beam.transforms package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.typehints.html">apache_beam.typehints package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.utils.html">apache_beam.utils package</a></li>
+</ul>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.error.html">apache_beam.error module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.pipeline.html">apache_beam.pipeline module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.pvalue.html">apache_beam.pvalue module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.version.html">apache_beam.version module</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../index.html">Apache Beam</a>
+        
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../index.html">Module code</a> &raquo;</li>
+        
+      <li>apache_beam.coders.avro_record</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+            
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for apache_beam.coders.avro_record</h1><div class="highlight"><pre>
+<span></span><span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one or more</span>
+<span class="c1"># contributor license agreements.  See the NOTICE file distributed with</span>
+<span class="c1"># this work for additional information regarding copyright ownership.</span>
+<span class="c1"># The ASF licenses this file to You under the Apache License, Version 2.0</span>
+<span class="c1"># (the &quot;License&quot;); you may not use this file except in compliance with</span>
+<span class="c1"># the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#    http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+<span class="c1">#</span>
+
+<span class="sd">&quot;&quot;&quot;AvroRecord for AvroGenericCoder.&quot;&quot;&quot;</span>
+
+<span class="c1"># pytype: skip-file</span>
+
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">absolute_import</span>
+
+<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;AvroRecord&#39;</span><span class="p">]</span>
+
+
+<div class="viewcode-block" id="AvroRecord"><a class="viewcode-back" href="../../../apache_beam.coders.avro_record.html#apache_beam.coders.avro_record.AvroRecord">[docs]</a><span class="k">class</span> <span class="nc">AvroRecord</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;Simple wrapper class for dictionary records.&quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">record</span> <span class="o">=</span> <span class="n">value</span>
+
+  <span class="k">def</span> <span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
+    <span class="k">return</span> <span class="p">(</span>
+        <span class="nb">issubclass</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="n">other</span><span class="p">),</span> <span class="n">AvroRecord</span><span class="p">)</span> <span class="ow">and</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">record</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">record</span>
+    <span class="p">)</span>
+
+  <span class="k">def</span> <span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">record</span><span class="p">)</span></div>
+</pre></div>
+
+           </div>
+           <div class="articleComments">
+            
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+        &copy; Copyright .
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file
diff --git a/pydoc/2.19.0/_modules/apache_beam/coders/coders.html b/pydoc/2.19.0/_modules/apache_beam/coders/coders.html
new file mode 100644
index 0000000..56aebd2
--- /dev/null
+++ b/pydoc/2.19.0/_modules/apache_beam/coders/coders.html
@@ -0,0 +1,1642 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>apache_beam.coders.coders &mdash; Apache Beam  documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../search.html"/>
+    <link rel="top" title="Apache Beam  documentation" href="../../../index.html"/>
+        <link rel="up" title="Module code" href="../../index.html"/> 
+
+  
+  <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../index.html" class="icon icon-home"> Apache Beam
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.coders.html">apache_beam.coders package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.internal.html">apache_beam.internal package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.io.html">apache_beam.io package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.metrics.html">apache_beam.metrics package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.options.html">apache_beam.options package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.portability.html">apache_beam.portability package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.runners.html">apache_beam.runners package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.testing.html">apache_beam.testing package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.tools.html">apache_beam.tools package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.transforms.html">apache_beam.transforms package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.typehints.html">apache_beam.typehints package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.utils.html">apache_beam.utils package</a></li>
+</ul>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.error.html">apache_beam.error module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.pipeline.html">apache_beam.pipeline module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.pvalue.html">apache_beam.pvalue module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.version.html">apache_beam.version module</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../index.html">Apache Beam</a>
+        
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../index.html">Module code</a> &raquo;</li>
+        
+      <li>apache_beam.coders.coders</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+            
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for apache_beam.coders.coders</h1><div class="highlight"><pre>
+<span></span><span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one or more</span>
+<span class="c1"># contributor license agreements.  See the NOTICE file distributed with</span>
+<span class="c1"># this work for additional information regarding copyright ownership.</span>
+<span class="c1"># The ASF licenses this file to You under the Apache License, Version 2.0</span>
+<span class="c1"># (the &quot;License&quot;); you may not use this file except in compliance with</span>
+<span class="c1"># the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#    http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+<span class="c1">#</span>
+
+<span class="sd">&quot;&quot;&quot;Collection of useful coders.</span>
+
+<span class="sd">Only those coders listed in __all__ are part of the public API of this module.</span>
+<span class="sd">&quot;&quot;&quot;</span>
+<span class="c1"># pytype: skip-file</span>
+
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">absolute_import</span>
+
+<span class="kn">import</span> <span class="nn">base64</span>
+<span class="kn">import</span> <span class="nn">sys</span>
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">object</span>
+<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">TYPE_CHECKING</span>
+<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">Any</span>
+<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">Callable</span>
+<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">Dict</span>
+<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">Iterable</span>
+<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">List</span>
+<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">Optional</span>
+<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">Sequence</span>
+<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">Tuple</span>
+<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">Type</span>
+<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">TypeVar</span>
+<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">overload</span>
+
+<span class="kn">import</span> <span class="nn">google.protobuf.wrappers_pb2</span>
+<span class="kn">from</span> <span class="nn">future.moves</span> <span class="kn">import</span> <span class="n">pickle</span>
+<span class="kn">from</span> <span class="nn">past.builtins</span> <span class="kn">import</span> <span class="n">unicode</span>
+
+<span class="kn">from</span> <span class="nn">apache_beam.coders</span> <span class="kn">import</span> <span class="n">coder_impl</span>
+<span class="kn">from</span> <span class="nn">apache_beam.coders.avro_record</span> <span class="kn">import</span> <span class="n">AvroRecord</span>
+<span class="kn">from</span> <span class="nn">apache_beam.portability</span> <span class="kn">import</span> <span class="n">common_urns</span>
+<span class="kn">from</span> <span class="nn">apache_beam.portability</span> <span class="kn">import</span> <span class="n">python_urns</span>
+<span class="kn">from</span> <span class="nn">apache_beam.portability.api</span> <span class="kn">import</span> <span class="n">beam_runner_api_pb2</span>
+<span class="kn">from</span> <span class="nn">apache_beam.typehints</span> <span class="kn">import</span> <span class="n">typehints</span>
+<span class="kn">from</span> <span class="nn">apache_beam.utils</span> <span class="kn">import</span> <span class="n">proto_utils</span>
+
+<span class="k">if</span> <span class="n">TYPE_CHECKING</span><span class="p">:</span>
+  <span class="kn">from</span> <span class="nn">google.protobuf</span> <span class="kn">import</span> <span class="n">message</span>  <span class="c1"># pylint: disable=ungrouped-imports</span>
+  <span class="kn">from</span> <span class="nn">apache_beam.coders.typecoders</span> <span class="kn">import</span> <span class="n">CoderRegistry</span>
+  <span class="kn">from</span> <span class="nn">apache_beam.runners.pipeline_context</span> <span class="kn">import</span> <span class="n">PipelineContext</span>
+
+<span class="c1"># pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports</span>
+<span class="k">try</span><span class="p">:</span>
+  <span class="kn">from</span> <span class="nn">.stream</span> <span class="kn">import</span> <span class="n">get_varint_size</span>
+<span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
+  <span class="kn">from</span> <span class="nn">.slow_stream</span> <span class="kn">import</span> <span class="n">get_varint_size</span>
+<span class="c1"># pylint: enable=wrong-import-order, wrong-import-position, ungrouped-imports</span>
+
+
+<span class="c1"># pylint: disable=wrong-import-order, wrong-import-position</span>
+<span class="c1"># Avoid dependencies on the full SDK.</span>
+<span class="k">try</span><span class="p">:</span>
+  <span class="c1"># Import dill from the pickler module to make sure our monkey-patching of dill</span>
+  <span class="c1"># occurs.</span>
+  <span class="kn">from</span> <span class="nn">apache_beam.internal.pickler</span> <span class="kn">import</span> <span class="n">dill</span>
+<span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
+  <span class="c1"># We fall back to using the stock dill library in tests that don&#39;t use the</span>
+  <span class="c1"># full Python SDK.</span>
+  <span class="kn">import</span> <span class="nn">dill</span>
+
+
+<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span>
+    <span class="s1">&#39;Coder&#39;</span><span class="p">,</span>
+    <span class="s1">&#39;AvroGenericCoder&#39;</span><span class="p">,</span> <span class="s1">&#39;BooleanCoder&#39;</span><span class="p">,</span> <span class="s1">&#39;BytesCoder&#39;</span><span class="p">,</span> <span class="s1">&#39;DillCoder&#39;</span><span class="p">,</span>
+    <span class="s1">&#39;FastPrimitivesCoder&#39;</span><span class="p">,</span> <span class="s1">&#39;FloatCoder&#39;</span><span class="p">,</span> <span class="s1">&#39;IterableCoder&#39;</span><span class="p">,</span> <span class="s1">&#39;PickleCoder&#39;</span><span class="p">,</span>
+    <span class="s1">&#39;ProtoCoder&#39;</span><span class="p">,</span> <span class="s1">&#39;SingletonCoder&#39;</span><span class="p">,</span> <span class="s1">&#39;StrUtf8Coder&#39;</span><span class="p">,</span> <span class="s1">&#39;TimestampCoder&#39;</span><span class="p">,</span>
+    <span class="s1">&#39;TupleCoder&#39;</span><span class="p">,</span> <span class="s1">&#39;TupleSequenceCoder&#39;</span><span class="p">,</span> <span class="s1">&#39;VarIntCoder&#39;</span><span class="p">,</span>
+    <span class="s1">&#39;WindowedValueCoder&#39;</span><span class="p">,</span> <span class="s1">&#39;ParamWindowedValueCoder&#39;</span>
+<span class="p">]</span>
+
+<span class="n">T</span> <span class="o">=</span> <span class="n">TypeVar</span><span class="p">(</span><span class="s1">&#39;T&#39;</span><span class="p">)</span>
+<span class="n">CoderT</span> <span class="o">=</span> <span class="n">TypeVar</span><span class="p">(</span><span class="s1">&#39;CoderT&#39;</span><span class="p">,</span> <span class="n">bound</span><span class="o">=</span><span class="s1">&#39;Coder&#39;</span><span class="p">)</span>
+<span class="n">ProtoCoderT</span> <span class="o">=</span> <span class="n">TypeVar</span><span class="p">(</span><span class="s1">&#39;ProtoCoderT&#39;</span><span class="p">,</span> <span class="n">bound</span><span class="o">=</span><span class="s1">&#39;ProtoCoder&#39;</span><span class="p">)</span>
+<span class="n">ConstructorFn</span> <span class="o">=</span> <span class="n">Callable</span><span class="p">[</span>
+    <span class="p">[</span><span class="n">Optional</span><span class="p">[</span><span class="n">Any</span><span class="p">],</span>
+     <span class="n">List</span><span class="p">[</span><span class="s1">&#39;Coder&#39;</span><span class="p">],</span>
+     <span class="s1">&#39;PipelineContext&#39;</span><span class="p">],</span>
+    <span class="n">Any</span><span class="p">]</span>
+
+
+<span class="k">def</span> <span class="nf">serialize_coder</span><span class="p">(</span><span class="n">coder</span><span class="p">):</span>
+  <span class="kn">from</span> <span class="nn">apache_beam.internal</span> <span class="kn">import</span> <span class="n">pickler</span>
+  <span class="k">return</span> <span class="sa">b</span><span class="s1">&#39;</span><span class="si">%s</span><span class="s1">$</span><span class="si">%s</span><span class="s1">&#39;</span> <span class="o">%</span> <span class="p">(</span><span class="n">coder</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">&#39;utf-8&#39; [...]
+                     <span class="n">pickler</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">coder</span><span class="p">))</span>
+
+
+<span class="k">def</span> <span class="nf">deserialize_coder</span><span class="p">(</span><span class="n">serialized</span><span class="p">):</span>
+  <span class="kn">from</span> <span class="nn">apache_beam.internal</span> <span class="kn">import</span> <span class="n">pickler</span>
+  <span class="k">return</span> <span class="n">pickler</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">serialized</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="sa">b</span><span class="s1">&#39;$&#39;</span><span class="p">,</span> <span class="mi">1</span><span class="p">)[</span><span class="mi">1</span><span class="p">])</span>
+<span class="c1"># pylint: enable=wrong-import-order, wrong-import-position</span>
+
+
+<div class="viewcode-block" id="Coder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.Coder">[docs]</a><span class="k">class</span> <span class="nc">Coder</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;Base class for coders.&quot;&quot;&quot;</span>
+
+<div class="viewcode-block" id="Coder.encode"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.Coder.encode">[docs]</a>  <span class="k">def</span> <span class="nf">encode</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
+    <span class="c1"># type: (Any) -&gt; bytes</span>
+    <span class="sd">&quot;&quot;&quot;Encodes the given object into a byte string.&quot;&quot;&quot;</span>
+    <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s1">&#39;Encode not implemented: </span><span class="si">%s</span><span class="s1">.&#39;</span> <span class="o">%</span> <span class="bp">self</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="Coder.decode"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.Coder.decode">[docs]</a>  <span class="k">def</span> <span class="nf">decode</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">encoded</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;Decodes the given byte string into the corresponding object.&quot;&quot;&quot;</span>
+    <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s1">&#39;Decode not implemented: </span><span class="si">%s</span><span class="s1">.&#39;</span> <span class="o">%</span> <span class="bp">self</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="Coder.encode_nested"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.Coder.encode_nested">[docs]</a>  <span class="k">def</span> <span class="nf">encode_nested</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;Uses the underlying implementation to encode in nested format.&quot;&quot;&quot;</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_impl</span><span class="p">()</span><span class="o">.</span><span class="n">encode_nested</span><span class="p">(</span><span class="n">value</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="Coder.decode_nested"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.Coder.decode_nested">[docs]</a>  <span class="k">def</span> <span class="nf">decode_nested</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">encoded</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;Uses the underlying implementation to decode in nested format.&quot;&quot;&quot;</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_impl</span><span class="p">()</span><span class="o">.</span><span class="n">decode_nested</span><span class="p">(</span><span class="n">encoded</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="Coder.is_deterministic"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.Coder.is_deterministic">[docs]</a>  <span class="k">def</span> <span class="nf">is_deterministic</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># type: () -&gt; bool</span>
+    <span class="sd">&quot;&quot;&quot;Whether this coder is guaranteed to encode values deterministically.</span>
+
+<span class="sd">    A deterministic coder is required for key coders in GroupByKey operations</span>
+<span class="sd">    to produce consistent results.</span>
+
+<span class="sd">    For example, note that the default coder, the PickleCoder, is not</span>
+<span class="sd">    deterministic: the ordering of picked entries in maps may vary across</span>
+<span class="sd">    executions since there is no defined order, and such a coder is not in</span>
+<span class="sd">    general suitable for usage as a key coder in GroupByKey operations, since</span>
+<span class="sd">    each instance of the same key may be encoded differently.</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">      Whether coder is deterministic.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">return</span> <span class="kc">False</span></div>
+
+<div class="viewcode-block" id="Coder.as_deterministic_coder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.Coder.as_deterministic_coder">[docs]</a>  <span class="k">def</span> <span class="nf">as_deterministic_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">step_label</span><span class="p">,</span> <span class="n">error_message</span><span class="o">=</span><span class="kc">None</ [...]
+    <span class="sd">&quot;&quot;&quot;Returns a deterministic version of self, if possible.</span>
+
+<span class="sd">    Otherwise raises a value error.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_deterministic</span><span class="p">():</span>
+      <span class="k">return</span> <span class="bp">self</span>
+    <span class="k">else</span><span class="p">:</span>
+      <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="n">error_message</span> <span class="ow">or</span> <span class="s2">&quot;&#39;</span><span class="si">%s</span><span class="s2">&#39; cannot be made deterministic.&quot;</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="Coder.estimate_size"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.Coder.estimate_size">[docs]</a>  <span class="k">def</span> <span class="nf">estimate_size</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;Estimates the encoded size of the given value, in bytes.</span>
+
+<span class="sd">    Dataflow estimates the encoded size of a PCollection processed in a pipeline</span>
+<span class="sd">    step by using the estimated size of a random sample of elements in that</span>
+<span class="sd">    PCollection.</span>
+
+<span class="sd">    The default implementation encodes the given value and returns its byte</span>
+<span class="sd">    size.  If a coder can provide a fast estimate of the encoded size of a value</span>
+<span class="sd">    (e.g., if the encoding has a fixed size), it can provide its estimate here</span>
+<span class="sd">    to improve performance.</span>
+
+<span class="sd">    Arguments:</span>
+<span class="sd">      value: the value whose encoded size is to be estimated.</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">      The estimated encoded size of the given value.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="n">value</span><span class="p">))</span></div>
+
+  <span class="c1"># ===========================================================================</span>
+  <span class="c1"># Methods below are internal SDK details that don&#39;t need to be modified for</span>
+  <span class="c1"># user-defined coders.</span>
+  <span class="c1"># ===========================================================================</span>
+
+  <span class="k">def</span> <span class="nf">_create_impl</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># type: () -&gt; coder_impl.CoderImpl</span>
+    <span class="sd">&quot;&quot;&quot;Creates a CoderImpl to do the actual encoding and decoding.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">return</span> <span class="n">coder_impl</span><span class="o">.</span><span class="n">CallbackCoderImpl</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">encode</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">decode</span><span class="p">,</span>
+                                        <span class="bp">self</span><span class="o">.</span><span class="n">estimate_size</span><span class="p">)</span>
+
+<div class="viewcode-block" id="Coder.get_impl"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.Coder.get_impl">[docs]</a>  <span class="k">def</span> <span class="nf">get_impl</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;For internal use only; no backwards-compatibility guarantees.</span>
+
+<span class="sd">    Returns the CoderImpl backing this Coder.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s1">&#39;_impl&#39;</span><span class="p">):</span>
+      <span class="bp">self</span><span class="o">.</span><span class="n">_impl</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_create_impl</span><span class="p">()</span>
+      <span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_impl</span><span class="p">,</span> <span class="n">coder_impl</span><span class="o">.</span><span class="n">CoderImpl</span><span class="p">)</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_impl</span></div>
+
+  <span class="k">def</span> <span class="nf">__getstate__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_dict_without_impl</span><span class="p">()</span>
+
+  <span class="k">def</span> <span class="nf">_dict_without_impl</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s1">&#39;_impl&#39;</span><span class="p">):</span>
+      <span class="n">d</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">)</span>
+      <span class="k">del</span> <span class="n">d</span><span class="p">[</span><span class="s1">&#39;_impl&#39;</span><span class="p">]</span>
+      <span class="k">return</span> <span class="n">d</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span>
+
+<div class="viewcode-block" id="Coder.to_type_hint"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.Coder.to_type_hint">[docs]</a>  <span class="k">def</span> <span class="nf">to_type_hint</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s1">&#39;BEAM-2717&#39;</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="Coder.from_type_hint"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.Coder.from_type_hint">[docs]</a>  <span class="nd">@classmethod</span>
+  <span class="k">def</span> <span class="nf">from_type_hint</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">unused_typehint</span><span class="p">,</span> <span class="n">unused_registry</span><span class="p">):</span>
+    <span class="c1"># type: (Type[CoderT], Any, CoderRegistry) -&gt; CoderT</span>
+    <span class="c1"># If not overridden, just construct the coder without arguments.</span>
+    <span class="k">return</span> <span class="bp">cls</span><span class="p">()</span></div>
+
+<div class="viewcode-block" id="Coder.is_kv_coder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.Coder.is_kv_coder">[docs]</a>  <span class="k">def</span> <span class="nf">is_kv_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># () -&gt; bool</span>
+    <span class="k">return</span> <span class="kc">False</span></div>
+
+<div class="viewcode-block" id="Coder.key_coder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.Coder.key_coder">[docs]</a>  <span class="k">def</span> <span class="nf">key_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># type: () -&gt; Coder</span>
+    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_kv_coder</span><span class="p">():</span>
+      <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s1">&#39;key_coder: </span><span class="si">%s</span><span class="s1">&#39;</span> <span class="o">%</span> <span class="bp">self</span><span class="p">)</span>
+    <span class="k">else</span><span class="p">:</span>
+      <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;Not a KV coder: </span><span class="si">%s</span><span class="s1">.&#39;</span> <span class="o">%</span> <span class="bp">self</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="Coder.value_coder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.Coder.value_coder">[docs]</a>  <span class="k">def</span> <span class="nf">value_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># type: () -&gt; Coder</span>
+    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_kv_coder</span><span class="p">():</span>
+      <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s1">&#39;value_coder: </span><span class="si">%s</span><span class="s1">&#39;</span> <span class="o">%</span> <span class="bp">self</span><span class="p">)</span>
+    <span class="k">else</span><span class="p">:</span>
+      <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;Not a KV coder: </span><span class="si">%s</span><span class="s1">.&#39;</span> <span class="o">%</span> <span class="bp">self</span><span class="p">)</span></div>
+
+  <span class="k">def</span> <span class="nf">_get_component_coders</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># type: () -&gt; Sequence[Coder]</span>
+    <span class="sd">&quot;&quot;&quot;For internal use only; no backwards-compatibility guarantees.</span>
+
+<span class="sd">    Returns the internal component coders of this coder.&quot;&quot;&quot;</span>
+    <span class="c1"># This is an internal detail of the Coder API and does not need to be</span>
+    <span class="c1"># refined in user-defined Coders.</span>
+    <span class="k">return</span> <span class="p">[]</span>
+
+<div class="viewcode-block" id="Coder.as_cloud_object"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.Coder.as_cloud_object">[docs]</a>  <span class="k">def</span> <span class="nf">as_cloud_object</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">coders_context</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;For internal use only; no backwards-compatibility guarantees.</span>
+
+<span class="sd">    Returns Google Cloud Dataflow API description of this coder.&quot;&quot;&quot;</span>
+    <span class="c1"># This is an internal detail of the Coder API and does not need to be</span>
+    <span class="c1"># refined in user-defined Coders.</span>
+
+    <span class="n">value</span> <span class="o">=</span> <span class="p">{</span>
+        <span class="c1"># We pass coders in the form &quot;&lt;coder_name&gt;$&lt;pickled_data&gt;&quot; to make the</span>
+        <span class="c1"># job description JSON more readable.  Data before the $ is ignored by</span>
+        <span class="c1"># the worker.</span>
+        <span class="s1">&#39;@type&#39;</span><span class="p">:</span>
+            <span class="n">serialize_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">),</span>
+        <span class="s1">&#39;component_encodings&#39;</span><span class="p">:</span> <span class="p">[</span>
+            <span class="n">component</span><span class="o">.</span><span class="n">as_cloud_object</span><span class="p">(</span><span class="n">coders_context</span><span class="p">)</span>
+            <span class="k">for</span> <span class="n">component</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_component_coders</span><span class="p">()</span>
+        <span class="p">],</span>
+    <span class="p">}</span>
+
+    <span class="k">if</span> <span class="n">coders_context</span><span class="p">:</span>
+      <span class="n">value</span><span class="p">[</span><span class="s1">&#39;pipeline_proto_coder_id&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">coders_context</span><span class="o">.</span><span class="n">get_id</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
+
+    <span class="k">return</span> <span class="n">value</span></div>
+
+  <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span>
+
+  <span class="c1"># pylint: disable=protected-access</span>
+  <span class="k">def</span> <span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
+    <span class="k">return</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="vm">__class__</span>
+            <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_dict_without_impl</span><span class="p">()</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">_dict_without_impl</span><span class="p">())</span>
+  <span class="c1"># pylint: enable=protected-access</span>
+
+  <span class="k">def</span> <span class="fm">__ne__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
+    <span class="c1"># TODO(BEAM-5949): Needed for Python 2 compatibility.</span>
+    <span class="k">return</span> <span class="ow">not</span> <span class="bp">self</span> <span class="o">==</span> <span class="n">other</span>
+
+  <span class="k">def</span> <span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">))</span>
+
+  <span class="n">_known_urns</span> <span class="o">=</span> <span class="p">{}</span>  <span class="c1"># type: Dict[str, Tuple[type, ConstructorFn]]</span>
+
+  <span class="nd">@classmethod</span>
+  <span class="nd">@overload</span>
+  <span class="k">def</span> <span class="nf">register_urn</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span>
+                   <span class="n">urn</span><span class="p">,</span>  <span class="c1"># type: str</span>
+                   <span class="n">parameter_type</span><span class="p">,</span>  <span class="c1"># type: Optional[Type[T]]</span>
+                  <span class="p">):</span>
+    <span class="c1"># type: (...) -&gt; Callable[[Callable[[T, List[Coder], PipelineContext], Any]], Callable[[T, List[Coder], PipelineContext], Any]]</span>
+    <span class="k">pass</span>
+
+  <span class="nd">@classmethod</span>
+  <span class="nd">@overload</span>
+  <span class="k">def</span> <span class="nf">register_urn</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span>
+                   <span class="n">urn</span><span class="p">,</span>  <span class="c1"># type: str</span>
+                   <span class="n">parameter_type</span><span class="p">,</span>  <span class="c1"># type: Optional[Type[T]]</span>
+                   <span class="n">fn</span>  <span class="c1"># type: Callable[[T, List[Coder], PipelineContext], Any]</span>
+                  <span class="p">):</span>
+    <span class="c1"># type: (...) -&gt; None</span>
+    <span class="k">pass</span>
+
+<div class="viewcode-block" id="Coder.register_urn"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.Coder.register_urn">[docs]</a>  <span class="nd">@classmethod</span>
+  <span class="k">def</span> <span class="nf">register_urn</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">urn</span><span class="p">,</span> <span class="n">parameter_type</span><span class="p">,</span> <span class="n">fn</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;Registers a urn with a constructor.</span>
+
+<span class="sd">    For example, if &#39;beam:fn:foo&#39; had parameter type FooPayload, one could</span>
+<span class="sd">    write `RunnerApiFn.register_urn(&#39;bean:fn:foo&#39;, FooPayload, foo_from_proto)`</span>
+<span class="sd">    where foo_from_proto took as arguments a FooPayload and a PipelineContext.</span>
+<span class="sd">    This function can also be used as a decorator rather than passing the</span>
+<span class="sd">    callable in as the final parameter.</span>
+
+<span class="sd">    A corresponding to_runner_api_parameter method would be expected that</span>
+<span class="sd">    returns the tuple (&#39;beam:fn:foo&#39;, FooPayload)</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">def</span> <span class="nf">register</span><span class="p">(</span><span class="n">fn</span><span class="p">):</span>
+      <span class="bp">cls</span><span class="o">.</span><span class="n">_known_urns</span><span class="p">[</span><span class="n">urn</span><span class="p">]</span> <span class="o">=</span> <span class="n">parameter_type</span><span class="p">,</span> <span class="n">fn</span>
+      <span class="k">return</span> <span class="nb">staticmethod</span><span class="p">(</span><span class="n">fn</span><span class="p">)</span>
+    <span class="k">if</span> <span class="n">fn</span><span class="p">:</span>
+      <span class="c1"># Used as a statement.</span>
+      <span class="n">register</span><span class="p">(</span><span class="n">fn</span><span class="p">)</span>
+    <span class="k">else</span><span class="p">:</span>
+      <span class="c1"># Used as a decorator.</span>
+      <span class="k">return</span> <span class="n">register</span></div>
+
+<div class="viewcode-block" id="Coder.to_runner_api"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.Coder.to_runner_api">[docs]</a>  <span class="k">def</span> <span class="nf">to_runner_api</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+    <span class="c1"># type: (PipelineContext) -&gt; beam_runner_api_pb2.Coder</span>
+    <span class="n">urn</span><span class="p">,</span> <span class="n">typed_param</span><span class="p">,</span> <span class="n">components</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">to_runner_api_parameter</span><span class="p">(</span><span class="n">context</span><span class="p">)</span>
+    <span class="k">return</span> <span class="n">beam_runner_api_pb2</span><span class="o">.</span><span class="n">Coder</span><span class="p">(</span>
+        <span class="n">spec</span><span class="o">=</span><span class="n">beam_runner_api_pb2</span><span class="o">.</span><span class="n">FunctionSpec</span><span class="p">(</span>
+            <span class="n">urn</span><span class="o">=</span><span class="n">urn</span><span class="p">,</span>
+            <span class="n">payload</span><span class="o">=</span><span class="n">typed_param</span>
+            <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">typed_param</span><span class="p">,</span> <span class="p">(</span><span class="nb">bytes</span><span class="p">,</span> <span class="nb">type</span><span class="p">(</span><span class="kc">None</span><span class="p">)))</span>
+            <span class="k">else</span> <span class="n">typed_param</span><span class="o">.</span><span class="n">SerializeToString</span><span class="p">()),</span>
+        <span class="n">component_coder_ids</span><span class="o">=</span><span class="p">[</span><span class="n">context</span><span class="o">.</span><span class="n">coders</span><span class="o">.</span><span class="n">get_id</span><span class="p">(</span><span class="n">c</span><span class="p">)</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">components</span><span class="p">])</span></div>
+
+<div class="viewcode-block" id="Coder.from_runner_api"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.Coder.from_runner_api">[docs]</a>  <span class="nd">@classmethod</span>
+  <span class="k">def</span> <span class="nf">from_runner_api</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="n">coder_proto</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+    <span class="c1"># type: (Type[CoderT], beam_runner_api_pb2.Coder, PipelineContext) -&gt; CoderT</span>
+    <span class="sd">&quot;&quot;&quot;Converts from an FunctionSpec to a Fn object.</span>
+
+<span class="sd">    Prefer registering a urn with its parameter type and constructor.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">parameter_type</span><span class="p">,</span> <span class="n">constructor</span> <span class="o">=</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_known_urns</span><span class="p">[</span><span class="n">coder_proto</span><span class="o">.</span><span class="n">spec</span><span class="o">.</span><span class="n">urn</span><span class="p">]</span>
+    <span class="k">try</span><span class="p">:</span>
+      <span class="k">return</span> <span class="n">constructor</span><span class="p">(</span>
+          <span class="n">proto_utils</span><span class="o">.</span><span class="n">parse_Bytes</span><span class="p">(</span>
+              <span class="n">coder_proto</span><span class="o">.</span><span class="n">spec</span><span class="o">.</span><span class="n">payload</span><span class="p">,</span> <span class="n">parameter_type</span><span class="p">),</span>
+          <span class="p">[</span><span class="n">context</span><span class="o">.</span><span class="n">coders</span><span class="o">.</span><span class="n">get_by_id</span><span class="p">(</span><span class="n">c</span><span class="p">)</span>
+           <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">coder_proto</span><span class="o">.</span><span class="n">component_coder_ids</span><span class="p">],</span>
+          <span class="n">context</span><span class="p">)</span>
+    <span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>
+      <span class="k">if</span> <span class="n">context</span><span class="o">.</span><span class="n">allow_proto_holders</span><span class="p">:</span>
+        <span class="k">return</span> <span class="n">RunnerAPICoderHolder</span><span class="p">(</span><span class="n">coder_proto</span><span class="p">)</span>  <span class="c1"># type: ignore  # too ambiguous</span>
+      <span class="k">raise</span></div>
+
+<div class="viewcode-block" id="Coder.to_runner_api_parameter"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.Coder.to_runner_api_parameter">[docs]</a>  <span class="k">def</span> <span class="nf">to_runner_api_parameter</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+    <span class="c1"># type: (Optional[PipelineContext]) -&gt; Tuple[str, Any, Sequence[Coder]]</span>
+    <span class="k">return</span> <span class="p">(</span>
+        <span class="n">python_urns</span><span class="o">.</span><span class="n">PICKLED_CODER</span><span class="p">,</span>
+        <span class="n">google</span><span class="o">.</span><span class="n">protobuf</span><span class="o">.</span><span class="n">wrappers_pb2</span><span class="o">.</span><span class="n">BytesValue</span><span class="p">(</span><span class="n">value</span><span class="o">=</span><span class="n">serialize_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">)),</span>
+        <span class="p">())</span></div>
+
+<div class="viewcode-block" id="Coder.register_structured_urn"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.Coder.register_structured_urn">[docs]</a>  <span class="nd">@staticmethod</span>
+  <span class="k">def</span> <span class="nf">register_structured_urn</span><span class="p">(</span><span class="n">urn</span><span class="p">,</span> <span class="bp">cls</span><span class="p">):</span>
+    <span class="c1"># type: (str, Type[Coder]) -&gt; None</span>
+    <span class="sd">&quot;&quot;&quot;Register a coder that&#39;s completely defined by its urn and its</span>
+<span class="sd">    component(s), if any, which are passed to construct the instance.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="bp">cls</span><span class="o">.</span><span class="n">to_runner_api_parameter</span> <span class="o">=</span> <span class="p">(</span>
+        <span class="k">lambda</span> <span class="bp">self</span><span class="p">,</span> <span class="n">unused_context</span><span class="p">:</span> <span class="p">(</span><span class="n">urn</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_component_coders</span><span class="p">()))</span>
+
+    <span class="c1"># pylint: disable=unused-variable</span>
+    <span class="nd">@Coder</span><span class="o">.</span><span class="n">register_urn</span><span class="p">(</span><span class="n">urn</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>
+    <span class="k">def</span> <span class="nf">from_runner_api_parameter</span><span class="p">(</span><span class="n">unused_payload</span><span class="p">,</span> <span class="n">components</span><span class="p">,</span> <span class="n">unused_context</span><span class="p">):</span>
+      <span class="k">if</span> <span class="n">components</span><span class="p">:</span>
+        <span class="k">return</span> <span class="bp">cls</span><span class="p">(</span><span class="o">*</span><span class="n">components</span><span class="p">)</span>
+      <span class="k">else</span><span class="p">:</span>
+        <span class="k">return</span> <span class="bp">cls</span><span class="p">()</span></div></div>
+
+
+<span class="nd">@Coder</span><span class="o">.</span><span class="n">register_urn</span><span class="p">(</span>
+    <span class="n">python_urns</span><span class="o">.</span><span class="n">PICKLED_CODER</span><span class="p">,</span> <span class="n">google</span><span class="o">.</span><span class="n">protobuf</span><span class="o">.</span><span class="n">wrappers_pb2</span><span class="o">.</span><span class="n">BytesValue</span><span class="p">)</span>
+<span class="k">def</span> <span class="nf">_pickle_from_runner_api_parameter</span><span class="p">(</span><span class="n">payload</span><span class="p">,</span> <span class="n">components</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+  <span class="k">return</span> <span class="n">deserialize_coder</span><span class="p">(</span><span class="n">payload</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
+
+
+<div class="viewcode-block" id="StrUtf8Coder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.StrUtf8Coder">[docs]</a><span class="k">class</span> <span class="nc">StrUtf8Coder</span><span class="p">(</span><span class="n">Coder</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;A coder used for reading and writing strings as UTF-8.&quot;&quot;&quot;</span>
+
+<div class="viewcode-block" id="StrUtf8Coder.encode"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.StrUtf8Coder.encode">[docs]</a>  <span class="k">def</span> <span class="nf">encode</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">value</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="StrUtf8Coder.decode"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.StrUtf8Coder.decode">[docs]</a>  <span class="k">def</span> <span class="nf">decode</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">value</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="StrUtf8Coder.is_deterministic"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.StrUtf8Coder.is_deterministic">[docs]</a>  <span class="k">def</span> <span class="nf">is_deterministic</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="kc">True</span></div>
+
+<div class="viewcode-block" id="StrUtf8Coder.to_type_hint"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.StrUtf8Coder.to_type_hint">[docs]</a>  <span class="k">def</span> <span class="nf">to_type_hint</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">unicode</span></div></div>
+
+
+<span class="n">Coder</span><span class="o">.</span><span class="n">register_structured_urn</span><span class="p">(</span>
+    <span class="n">common_urns</span><span class="o">.</span><span class="n">coders</span><span class="o">.</span><span class="n">STRING_UTF8</span><span class="o">.</span><span class="n">urn</span><span class="p">,</span> <span class="n">StrUtf8Coder</span><span class="p">)</span>
+
+
+<span class="k">class</span> <span class="nc">ToStringCoder</span><span class="p">(</span><span class="n">Coder</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;A default string coder used if no sink coder is specified.&quot;&quot;&quot;</span>
+
+  <span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="o">.</span><span class="n">major</span> <span class="o">==</span> <span class="mi">2</span><span class="p">:</span>
+
+    <span class="k">def</span> <span class="nf">encode</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
+      <span class="c1"># pylint: disable=unicode-builtin</span>
+      <span class="k">return</span> <span class="p">(</span><span class="n">value</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="n">unicode</span><span class="p">)</span>  <span class="c1"># noqa: F821</span>
+              <span class="k">else</span> <span class="nb">str</span><span class="p">(</span><span class="n">value</span><span class="p">))</span>
+
+  <span class="k">else</span><span class="p">:</span>
+
+    <span class="k">def</span> <span class="nf">encode</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
+      <span class="k">return</span> <span class="n">value</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="nb">bytes</span><span class="p">)</span> <span class="k">else</span> <span class="nb">str</span><span class="p">(</span><span class="n">value</span><span class="p">)</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">&#39;utf-8& [...]
+
+  <span class="k">def</span> <span class="nf">decode</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">_</span><span class="p">):</span>
+    <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s1">&#39;ToStringCoder cannot be used for decoding.&#39;</span><span class="p">)</span>
+
+  <span class="k">def</span> <span class="nf">is_deterministic</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="kc">True</span>
+
+
+<span class="k">class</span> <span class="nc">FastCoder</span><span class="p">(</span><span class="n">Coder</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;Coder subclass used when a (faster) CoderImpl is supplied directly.</span>
+
+<span class="sd">  The Coder class defines _create_impl in terms of encode() and decode();</span>
+<span class="sd">  this class inverts that by defining encode() and decode() in terms of</span>
+<span class="sd">  _create_impl().</span>
+<span class="sd">  &quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="nf">encode</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;Encodes the given object into a byte string.&quot;&quot;&quot;</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_impl</span><span class="p">()</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
+
+  <span class="k">def</span> <span class="nf">decode</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">encoded</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;Decodes the given byte string into the corresponding object.&quot;&quot;&quot;</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_impl</span><span class="p">()</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">encoded</span><span class="p">)</span>
+
+  <span class="k">def</span> <span class="nf">estimate_size</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_impl</span><span class="p">()</span><span class="o">.</span><span class="n">estimate_size</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
+
+  <span class="k">def</span> <span class="nf">_create_impl</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">raise</span> <span class="ne">NotImplementedError</span>
+
+
+<div class="viewcode-block" id="BytesCoder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.BytesCoder">[docs]</a><span class="k">class</span> <span class="nc">BytesCoder</span><span class="p">(</span><span class="n">FastCoder</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;Byte string coder.&quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="nf">_create_impl</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">coder_impl</span><span class="o">.</span><span class="n">BytesCoderImpl</span><span class="p">()</span>
+
+<div class="viewcode-block" id="BytesCoder.is_deterministic"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.BytesCoder.is_deterministic">[docs]</a>  <span class="k">def</span> <span class="nf">is_deterministic</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># type: () -&gt; bool</span>
+    <span class="k">return</span> <span class="kc">True</span></div>
+
+<div class="viewcode-block" id="BytesCoder.to_type_hint"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.BytesCoder.to_type_hint">[docs]</a>  <span class="k">def</span> <span class="nf">to_type_hint</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">bytes</span></div>
+
+<div class="viewcode-block" id="BytesCoder.as_cloud_object"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.BytesCoder.as_cloud_object">[docs]</a>  <span class="k">def</span> <span class="nf">as_cloud_object</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">coders_context</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+    <span class="k">return</span> <span class="p">{</span>
+        <span class="s1">&#39;@type&#39;</span><span class="p">:</span> <span class="s1">&#39;kind:bytes&#39;</span><span class="p">,</span>
+    <span class="p">}</span></div>
+
+  <span class="k">def</span> <span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="nb">type</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
+
+  <span class="k">def</span> <span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">))</span></div>
+
+
+<span class="n">Coder</span><span class="o">.</span><span class="n">register_structured_urn</span><span class="p">(</span><span class="n">common_urns</span><span class="o">.</span><span class="n">coders</span><span class="o">.</span><span class="n">BYTES</span><span class="o">.</span><span class="n">urn</span><span class="p">,</span> <span class="n">BytesCoder</span><span class="p">)</span>
+
+
+<div class="viewcode-block" id="BooleanCoder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.BooleanCoder">[docs]</a><span class="k">class</span> <span class="nc">BooleanCoder</span><span class="p">(</span><span class="n">FastCoder</span><span class="p">):</span>
+  <span class="k">def</span> <span class="nf">_create_impl</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">coder_impl</span><span class="o">.</span><span class="n">BooleanCoderImpl</span><span class="p">()</span>
+
+<div class="viewcode-block" id="BooleanCoder.is_deterministic"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.BooleanCoder.is_deterministic">[docs]</a>  <span class="k">def</span> <span class="nf">is_deterministic</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="kc">True</span></div>
+
+<div class="viewcode-block" id="BooleanCoder.to_type_hint"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.BooleanCoder.to_type_hint">[docs]</a>  <span class="k">def</span> <span class="nf">to_type_hint</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">bool</span></div>
+
+  <span class="k">def</span> <span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="nb">type</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
+
+  <span class="k">def</span> <span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">))</span></div>
+
+
+<span class="n">Coder</span><span class="o">.</span><span class="n">register_structured_urn</span><span class="p">(</span><span class="n">common_urns</span><span class="o">.</span><span class="n">coders</span><span class="o">.</span><span class="n">BOOL</span><span class="o">.</span><span class="n">urn</span><span class="p">,</span> <span class="n">BooleanCoder</span><span class="p">)</span>
+
+
+<div class="viewcode-block" id="VarIntCoder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.VarIntCoder">[docs]</a><span class="k">class</span> <span class="nc">VarIntCoder</span><span class="p">(</span><span class="n">FastCoder</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;Variable-length integer coder.&quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="nf">_create_impl</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">coder_impl</span><span class="o">.</span><span class="n">VarIntCoderImpl</span><span class="p">()</span>
+
+<div class="viewcode-block" id="VarIntCoder.is_deterministic"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.VarIntCoder.is_deterministic">[docs]</a>  <span class="k">def</span> <span class="nf">is_deterministic</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># type: () -&gt; bool</span>
+    <span class="k">return</span> <span class="kc">True</span></div>
+
+<div class="viewcode-block" id="VarIntCoder.to_type_hint"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.VarIntCoder.to_type_hint">[docs]</a>  <span class="k">def</span> <span class="nf">to_type_hint</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">int</span></div>
+
+<div class="viewcode-block" id="VarIntCoder.as_cloud_object"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.VarIntCoder.as_cloud_object">[docs]</a>  <span class="k">def</span> <span class="nf">as_cloud_object</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">coders_context</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+    <span class="k">return</span> <span class="p">{</span>
+        <span class="s1">&#39;@type&#39;</span><span class="p">:</span> <span class="s1">&#39;kind:varint&#39;</span><span class="p">,</span>
+    <span class="p">}</span></div>
+
+  <span class="k">def</span> <span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="nb">type</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
+
+  <span class="k">def</span> <span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">))</span></div>
+
+
+<span class="n">Coder</span><span class="o">.</span><span class="n">register_structured_urn</span><span class="p">(</span><span class="n">common_urns</span><span class="o">.</span><span class="n">coders</span><span class="o">.</span><span class="n">VARINT</span><span class="o">.</span><span class="n">urn</span><span class="p">,</span> <span class="n">VarIntCoder</span><span class="p">)</span>
+
+
+<div class="viewcode-block" id="FloatCoder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.FloatCoder">[docs]</a><span class="k">class</span> <span class="nc">FloatCoder</span><span class="p">(</span><span class="n">FastCoder</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;A coder used for floating-point values.&quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="nf">_create_impl</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">coder_impl</span><span class="o">.</span><span class="n">FloatCoderImpl</span><span class="p">()</span>
+
+<div class="viewcode-block" id="FloatCoder.is_deterministic"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.FloatCoder.is_deterministic">[docs]</a>  <span class="k">def</span> <span class="nf">is_deterministic</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># type: () -&gt; bool</span>
+    <span class="k">return</span> <span class="kc">True</span></div>
+
+<div class="viewcode-block" id="FloatCoder.to_type_hint"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.FloatCoder.to_type_hint">[docs]</a>  <span class="k">def</span> <span class="nf">to_type_hint</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">float</span></div>
+
+  <span class="k">def</span> <span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="nb">type</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
+
+  <span class="k">def</span> <span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">))</span></div>
+
+
+<span class="n">Coder</span><span class="o">.</span><span class="n">register_structured_urn</span><span class="p">(</span><span class="n">common_urns</span><span class="o">.</span><span class="n">coders</span><span class="o">.</span><span class="n">DOUBLE</span><span class="o">.</span><span class="n">urn</span><span class="p">,</span> <span class="n">FloatCoder</span><span class="p">)</span>
+
+
+<div class="viewcode-block" id="TimestampCoder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.TimestampCoder">[docs]</a><span class="k">class</span> <span class="nc">TimestampCoder</span><span class="p">(</span><span class="n">FastCoder</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;A coder used for timeutil.Timestamp values.&quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="nf">_create_impl</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">coder_impl</span><span class="o">.</span><span class="n">TimestampCoderImpl</span><span class="p">()</span>
+
+<div class="viewcode-block" id="TimestampCoder.is_deterministic"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.TimestampCoder.is_deterministic">[docs]</a>  <span class="k">def</span> <span class="nf">is_deterministic</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># () -&gt; bool</span>
+    <span class="k">return</span> <span class="kc">True</span></div>
+
+  <span class="k">def</span> <span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="nb">type</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
+
+  <span class="k">def</span> <span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">))</span></div>
+
+
+<span class="k">class</span> <span class="nc">_TimerCoder</span><span class="p">(</span><span class="n">FastCoder</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;A coder used for timer values.</span>
+
+<span class="sd">  For internal use.&quot;&quot;&quot;</span>
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">payload_coder</span><span class="p">):</span>
+    <span class="c1"># type: (Coder) -&gt; None</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">_payload_coder</span> <span class="o">=</span> <span class="n">payload_coder</span>
+
+  <span class="k">def</span> <span class="nf">_get_component_coders</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># type: () -&gt; List[Coder]</span>
+    <span class="k">return</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_payload_coder</span><span class="p">]</span>
+
+  <span class="k">def</span> <span class="nf">_create_impl</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">coder_impl</span><span class="o">.</span><span class="n">TimerCoderImpl</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_payload_coder</span><span class="o">.</span><span class="n">get_impl</span><span class="p">())</span>
+
+  <span class="k">def</span> <span class="nf">is_deterministic</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># () -&gt; bool</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_payload_coder</span><span class="o">.</span><span class="n">is_deterministic</span><span class="p">()</span>
+
+  <span class="k">def</span> <span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
+    <span class="k">return</span> <span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="nb">type</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
+            <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_payload_coder</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">_payload_coder</span><span class="p">)</span>
+
+  <span class="k">def</span> <span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">))</span> <span class="o">+</span> <span class="nb">hash</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_payload_coder</span><span class="p">)</span>
+
+
+<span class="n">Coder</span><span class="o">.</span><span class="n">register_structured_urn</span><span class="p">(</span>
+    <span class="n">common_urns</span><span class="o">.</span><span class="n">coders</span><span class="o">.</span><span class="n">TIMER</span><span class="o">.</span><span class="n">urn</span><span class="p">,</span> <span class="n">_TimerCoder</span><span class="p">)</span>
+
+
+<div class="viewcode-block" id="SingletonCoder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.SingletonCoder">[docs]</a><span class="k">class</span> <span class="nc">SingletonCoder</span><span class="p">(</span><span class="n">FastCoder</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;A coder that always encodes exactly one value.&quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">_value</span> <span class="o">=</span> <span class="n">value</span>
+
+  <span class="k">def</span> <span class="nf">_create_impl</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">coder_impl</span><span class="o">.</span><span class="n">SingletonCoderImpl</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_value</span><span class="p">)</span>
+
+<div class="viewcode-block" id="SingletonCoder.is_deterministic"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.SingletonCoder.is_deterministic">[docs]</a>  <span class="k">def</span> <span class="nf">is_deterministic</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># () -&gt; bool</span>
+    <span class="k">return</span> <span class="kc">True</span></div>
+
+  <span class="k">def</span> <span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="nb">type</span><span class="p">(</span><span class="n">other</span><span class="p">)</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_value</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">_value</span>
+
+  <span class="k">def</span> <span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_value</span><span class="p">)</span></div>
+
+
+<span class="k">def</span> <span class="nf">maybe_dill_dumps</span><span class="p">(</span><span class="n">o</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;Pickle using cPickle or the Dill pickler as a fallback.&quot;&quot;&quot;</span>
+  <span class="c1"># We need to use the dill pickler for objects of certain custom classes,</span>
+  <span class="c1"># including, for example, ones that contain lambdas.</span>
+  <span class="k">try</span><span class="p">:</span>
+    <span class="k">return</span> <span class="n">pickle</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">o</span><span class="p">,</span> <span class="n">pickle</span><span class="o">.</span><span class="n">HIGHEST_PROTOCOL</span><span class="p">)</span>
+  <span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>  <span class="c1"># pylint: disable=broad-except</span>
+    <span class="k">return</span> <span class="n">dill</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">o</span><span class="p">)</span>
+
+
+<span class="k">def</span> <span class="nf">maybe_dill_loads</span><span class="p">(</span><span class="n">o</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;Unpickle using cPickle or the Dill pickler as a fallback.&quot;&quot;&quot;</span>
+  <span class="k">try</span><span class="p">:</span>
+    <span class="k">return</span> <span class="n">pickle</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">o</span><span class="p">)</span>
+  <span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>  <span class="c1"># pylint: disable=broad-except</span>
+    <span class="k">return</span> <span class="n">dill</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">o</span><span class="p">)</span>
+
+
+<span class="k">class</span> <span class="nc">_PickleCoderBase</span><span class="p">(</span><span class="n">FastCoder</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;Base class for pickling coders.&quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="nf">is_deterministic</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># () -&gt; bool</span>
+    <span class="c1"># Note that the default coder, the PickleCoder, is not deterministic (for</span>
+    <span class="c1"># example, the ordering of picked entries in maps may vary across</span>
+    <span class="c1"># executions), and so is not in general suitable for usage as a key coder in</span>
+    <span class="c1"># GroupByKey operations.</span>
+    <span class="k">return</span> <span class="kc">False</span>
+
+  <span class="k">def</span> <span class="nf">as_cloud_object</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">coders_context</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">is_pair_like</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
+    <span class="n">value</span> <span class="o">=</span> <span class="nb">super</span><span class="p">(</span><span class="n">_PickleCoderBase</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">as_cloud_object</span><span class="p">(</span><span class="n">coders_context</span><span class="p">)</span>
+    <span class="c1"># We currently use this coder in places where we cannot infer the coder to</span>
+    <span class="c1"># use for the value type in a more granular way.  In places where the</span>
+    <span class="c1"># service expects a pair, it checks for the &quot;is_pair_like&quot; key, in which</span>
+    <span class="c1"># case we would fail without the hack below.</span>
+    <span class="k">if</span> <span class="n">is_pair_like</span><span class="p">:</span>
+      <span class="n">value</span><span class="p">[</span><span class="s1">&#39;is_pair_like&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
+      <span class="n">value</span><span class="p">[</span><span class="s1">&#39;component_encodings&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span>
+          <span class="bp">self</span><span class="o">.</span><span class="n">as_cloud_object</span><span class="p">(</span><span class="n">coders_context</span><span class="p">,</span> <span class="n">is_pair_like</span><span class="o">=</span><span class="kc">False</span><span class="p">),</span>
+          <span class="bp">self</span><span class="o">.</span><span class="n">as_cloud_object</span><span class="p">(</span><span class="n">coders_context</span><span class="p">,</span> <span class="n">is_pair_like</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
+      <span class="p">]</span>
+
+    <span class="k">return</span> <span class="n">value</span>
+
+  <span class="c1"># We allow .key_coder() and .value_coder() to be called on PickleCoder since</span>
+  <span class="c1"># we can&#39;t always infer the return values of lambdas in ParDo operations, the</span>
+  <span class="c1"># result of which may be used in a GroupBykey.</span>
+  <span class="k">def</span> <span class="nf">is_kv_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># () -&gt; bool</span>
+    <span class="k">return</span> <span class="kc">True</span>
+
+  <span class="k">def</span> <span class="nf">key_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="bp">self</span>
+
+  <span class="k">def</span> <span class="nf">value_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="bp">self</span>
+
+  <span class="k">def</span> <span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="nb">type</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
+
+  <span class="k">def</span> <span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">))</span>
+
+
+<div class="viewcode-block" id="PickleCoder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.PickleCoder">[docs]</a><span class="k">class</span> <span class="nc">PickleCoder</span><span class="p">(</span><span class="n">_PickleCoderBase</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;Coder using Python&#39;s pickle functionality.&quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="nf">_create_impl</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="n">dumps</span> <span class="o">=</span> <span class="n">pickle</span><span class="o">.</span><span class="n">dumps</span>
+    <span class="n">HIGHEST_PROTOCOL</span> <span class="o">=</span> <span class="n">pickle</span><span class="o">.</span><span class="n">HIGHEST_PROTOCOL</span>
+    <span class="k">return</span> <span class="n">coder_impl</span><span class="o">.</span><span class="n">CallbackCoderImpl</span><span class="p">(</span>
+        <span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">dumps</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">HIGHEST_PROTOCOL</span><span class="p">),</span> <span class="n">pickle</span><span class="o">.</span><span class="n">loads</span><span class="p">)</span>
+
+<div class="viewcode-block" id="PickleCoder.as_deterministic_coder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.PickleCoder.as_deterministic_coder">[docs]</a>  <span class="k">def</span> <span class="nf">as_deterministic_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">step_label</span><span class="p">,</span> <span class="n">error_message</span><span class="o">=</span><span class [...]
+    <span class="k">return</span> <span class="n">DeterministicFastPrimitivesCoder</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">step_label</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="PickleCoder.to_type_hint"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.PickleCoder.to_type_hint">[docs]</a>  <span class="k">def</span> <span class="nf">to_type_hint</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">Any</span></div></div>
+
+
+<div class="viewcode-block" id="DillCoder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.DillCoder">[docs]</a><span class="k">class</span> <span class="nc">DillCoder</span><span class="p">(</span><span class="n">_PickleCoderBase</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;Coder using dill&#39;s pickle functionality.&quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="nf">_create_impl</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">coder_impl</span><span class="o">.</span><span class="n">CallbackCoderImpl</span><span class="p">(</span><span class="n">maybe_dill_dumps</span><span class="p">,</span> <span class="n">maybe_dill_loads</span><span class="p">)</span></div>
+
+
+<span class="k">class</span> <span class="nc">DeterministicFastPrimitivesCoder</span><span class="p">(</span><span class="n">FastCoder</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;Throws runtime errors when encoding non-deterministic values.&quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">coder</span><span class="p">,</span> <span class="n">step_label</span><span class="p">):</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">_underlying_coder</span> <span class="o">=</span> <span class="n">coder</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">_step_label</span> <span class="o">=</span> <span class="n">step_label</span>
+
+  <span class="k">def</span> <span class="nf">_create_impl</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">coder_impl</span><span class="o">.</span><span class="n">DeterministicFastPrimitivesCoderImpl</span><span class="p">(</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_underlying_coder</span><span class="o">.</span><span class="n">get_impl</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">_step_label</span><span class="p">)</span>
+
+  <span class="k">def</span> <span class="nf">is_deterministic</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># () -&gt; bool</span>
+    <span class="k">return</span> <span class="kc">True</span>
+
+  <span class="k">def</span> <span class="nf">is_kv_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># () -&gt; bool</span>
+    <span class="k">return</span> <span class="kc">True</span>
+
+  <span class="k">def</span> <span class="nf">key_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="bp">self</span>
+
+  <span class="k">def</span> <span class="nf">value_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="bp">self</span>
+
+  <span class="k">def</span> <span class="nf">to_type_hint</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">Any</span>
+
+
+<div class="viewcode-block" id="FastPrimitivesCoder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.FastPrimitivesCoder">[docs]</a><span class="k">class</span> <span class="nc">FastPrimitivesCoder</span><span class="p">(</span><span class="n">FastCoder</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;Encodes simple primitives (e.g. str, int) efficiently.</span>
+
+<span class="sd">  For unknown types, falls back to another coder (e.g. PickleCoder).</span>
+<span class="sd">  &quot;&quot;&quot;</span>
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">fallback_coder</span><span class="o">=</span><span class="n">PickleCoder</span><span class="p">()):</span>
+    <span class="c1"># type: (Coder) -&gt; None</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">_fallback_coder</span> <span class="o">=</span> <span class="n">fallback_coder</span>
+
+  <span class="k">def</span> <span class="nf">_create_impl</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">coder_impl</span><span class="o">.</span><span class="n">FastPrimitivesCoderImpl</span><span class="p">(</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_fallback_coder</span><span class="o">.</span><span class="n">get_impl</span><span class="p">())</span>
+
+<div class="viewcode-block" id="FastPrimitivesCoder.is_deterministic"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.FastPrimitivesCoder.is_deterministic">[docs]</a>  <span class="k">def</span> <span class="nf">is_deterministic</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># () -&gt; bool</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_fallback_coder</span><span class="o">.</span><span class="n">is_deterministic</span><span class="p">()</span></div>
+
+<div class="viewcode-block" id="FastPrimitivesCoder.as_deterministic_coder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.FastPrimitivesCoder.as_deterministic_coder">[docs]</a>  <span class="k">def</span> <span class="nf">as_deterministic_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">step_label</span><span class="p">,</span> <span class="n">error_message</span><span class="o">=</ [...]
+    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_deterministic</span><span class="p">():</span>
+      <span class="k">return</span> <span class="bp">self</span>
+    <span class="k">else</span><span class="p">:</span>
+      <span class="k">return</span> <span class="n">DeterministicFastPrimitivesCoder</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">step_label</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="FastPrimitivesCoder.to_type_hint"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.FastPrimitivesCoder.to_type_hint">[docs]</a>  <span class="k">def</span> <span class="nf">to_type_hint</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">Any</span></div>
+
+<div class="viewcode-block" id="FastPrimitivesCoder.as_cloud_object"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.FastPrimitivesCoder.as_cloud_object">[docs]</a>  <span class="k">def</span> <span class="nf">as_cloud_object</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">coders_context</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">is_p [...]
+    <span class="n">value</span> <span class="o">=</span> <span class="nb">super</span><span class="p">(</span><span class="n">FastCoder</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">as_cloud_object</span><span class="p">(</span><span class="n">coders_context</span><span class="p">)</span>
+    <span class="c1"># We currently use this coder in places where we cannot infer the coder to</span>
+    <span class="c1"># use for the value type in a more granular way.  In places where the</span>
+    <span class="c1"># service expects a pair, it checks for the &quot;is_pair_like&quot; key, in which</span>
+    <span class="c1"># case we would fail without the hack below.</span>
+    <span class="k">if</span> <span class="n">is_pair_like</span><span class="p">:</span>
+      <span class="n">value</span><span class="p">[</span><span class="s1">&#39;is_pair_like&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>
+      <span class="n">value</span><span class="p">[</span><span class="s1">&#39;component_encodings&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span>
+          <span class="bp">self</span><span class="o">.</span><span class="n">as_cloud_object</span><span class="p">(</span><span class="n">coders_context</span><span class="p">,</span> <span class="n">is_pair_like</span><span class="o">=</span><span class="kc">False</span><span class="p">),</span>
+          <span class="bp">self</span><span class="o">.</span><span class="n">as_cloud_object</span><span class="p">(</span><span class="n">coders_context</span><span class="p">,</span> <span class="n">is_pair_like</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
+      <span class="p">]</span>
+
+    <span class="k">return</span> <span class="n">value</span></div>
+
+  <span class="c1"># We allow .key_coder() and .value_coder() to be called on FastPrimitivesCoder</span>
+  <span class="c1"># since we can&#39;t always infer the return values of lambdas in ParDo</span>
+  <span class="c1"># operations, the result of which may be used in a GroupBykey.</span>
+<div class="viewcode-block" id="FastPrimitivesCoder.is_kv_coder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.FastPrimitivesCoder.is_kv_coder">[docs]</a>  <span class="k">def</span> <span class="nf">is_kv_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># () -&gt; bool</span>
+    <span class="k">return</span> <span class="kc">True</span></div>
+
+<div class="viewcode-block" id="FastPrimitivesCoder.key_coder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.FastPrimitivesCoder.key_coder">[docs]</a>  <span class="k">def</span> <span class="nf">key_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="bp">self</span></div>
+
+<div class="viewcode-block" id="FastPrimitivesCoder.value_coder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.FastPrimitivesCoder.value_coder">[docs]</a>  <span class="k">def</span> <span class="nf">value_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="bp">self</span></div>
+
+  <span class="k">def</span> <span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="nb">type</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
+
+  <span class="k">def</span> <span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">))</span></div>
+
+
+<span class="k">class</span> <span class="nc">Base64PickleCoder</span><span class="p">(</span><span class="n">Coder</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;Coder of objects by Python pickle, then base64 encoding.&quot;&quot;&quot;</span>
+  <span class="c1"># TODO(robertwb): Do base64 encoding where it&#39;s needed (e.g. in json) rather</span>
+  <span class="c1"># than via a special Coder.</span>
+
+  <span class="k">def</span> <span class="nf">encode</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">base64</span><span class="o">.</span><span class="n">b64encode</span><span class="p">(</span><span class="n">pickle</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="n">pickle</span><span class="o">.</span><span class="n">HIGHEST_PROTOCOL</span><span class="p">))</span>
+
+  <span class="k">def</span> <span class="nf">decode</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">encoded</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">pickle</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">base64</span><span class="o">.</span><span class="n">b64decode</span><span class="p">(</span><span class="n">encoded</span><span class="p">))</span>
+
+  <span class="k">def</span> <span class="nf">is_deterministic</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># () -&gt; bool</span>
+    <span class="c1"># Note that the Base64PickleCoder is not deterministic.  See the</span>
+    <span class="c1"># corresponding comments for PickleCoder above.</span>
+    <span class="k">return</span> <span class="kc">False</span>
+
+  <span class="c1"># We allow .key_coder() and .value_coder() to be called on Base64PickleCoder</span>
+  <span class="c1"># since we can&#39;t always infer the return values of lambdas in ParDo</span>
+  <span class="c1"># operations, the result of which may be used in a GroupBykey.</span>
+  <span class="c1">#</span>
+  <span class="c1"># TODO(ccy): this is currently only used for KV values from Create transforms.</span>
+  <span class="c1"># Investigate a way to unify this with PickleCoder.</span>
+  <span class="k">def</span> <span class="nf">is_kv_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="kc">True</span>
+
+  <span class="k">def</span> <span class="nf">key_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="bp">self</span>
+
+  <span class="k">def</span> <span class="nf">value_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="bp">self</span>
+
+
+<div class="viewcode-block" id="ProtoCoder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.ProtoCoder">[docs]</a><span class="k">class</span> <span class="nc">ProtoCoder</span><span class="p">(</span><span class="n">FastCoder</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;A Coder for Google Protocol Buffers.</span>
+
+<span class="sd">  It supports both Protocol Buffers syntax versions 2 and 3. However,</span>
+<span class="sd">  the runtime version of the python protobuf library must exactly match the</span>
+<span class="sd">  version of the protoc compiler what was used to generate the protobuf</span>
+<span class="sd">  messages.</span>
+
+<span class="sd">  ProtoCoder is registered in the global CoderRegistry as the default coder for</span>
+<span class="sd">  any protobuf Message object.</span>
+
+<span class="sd">  &quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">proto_message_type</span><span class="p">):</span>
+    <span class="c1"># type: (google.protobuf.message.Message) -&gt; None</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">proto_message_type</span> <span class="o">=</span> <span class="n">proto_message_type</span>
+
+  <span class="k">def</span> <span class="nf">_create_impl</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">coder_impl</span><span class="o">.</span><span class="n">ProtoCoderImpl</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">proto_message_type</span><span class="p">)</span>
+
+<div class="viewcode-block" id="ProtoCoder.is_deterministic"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.ProtoCoder.is_deterministic">[docs]</a>  <span class="k">def</span> <span class="nf">is_deterministic</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># () -&gt; bool</span>
+    <span class="c1"># TODO(vikasrk): A proto message can be deterministic if it does not contain</span>
+    <span class="c1"># a Map.</span>
+    <span class="k">return</span> <span class="kc">False</span></div>
+
+<div class="viewcode-block" id="ProtoCoder.as_deterministic_coder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.ProtoCoder.as_deterministic_coder">[docs]</a>  <span class="k">def</span> <span class="nf">as_deterministic_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">step_label</span><span class="p">,</span> <span class="n">error_message</span><span class="o">=</span><span class=" [...]
+    <span class="k">return</span> <span class="n">DeterministicProtoCoder</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">proto_message_type</span><span class="p">)</span></div>
+
+  <span class="k">def</span> <span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
+    <span class="k">return</span> <span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="nb">type</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
+            <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">proto_message_type</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">proto_message_type</span><span class="p">)</span>
+
+  <span class="k">def</span> <span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">proto_message_type</span><span class="p">)</span>
+
+<div class="viewcode-block" id="ProtoCoder.from_type_hint"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.ProtoCoder.from_type_hint">[docs]</a>  <span class="nd">@staticmethod</span>
+  <span class="k">def</span> <span class="nf">from_type_hint</span><span class="p">(</span><span class="n">typehint</span><span class="p">,</span> <span class="n">unused_registry</span><span class="p">):</span>
+    <span class="k">if</span> <span class="nb">issubclass</span><span class="p">(</span><span class="n">typehint</span><span class="p">,</span> <span class="n">google</span><span class="o">.</span><span class="n">protobuf</span><span class="o">.</span><span class="n">message</span><span class="o">.</span><span class="n">Message</span><span class="p">):</span>
+      <span class="k">return</span> <span class="n">ProtoCoder</span><span class="p">(</span><span class="n">typehint</span><span class="p">)</span>
+    <span class="k">else</span><span class="p">:</span>
+      <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">((</span><span class="s1">&#39;Expected a subclass of google.protobuf.message.Message&#39;</span>
+                        <span class="s1">&#39;, but got a </span><span class="si">%s</span><span class="s1">&#39;</span> <span class="o">%</span> <span class="n">typehint</span><span class="p">))</span></div></div>
+
+
+<span class="k">class</span> <span class="nc">DeterministicProtoCoder</span><span class="p">(</span><span class="n">ProtoCoder</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;A deterministic Coder for Google Protocol Buffers.</span>
+
+<span class="sd">  It supports both Protocol Buffers syntax versions 2 and 3. However,</span>
+<span class="sd">  the runtime version of the python protobuf library must exactly match the</span>
+<span class="sd">  version of the protoc compiler what was used to generate the protobuf</span>
+<span class="sd">  messages.</span>
+<span class="sd">  &quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="nf">_create_impl</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">coder_impl</span><span class="o">.</span><span class="n">DeterministicProtoCoderImpl</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">proto_message_type</span><span class="p">)</span>
+
+  <span class="k">def</span> <span class="nf">is_deterministic</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># () -&gt; bool</span>
+    <span class="k">return</span> <span class="kc">True</span>
+
+  <span class="k">def</span> <span class="nf">as_deterministic_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">step_label</span><span class="p">,</span> <span class="n">error_message</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+    <span class="k">return</span> <span class="bp">self</span>
+
+
+<span class="n">AVRO_GENERIC_CODER_URN</span> <span class="o">=</span> <span class="s2">&quot;beam:coder:avro:generic:v1&quot;</span>
+
+
+<div class="viewcode-block" id="AvroGenericCoder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.AvroGenericCoder">[docs]</a><span class="k">class</span> <span class="nc">AvroGenericCoder</span><span class="p">(</span><span class="n">FastCoder</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;A coder used for AvroRecord values.&quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">schema</span><span class="p">):</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">schema</span> <span class="o">=</span> <span class="n">schema</span>
+
+  <span class="k">def</span> <span class="nf">_create_impl</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">coder_impl</span><span class="o">.</span><span class="n">AvroCoderImpl</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">)</span>
+
+<div class="viewcode-block" id="AvroGenericCoder.is_deterministic"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.AvroGenericCoder.is_deterministic">[docs]</a>  <span class="k">def</span> <span class="nf">is_deterministic</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># TODO(BEAM-7903): need to confirm if it&#39;s deterministic</span>
+    <span class="k">return</span> <span class="kc">False</span></div>
+
+  <span class="k">def</span> <span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
+    <span class="k">return</span> <span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="nb">type</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
+            <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">schema</span><span class="p">)</span>
+
+  <span class="k">def</span> <span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">)</span>
+
+<div class="viewcode-block" id="AvroGenericCoder.to_type_hint"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.AvroGenericCoder.to_type_hint">[docs]</a>  <span class="k">def</span> <span class="nf">to_type_hint</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">AvroRecord</span></div>
+
+<div class="viewcode-block" id="AvroGenericCoder.to_runner_api_parameter"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.AvroGenericCoder.to_runner_api_parameter">[docs]</a>  <span class="k">def</span> <span class="nf">to_runner_api_parameter</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">AVRO_GENERIC_CODER_URN</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> <span class="p">()</span></div>
+
+<div class="viewcode-block" id="AvroGenericCoder.from_runner_api_parameter"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.AvroGenericCoder.from_runner_api_parameter">[docs]</a>  <span class="nd">@Coder</span><span class="o">.</span><span class="n">register_urn</span><span class="p">(</span><span class="n">AVRO_GENERIC_CODER_URN</span><span class="p">,</span> <span class="nb">bytes</span><span class="p">)</span>
+  <span class="k">def</span> <span class="nf">from_runner_api_parameter</span><span class="p">(</span><span class="n">payload</span><span class="p">,</span> <span class="n">unused_components</span><span class="p">,</span> <span class="n">unused_context</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">AvroGenericCoder</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span></div></div>
+
+
+<div class="viewcode-block" id="TupleCoder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.TupleCoder">[docs]</a><span class="k">class</span> <span class="nc">TupleCoder</span><span class="p">(</span><span class="n">FastCoder</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;Coder of tuple objects.&quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">components</span><span class="p">):</span>
+    <span class="c1"># type: (Iterable[Coder]) -&gt; None</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">_coders</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">components</span><span class="p">)</span>
+
+  <span class="k">def</span> <span class="nf">_create_impl</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">coder_impl</span><span class="o">.</span><span class="n">TupleCoderImpl</span><span class="p">([</span><span class="n">c</span><span class="o">.</span><span class="n">get_impl</span><span class="p">()</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_coders</span><span class="p">])</span>
+
+<div class="viewcode-block" id="TupleCoder.is_deterministic"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.TupleCoder.is_deterministic">[docs]</a>  <span class="k">def</span> <span class="nf">is_deterministic</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># () -&gt; bool</span>
+    <span class="k">return</span> <span class="nb">all</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">is_deterministic</span><span class="p">()</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_coders</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="TupleCoder.as_deterministic_coder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.TupleCoder.as_deterministic_coder">[docs]</a>  <span class="k">def</span> <span class="nf">as_deterministic_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">step_label</span><span class="p">,</span> <span class="n">error_message</span><span class="o">=</span><span class=" [...]
+    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_deterministic</span><span class="p">():</span>
+      <span class="k">return</span> <span class="bp">self</span>
+    <span class="k">else</span><span class="p">:</span>
+      <span class="k">return</span> <span class="n">TupleCoder</span><span class="p">([</span><span class="n">c</span><span class="o">.</span><span class="n">as_deterministic_coder</span><span class="p">(</span><span class="n">step_label</span><span class="p">,</span> <span class="n">error_message</span><span class="p">)</span>
+                         <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_coders</span><span class="p">])</span></div>
+
+<div class="viewcode-block" id="TupleCoder.to_type_hint"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.TupleCoder.to_type_hint">[docs]</a>  <span class="k">def</span> <span class="nf">to_type_hint</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">typehints</span><span class="o">.</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">tuple</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">to_type_hint</span><span class="p">()</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_coders</span><span class="p">)]</span></div>
+
+<div class="viewcode-block" id="TupleCoder.from_type_hint"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.TupleCoder.from_type_hint">[docs]</a>  <span class="nd">@staticmethod</span>
+  <span class="k">def</span> <span class="nf">from_type_hint</span><span class="p">(</span><span class="n">typehint</span><span class="p">,</span> <span class="n">registry</span><span class="p">):</span>
+    <span class="c1"># type: (typehints.TupleConstraint, CoderRegistry) -&gt; TupleCoder</span>
+    <span class="k">return</span> <span class="n">TupleCoder</span><span class="p">([</span><span class="n">registry</span><span class="o">.</span><span class="n">get_coder</span><span class="p">(</span><span class="n">t</span><span class="p">)</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">typehint</span><span class="o">.</span><span class="n">tuple_types</span><span class="p">])</span></div>
+
+<div class="viewcode-block" id="TupleCoder.as_cloud_object"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.TupleCoder.as_cloud_object">[docs]</a>  <span class="k">def</span> <span class="nf">as_cloud_object</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">coders_context</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_kv_coder</span><span class="p">():</span>
+      <span class="k">return</span> <span class="p">{</span>
+          <span class="s1">&#39;@type&#39;</span><span class="p">:</span>
+              <span class="s1">&#39;kind:pair&#39;</span><span class="p">,</span>
+          <span class="s1">&#39;is_pair_like&#39;</span><span class="p">:</span>
+              <span class="kc">True</span><span class="p">,</span>
+          <span class="s1">&#39;component_encodings&#39;</span><span class="p">:</span> <span class="p">[</span>
+              <span class="n">component</span><span class="o">.</span><span class="n">as_cloud_object</span><span class="p">(</span><span class="n">coders_context</span><span class="p">)</span>
+              <span class="k">for</span> <span class="n">component</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_component_coders</span><span class="p">()</span>
+          <span class="p">],</span>
+      <span class="p">}</span>
+
+    <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">TupleCoder</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">as_cloud_object</span><span class="p">(</span><span class="n">coders_context</span><span class="p">)</span></div>
+
+  <span class="k">def</span> <span class="nf">_get_component_coders</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># type: () -&gt; Tuple[Coder, ...]</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">coders</span><span class="p">()</span>
+
+<div class="viewcode-block" id="TupleCoder.coders"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.TupleCoder.coders">[docs]</a>  <span class="k">def</span> <span class="nf">coders</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># type: () -&gt; Tuple[Coder, ...]</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_coders</span></div>
+
+<div class="viewcode-block" id="TupleCoder.is_kv_coder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.TupleCoder.is_kv_coder">[docs]</a>  <span class="k">def</span> <span class="nf">is_kv_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># () -&gt; bool</span>
+    <span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_coders</span><span class="p">)</span> <span class="o">==</span> <span class="mi">2</span></div>
+
+<div class="viewcode-block" id="TupleCoder.key_coder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.TupleCoder.key_coder">[docs]</a>  <span class="k">def</span> <span class="nf">key_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># type: () -&gt; Coder</span>
+    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_coders</span><span class="p">)</span> <span class="o">!=</span> <span class="mi">2</span><span class="p">:</span>
+      <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;TupleCoder does not have exactly 2 components.&#39;</span><span class="p">)</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_coders</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span></div>
+
+<div class="viewcode-block" id="TupleCoder.value_coder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.TupleCoder.value_coder">[docs]</a>  <span class="k">def</span> <span class="nf">value_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># type: () -&gt; Coder</span>
+    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_coders</span><span class="p">)</span> <span class="o">!=</span> <span class="mi">2</span><span class="p">:</span>
+      <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;TupleCoder does not have exactly 2 components.&#39;</span><span class="p">)</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_coders</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span></div>
+
+  <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="s1">&#39;TupleCoder[</span><span class="si">%s</span><span class="s1">]&#39;</span> <span class="o">%</span> <span class="s1">&#39;, &#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">c</span><span class="p">)</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.< [...]
+
+  <span class="k">def</span> <span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
+    <span class="k">return</span> <span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="nb">type</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
+            <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_coders</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">coders</span><span class="p">())</span>
+
+  <span class="k">def</span> <span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_coders</span><span class="p">)</span>
+
+<div class="viewcode-block" id="TupleCoder.to_runner_api_parameter"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.TupleCoder.to_runner_api_parameter">[docs]</a>  <span class="k">def</span> <span class="nf">to_runner_api_parameter</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_kv_coder</span><span class="p">():</span>
+      <span class="k">return</span> <span class="n">common_urns</span><span class="o">.</span><span class="n">coders</span><span class="o">.</span><span class="n">KV</span><span class="o">.</span><span class="n">urn</span><span class="p">,</span> <span class="kc">None</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">coders</span><span class="p">()</span>
+    <span class="k">else</span><span class="p">:</span>
+      <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">TupleCoder</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">to_runner_api_parameter</span><span class="p">(</span><span class="n">context</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="TupleCoder.from_runner_api_parameter"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.TupleCoder.from_runner_api_parameter">[docs]</a>  <span class="nd">@Coder</span><span class="o">.</span><span class="n">register_urn</span><span class="p">(</span><span class="n">common_urns</span><span class="o">.</span><span class="n">coders</span><span class="o">.</span><span class="n">KV</span><span class="o">.</span><s [...]
+  <span class="k">def</span> <span class="nf">from_runner_api_parameter</span><span class="p">(</span><span class="n">unused_payload</span><span class="p">,</span> <span class="n">components</span><span class="p">,</span> <span class="n">unused_context</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">TupleCoder</span><span class="p">(</span><span class="n">components</span><span class="p">)</span></div></div>
+
+
+<div class="viewcode-block" id="TupleSequenceCoder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.TupleSequenceCoder">[docs]</a><span class="k">class</span> <span class="nc">TupleSequenceCoder</span><span class="p">(</span><span class="n">FastCoder</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;Coder of homogeneous tuple objects.&quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">elem_coder</span><span class="p">):</span>
+    <span class="c1"># type: (Coder) -&gt; None</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">_elem_coder</span> <span class="o">=</span> <span class="n">elem_coder</span>
+
+<div class="viewcode-block" id="TupleSequenceCoder.value_coder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.TupleSequenceCoder.value_coder">[docs]</a>  <span class="k">def</span> <span class="nf">value_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_elem_coder</span></div>
+
+  <span class="k">def</span> <span class="nf">_create_impl</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">coder_impl</span><span class="o">.</span><span class="n">TupleSequenceCoderImpl</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_elem_coder</span><span class="o">.</span><span class="n">get_impl</span><span class="p">())</span>
+
+<div class="viewcode-block" id="TupleSequenceCoder.is_deterministic"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.TupleSequenceCoder.is_deterministic">[docs]</a>  <span class="k">def</span> <span class="nf">is_deterministic</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># () -&gt; bool</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_elem_coder</span><span class="o">.</span><span class="n">is_deterministic</span><span class="p">()</span></div>
+
+<div class="viewcode-block" id="TupleSequenceCoder.as_deterministic_coder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.TupleSequenceCoder.as_deterministic_coder">[docs]</a>  <span class="k">def</span> <span class="nf">as_deterministic_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">step_label</span><span class="p">,</span> <span class="n">error_message</span><span class="o">=</sp [...]
+    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_deterministic</span><span class="p">():</span>
+      <span class="k">return</span> <span class="bp">self</span>
+    <span class="k">else</span><span class="p">:</span>
+      <span class="k">return</span> <span class="n">TupleSequenceCoder</span><span class="p">(</span>
+          <span class="bp">self</span><span class="o">.</span><span class="n">_elem_coder</span><span class="o">.</span><span class="n">as_deterministic_coder</span><span class="p">(</span><span class="n">step_label</span><span class="p">,</span> <span class="n">error_message</span><span class="p">))</span></div>
+
+<div class="viewcode-block" id="TupleSequenceCoder.from_type_hint"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.TupleSequenceCoder.from_type_hint">[docs]</a>  <span class="nd">@staticmethod</span>
+  <span class="k">def</span> <span class="nf">from_type_hint</span><span class="p">(</span><span class="n">typehint</span><span class="p">,</span> <span class="n">registry</span><span class="p">):</span>
+    <span class="c1"># type: (Any, CoderRegistry) -&gt; TupleSequenceCoder</span>
+    <span class="k">return</span> <span class="n">TupleSequenceCoder</span><span class="p">(</span><span class="n">registry</span><span class="o">.</span><span class="n">get_coder</span><span class="p">(</span><span class="n">typehint</span><span class="o">.</span><span class="n">inner_type</span><span class="p">))</span></div>
+
+  <span class="k">def</span> <span class="nf">_get_component_coders</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># type: () -&gt; Tuple[Coder, ...]</span>
+    <span class="k">return</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_elem_coder</span><span class="p">,)</span>
+
+  <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="s1">&#39;TupleSequenceCoder[</span><span class="si">%r</span><span class="s1">]&#39;</span> <span class="o">%</span> <span class="bp">self</span><span class="o">.</span><span class="n">_elem_coder</span>
+
+  <span class="k">def</span> <span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
+    <span class="k">return</span> <span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="nb">type</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
+            <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_elem_coder</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">value_coder</span><span class="p">())</span>
+
+  <span class="k">def</span> <span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">hash</span><span class="p">((</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">_elem_coder</span><span class="p">))</span></div>
+
+
+<div class="viewcode-block" id="IterableCoder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.IterableCoder">[docs]</a><span class="k">class</span> <span class="nc">IterableCoder</span><span class="p">(</span><span class="n">FastCoder</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;Coder of iterables of homogeneous objects.&quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">elem_coder</span><span class="p">):</span>
+    <span class="c1"># type: (Coder) -&gt; None</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">_elem_coder</span> <span class="o">=</span> <span class="n">elem_coder</span>
+
+  <span class="k">def</span> <span class="nf">_create_impl</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">coder_impl</span><span class="o">.</span><span class="n">IterableCoderImpl</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_elem_coder</span><span class="o">.</span><span class="n">get_impl</span><span class="p">())</span>
+
+<div class="viewcode-block" id="IterableCoder.is_deterministic"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.IterableCoder.is_deterministic">[docs]</a>  <span class="k">def</span> <span class="nf">is_deterministic</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># () -&gt; bool</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_elem_coder</span><span class="o">.</span><span class="n">is_deterministic</span><span class="p">()</span></div>
+
+<div class="viewcode-block" id="IterableCoder.as_deterministic_coder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.IterableCoder.as_deterministic_coder">[docs]</a>  <span class="k">def</span> <span class="nf">as_deterministic_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">step_label</span><span class="p">,</span> <span class="n">error_message</span><span class="o">=</span><span c [...]
+    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_deterministic</span><span class="p">():</span>
+      <span class="k">return</span> <span class="bp">self</span>
+    <span class="k">else</span><span class="p">:</span>
+      <span class="k">return</span> <span class="n">IterableCoder</span><span class="p">(</span>
+          <span class="bp">self</span><span class="o">.</span><span class="n">_elem_coder</span><span class="o">.</span><span class="n">as_deterministic_coder</span><span class="p">(</span><span class="n">step_label</span><span class="p">,</span> <span class="n">error_message</span><span class="p">))</span></div>
+
+<div class="viewcode-block" id="IterableCoder.as_cloud_object"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.IterableCoder.as_cloud_object">[docs]</a>  <span class="k">def</span> <span class="nf">as_cloud_object</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">coders_context</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+    <span class="k">return</span> <span class="p">{</span>
+        <span class="s1">&#39;@type&#39;</span><span class="p">:</span>
+            <span class="s1">&#39;kind:stream&#39;</span><span class="p">,</span>
+        <span class="s1">&#39;is_stream_like&#39;</span><span class="p">:</span>
+            <span class="kc">True</span><span class="p">,</span>
+        <span class="s1">&#39;component_encodings&#39;</span><span class="p">:</span> <span class="p">[</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">_elem_coder</span><span class="o">.</span><span class="n">as_cloud_object</span><span class="p">(</span><span class="n">coders_context</span><span class="p">)</span>
+        <span class="p">],</span>
+    <span class="p">}</span></div>
+
+<div class="viewcode-block" id="IterableCoder.value_coder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.IterableCoder.value_coder">[docs]</a>  <span class="k">def</span> <span class="nf">value_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_elem_coder</span></div>
+
+<div class="viewcode-block" id="IterableCoder.to_type_hint"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.IterableCoder.to_type_hint">[docs]</a>  <span class="k">def</span> <span class="nf">to_type_hint</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">typehints</span><span class="o">.</span><span class="n">Iterable</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_elem_coder</span><span class="o">.</span><span class="n">to_type_hint</span><span class="p">()]</span></div>
+
+<div class="viewcode-block" id="IterableCoder.from_type_hint"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.IterableCoder.from_type_hint">[docs]</a>  <span class="nd">@staticmethod</span>
+  <span class="k">def</span> <span class="nf">from_type_hint</span><span class="p">(</span><span class="n">typehint</span><span class="p">,</span> <span class="n">registry</span><span class="p">):</span>
+    <span class="c1"># type: (Any, CoderRegistry) -&gt; IterableCoder</span>
+    <span class="k">return</span> <span class="n">IterableCoder</span><span class="p">(</span><span class="n">registry</span><span class="o">.</span><span class="n">get_coder</span><span class="p">(</span><span class="n">typehint</span><span class="o">.</span><span class="n">inner_type</span><span class="p">))</span></div>
+
+  <span class="k">def</span> <span class="nf">_get_component_coders</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># type: () -&gt; Tuple[Coder, ...]</span>
+    <span class="k">return</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_elem_coder</span><span class="p">,)</span>
+
+  <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="s1">&#39;IterableCoder[</span><span class="si">%r</span><span class="s1">]&#39;</span> <span class="o">%</span> <span class="bp">self</span><span class="o">.</span><span class="n">_elem_coder</span>
+
+  <span class="k">def</span> <span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
+    <span class="k">return</span> <span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="nb">type</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
+            <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_elem_coder</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">value_coder</span><span class="p">())</span>
+
+  <span class="k">def</span> <span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">hash</span><span class="p">((</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">_elem_coder</span><span class="p">))</span></div>
+
+
+<span class="n">Coder</span><span class="o">.</span><span class="n">register_structured_urn</span><span class="p">(</span><span class="n">common_urns</span><span class="o">.</span><span class="n">coders</span><span class="o">.</span><span class="n">ITERABLE</span><span class="o">.</span><span class="n">urn</span><span class="p">,</span> <span class="n">IterableCoder</span><span class="p">)</span>
+
+
+<span class="k">class</span> <span class="nc">GlobalWindowCoder</span><span class="p">(</span><span class="n">SingletonCoder</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;Coder for global windows.&quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="kn">from</span> <span class="nn">apache_beam.transforms</span> <span class="kn">import</span> <span class="n">window</span>
+    <span class="nb">super</span><span class="p">(</span><span class="n">GlobalWindowCoder</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">window</span><span class="o">.</span><span class="n">GlobalWindow</span><span class="p">())</span>
+
+  <span class="k">def</span> <span class="nf">as_cloud_object</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">coders_context</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+    <span class="k">return</span> <span class="p">{</span>
+        <span class="s1">&#39;@type&#39;</span><span class="p">:</span> <span class="s1">&#39;kind:global_window&#39;</span><span class="p">,</span>
+    <span class="p">}</span>
+
+
+<span class="n">Coder</span><span class="o">.</span><span class="n">register_structured_urn</span><span class="p">(</span>
+    <span class="n">common_urns</span><span class="o">.</span><span class="n">coders</span><span class="o">.</span><span class="n">GLOBAL_WINDOW</span><span class="o">.</span><span class="n">urn</span><span class="p">,</span> <span class="n">GlobalWindowCoder</span><span class="p">)</span>
+
+
+<span class="k">class</span> <span class="nc">IntervalWindowCoder</span><span class="p">(</span><span class="n">FastCoder</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;Coder for an window defined by a start timestamp and a duration.&quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="nf">_create_impl</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">coder_impl</span><span class="o">.</span><span class="n">IntervalWindowCoderImpl</span><span class="p">()</span>
+
+  <span class="k">def</span> <span class="nf">is_deterministic</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># () -&gt; bool</span>
+    <span class="k">return</span> <span class="kc">True</span>
+
+  <span class="k">def</span> <span class="nf">as_cloud_object</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">coders_context</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+    <span class="k">return</span> <span class="p">{</span>
+        <span class="s1">&#39;@type&#39;</span><span class="p">:</span> <span class="s1">&#39;kind:interval_window&#39;</span><span class="p">,</span>
+    <span class="p">}</span>
+
+  <span class="k">def</span> <span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="nb">type</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
+
+  <span class="k">def</span> <span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">))</span>
+
+
+<span class="n">Coder</span><span class="o">.</span><span class="n">register_structured_urn</span><span class="p">(</span>
+    <span class="n">common_urns</span><span class="o">.</span><span class="n">coders</span><span class="o">.</span><span class="n">INTERVAL_WINDOW</span><span class="o">.</span><span class="n">urn</span><span class="p">,</span> <span class="n">IntervalWindowCoder</span><span class="p">)</span>
+
+
+<div class="viewcode-block" id="WindowedValueCoder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.WindowedValueCoder">[docs]</a><span class="k">class</span> <span class="nc">WindowedValueCoder</span><span class="p">(</span><span class="n">FastCoder</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;Coder for windowed values.&quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">wrapped_value_coder</span><span class="p">,</span> <span class="n">window_coder</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+    <span class="c1"># type: (Coder, Optional[Coder]) -&gt; None</span>
+    <span class="k">if</span> <span class="ow">not</span> <span class="n">window_coder</span><span class="p">:</span>
+      <span class="n">window_coder</span> <span class="o">=</span> <span class="n">PickleCoder</span><span class="p">()</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">wrapped_value_coder</span> <span class="o">=</span> <span class="n">wrapped_value_coder</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">timestamp_coder</span> <span class="o">=</span> <span class="n">TimestampCoder</span><span class="p">()</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">window_coder</span> <span class="o">=</span> <span class="n">window_coder</span>
+
+  <span class="k">def</span> <span class="nf">_create_impl</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">coder_impl</span><span class="o">.</span><span class="n">WindowedValueCoderImpl</span><span class="p">(</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">wrapped_value_coder</span><span class="o">.</span><span class="n">get_impl</span><span class="p">(),</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">timestamp_coder</span><span class="o">.</span><span class="n">get_impl</span><span class="p">(),</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">window_coder</span><span class="o">.</span><span class="n">get_impl</span><span class="p">())</span>
+
+<div class="viewcode-block" id="WindowedValueCoder.is_deterministic"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.WindowedValueCoder.is_deterministic">[docs]</a>  <span class="k">def</span> <span class="nf">is_deterministic</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># () -&gt; bool</span>
+    <span class="k">return</span> <span class="nb">all</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">is_deterministic</span><span class="p">()</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">wrapped_value_coder</span><span class="p">,</span>
+                                              <span class="bp">self</span><span class="o">.</span><span class="n">timestamp_coder</span><span class="p">,</span>
+                                              <span class="bp">self</span><span class="o">.</span><span class="n">window_coder</span><span class="p">])</span></div>
+
+<div class="viewcode-block" id="WindowedValueCoder.as_cloud_object"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.WindowedValueCoder.as_cloud_object">[docs]</a>  <span class="k">def</span> <span class="nf">as_cloud_object</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">coders_context</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+    <span class="k">return</span> <span class="p">{</span>
+        <span class="s1">&#39;@type&#39;</span><span class="p">:</span>
+            <span class="s1">&#39;kind:windowed_value&#39;</span><span class="p">,</span>
+        <span class="s1">&#39;is_wrapper&#39;</span><span class="p">:</span>
+            <span class="kc">True</span><span class="p">,</span>
+        <span class="s1">&#39;component_encodings&#39;</span><span class="p">:</span> <span class="p">[</span>
+            <span class="n">component</span><span class="o">.</span><span class="n">as_cloud_object</span><span class="p">(</span><span class="n">coders_context</span><span class="p">)</span>
+            <span class="k">for</span> <span class="n">component</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_component_coders</span><span class="p">()</span>
+        <span class="p">],</span>
+    <span class="p">}</span></div>
+
+  <span class="k">def</span> <span class="nf">_get_component_coders</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># type: () -&gt; List[Coder]</span>
+    <span class="k">return</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">wrapped_value_coder</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">window_coder</span><span class="p">]</span>
+
+<div class="viewcode-block" id="WindowedValueCoder.is_kv_coder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.WindowedValueCoder.is_kv_coder">[docs]</a>  <span class="k">def</span> <span class="nf">is_kv_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># () -&gt; bool</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">wrapped_value_coder</span><span class="o">.</span><span class="n">is_kv_coder</span><span class="p">()</span></div>
+
+<div class="viewcode-block" id="WindowedValueCoder.key_coder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.WindowedValueCoder.key_coder">[docs]</a>  <span class="k">def</span> <span class="nf">key_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># type: () -&gt; Coder</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">wrapped_value_coder</span><span class="o">.</span><span class="n">key_coder</span><span class="p">()</span></div>
+
+<div class="viewcode-block" id="WindowedValueCoder.value_coder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.WindowedValueCoder.value_coder">[docs]</a>  <span class="k">def</span> <span class="nf">value_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># type: () -&gt; Coder</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">wrapped_value_coder</span><span class="o">.</span><span class="n">value_coder</span><span class="p">()</span></div>
+
+  <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="s1">&#39;WindowedValueCoder[</span><span class="si">%s</span><span class="s1">]&#39;</span> <span class="o">%</span> <span class="bp">self</span><span class="o">.</span><span class="n">wrapped_value_coder</span>
+
+  <span class="k">def</span> <span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
+    <span class="k">return</span> <span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="nb">type</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
+            <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">wrapped_value_coder</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">wrapped_value_coder</span>
+            <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">timestamp_coder</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">timestamp_coder</span>
+            <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">window_coder</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">window_coder</span><span class="p">)</span>
+
+  <span class="k">def</span> <span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">hash</span><span class="p">(</span>
+        <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">wrapped_value_coder</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">timestamp_coder</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">window_coder</span><span class="p">))</span></div>
+
+
+<span class="n">Coder</span><span class="o">.</span><span class="n">register_structured_urn</span><span class="p">(</span>
+    <span class="n">common_urns</span><span class="o">.</span><span class="n">coders</span><span class="o">.</span><span class="n">WINDOWED_VALUE</span><span class="o">.</span><span class="n">urn</span><span class="p">,</span> <span class="n">WindowedValueCoder</span><span class="p">)</span>
+
+
+<div class="viewcode-block" id="ParamWindowedValueCoder"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.ParamWindowedValueCoder">[docs]</a><span class="k">class</span> <span class="nc">ParamWindowedValueCoder</span><span class="p">(</span><span class="n">WindowedValueCoder</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;A coder used for parameterized windowed values.&quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">payload</span><span class="p">,</span> <span class="n">components</span><span class="p">):</span>
+    <span class="nb">super</span><span class="p">(</span><span class="n">ParamWindowedValueCoder</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">components</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">components</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">payload</span> <span class="o">=</span> <span class="n">payload</span>
+
+  <span class="k">def</span> <span class="nf">_create_impl</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">coder_impl</span><span class="o">.</span><span class="n">ParamWindowedValueCoderImpl</span><span class="p">(</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">wrapped_value_coder</span><span class="o">.</span><span class="n">get_impl</span><span class="p">(),</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">window_coder</span><span class="o">.</span><span class="n">get_impl</span><span class="p">(),</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">payload</span><span class="p">)</span>
+
+<div class="viewcode-block" id="ParamWindowedValueCoder.is_deterministic"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.ParamWindowedValueCoder.is_deterministic">[docs]</a>  <span class="k">def</span> <span class="nf">is_deterministic</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">wrapped_value_coder</span><span class="o">.</span><span class="n">is_deterministic</span><span class="p">()</span></div>
+
+<div class="viewcode-block" id="ParamWindowedValueCoder.as_cloud_object"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.ParamWindowedValueCoder.as_cloud_object">[docs]</a>  <span class="k">def</span> <span class="nf">as_cloud_object</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">coders_context</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+    <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span>
+        <span class="s2">&quot;as_cloud_object not supported for ParamWindowedValueCoder&quot;</span><span class="p">)</span></div>
+
+  <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="s1">&#39;ParamWindowedValueCoder[</span><span class="si">%s</span><span class="s1">]&#39;</span> <span class="o">%</span> <span class="bp">self</span><span class="o">.</span><span class="n">wrapped_value_coder</span>
+
+  <span class="k">def</span> <span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
+    <span class="k">return</span> <span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="nb">type</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
+            <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">wrapped_value_coder</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">wrapped_value_coder</span>
+            <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">window_coder</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">window_coder</span>
+            <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">payload</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">payload</span><span class="p">)</span>
+
+  <span class="k">def</span> <span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">hash</span><span class="p">((</span><span class="bp">self</span><span class="o">.</span><span class="n">wrapped_value_coder</span><span class="p">,</span>
+                 <span class="bp">self</span><span class="o">.</span><span class="n">window_coder</span><span class="p">,</span>
+                 <span class="bp">self</span><span class="o">.</span><span class="n">payload</span><span class="p">))</span>
+
+<div class="viewcode-block" id="ParamWindowedValueCoder.from_runner_api_parameter"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.ParamWindowedValueCoder.from_runner_api_parameter">[docs]</a>  <span class="nd">@Coder</span><span class="o">.</span><span class="n">register_urn</span><span class="p">(</span><span class="n">common_urns</span><span class="o">.</span><span class="n">coders</span><span class="o">.</span><span class="n">PARAM_WIN [...]
+  <span class="k">def</span> <span class="nf">from_runner_api_parameter</span><span class="p">(</span><span class="n">payload</span><span class="p">,</span> <span class="n">components</span><span class="p">,</span> <span class="n">unused_context</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">ParamWindowedValueCoder</span><span class="p">(</span><span class="n">payload</span><span class="p">,</span> <span class="n">components</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="ParamWindowedValueCoder.to_runner_api_parameter"><a class="viewcode-back" href="../../../apache_beam.coders.coders.html#apache_beam.coders.coders.ParamWindowedValueCoder.to_runner_api_parameter">[docs]</a>  <span class="k">def</span> <span class="nf">to_runner_api_parameter</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+    <span class="k">return</span> <span class="p">(</span><span class="n">common_urns</span><span class="o">.</span><span class="n">coders</span><span class="o">.</span><span class="n">PARAM_WINDOWED_VALUE</span><span class="o">.</span><span class="n">urn</span><span class="p">,</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">payload</span><span class="p">,</span>
+            <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">wrapped_value_coder</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">window_coder</span><span class="p">))</span></div></div>
+
+
+<span class="k">class</span> <span class="nc">LengthPrefixCoder</span><span class="p">(</span><span class="n">FastCoder</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;For internal use only; no backwards-compatibility guarantees.</span>
+
+<span class="sd">  Coder which prefixes the length of the encoded object in the stream.&quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value_coder</span><span class="p">):</span>
+    <span class="c1"># type: (Coder) -&gt; None</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">_value_coder</span> <span class="o">=</span> <span class="n">value_coder</span>
+
+  <span class="k">def</span> <span class="nf">_create_impl</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">coder_impl</span><span class="o">.</span><span class="n">LengthPrefixCoderImpl</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_value_coder</span><span class="o">.</span><span class="n">get_impl</span><span class="p">())</span>
+
+  <span class="k">def</span> <span class="nf">is_deterministic</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># () -&gt; bool</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_value_coder</span><span class="o">.</span><span class="n">is_deterministic</span><span class="p">()</span>
+
+  <span class="k">def</span> <span class="nf">estimate_size</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
+    <span class="n">value_size</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_value_coder</span><span class="o">.</span><span class="n">estimate_size</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
+    <span class="k">return</span> <span class="n">get_varint_size</span><span class="p">(</span><span class="n">value_size</span><span class="p">)</span> <span class="o">+</span> <span class="n">value_size</span>
+
+  <span class="k">def</span> <span class="nf">value_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_value_coder</span>
+
+  <span class="k">def</span> <span class="nf">as_cloud_object</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">coders_context</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+    <span class="k">return</span> <span class="p">{</span>
+        <span class="s1">&#39;@type&#39;</span><span class="p">:</span>
+            <span class="s1">&#39;kind:length_prefix&#39;</span><span class="p">,</span>
+        <span class="s1">&#39;component_encodings&#39;</span><span class="p">:</span> <span class="p">[</span>
+            <span class="bp">self</span><span class="o">.</span><span class="n">_value_coder</span><span class="o">.</span><span class="n">as_cloud_object</span><span class="p">(</span><span class="n">coders_context</span><span class="p">)</span>
+        <span class="p">],</span>
+    <span class="p">}</span>
+
+  <span class="k">def</span> <span class="nf">_get_component_coders</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># type: () -&gt; Tuple[Coder, ...]</span>
+    <span class="k">return</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_value_coder</span><span class="p">,)</span>
+
+  <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="s1">&#39;LengthPrefixCoder[</span><span class="si">%r</span><span class="s1">]&#39;</span> <span class="o">%</span> <span class="bp">self</span><span class="o">.</span><span class="n">_value_coder</span>
+
+  <span class="k">def</span> <span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
+    <span class="k">return</span> <span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="nb">type</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
+            <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_value_coder</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">_value_coder</span><span class="p">)</span>
+
+  <span class="k">def</span> <span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">hash</span><span class="p">((</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">_value_coder</span><span class="p">))</span>
+
+
+<span class="n">Coder</span><span class="o">.</span><span class="n">register_structured_urn</span><span class="p">(</span>
+    <span class="n">common_urns</span><span class="o">.</span><span class="n">coders</span><span class="o">.</span><span class="n">LENGTH_PREFIX</span><span class="o">.</span><span class="n">urn</span><span class="p">,</span> <span class="n">LengthPrefixCoder</span><span class="p">)</span>
+
+
+<span class="k">class</span> <span class="nc">StateBackedIterableCoder</span><span class="p">(</span><span class="n">FastCoder</span><span class="p">):</span>
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
+      <span class="bp">self</span><span class="p">,</span>
+      <span class="n">element_coder</span><span class="p">,</span>  <span class="c1"># type: Coder</span>
+      <span class="n">read_state</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>  <span class="c1"># type: Optional[coder_impl.IterableStateReader]</span>
+      <span class="n">write_state</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>  <span class="c1"># type: Optional[coder_impl.IterableStateWriter]</span>
+      <span class="n">write_state_threshold</span><span class="o">=</span><span class="mi">1</span><span class="p">):</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">_element_coder</span> <span class="o">=</span> <span class="n">element_coder</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">_read_state</span> <span class="o">=</span> <span class="n">read_state</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">_write_state</span> <span class="o">=</span> <span class="n">write_state</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">_write_state_threshold</span> <span class="o">=</span> <span class="n">write_state_threshold</span>
+
+  <span class="k">def</span> <span class="nf">_create_impl</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">coder_impl</span><span class="o">.</span><span class="n">IterableCoderImpl</span><span class="p">(</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_element_coder</span><span class="o">.</span><span class="n">get_impl</span><span class="p">(),</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_read_state</span><span class="p">,</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_write_state</span><span class="p">,</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_write_state_threshold</span><span class="p">)</span>
+
+  <span class="k">def</span> <span class="nf">is_deterministic</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># () -&gt; bool</span>
+    <span class="k">return</span> <span class="kc">False</span>
+
+  <span class="k">def</span> <span class="nf">_get_component_coders</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># type: () -&gt; Tuple[Coder, ...]</span>
+    <span class="k">return</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_element_coder</span><span class="p">,)</span>
+
+  <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="s1">&#39;StateBackedIterableCoder[</span><span class="si">%r</span><span class="s1">]&#39;</span> <span class="o">%</span> <span class="bp">self</span><span class="o">.</span><span class="n">_element_coder</span>
+
+  <span class="k">def</span> <span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
+    <span class="k">return</span> <span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="nb">type</span><span class="p">(</span><span class="n">other</span><span class="p">)</span>
+            <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_element_coder</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">_element_coder</span>
+            <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">_write_state_threshold</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">_write_state_threshold</span><span class="p">)</span>
+
+  <span class="k">def</span> <span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">hash</span><span class="p">((</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">_element_coder</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_write_state_threshold</span><span class="p">))</span>
+
+  <span class="k">def</span> <span class="nf">to_runner_api_parameter</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+    <span class="c1"># type: (Optional[PipelineContext]) -&gt; Tuple[str, Any, Sequence[Coder]]</span>
+    <span class="k">return</span> <span class="p">(</span>
+        <span class="n">common_urns</span><span class="o">.</span><span class="n">coders</span><span class="o">.</span><span class="n">STATE_BACKED_ITERABLE</span><span class="o">.</span><span class="n">urn</span><span class="p">,</span>
+        <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_write_state_threshold</span><span class="p">)</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">&#39;ascii&#39;</span><span class="p">),</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">_get_component_coders</span><span class="p">())</span>
+
+  <span class="nd">@Coder</span><span class="o">.</span><span class="n">register_urn</span><span class="p">(</span><span class="n">common_urns</span><span class="o">.</span><span class="n">coders</span><span class="o">.</span><span class="n">STATE_BACKED_ITERABLE</span><span class="o">.</span><span class="n">urn</span><span class="p">,</span> <span class="nb">bytes</span><span class="p">)</span>
+  <span class="k">def</span> <span class="nf">from_runner_api_parameter</span><span class="p">(</span><span class="n">payload</span><span class="p">,</span> <span class="n">components</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">StateBackedIterableCoder</span><span class="p">(</span>
+        <span class="n">components</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span>
+        <span class="n">read_state</span><span class="o">=</span><span class="n">context</span><span class="o">.</span><span class="n">iterable_state_read</span><span class="p">,</span>
+        <span class="n">write_state</span><span class="o">=</span><span class="n">context</span><span class="o">.</span><span class="n">iterable_state_write</span><span class="p">,</span>
+        <span class="n">write_state_threshold</span><span class="o">=</span><span class="nb">int</span><span class="p">(</span><span class="n">payload</span><span class="p">))</span>
+
+
+<span class="k">class</span> <span class="nc">RunnerAPICoderHolder</span><span class="p">(</span><span class="n">Coder</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;A `Coder` that holds a runner API `Coder` proto.</span>
+
+<span class="sd">  This is used for coders for which corresponding objects cannot be</span>
+<span class="sd">  initialized in Python SDK. For example, coders for remote SDKs that may</span>
+<span class="sd">  be available in Python SDK transform graph when expanding a cross-language</span>
+<span class="sd">  transform.</span>
+<span class="sd">  &quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">proto</span><span class="p">):</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">_proto</span> <span class="o">=</span> <span class="n">proto</span>
+
+  <span class="k">def</span> <span class="nf">proto</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_proto</span>
+
+  <span class="k">def</span> <span class="nf">to_runner_api</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">context</span><span class="p">):</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_proto</span>
+
+  <span class="k">def</span> <span class="nf">to_type_hint</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">Any</span>
+</pre></div>
+
+           </div>
+           <div class="articleComments">
+            
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+        &copy; Copyright .
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file
diff --git a/pydoc/2.19.0/_modules/apache_beam/coders/observable.html b/pydoc/2.19.0/_modules/apache_beam/coders/observable.html
new file mode 100644
index 0000000..ace8285
--- /dev/null
+++ b/pydoc/2.19.0/_modules/apache_beam/coders/observable.html
@@ -0,0 +1,283 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>apache_beam.coders.observable &mdash; Apache Beam  documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../search.html"/>
+    <link rel="top" title="Apache Beam  documentation" href="../../../index.html"/>
+        <link rel="up" title="Module code" href="../../index.html"/> 
+
+  
+  <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../index.html" class="icon icon-home"> Apache Beam
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.coders.html">apache_beam.coders package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.internal.html">apache_beam.internal package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.io.html">apache_beam.io package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.metrics.html">apache_beam.metrics package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.options.html">apache_beam.options package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.portability.html">apache_beam.portability package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.runners.html">apache_beam.runners package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.testing.html">apache_beam.testing package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.tools.html">apache_beam.tools package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.transforms.html">apache_beam.transforms package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.typehints.html">apache_beam.typehints package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.utils.html">apache_beam.utils package</a></li>
+</ul>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.error.html">apache_beam.error module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.pipeline.html">apache_beam.pipeline module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.pvalue.html">apache_beam.pvalue module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.version.html">apache_beam.version module</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../index.html">Apache Beam</a>
+        
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../index.html">Module code</a> &raquo;</li>
+        
+      <li>apache_beam.coders.observable</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+            
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for apache_beam.coders.observable</h1><div class="highlight"><pre>
+<span></span><span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one or more</span>
+<span class="c1"># contributor license agreements.  See the NOTICE file distributed with</span>
+<span class="c1"># this work for additional information regarding copyright ownership.</span>
+<span class="c1"># The ASF licenses this file to You under the Apache License, Version 2.0</span>
+<span class="c1"># (the &quot;License&quot;); you may not use this file except in compliance with</span>
+<span class="c1"># the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#    http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+<span class="c1">#</span>
+
+
+<span class="sd">&quot;&quot;&quot;Observable base class for iterables.</span>
+
+<span class="sd">For internal use only; no backwards-compatibility guarantees.</span>
+<span class="sd">&quot;&quot;&quot;</span>
+<span class="c1"># pytype: skip-file</span>
+
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">absolute_import</span>
+
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">object</span>
+
+
+<div class="viewcode-block" id="ObservableMixin"><a class="viewcode-back" href="../../../apache_beam.coders.observable.html#apache_beam.coders.observable.ObservableMixin">[docs]</a><span class="k">class</span> <span class="nc">ObservableMixin</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;For internal use only; no backwards-compatibility guarantees.</span>
+
+<span class="sd">  An observable iterable.</span>
+
+<span class="sd">  Subclasses need to call self.notify_observers with any object yielded.</span>
+<span class="sd">  &quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">observers</span> <span class="o">=</span> <span class="p">[]</span>
+
+<div class="viewcode-block" id="ObservableMixin.register_observer"><a class="viewcode-back" href="../../../apache_beam.coders.observable.html#apache_beam.coders.observable.ObservableMixin.register_observer">[docs]</a>  <span class="k">def</span> <span class="nf">register_observer</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">callback</span><span class="p">):</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">observers</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">callback</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="ObservableMixin.notify_observers"><a class="viewcode-back" href="../../../apache_beam.coders.observable.html#apache_beam.coders.observable.ObservableMixin.notify_observers">[docs]</a>  <span class="k">def</span> <span class="nf">notify_observers</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+    <span class="c1"># self.observers is almost always empty</span>
+    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">observers</span><span class="p">:</span>
+      <span class="k">for</span> <span class="n">o</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">observers</span><span class="p">:</span>
+        <span class="n">o</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span></div></div>
+</pre></div>
+
+           </div>
+           <div class="articleComments">
+            
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+        &copy; Copyright .
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file
diff --git a/pydoc/2.19.0/_modules/apache_beam/coders/row_coder.html b/pydoc/2.19.0/_modules/apache_beam/coders/row_coder.html
new file mode 100644
index 0000000..f50dc11
--- /dev/null
+++ b/pydoc/2.19.0/_modules/apache_beam/coders/row_coder.html
@@ -0,0 +1,411 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>apache_beam.coders.row_coder &mdash; Apache Beam  documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../search.html"/>
+    <link rel="top" title="Apache Beam  documentation" href="../../../index.html"/>
+        <link rel="up" title="Module code" href="../../index.html"/> 
+
+  
+  <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../index.html" class="icon icon-home"> Apache Beam
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.coders.html">apache_beam.coders package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.internal.html">apache_beam.internal package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.io.html">apache_beam.io package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.metrics.html">apache_beam.metrics package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.options.html">apache_beam.options package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.portability.html">apache_beam.portability package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.runners.html">apache_beam.runners package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.testing.html">apache_beam.testing package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.tools.html">apache_beam.tools package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.transforms.html">apache_beam.transforms package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.typehints.html">apache_beam.typehints package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.utils.html">apache_beam.utils package</a></li>
+</ul>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.error.html">apache_beam.error module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.pipeline.html">apache_beam.pipeline module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.pvalue.html">apache_beam.pvalue module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.version.html">apache_beam.version module</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../index.html">Apache Beam</a>
+        
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../index.html">Module code</a> &raquo;</li>
+        
+      <li>apache_beam.coders.row_coder</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+            
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for apache_beam.coders.row_coder</h1><div class="highlight"><pre>
+<span></span><span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one or more</span>
+<span class="c1"># contributor license agreements.  See the NOTICE file distributed with</span>
+<span class="c1"># this work for additional information regarding copyright ownership.</span>
+<span class="c1"># The ASF licenses this file to You under the Apache License, Version 2.0</span>
+<span class="c1"># (the &quot;License&quot;); you may not use this file except in compliance with</span>
+<span class="c1"># the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#    http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+<span class="c1">#</span>
+
+<span class="c1"># pytype: skip-file</span>
+
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">absolute_import</span>
+
+<span class="kn">import</span> <span class="nn">itertools</span>
+<span class="kn">from</span> <span class="nn">array</span> <span class="kn">import</span> <span class="n">array</span>
+
+<span class="kn">from</span> <span class="nn">apache_beam.coders.coder_impl</span> <span class="kn">import</span> <span class="n">StreamCoderImpl</span>
+<span class="kn">from</span> <span class="nn">apache_beam.coders.coders</span> <span class="kn">import</span> <span class="n">BytesCoder</span>
+<span class="kn">from</span> <span class="nn">apache_beam.coders.coders</span> <span class="kn">import</span> <span class="n">Coder</span>
+<span class="kn">from</span> <span class="nn">apache_beam.coders.coders</span> <span class="kn">import</span> <span class="n">FastCoder</span>
+<span class="kn">from</span> <span class="nn">apache_beam.coders.coders</span> <span class="kn">import</span> <span class="n">FloatCoder</span>
+<span class="kn">from</span> <span class="nn">apache_beam.coders.coders</span> <span class="kn">import</span> <span class="n">IterableCoder</span>
+<span class="kn">from</span> <span class="nn">apache_beam.coders.coders</span> <span class="kn">import</span> <span class="n">StrUtf8Coder</span>
+<span class="kn">from</span> <span class="nn">apache_beam.coders.coders</span> <span class="kn">import</span> <span class="n">TupleCoder</span>
+<span class="kn">from</span> <span class="nn">apache_beam.coders.coders</span> <span class="kn">import</span> <span class="n">VarIntCoder</span>
+<span class="kn">from</span> <span class="nn">apache_beam.portability</span> <span class="kn">import</span> <span class="n">common_urns</span>
+<span class="kn">from</span> <span class="nn">apache_beam.portability.api</span> <span class="kn">import</span> <span class="n">schema_pb2</span>
+<span class="kn">from</span> <span class="nn">apache_beam.typehints.schemas</span> <span class="kn">import</span> <span class="n">named_tuple_from_schema</span>
+<span class="kn">from</span> <span class="nn">apache_beam.typehints.schemas</span> <span class="kn">import</span> <span class="n">named_tuple_to_schema</span>
+
+<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;RowCoder&quot;</span><span class="p">]</span>
+
+
+<div class="viewcode-block" id="RowCoder"><a class="viewcode-back" href="../../../apache_beam.coders.row_coder.html#apache_beam.coders.row_coder.RowCoder">[docs]</a><span class="k">class</span> <span class="nc">RowCoder</span><span class="p">(</span><span class="n">FastCoder</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot; Coder for `typing.NamedTuple` instances.</span>
+
+<span class="sd">  Implements the beam:coder:row:v1 standard coder spec.</span>
+<span class="sd">  &quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">schema</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;Initializes a :class:`RowCoder`.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">      schema (apache_beam.portability.api.schema_pb2.Schema): The protobuf</span>
+<span class="sd">        representation of the schema of the data that the RowCoder will be used</span>
+<span class="sd">        to encode/decode.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">schema</span> <span class="o">=</span> <span class="n">schema</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">components</span> <span class="o">=</span> <span class="p">[</span>
+        <span class="n">RowCoder</span><span class="o">.</span><span class="n">coder_from_type</span><span class="p">(</span><span class="n">field</span><span class="o">.</span><span class="n">type</span><span class="p">)</span> <span class="k">for</span> <span class="n">field</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="o">.</span><span class="n">fields</span>
+    <span class="p">]</span>
+
+  <span class="k">def</span> <span class="nf">_create_impl</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">RowCoderImpl</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">components</span><span class="p">)</span>
+
+<div class="viewcode-block" id="RowCoder.is_deterministic"><a class="viewcode-back" href="../../../apache_beam.coders.row_coder.html#apache_beam.coders.row_coder.RowCoder.is_deterministic">[docs]</a>  <span class="k">def</span> <span class="nf">is_deterministic</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">all</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">is_deterministic</span><span class="p">()</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">components</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="RowCoder.to_type_hint"><a class="viewcode-back" href="../../../apache_beam.coders.row_coder.html#apache_beam.coders.row_coder.RowCoder.to_type_hint">[docs]</a>  <span class="k">def</span> <span class="nf">to_type_hint</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">named_tuple_from_schema</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="RowCoder.as_cloud_object"><a class="viewcode-back" href="../../../apache_beam.coders.row_coder.html#apache_beam.coders.row_coder.RowCoder.as_cloud_object">[docs]</a>  <span class="k">def</span> <span class="nf">as_cloud_object</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">coders_context</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+    <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span><span class="s2">&quot;as_cloud_object not supported for RowCoder&quot;</span><span class="p">)</span></div>
+
+  <span class="fm">__hash__</span> <span class="o">=</span> <span class="kc">None</span>  <span class="c1"># type: ignore[assignment]</span>
+
+  <span class="k">def</span> <span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">==</span> <span class="nb">type</span><span class="p">(</span><span class="n">other</span><span class="p">)</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema</span> <span class="o">==</span> <span class="n">other</span><span class="o">.</span><span class="n">schema</span>
+
+<div class="viewcode-block" id="RowCoder.to_runner_api_parameter"><a class="viewcode-back" href="../../../apache_beam.coders.row_coder.html#apache_beam.coders.row_coder.RowCoder.to_runner_api_parameter">[docs]</a>  <span class="k">def</span> <span class="nf">to_runner_api_parameter</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">unused_context</span><span class="p">):</span>
+    <span class="k">return</span> <span class="p">(</span><span class="n">common_urns</span><span class="o">.</span><span class="n">coders</span><span class="o">.</span><span class="n">ROW</span><span class="o">.</span><span class="n">urn</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="p">,</span> <span class="p">[])</span></div>
+
+<div class="viewcode-block" id="RowCoder.from_runner_api_parameter"><a class="viewcode-back" href="../../../apache_beam.coders.row_coder.html#apache_beam.coders.row_coder.RowCoder.from_runner_api_parameter">[docs]</a>  <span class="nd">@Coder</span><span class="o">.</span><span class="n">register_urn</span><span class="p">(</span><span class="n">common_urns</span><span class="o">.</span><span class="n">coders</span><span class="o">.</span><span class="n">ROW</span><span class="o">.</span [...]
+  <span class="k">def</span> <span class="nf">from_runner_api_parameter</span><span class="p">(</span><span class="n">payload</span><span class="p">,</span> <span class="n">components</span><span class="p">,</span> <span class="n">unused_context</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">RowCoder</span><span class="p">(</span><span class="n">payload</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="RowCoder.from_type_hint"><a class="viewcode-back" href="../../../apache_beam.coders.row_coder.html#apache_beam.coders.row_coder.RowCoder.from_type_hint">[docs]</a>  <span class="nd">@staticmethod</span>
+  <span class="k">def</span> <span class="nf">from_type_hint</span><span class="p">(</span><span class="n">named_tuple_type</span><span class="p">,</span> <span class="n">registry</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">RowCoder</span><span class="p">(</span><span class="n">named_tuple_to_schema</span><span class="p">(</span><span class="n">named_tuple_type</span><span class="p">))</span></div>
+
+<div class="viewcode-block" id="RowCoder.coder_from_type"><a class="viewcode-back" href="../../../apache_beam.coders.row_coder.html#apache_beam.coders.row_coder.RowCoder.coder_from_type">[docs]</a>  <span class="nd">@staticmethod</span>
+  <span class="k">def</span> <span class="nf">coder_from_type</span><span class="p">(</span><span class="n">field_type</span><span class="p">):</span>
+    <span class="n">type_info</span> <span class="o">=</span> <span class="n">field_type</span><span class="o">.</span><span class="n">WhichOneof</span><span class="p">(</span><span class="s2">&quot;type_info&quot;</span><span class="p">)</span>
+    <span class="k">if</span> <span class="n">type_info</span> <span class="o">==</span> <span class="s2">&quot;atomic_type&quot;</span><span class="p">:</span>
+      <span class="k">if</span> <span class="n">field_type</span><span class="o">.</span><span class="n">atomic_type</span> <span class="ow">in</span> <span class="p">(</span><span class="n">schema_pb2</span><span class="o">.</span><span class="n">INT32</span><span class="p">,</span>
+                                    <span class="n">schema_pb2</span><span class="o">.</span><span class="n">INT64</span><span class="p">):</span>
+        <span class="k">return</span> <span class="n">VarIntCoder</span><span class="p">()</span>
+      <span class="k">elif</span> <span class="n">field_type</span><span class="o">.</span><span class="n">atomic_type</span> <span class="o">==</span> <span class="n">schema_pb2</span><span class="o">.</span><span class="n">DOUBLE</span><span class="p">:</span>
+        <span class="k">return</span> <span class="n">FloatCoder</span><span class="p">()</span>
+      <span class="k">elif</span> <span class="n">field_type</span><span class="o">.</span><span class="n">atomic_type</span> <span class="o">==</span> <span class="n">schema_pb2</span><span class="o">.</span><span class="n">STRING</span><span class="p">:</span>
+        <span class="k">return</span> <span class="n">StrUtf8Coder</span><span class="p">()</span>
+    <span class="k">elif</span> <span class="n">type_info</span> <span class="o">==</span> <span class="s2">&quot;array_type&quot;</span><span class="p">:</span>
+      <span class="k">return</span> <span class="n">IterableCoder</span><span class="p">(</span>
+          <span class="n">RowCoder</span><span class="o">.</span><span class="n">coder_from_type</span><span class="p">(</span><span class="n">field_type</span><span class="o">.</span><span class="n">array_type</span><span class="o">.</span><span class="n">element_type</span><span class="p">))</span>
+
+    <span class="c1"># The Java SDK supports several more types, but the coders are not yet</span>
+    <span class="c1"># standard, and are not implemented in Python.</span>
+    <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+        <span class="s2">&quot;Encountered a type that is not currently supported by RowCoder: </span><span class="si">%s</span><span class="s2">&quot;</span> <span class="o">%</span>
+        <span class="n">field_type</span><span class="p">)</span></div></div>
+
+
+<span class="k">class</span> <span class="nc">RowCoderImpl</span><span class="p">(</span><span class="n">StreamCoderImpl</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;For internal use only; no backwards-compatibility guarantees.&quot;&quot;&quot;</span>
+  <span class="n">SIZE_CODER</span> <span class="o">=</span> <span class="n">VarIntCoder</span><span class="p">()</span><span class="o">.</span><span class="n">get_impl</span><span class="p">()</span>
+  <span class="n">NULL_MARKER_CODER</span> <span class="o">=</span> <span class="n">BytesCoder</span><span class="p">()</span><span class="o">.</span><span class="n">get_impl</span><span class="p">()</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">schema</span><span class="p">,</span> <span class="n">components</span><span class="p">):</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">schema</span> <span class="o">=</span> <span class="n">schema</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">constructor</span> <span class="o">=</span> <span class="n">named_tuple_from_schema</span><span class="p">(</span><span class="n">schema</span><span class="p">)</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">components</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">c</span><span class="o">.</span><span class="n">get_impl</span><span class="p">()</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">components</span><span class="p">)</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">has_nullable_fields</span> <span class="o">=</span> <span class="nb">any</span><span class="p">(</span>
+        <span class="n">field</span><span class="o">.</span><span class="n">type</span><span class="o">.</span><span class="n">nullable</span> <span class="k">for</span> <span class="n">field</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="o">.</span><span class="n">fields</span><span class="p">)</span>
+
+  <span class="k">def</span> <span class="nf">encode_to_stream</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">out</span><span class="p">,</span> <span class="n">nested</span><span class="p">):</span>
+    <span class="n">nvals</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="o">.</span><span class="n">fields</span><span class="p">)</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">SIZE_CODER</span><span class="o">.</span><span class="n">encode_to_stream</span><span class="p">(</span><span class="n">nvals</span><span class="p">,</span> <span class="n">out</span><span class="p">,</span> <span class="kc">True</span><span class="p">)</span>
+    <span class="n">attrs</span> <span class="o">=</span> <span class="p">[</span><span class="nb">getattr</span><span class="p">(</span><span class="n">value</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">name</span><span class="p">)</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="o">.</span><span class="n">fi [...]
+
+    <span class="n">words</span> <span class="o">=</span> <span class="n">array</span><span class="p">(</span><span class="s1">&#39;B&#39;</span><span class="p">)</span>
+    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">has_nullable_fields</span><span class="p">:</span>
+      <span class="n">nulls</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">attr</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">for</span> <span class="n">attr</span> <span class="ow">in</span> <span class="n">attrs</span><span class="p">)</span>
+      <span class="k">if</span> <span class="nb">any</span><span class="p">(</span><span class="n">nulls</span><span class="p">):</span>
+        <span class="n">words</span> <span class="o">=</span> <span class="n">array</span><span class="p">(</span><span class="s1">&#39;B&#39;</span><span class="p">,</span> <span class="n">itertools</span><span class="o">.</span><span class="n">repeat</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="p">(</span><span class="n">nvals</span><span class="o">+</span><span class="mi">7</span><span class="p">)</span><span class="o">//</span><span cla [...]
+        <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">is_null</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">nulls</span><span class="p">):</span>
+          <span class="n">words</span><span class="p">[</span><span class="n">i</span><span class="o">//</span><span class="mi">8</span><span class="p">]</span> <span class="o">|=</span> <span class="n">is_null</span> <span class="o">&lt;&lt;</span> <span class="p">(</span><span class="n">i</span> <span class="o">%</span> <span class="mi">8</span><span class="p">)</span>
+
+    <span class="bp">self</span><span class="o">.</span><span class="n">NULL_MARKER_CODER</span><span class="o">.</span><span class="n">encode_to_stream</span><span class="p">(</span><span class="n">words</span><span class="o">.</span><span class="n">tostring</span><span class="p">(),</span> <span class="n">out</span><span class="p">,</span> <span class="kc">True</span><span class="p">)</span>
+
+    <span class="k">for</span> <span class="n">c</span><span class="p">,</span> <span class="n">field</span><span class="p">,</span> <span class="n">attr</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">components</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">schema</span><span class="o">.</span><span class="n">fields</span><span cla [...]
+      <span class="k">if</span> <span class="n">attr</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">field</span><span class="o">.</span><span class="n">type</span><span class="o">.</span><span class="n">nullable</span><span class="p">:</span>
+          <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
+              <span class="s2">&quot;Attempted to encode null for non-nullable field </span><span class="se">\&quot;</span><span class="si">{}</span><span class="se">\&quot;</span><span class="s2">.&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+                  <span class="n">field</span><span class="o">.</span><span class="n">name</span><span class="p">))</span>
+        <span class="k">continue</span>
+      <span class="n">c</span><span class="o">.</span><span class="n">encode_to_stream</span><span class="p">(</span><span class="n">attr</span><span class="p">,</span> <span class="n">out</span><span class="p">,</span> <span class="kc">True</span><span class="p">)</span>
+
+  <span class="k">def</span> <span class="nf">decode_from_stream</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">in_stream</span><span class="p">,</span> <span class="n">nested</span><span class="p">):</span>
+    <span class="n">nvals</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">SIZE_CODER</span><span class="o">.</span><span class="n">decode_from_stream</span><span class="p">(</span><span class="n">in_stream</span><span class="p">,</span> <span class="kc">True</span><span class="p">)</span>
+    <span class="n">words</span> <span class="o">=</span> <span class="n">array</span><span class="p">(</span><span class="s1">&#39;B&#39;</span><span class="p">)</span>
+    <span class="n">words</span><span class="o">.</span><span class="n">fromstring</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">NULL_MARKER_CODER</span><span class="o">.</span><span class="n">decode_from_stream</span><span class="p">(</span><span class="n">in_stream</span><span class="p">,</span> <span class="kc">True</span><span class="p">))</span>
+
+    <span class="k">if</span> <span class="n">words</span><span class="p">:</span>
+      <span class="n">nulls</span> <span class="o">=</span> <span class="p">((</span><span class="n">words</span><span class="p">[</span><span class="n">i</span> <span class="o">//</span> <span class="mi">8</span><span class="p">]</span> <span class="o">&gt;&gt;</span> <span class="p">(</span><span class="n">i</span> <span class="o">%</span> <span class="mi">8</span><span class="p">))</span> <span class="o">&amp;</span> <span class="mh">0x01</span> <span class="k">for</span> <span class= [...]
+    <span class="k">else</span><span class="p">:</span>
+      <span class="n">nulls</span> <span class="o">=</span> <span class="n">itertools</span><span class="o">.</span><span class="n">repeat</span><span class="p">(</span><span class="kc">False</span><span class="p">,</span> <span class="n">nvals</span><span class="p">)</span>
+
+    <span class="c1"># If this coder&#39;s schema has more attributes than the encoded value, then</span>
+    <span class="c1"># the schema must have changed. Populate the unencoded fields with nulls.</span>
+    <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">components</span><span class="p">)</span> <span class="o">&gt;</span> <span class="n">nvals</span><span class="p">:</span>
+      <span class="n">nulls</span> <span class="o">=</span> <span class="n">itertools</span><span class="o">.</span><span class="n">chain</span><span class="p">(</span>
+          <span class="n">nulls</span><span class="p">,</span>
+          <span class="n">itertools</span><span class="o">.</span><span class="n">repeat</span><span class="p">(</span><span class="kc">True</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">components</span><span class="p">)</span> <span class="o">-</span> <span class="n">nvals</span><span class="p">))</span>
+
+    <span class="c1"># Note that if this coder&#39;s schema has *fewer* attributes than the encoded</span>
+    <span class="c1"># value, we just need to ignore the additional values, which will occur</span>
+    <span class="c1"># here because we only decode as many values as we have coders for.</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">constructor</span><span class="p">(</span><span class="o">*</span><span class="p">(</span>
+        <span class="kc">None</span> <span class="k">if</span> <span class="n">is_null</span> <span class="k">else</span> <span class="n">c</span><span class="o">.</span><span class="n">decode_from_stream</span><span class="p">(</span><span class="n">in_stream</span><span class="p">,</span> <span class="kc">True</span><span class="p">)</span>
+        <span class="k">for</span> <span class="n">c</span><span class="p">,</span> <span class="n">is_null</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">components</span><span class="p">,</span> <span class="n">nulls</span><span class="p">)))</span>
+
+  <span class="k">def</span> <span class="nf">_make_value_coder</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">nulls</span><span class="o">=</span><span class="n">itertools</span><span class="o">.</span><span class="n">repeat</span><span class="p">(</span><span class="kc">False</span><span class="p">)):</span>
+    <span class="n">components</span> <span class="o">=</span> <span class="p">[</span>
+        <span class="n">component</span> <span class="k">for</span> <span class="n">component</span><span class="p">,</span> <span class="n">is_null</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">components</span><span class="p">,</span> <span class="n">nulls</span><span class="p">)</span>
+        <span class="k">if</span> <span class="ow">not</span> <span class="n">is_null</span>
+    <span class="p">]</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">has_nullable_fields</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">components</span>
+    <span class="k">return</span> <span class="n">TupleCoder</span><span class="p">(</span><span class="n">components</span><span class="p">)</span><span class="o">.</span><span class="n">get_impl</span><span class="p">()</span>
+</pre></div>
+
+           </div>
+           <div class="articleComments">
+            
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+        &copy; Copyright .
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file
diff --git a/pydoc/2.19.0/_modules/apache_beam/coders/slow_stream.html b/pydoc/2.19.0/_modules/apache_beam/coders/slow_stream.html
new file mode 100644
index 0000000..7328bdc
--- /dev/null
+++ b/pydoc/2.19.0/_modules/apache_beam/coders/slow_stream.html
@@ -0,0 +1,452 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>apache_beam.coders.slow_stream &mdash; Apache Beam  documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../search.html"/>
+    <link rel="top" title="Apache Beam  documentation" href="../../../index.html"/>
+        <link rel="up" title="Module code" href="../../index.html"/> 
+
+  
+  <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../index.html" class="icon icon-home"> Apache Beam
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.coders.html">apache_beam.coders package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.internal.html">apache_beam.internal package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.io.html">apache_beam.io package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.metrics.html">apache_beam.metrics package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.options.html">apache_beam.options package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.portability.html">apache_beam.portability package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.runners.html">apache_beam.runners package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.testing.html">apache_beam.testing package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.tools.html">apache_beam.tools package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.transforms.html">apache_beam.transforms package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.typehints.html">apache_beam.typehints package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.utils.html">apache_beam.utils package</a></li>
+</ul>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.error.html">apache_beam.error module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.pipeline.html">apache_beam.pipeline module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.pvalue.html">apache_beam.pvalue module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.version.html">apache_beam.version module</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../index.html">Apache Beam</a>
+        
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../index.html">Module code</a> &raquo;</li>
+        
+      <li>apache_beam.coders.slow_stream</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+            
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for apache_beam.coders.slow_stream</h1><div class="highlight"><pre>
+<span></span><span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one or more</span>
+<span class="c1"># contributor license agreements.  See the NOTICE file distributed with</span>
+<span class="c1"># this work for additional information regarding copyright ownership.</span>
+<span class="c1"># The ASF licenses this file to You under the Apache License, Version 2.0</span>
+<span class="c1"># (the &quot;License&quot;); you may not use this file except in compliance with</span>
+<span class="c1"># the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#    http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+<span class="c1">#</span>
+
+<span class="sd">&quot;&quot;&quot;A pure Python implementation of stream.pyx.</span>
+
+<span class="sd">For internal use only; no backwards-compatibility guarantees.</span>
+<span class="sd">&quot;&quot;&quot;</span>
+<span class="c1"># pytype: skip-file</span>
+
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">absolute_import</span>
+
+<span class="kn">import</span> <span class="nn">struct</span>
+<span class="kn">import</span> <span class="nn">sys</span>
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">chr</span>
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">object</span>
+<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">List</span>
+
+
+<div class="viewcode-block" id="OutputStream"><a class="viewcode-back" href="../../../apache_beam.coders.slow_stream.html#apache_beam.coders.slow_stream.OutputStream">[docs]</a><span class="k">class</span> <span class="nc">OutputStream</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;For internal use only; no backwards-compatibility guarantees.</span>
+
+<span class="sd">  A pure Python implementation of stream.OutputStream.&quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">data</span> <span class="o">=</span> <span class="p">[]</span>  <span class="c1"># type: List[bytes]</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">byte_count</span> <span class="o">=</span> <span class="mi">0</span>
+
+<div class="viewcode-block" id="OutputStream.write"><a class="viewcode-back" href="../../../apache_beam.coders.slow_stream.html#apache_beam.coders.slow_stream.OutputStream.write">[docs]</a>  <span class="k">def</span> <span class="nf">write</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">b</span><span class="p">,</span> <span class="n">nested</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+    <span class="c1"># type: (bytes, bool) -&gt; None</span>
+    <span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">b</span><span class="p">,</span> <span class="nb">bytes</span><span class="p">)</span>
+    <span class="k">if</span> <span class="n">nested</span><span class="p">:</span>
+      <span class="bp">self</span><span class="o">.</span><span class="n">write_var_int64</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">b</span><span class="p">))</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">b</span><span class="p">)</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">byte_count</span> <span class="o">+=</span> <span class="nb">len</span><span class="p">(</span><span class="n">b</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="OutputStream.write_byte"><a class="viewcode-back" href="../../../apache_beam.coders.slow_stream.html#apache_beam.coders.slow_stream.OutputStream.write_byte">[docs]</a>  <span class="k">def</span> <span class="nf">write_byte</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">val</span><span class="p">):</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="nb">chr</span><span class="p">(</span><span class="n">val</span><span class="p">)</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="s1">&#39;latin-1&#39;</span><span class="p">))</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">byte_count</span> <span class="o">+=</span> <span class="mi">1</span></div>
+
+<div class="viewcode-block" id="OutputStream.write_var_int64"><a class="viewcode-back" href="../../../apache_beam.coders.slow_stream.html#apache_beam.coders.slow_stream.OutputStream.write_var_int64">[docs]</a>  <span class="k">def</span> <span class="nf">write_var_int64</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">v</span><span class="p">):</span>
+    <span class="c1"># type: (int) -&gt; None</span>
+    <span class="k">if</span> <span class="n">v</span> <span class="o">&lt;</span> <span class="mi">0</span><span class="p">:</span>
+      <span class="n">v</span> <span class="o">+=</span> <span class="mi">1</span> <span class="o">&lt;&lt;</span> <span class="mi">64</span>
+      <span class="k">if</span> <span class="n">v</span> <span class="o">&lt;=</span> <span class="mi">0</span><span class="p">:</span>
+        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;Value too large (negative).&#39;</span><span class="p">)</span>
+    <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
+      <span class="n">bits</span> <span class="o">=</span> <span class="n">v</span> <span class="o">&amp;</span> <span class="mh">0x7F</span>
+      <span class="n">v</span> <span class="o">&gt;&gt;=</span> <span class="mi">7</span>
+      <span class="k">if</span> <span class="n">v</span><span class="p">:</span>
+        <span class="n">bits</span> <span class="o">|=</span> <span class="mh">0x80</span>
+      <span class="bp">self</span><span class="o">.</span><span class="n">write_byte</span><span class="p">(</span><span class="n">bits</span><span class="p">)</span>
+      <span class="k">if</span> <span class="ow">not</span> <span class="n">v</span><span class="p">:</span>
+        <span class="k">break</span></div>
+
+<div class="viewcode-block" id="OutputStream.write_bigendian_int64"><a class="viewcode-back" href="../../../apache_beam.coders.slow_stream.html#apache_beam.coders.slow_stream.OutputStream.write_bigendian_int64">[docs]</a>  <span class="k">def</span> <span class="nf">write_bigendian_int64</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">v</span><span class="p">):</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">struct</span><span class="o">.</span><span class="n">pack</span><span class="p">(</span><span class="s1">&#39;&gt;q&#39;</span><span class="p">,</span> <span class="n">v</span><span class="p">))</span></div>
+
+<div class="viewcode-block" id="OutputStream.write_bigendian_uint64"><a class="viewcode-back" href="../../../apache_beam.coders.slow_stream.html#apache_beam.coders.slow_stream.OutputStream.write_bigendian_uint64">[docs]</a>  <span class="k">def</span> <span class="nf">write_bigendian_uint64</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">v</span><span class="p">):</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">struct</span><span class="o">.</span><span class="n">pack</span><span class="p">(</span><span class="s1">&#39;&gt;Q&#39;</span><span class="p">,</span> <span class="n">v</span><span class="p">))</span></div>
+
+<div class="viewcode-block" id="OutputStream.write_bigendian_int32"><a class="viewcode-back" href="../../../apache_beam.coders.slow_stream.html#apache_beam.coders.slow_stream.OutputStream.write_bigendian_int32">[docs]</a>  <span class="k">def</span> <span class="nf">write_bigendian_int32</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">v</span><span class="p">):</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">struct</span><span class="o">.</span><span class="n">pack</span><span class="p">(</span><span class="s1">&#39;&gt;i&#39;</span><span class="p">,</span> <span class="n">v</span><span class="p">))</span></div>
+
+<div class="viewcode-block" id="OutputStream.write_bigendian_double"><a class="viewcode-back" href="../../../apache_beam.coders.slow_stream.html#apache_beam.coders.slow_stream.OutputStream.write_bigendian_double">[docs]</a>  <span class="k">def</span> <span class="nf">write_bigendian_double</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">v</span><span class="p">):</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">struct</span><span class="o">.</span><span class="n">pack</span><span class="p">(</span><span class="s1">&#39;&gt;d&#39;</span><span class="p">,</span> <span class="n">v</span><span class="p">))</span></div>
+
+<div class="viewcode-block" id="OutputStream.get"><a class="viewcode-back" href="../../../apache_beam.coders.slow_stream.html#apache_beam.coders.slow_stream.OutputStream.get">[docs]</a>  <span class="k">def</span> <span class="nf">get</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># type: () -&gt; bytes</span>
+    <span class="k">return</span> <span class="sa">b</span><span class="s1">&#39;&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="OutputStream.size"><a class="viewcode-back" href="../../../apache_beam.coders.slow_stream.html#apache_beam.coders.slow_stream.OutputStream.size">[docs]</a>  <span class="k">def</span> <span class="nf">size</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># type: () -&gt; int</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">byte_count</span></div>
+
+  <span class="k">def</span> <span class="nf">_clear</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># type: () -&gt; None</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">data</span> <span class="o">=</span> <span class="p">[]</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">byte_count</span> <span class="o">=</span> <span class="mi">0</span></div>
+
+
+<div class="viewcode-block" id="ByteCountingOutputStream"><a class="viewcode-back" href="../../../apache_beam.coders.slow_stream.html#apache_beam.coders.slow_stream.ByteCountingOutputStream">[docs]</a><span class="k">class</span> <span class="nc">ByteCountingOutputStream</span><span class="p">(</span><span class="n">OutputStream</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;For internal use only; no backwards-compatibility guarantees.</span>
+
+<span class="sd">  A pure Python implementation of stream.ByteCountingOutputStream.&quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># Note that we don&#39;t actually use any of the data initialized by our super.</span>
+    <span class="nb">super</span><span class="p">(</span><span class="n">ByteCountingOutputStream</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">count</span> <span class="o">=</span> <span class="mi">0</span>
+
+<div class="viewcode-block" id="ByteCountingOutputStream.write"><a class="viewcode-back" href="../../../apache_beam.coders.slow_stream.html#apache_beam.coders.slow_stream.ByteCountingOutputStream.write">[docs]</a>  <span class="k">def</span> <span class="nf">write</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">byte_array</span><span class="p">,</span> <span class="n">nested</span><span class="o">=</span><span class="kc">False</span><spa [...]
+    <span class="c1"># type: (bytes, bool) -&gt; None</span>
+    <span class="n">blen</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">byte_array</span><span class="p">)</span>
+    <span class="k">if</span> <span class="n">nested</span><span class="p">:</span>
+      <span class="bp">self</span><span class="o">.</span><span class="n">write_var_int64</span><span class="p">(</span><span class="n">blen</span><span class="p">)</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">count</span> <span class="o">+=</span> <span class="n">blen</span></div>
+
+<div class="viewcode-block" id="ByteCountingOutputStream.write_byte"><a class="viewcode-back" href="../../../apache_beam.coders.slow_stream.html#apache_beam.coders.slow_stream.ByteCountingOutputStream.write_byte">[docs]</a>  <span class="k">def</span> <span class="nf">write_byte</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">_</span><span class="p">):</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">count</span> <span class="o">+=</span> <span class="mi">1</span></div>
+
+<div class="viewcode-block" id="ByteCountingOutputStream.get_count"><a class="viewcode-back" href="../../../apache_beam.coders.slow_stream.html#apache_beam.coders.slow_stream.ByteCountingOutputStream.get_count">[docs]</a>  <span class="k">def</span> <span class="nf">get_count</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">count</span></div>
+
+<div class="viewcode-block" id="ByteCountingOutputStream.get"><a class="viewcode-back" href="../../../apache_beam.coders.slow_stream.html#apache_beam.coders.slow_stream.ByteCountingOutputStream.get">[docs]</a>  <span class="k">def</span> <span class="nf">get</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">raise</span> <span class="ne">NotImplementedError</span></div>
+
+  <span class="k">def</span> <span class="fm">__str__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="s1">&#39;&lt;</span><span class="si">%s</span><span class="s1"> </span><span class="si">%s</span><span class="s1">&gt;&#39;</span> <span class="o">%</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="vm">__class__</span><span class="o">.</span><span class="vm">__name__</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">count</span><span class="p">)</ [...]
+
+
+<div class="viewcode-block" id="InputStream"><a class="viewcode-back" href="../../../apache_beam.coders.slow_stream.html#apache_beam.coders.slow_stream.InputStream">[docs]</a><span class="k">class</span> <span class="nc">InputStream</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;For internal use only; no backwards-compatibility guarantees.</span>
+
+<span class="sd">  A pure Python implementation of stream.InputStream.&quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data</span><span class="p">):</span>
+    <span class="c1"># type: (bytes) -&gt; None</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">data</span> <span class="o">=</span> <span class="n">data</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">pos</span> <span class="o">=</span> <span class="mi">0</span>
+
+    <span class="c1"># The behavior of looping over a byte-string and obtaining byte characters</span>
+    <span class="c1"># has been changed between python 2 and 3.</span>
+    <span class="c1"># b = b&#39;\xff\x01&#39;</span>
+    <span class="c1"># Python 2:</span>
+    <span class="c1"># b[0] = &#39;\xff&#39;</span>
+    <span class="c1"># ord(b[0]) = 255</span>
+    <span class="c1"># Python 3:</span>
+    <span class="c1"># b[0] = 255</span>
+    <span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">&gt;=</span> <span class="mi">3</span><span class="p">:</span>
+      <span class="bp">self</span><span class="o">.</span><span class="n">read_byte</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">read_byte_py3</span>
+    <span class="k">else</span><span class="p">:</span>
+      <span class="bp">self</span><span class="o">.</span><span class="n">read_byte</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">read_byte_py2</span>
+
+<div class="viewcode-block" id="InputStream.size"><a class="viewcode-back" href="../../../apache_beam.coders.slow_stream.html#apache_beam.coders.slow_stream.InputStream.size">[docs]</a>  <span class="k">def</span> <span class="nf">size</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">)</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">pos</span></div>
+
+<div class="viewcode-block" id="InputStream.read"><a class="viewcode-back" href="../../../apache_beam.coders.slow_stream.html#apache_beam.coders.slow_stream.InputStream.read">[docs]</a>  <span class="k">def</span> <span class="nf">read</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">size</span><span class="p">):</span>
+    <span class="c1"># type: (int) -&gt; bytes</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">pos</span> <span class="o">+=</span> <span class="n">size</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">pos</span> <span class="o">-</span> <span class="n">size</span> <span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">pos</span><span class="p">]</span></div>
+
+<div class="viewcode-block" id="InputStream.read_all"><a class="viewcode-back" href="../../../apache_beam.coders.slow_stream.html#apache_beam.coders.slow_stream.InputStream.read_all">[docs]</a>  <span class="k">def</span> <span class="nf">read_all</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">nested</span><span class="p">):</span>
+    <span class="c1"># type: (bool) -&gt; bytes</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">read_var_int64</span><span class="p">()</span> <span class="k">if</span> <span class="n">nested</span> <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">size</span><span class="p">())</span></div>
+
+<div class="viewcode-block" id="InputStream.read_byte_py2"><a class="viewcode-back" href="../../../apache_beam.coders.slow_stream.html#apache_beam.coders.slow_stream.InputStream.read_byte_py2">[docs]</a>  <span class="k">def</span> <span class="nf">read_byte_py2</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># type: () -&gt; int</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">pos</span> <span class="o">+=</span> <span class="mi">1</span>
+    <span class="c1"># mypy tests against python 3.x, where this is an error:</span>
+    <span class="k">return</span> <span class="nb">ord</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">pos</span> <span class="o">-</span> <span class="mi">1</span><span class="p">])</span>  <span class="c1"># type: ignore[arg-type]</span></div>
+
+<div class="viewcode-block" id="InputStream.read_byte_py3"><a class="viewcode-back" href="../../../apache_beam.coders.slow_stream.html#apache_beam.coders.slow_stream.InputStream.read_byte_py3">[docs]</a>  <span class="k">def</span> <span class="nf">read_byte_py3</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="c1"># type: () -&gt; int</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">pos</span> <span class="o">+=</span> <span class="mi">1</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">data</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">pos</span> <span class="o">-</span> <span class="mi">1</span><span class="p">]</span></div>
+
+<div class="viewcode-block" id="InputStream.read_var_int64"><a class="viewcode-back" href="../../../apache_beam.coders.slow_stream.html#apache_beam.coders.slow_stream.InputStream.read_var_int64">[docs]</a>  <span class="k">def</span> <span class="nf">read_var_int64</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="n">shift</span> <span class="o">=</span> <span class="mi">0</span>
+    <span class="n">result</span> <span class="o">=</span> <span class="mi">0</span>
+    <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
+      <span class="n">byte</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">read_byte</span><span class="p">()</span>
+      <span class="k">if</span> <span class="n">byte</span> <span class="o">&lt;</span> <span class="mi">0</span><span class="p">:</span>
+        <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s1">&#39;VarLong not terminated.&#39;</span><span class="p">)</span>
+
+      <span class="n">bits</span> <span class="o">=</span> <span class="n">byte</span> <span class="o">&amp;</span> <span class="mh">0x7F</span>
+      <span class="k">if</span> <span class="n">shift</span> <span class="o">&gt;=</span> <span class="mi">64</span> <span class="ow">or</span> <span class="p">(</span><span class="n">shift</span> <span class="o">&gt;=</span> <span class="mi">63</span> <span class="ow">and</span> <span class="n">bits</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">):</span>
+        <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s1">&#39;VarLong too long.&#39;</span><span class="p">)</span>
+      <span class="n">result</span> <span class="o">|=</span> <span class="n">bits</span> <span class="o">&lt;&lt;</span> <span class="n">shift</span>
+      <span class="n">shift</span> <span class="o">+=</span> <span class="mi">7</span>
+      <span class="k">if</span> <span class="ow">not</span> <span class="n">byte</span> <span class="o">&amp;</span> <span class="mh">0x80</span><span class="p">:</span>
+        <span class="k">break</span>
+    <span class="k">if</span> <span class="n">result</span> <span class="o">&gt;=</span> <span class="mi">1</span> <span class="o">&lt;&lt;</span> <span class="mi">63</span><span class="p">:</span>
+      <span class="n">result</span> <span class="o">-=</span> <span class="mi">1</span> <span class="o">&lt;&lt;</span> <span class="mi">64</span>
+    <span class="k">return</span> <span class="n">result</span></div>
+
+<div class="viewcode-block" id="InputStream.read_bigendian_int64"><a class="viewcode-back" href="../../../apache_beam.coders.slow_stream.html#apache_beam.coders.slow_stream.InputStream.read_bigendian_int64">[docs]</a>  <span class="k">def</span> <span class="nf">read_bigendian_int64</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">struct</span><span class="o">.</span><span class="n">unpack</span><span class="p">(</span><span class="s1">&#39;&gt;q&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="mi">8</span><span class="p">))[</span><span class="mi">0</span><span class="p">]</span></div>
+
+<div class="viewcode-block" id="InputStream.read_bigendian_uint64"><a class="viewcode-back" href="../../../apache_beam.coders.slow_stream.html#apache_beam.coders.slow_stream.InputStream.read_bigendian_uint64">[docs]</a>  <span class="k">def</span> <span class="nf">read_bigendian_uint64</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">struct</span><span class="o">.</span><span class="n">unpack</span><span class="p">(</span><span class="s1">&#39;&gt;Q&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="mi">8</span><span class="p">))[</span><span class="mi">0</span><span class="p">]</span></div>
+
+<div class="viewcode-block" id="InputStream.read_bigendian_int32"><a class="viewcode-back" href="../../../apache_beam.coders.slow_stream.html#apache_beam.coders.slow_stream.InputStream.read_bigendian_int32">[docs]</a>  <span class="k">def</span> <span class="nf">read_bigendian_int32</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">struct</span><span class="o">.</span><span class="n">unpack</span><span class="p">(</span><span class="s1">&#39;&gt;i&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="mi">4</span><span class="p">))[</span><span class="mi">0</span><span class="p">]</span></div>
+
+<div class="viewcode-block" id="InputStream.read_bigendian_double"><a class="viewcode-back" href="../../../apache_beam.coders.slow_stream.html#apache_beam.coders.slow_stream.InputStream.read_bigendian_double">[docs]</a>  <span class="k">def</span> <span class="nf">read_bigendian_double</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">struct</span><span class="o">.</span><span class="n">unpack</span><span class="p">(</span><span class="s1">&#39;&gt;d&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="mi">8</span><span class="p">))[</span><span class="mi">0</span><span class="p">]</span></div></div>
+
+
+<div class="viewcode-block" id="get_varint_size"><a class="viewcode-back" href="../../../apache_beam.coders.slow_stream.html#apache_beam.coders.slow_stream.get_varint_size">[docs]</a><span class="k">def</span> <span class="nf">get_varint_size</span><span class="p">(</span><span class="n">v</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;For internal use only; no backwards-compatibility guarantees.</span>
+
+<span class="sd">  Returns the size of the given integer value when encode as a VarInt.&quot;&quot;&quot;</span>
+  <span class="k">if</span> <span class="n">v</span> <span class="o">&lt;</span> <span class="mi">0</span><span class="p">:</span>
+    <span class="n">v</span> <span class="o">+=</span> <span class="mi">1</span> <span class="o">&lt;&lt;</span> <span class="mi">64</span>
+    <span class="k">if</span> <span class="n">v</span> <span class="o">&lt;=</span> <span class="mi">0</span><span class="p">:</span>
+      <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;Value too large (negative).&#39;</span><span class="p">)</span>
+  <span class="n">varint_size</span> <span class="o">=</span> <span class="mi">0</span>
+  <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
+    <span class="n">varint_size</span> <span class="o">+=</span> <span class="mi">1</span>
+    <span class="n">v</span> <span class="o">&gt;&gt;=</span> <span class="mi">7</span>
+    <span class="k">if</span> <span class="ow">not</span> <span class="n">v</span><span class="p">:</span>
+      <span class="k">break</span>
+  <span class="k">return</span> <span class="n">varint_size</span></div>
+</pre></div>
+
+           </div>
+           <div class="articleComments">
+            
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+        &copy; Copyright .
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file
diff --git a/pydoc/2.19.0/_modules/apache_beam/error.html b/pydoc/2.19.0/_modules/apache_beam/error.html
new file mode 100644
index 0000000..78a95cf
--- /dev/null
+++ b/pydoc/2.19.0/_modules/apache_beam/error.html
@@ -0,0 +1,285 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>apache_beam.error &mdash; Apache Beam  documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="../../genindex.html"/>
+        <link rel="search" title="Search" href="../../search.html"/>
+    <link rel="top" title="Apache Beam  documentation" href="../../index.html"/>
+        <link rel="up" title="Module code" href="../index.html"/> 
+
+  
+  <script src="../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../index.html" class="icon icon-home"> Apache Beam
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../apache_beam.coders.html">apache_beam.coders package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../apache_beam.internal.html">apache_beam.internal package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../apache_beam.io.html">apache_beam.io package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../apache_beam.metrics.html">apache_beam.metrics package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../apache_beam.options.html">apache_beam.options package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../apache_beam.portability.html">apache_beam.portability package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../apache_beam.runners.html">apache_beam.runners package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../apache_beam.testing.html">apache_beam.testing package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../apache_beam.tools.html">apache_beam.tools package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../apache_beam.transforms.html">apache_beam.transforms package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../apache_beam.typehints.html">apache_beam.typehints package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../apache_beam.utils.html">apache_beam.utils package</a></li>
+</ul>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../apache_beam.error.html">apache_beam.error module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../apache_beam.pipeline.html">apache_beam.pipeline module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../apache_beam.pvalue.html">apache_beam.pvalue module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../apache_beam.version.html">apache_beam.version module</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../index.html">Apache Beam</a>
+        
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../index.html">Module code</a> &raquo;</li>
+        
+      <li>apache_beam.error</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+            
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for apache_beam.error</h1><div class="highlight"><pre>
+<span></span><span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one or more</span>
+<span class="c1"># contributor license agreements.  See the NOTICE file distributed with</span>
+<span class="c1"># this work for additional information regarding copyright ownership.</span>
+<span class="c1"># The ASF licenses this file to You under the Apache License, Version 2.0</span>
+<span class="c1"># (the &quot;License&quot;); you may not use this file except in compliance with</span>
+<span class="c1"># the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#    http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+<span class="c1">#</span>
+
+<span class="sd">&quot;&quot;&quot;Python Dataflow error classes.&quot;&quot;&quot;</span>
+
+<span class="c1"># pytype: skip-file</span>
+
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">absolute_import</span>
+
+
+<div class="viewcode-block" id="BeamError"><a class="viewcode-back" href="../../apache_beam.error.html#apache_beam.error.BeamError">[docs]</a><span class="k">class</span> <span class="nc">BeamError</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;Base class for all Beam errors.&quot;&quot;&quot;</span></div>
+
+
+<div class="viewcode-block" id="PipelineError"><a class="viewcode-back" href="../../apache_beam.error.html#apache_beam.error.PipelineError">[docs]</a><span class="k">class</span> <span class="nc">PipelineError</span><span class="p">(</span><span class="n">BeamError</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;An error in the pipeline object (e.g. a PValue not linked to it).&quot;&quot;&quot;</span></div>
+
+
+<div class="viewcode-block" id="PValueError"><a class="viewcode-back" href="../../apache_beam.error.html#apache_beam.error.PValueError">[docs]</a><span class="k">class</span> <span class="nc">PValueError</span><span class="p">(</span><span class="n">BeamError</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;An error related to a PValue object (e.g. value is not computed).&quot;&quot;&quot;</span></div>
+
+
+<div class="viewcode-block" id="RunnerError"><a class="viewcode-back" href="../../apache_beam.error.html#apache_beam.error.RunnerError">[docs]</a><span class="k">class</span> <span class="nc">RunnerError</span><span class="p">(</span><span class="n">BeamError</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;An error related to a Runner object (e.g. cannot find a runner to run).&quot;&quot;&quot;</span></div>
+
+
+<div class="viewcode-block" id="RuntimeValueProviderError"><a class="viewcode-back" href="../../apache_beam.error.html#apache_beam.error.RuntimeValueProviderError">[docs]</a><span class="k">class</span> <span class="nc">RuntimeValueProviderError</span><span class="p">(</span><span class="ne">RuntimeError</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;An error related to a ValueProvider object raised during runtime.&quot;&quot;&quot;</span></div>
+
+
+<div class="viewcode-block" id="SideInputError"><a class="viewcode-back" href="../../apache_beam.error.html#apache_beam.error.SideInputError">[docs]</a><span class="k">class</span> <span class="nc">SideInputError</span><span class="p">(</span><span class="n">BeamError</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;An error related to a side input to a parallel Do operation.&quot;&quot;&quot;</span></div>
+
+
+<div class="viewcode-block" id="TransformError"><a class="viewcode-back" href="../../apache_beam.error.html#apache_beam.error.TransformError">[docs]</a><span class="k">class</span> <span class="nc">TransformError</span><span class="p">(</span><span class="n">BeamError</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;An error related to a PTransform object.&quot;&quot;&quot;</span></div>
+</pre></div>
+
+           </div>
+           <div class="articleComments">
+            
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+        &copy; Copyright .
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file
diff --git a/pydoc/2.19.0/_modules/apache_beam/internal/gcp/auth.html b/pydoc/2.19.0/_modules/apache_beam/internal/gcp/auth.html
new file mode 100644
index 0000000..1b75c7d
--- /dev/null
+++ b/pydoc/2.19.0/_modules/apache_beam/internal/gcp/auth.html
@@ -0,0 +1,378 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>apache_beam.internal.gcp.auth &mdash; Apache Beam  documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="../../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../../search.html"/>
+    <link rel="top" title="Apache Beam  documentation" href="../../../../index.html"/>
+        <link rel="up" title="Module code" href="../../../index.html"/> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Apache Beam
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.coders.html">apache_beam.coders package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.internal.html">apache_beam.internal package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.io.html">apache_beam.io package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.metrics.html">apache_beam.metrics package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.options.html">apache_beam.options package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.portability.html">apache_beam.portability package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.runners.html">apache_beam.runners package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.testing.html">apache_beam.testing package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.tools.html">apache_beam.tools package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.transforms.html">apache_beam.transforms package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.typehints.html">apache_beam.typehints package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.utils.html">apache_beam.utils package</a></li>
+</ul>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.error.html">apache_beam.error module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.pipeline.html">apache_beam.pipeline module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.pvalue.html">apache_beam.pvalue module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.version.html">apache_beam.version module</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Apache Beam</a>
+        
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>apache_beam.internal.gcp.auth</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+            
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for apache_beam.internal.gcp.auth</h1><div class="highlight"><pre>
+<span></span><span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one or more</span>
+<span class="c1"># contributor license agreements.  See the NOTICE file distributed with</span>
+<span class="c1"># this work for additional information regarding copyright ownership.</span>
+<span class="c1"># The ASF licenses this file to You under the Apache License, Version 2.0</span>
+<span class="c1"># (the &quot;License&quot;); you may not use this file except in compliance with</span>
+<span class="c1"># the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#    http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+<span class="c1">#</span>
+
+<span class="sd">&quot;&quot;&quot;Dataflow credentials and authentication.&quot;&quot;&quot;</span>
+
+<span class="c1"># pytype: skip-file</span>
+
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">absolute_import</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
+<span class="kn">import</span> <span class="nn">socket</span>
+<span class="kn">import</span> <span class="nn">threading</span>
+
+<span class="kn">from</span> <span class="nn">oauth2client.client</span> <span class="kn">import</span> <span class="n">GoogleCredentials</span>
+
+<span class="kn">from</span> <span class="nn">apache_beam.utils</span> <span class="kn">import</span> <span class="n">retry</span>
+
+<span class="c1"># Protect against environments where apitools library is not available.</span>
+<span class="k">try</span><span class="p">:</span>
+  <span class="kn">from</span> <span class="nn">apitools.base.py.credentials_lib</span> <span class="kn">import</span> <span class="n">GceAssertionCredentials</span>
+<span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
+  <span class="n">GceAssertionCredentials</span> <span class="o">=</span> <span class="kc">None</span>
+
+<span class="c1"># When we are running in GCE, we can authenticate with VM credentials.</span>
+<span class="n">is_running_in_gce</span> <span class="o">=</span> <span class="kc">False</span>
+
+<span class="c1"># When we are running in GCE, this value is set based on worker startup</span>
+<span class="c1"># information.</span>
+<span class="n">executing_project</span> <span class="o">=</span> <span class="kc">None</span>
+
+
+<span class="n">_LOGGER</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="vm">__name__</span><span class="p">)</span>
+
+
+<span class="k">if</span> <span class="n">GceAssertionCredentials</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+  <span class="k">class</span> <span class="nc">_GceAssertionCredentials</span><span class="p">(</span><span class="n">GceAssertionCredentials</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;GceAssertionCredentials with retry wrapper.</span>
+
+<span class="sd">    For internal use only; no backwards-compatibility guarantees.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+
+    <span class="nd">@retry</span><span class="o">.</span><span class="n">with_exponential_backoff</span><span class="p">(</span>
+        <span class="n">retry_filter</span><span class="o">=</span><span class="n">retry</span><span class="o">.</span><span class="n">retry_on_server_errors_and_timeout_filter</span><span class="p">)</span>
+    <span class="k">def</span> <span class="nf">_do_refresh_request</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">http_request</span><span class="p">):</span>
+      <span class="k">return</span> <span class="nb">super</span><span class="p">(</span><span class="n">_GceAssertionCredentials</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">_do_refresh_request</span><span class="p">(</span>
+          <span class="n">http_request</span><span class="p">)</span>
+
+
+<div class="viewcode-block" id="set_running_in_gce"><a class="viewcode-back" href="../../../../apache_beam.internal.gcp.auth.html#apache_beam.internal.gcp.auth.set_running_in_gce">[docs]</a><span class="k">def</span> <span class="nf">set_running_in_gce</span><span class="p">(</span><span class="n">worker_executing_project</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;For internal use only; no backwards-compatibility guarantees.</span>
+
+<span class="sd">  Informs the authentication library that we are running in GCE.</span>
+
+<span class="sd">  When we are running in GCE, we have the option of using the VM metadata</span>
+<span class="sd">  credentials for authentication to Google services.</span>
+
+<span class="sd">  Args:</span>
+<span class="sd">    worker_executing_project: The project running the workflow. This information</span>
+<span class="sd">      comes from worker startup information.</span>
+<span class="sd">  &quot;&quot;&quot;</span>
+  <span class="k">global</span> <span class="n">is_running_in_gce</span>
+  <span class="k">global</span> <span class="n">executing_project</span>
+  <span class="n">is_running_in_gce</span> <span class="o">=</span> <span class="kc">True</span>
+  <span class="n">executing_project</span> <span class="o">=</span> <span class="n">worker_executing_project</span></div>
+
+
+<div class="viewcode-block" id="get_service_credentials"><a class="viewcode-back" href="../../../../apache_beam.internal.gcp.auth.html#apache_beam.internal.gcp.auth.get_service_credentials">[docs]</a><span class="k">def</span> <span class="nf">get_service_credentials</span><span class="p">():</span>
+  <span class="sd">&quot;&quot;&quot;For internal use only; no backwards-compatibility guarantees.</span>
+
+<span class="sd">  Get credentials to access Google services.</span>
+
+<span class="sd">  Returns:</span>
+<span class="sd">    A ``oauth2client.client.OAuth2Credentials`` object or None if credentials</span>
+<span class="sd">    not found. Returned object is thread-safe.</span>
+<span class="sd">  &quot;&quot;&quot;</span>
+  <span class="k">return</span> <span class="n">_Credentials</span><span class="o">.</span><span class="n">get_service_credentials</span><span class="p">()</span></div>
+
+
+<span class="k">class</span> <span class="nc">_Credentials</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
+  <span class="n">_credentials_lock</span> <span class="o">=</span> <span class="n">threading</span><span class="o">.</span><span class="n">Lock</span><span class="p">()</span>
+  <span class="n">_credentials_init</span> <span class="o">=</span> <span class="kc">False</span>
+  <span class="n">_credentials</span> <span class="o">=</span> <span class="kc">None</span>
+
+  <span class="nd">@classmethod</span>
+  <span class="k">def</span> <span class="nf">get_service_credentials</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
+    <span class="k">if</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_credentials_init</span><span class="p">:</span>
+      <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_credentials</span>
+
+    <span class="k">with</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_credentials_lock</span><span class="p">:</span>
+      <span class="k">if</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_credentials_init</span><span class="p">:</span>
+        <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_credentials</span>
+
+      <span class="c1"># apitools use urllib with the global timeout. Set it to 60 seconds</span>
+      <span class="c1"># to prevent network related stuckness issues.</span>
+      <span class="k">if</span> <span class="ow">not</span> <span class="n">socket</span><span class="o">.</span><span class="n">getdefaulttimeout</span><span class="p">():</span>
+        <span class="n">_LOGGER</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Setting socket default timeout to 60 seconds.&quot;</span><span class="p">)</span>
+        <span class="n">socket</span><span class="o">.</span><span class="n">setdefaulttimeout</span><span class="p">(</span><span class="mi">60</span><span class="p">)</span>
+      <span class="n">_LOGGER</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+          <span class="s2">&quot;socket default timeout is </span><span class="si">%s</span><span class="s2"> seconds.&quot;</span><span class="p">,</span> <span class="n">socket</span><span class="o">.</span><span class="n">getdefaulttimeout</span><span class="p">())</span>
+
+      <span class="bp">cls</span><span class="o">.</span><span class="n">_credentials</span> <span class="o">=</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_get_service_credentials</span><span class="p">()</span>
+      <span class="bp">cls</span><span class="o">.</span><span class="n">_credentials_init</span> <span class="o">=</span> <span class="kc">True</span>
+
+    <span class="k">return</span> <span class="bp">cls</span><span class="o">.</span><span class="n">_credentials</span>
+
+  <span class="nd">@staticmethod</span>
+  <span class="k">def</span> <span class="nf">_get_service_credentials</span><span class="p">():</span>
+    <span class="k">if</span> <span class="n">is_running_in_gce</span><span class="p">:</span>
+      <span class="c1"># We are currently running as a GCE taskrunner worker.</span>
+      <span class="k">return</span> <span class="n">_GceAssertionCredentials</span><span class="p">(</span><span class="n">user_agent</span><span class="o">=</span><span class="s1">&#39;beam-python-sdk/1.0&#39;</span><span class="p">)</span>
+    <span class="k">else</span><span class="p">:</span>
+      <span class="n">client_scopes</span> <span class="o">=</span> <span class="p">[</span>
+          <span class="s1">&#39;https://www.googleapis.com/auth/bigquery&#39;</span><span class="p">,</span>
+          <span class="s1">&#39;https://www.googleapis.com/auth/cloud-platform&#39;</span><span class="p">,</span>
+          <span class="s1">&#39;https://www.googleapis.com/auth/devstorage.full_control&#39;</span><span class="p">,</span>
+          <span class="s1">&#39;https://www.googleapis.com/auth/userinfo.email&#39;</span><span class="p">,</span>
+          <span class="s1">&#39;https://www.googleapis.com/auth/datastore&#39;</span>
+      <span class="p">]</span>
+      <span class="k">try</span><span class="p">:</span>
+        <span class="n">credentials</span> <span class="o">=</span> <span class="n">GoogleCredentials</span><span class="o">.</span><span class="n">get_application_default</span><span class="p">()</span>
+        <span class="n">credentials</span> <span class="o">=</span> <span class="n">credentials</span><span class="o">.</span><span class="n">create_scoped</span><span class="p">(</span><span class="n">client_scopes</span><span class="p">)</span>
+        <span class="n">logging</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">&#39;Connecting using Google Application Default &#39;</span>
+                      <span class="s1">&#39;Credentials.&#39;</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">credentials</span>
+      <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+        <span class="n">_LOGGER</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
+            <span class="s1">&#39;Unable to find default credentials to use: </span><span class="si">%s</span><span class="se">\n</span><span class="s1">&#39;</span>
+            <span class="s1">&#39;Connecting anonymously.&#39;</span><span class="p">,</span> <span class="n">e</span><span class="p">)</span>
+        <span class="k">return</span> <span class="kc">None</span>
+</pre></div>
+
+           </div>
+           <div class="articleComments">
+            
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+        &copy; Copyright .
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../../../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file
diff --git a/pydoc/2.19.0/_modules/apache_beam/internal/gcp/json_value.html b/pydoc/2.19.0/_modules/apache_beam/internal/gcp/json_value.html
new file mode 100644
index 0000000..f727d30
--- /dev/null
+++ b/pydoc/2.19.0/_modules/apache_beam/internal/gcp/json_value.html
@@ -0,0 +1,406 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>apache_beam.internal.gcp.json_value &mdash; Apache Beam  documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="../../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../../search.html"/>
+    <link rel="top" title="Apache Beam  documentation" href="../../../../index.html"/>
+        <link rel="up" title="Module code" href="../../../index.html"/> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Apache Beam
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.coders.html">apache_beam.coders package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.internal.html">apache_beam.internal package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.io.html">apache_beam.io package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.metrics.html">apache_beam.metrics package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.options.html">apache_beam.options package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.portability.html">apache_beam.portability package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.runners.html">apache_beam.runners package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.testing.html">apache_beam.testing package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.tools.html">apache_beam.tools package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.transforms.html">apache_beam.transforms package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.typehints.html">apache_beam.typehints package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.utils.html">apache_beam.utils package</a></li>
+</ul>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.error.html">apache_beam.error module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.pipeline.html">apache_beam.pipeline module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.pvalue.html">apache_beam.pvalue module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.version.html">apache_beam.version module</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Apache Beam</a>
+        
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>apache_beam.internal.gcp.json_value</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+            
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for apache_beam.internal.gcp.json_value</h1><div class="highlight"><pre>
+<span></span><span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one or more</span>
+<span class="c1"># contributor license agreements.  See the NOTICE file distributed with</span>
+<span class="c1"># this work for additional information regarding copyright ownership.</span>
+<span class="c1"># The ASF licenses this file to You under the Apache License, Version 2.0</span>
+<span class="c1"># (the &quot;License&quot;); you may not use this file except in compliance with</span>
+<span class="c1"># the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#    http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+<span class="c1">#</span>
+
+<span class="sd">&quot;&quot;&quot;JSON conversion utility functions.&quot;&quot;&quot;</span>
+
+<span class="c1"># pytype: skip-file</span>
+
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">absolute_import</span>
+
+<span class="kn">from</span> <span class="nn">past.builtins</span> <span class="kn">import</span> <span class="n">long</span>
+<span class="kn">from</span> <span class="nn">past.builtins</span> <span class="kn">import</span> <span class="n">unicode</span>
+
+<span class="kn">from</span> <span class="nn">apache_beam.options.value_provider</span> <span class="kn">import</span> <span class="n">ValueProvider</span>
+
+<span class="c1"># Protect against environments where apitools library is not available.</span>
+<span class="c1"># pylint: disable=wrong-import-order, wrong-import-position</span>
+<span class="k">try</span><span class="p">:</span>
+  <span class="kn">from</span> <span class="nn">apitools.base.py</span> <span class="kn">import</span> <span class="n">extra_types</span>
+<span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
+  <span class="n">extra_types</span> <span class="o">=</span> <span class="kc">None</span>
+<span class="c1"># pylint: enable=wrong-import-order, wrong-import-position</span>
+
+
+<span class="n">_MAXINT64</span> <span class="o">=</span> <span class="p">(</span><span class="mi">1</span> <span class="o">&lt;&lt;</span> <span class="mi">63</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span>
+<span class="n">_MININT64</span> <span class="o">=</span> <span class="o">-</span> <span class="p">(</span><span class="mi">1</span> <span class="o">&lt;&lt;</span> <span class="mi">63</span><span class="p">)</span>
+
+
+<div class="viewcode-block" id="get_typed_value_descriptor"><a class="viewcode-back" href="../../../../apache_beam.internal.gcp.json_value.html#apache_beam.internal.gcp.json_value.get_typed_value_descriptor">[docs]</a><span class="k">def</span> <span class="nf">get_typed_value_descriptor</span><span class="p">(</span><span class="n">obj</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;For internal use only; no backwards-compatibility guarantees.</span>
+
+<span class="sd">  Converts a basic type into a @type/value dictionary.</span>
+
+<span class="sd">  Args:</span>
+<span class="sd">    obj: A bytes, unicode, bool, int, or float to be converted.</span>
+
+<span class="sd">  Returns:</span>
+<span class="sd">    A dictionary containing the keys ``@type`` and ``value`` with the value for</span>
+<span class="sd">    the ``@type`` of appropriate type.</span>
+
+<span class="sd">  Raises:</span>
+<span class="sd">    ~exceptions.TypeError: if the Python object has a type that is not</span>
+<span class="sd">      supported.</span>
+<span class="sd">  &quot;&quot;&quot;</span>
+  <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="p">(</span><span class="nb">bytes</span><span class="p">,</span> <span class="n">unicode</span><span class="p">)):</span>
+    <span class="n">type_name</span> <span class="o">=</span> <span class="s1">&#39;Text&#39;</span>
+  <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="nb">bool</span><span class="p">):</span>
+    <span class="n">type_name</span> <span class="o">=</span> <span class="s1">&#39;Boolean&#39;</span>
+  <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="nb">int</span><span class="p">):</span>
+    <span class="n">type_name</span> <span class="o">=</span> <span class="s1">&#39;Integer&#39;</span>
+  <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="nb">float</span><span class="p">):</span>
+    <span class="n">type_name</span> <span class="o">=</span> <span class="s1">&#39;Float&#39;</span>
+  <span class="k">else</span><span class="p">:</span>
+    <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s1">&#39;Cannot get a type descriptor for </span><span class="si">%s</span><span class="s1">.&#39;</span> <span class="o">%</span> <span class="nb">repr</span><span class="p">(</span><span class="n">obj</span><span class="p">))</span>
+  <span class="k">return</span> <span class="p">{</span><span class="s1">&#39;@type&#39;</span><span class="p">:</span> <span class="s1">&#39;http://schema.org/</span><span class="si">%s</span><span class="s1">&#39;</span> <span class="o">%</span> <span class="n">type_name</span><span class="p">,</span> <span class="s1">&#39;value&#39;</span><span class="p">:</span> <span class="n">obj</span><span class="p">}</span></div>
+
+
+<div class="viewcode-block" id="to_json_value"><a class="viewcode-back" href="../../../../apache_beam.internal.gcp.json_value.html#apache_beam.internal.gcp.json_value.to_json_value">[docs]</a><span class="k">def</span> <span class="nf">to_json_value</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="n">with_type</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;For internal use only; no backwards-compatibility guarantees.</span>
+
+<span class="sd">  Converts Python objects into extra_types.JsonValue objects.</span>
+
+<span class="sd">  Args:</span>
+<span class="sd">    obj: Python object to be converted. Can be :data:`None`.</span>
+<span class="sd">    with_type: If true then the basic types (``bytes``, ``unicode``, ``int``,</span>
+<span class="sd">      ``float``, ``bool``) will be wrapped in ``@type:value`` dictionaries.</span>
+<span class="sd">      Otherwise the straight value is encoded into a ``JsonValue``.</span>
+
+<span class="sd">  Returns:</span>
+<span class="sd">    A ``JsonValue`` object using ``JsonValue``, ``JsonArray`` and ``JsonObject``</span>
+<span class="sd">    types for the corresponding values, lists, or dictionaries.</span>
+
+<span class="sd">  Raises:</span>
+<span class="sd">    ~exceptions.TypeError: if the Python object contains a type that is not</span>
+<span class="sd">      supported.</span>
+
+<span class="sd">  The types supported are ``str``, ``bool``, ``list``, ``tuple``, ``dict``, and</span>
+<span class="sd">  ``None``. The Dataflow API requires JsonValue(s) in many places, and it is</span>
+<span class="sd">  quite convenient to be able to specify these hierarchical objects using</span>
+<span class="sd">  Python syntax.</span>
+<span class="sd">  &quot;&quot;&quot;</span>
+  <span class="k">if</span> <span class="n">obj</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+    <span class="k">return</span> <span class="n">extra_types</span><span class="o">.</span><span class="n">JsonValue</span><span class="p">(</span><span class="n">is_null</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+  <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="p">(</span><span class="nb">list</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)):</span>
+    <span class="k">return</span> <span class="n">extra_types</span><span class="o">.</span><span class="n">JsonValue</span><span class="p">(</span>
+        <span class="n">array_value</span><span class="o">=</span><span class="n">extra_types</span><span class="o">.</span><span class="n">JsonArray</span><span class="p">(</span>
+            <span class="n">entries</span><span class="o">=</span><span class="p">[</span><span class="n">to_json_value</span><span class="p">(</span><span class="n">o</span><span class="p">,</span> <span class="n">with_type</span><span class="o">=</span><span class="n">with_type</span><span class="p">)</span> <span class="k">for</span> <span class="n">o</span> <span class="ow">in</span> <span class="n">obj</span><span class="p">]))</span>
+  <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
+    <span class="n">json_object</span> <span class="o">=</span> <span class="n">extra_types</span><span class="o">.</span><span class="n">JsonObject</span><span class="p">()</span>
+    <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">obj</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+      <span class="n">json_object</span><span class="o">.</span><span class="n">properties</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
+          <span class="n">extra_types</span><span class="o">.</span><span class="n">JsonObject</span><span class="o">.</span><span class="n">Property</span><span class="p">(</span>
+              <span class="n">key</span><span class="o">=</span><span class="n">k</span><span class="p">,</span> <span class="n">value</span><span class="o">=</span><span class="n">to_json_value</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="n">with_type</span><span class="o">=</span><span class="n">with_type</span><span class="p">)))</span>
+    <span class="k">return</span> <span class="n">extra_types</span><span class="o">.</span><span class="n">JsonValue</span><span class="p">(</span><span class="n">object_value</span><span class="o">=</span><span class="n">json_object</span><span class="p">)</span>
+  <span class="k">elif</span> <span class="n">with_type</span><span class="p">:</span>
+    <span class="k">return</span> <span class="n">to_json_value</span><span class="p">(</span><span class="n">get_typed_value_descriptor</span><span class="p">(</span><span class="n">obj</span><span class="p">),</span> <span class="n">with_type</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
+  <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="p">(</span><span class="nb">str</span><span class="p">,</span> <span class="n">unicode</span><span class="p">)):</span>
+    <span class="k">return</span> <span class="n">extra_types</span><span class="o">.</span><span class="n">JsonValue</span><span class="p">(</span><span class="n">string_value</span><span class="o">=</span><span class="n">obj</span><span class="p">)</span>
+  <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="nb">bytes</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">extra_types</span><span class="o">.</span><span class="n">JsonValue</span><span class="p">(</span><span class="n">string_value</span><span class="o">=</span><span class="n">obj</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">&#39;utf8&#39;</span><span class="p">))</span>
+  <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="nb">bool</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">extra_types</span><span class="o">.</span><span class="n">JsonValue</span><span class="p">(</span><span class="n">boolean_value</span><span class="o">=</span><span class="n">obj</span><span class="p">)</span>
+  <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="p">(</span><span class="nb">int</span><span class="p">,</span> <span class="n">long</span><span class="p">)):</span>
+    <span class="k">if</span> <span class="n">_MININT64</span> <span class="o">&lt;=</span> <span class="n">obj</span> <span class="o">&lt;=</span> <span class="n">_MAXINT64</span><span class="p">:</span>
+      <span class="k">return</span> <span class="n">extra_types</span><span class="o">.</span><span class="n">JsonValue</span><span class="p">(</span><span class="n">integer_value</span><span class="o">=</span><span class="n">obj</span><span class="p">)</span>
+    <span class="k">else</span><span class="p">:</span>
+      <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s1">&#39;Can not encode </span><span class="si">{}</span><span class="s1"> as a 64-bit integer&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">obj</span><span class="p">))</span>
+  <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="nb">float</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">extra_types</span><span class="o">.</span><span class="n">JsonValue</span><span class="p">(</span><span class="n">double_value</span><span class="o">=</span><span class="n">obj</span><span class="p">)</span>
+  <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="n">ValueProvider</span><span class="p">):</span>
+    <span class="k">if</span> <span class="n">obj</span><span class="o">.</span><span class="n">is_accessible</span><span class="p">():</span>
+      <span class="k">return</span> <span class="n">to_json_value</span><span class="p">(</span><span class="n">obj</span><span class="o">.</span><span class="n">get</span><span class="p">())</span>
+    <span class="k">return</span> <span class="n">extra_types</span><span class="o">.</span><span class="n">JsonValue</span><span class="p">(</span><span class="n">is_null</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+  <span class="k">else</span><span class="p">:</span>
+    <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s1">&#39;Cannot convert </span><span class="si">%s</span><span class="s1"> to a JSON value.&#39;</span> <span class="o">%</span> <span class="nb">repr</span><span class="p">(</span><span class="n">obj</span><span class="p">))</span></div>
+
+
+<div class="viewcode-block" id="from_json_value"><a class="viewcode-back" href="../../../../apache_beam.internal.gcp.json_value.html#apache_beam.internal.gcp.json_value.from_json_value">[docs]</a><span class="k">def</span> <span class="nf">from_json_value</span><span class="p">(</span><span class="n">v</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;For internal use only; no backwards-compatibility guarantees.</span>
+
+<span class="sd">  Converts ``extra_types.JsonValue`` objects into Python objects.</span>
+
+<span class="sd">  Args:</span>
+<span class="sd">    v: ``JsonValue`` object to be converted.</span>
+
+<span class="sd">  Returns:</span>
+<span class="sd">    A Python object structured as values, lists, and dictionaries corresponding</span>
+<span class="sd">    to ``JsonValue``, ``JsonArray`` and ``JsonObject`` types.</span>
+
+<span class="sd">  Raises:</span>
+<span class="sd">    ~exceptions.TypeError: if the ``JsonValue`` object contains a type that is</span>
+<span class="sd">      not supported.</span>
+
+<span class="sd">  The types supported are ``str``, ``bool``, ``list``, ``dict``, and ``None``.</span>
+<span class="sd">  The Dataflow API returns JsonValue(s) in many places and it is quite</span>
+<span class="sd">  convenient to be able to convert these hierarchical objects to much simpler</span>
+<span class="sd">  Python objects.</span>
+<span class="sd">  &quot;&quot;&quot;</span>
+  <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="n">extra_types</span><span class="o">.</span><span class="n">JsonValue</span><span class="p">):</span>
+    <span class="k">if</span> <span class="n">v</span><span class="o">.</span><span class="n">string_value</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+      <span class="k">return</span> <span class="n">v</span><span class="o">.</span><span class="n">string_value</span>
+    <span class="k">elif</span> <span class="n">v</span><span class="o">.</span><span class="n">boolean_value</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+      <span class="k">return</span> <span class="n">v</span><span class="o">.</span><span class="n">boolean_value</span>
+    <span class="k">elif</span> <span class="n">v</span><span class="o">.</span><span class="n">integer_value</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+      <span class="k">return</span> <span class="n">v</span><span class="o">.</span><span class="n">integer_value</span>
+    <span class="k">elif</span> <span class="n">v</span><span class="o">.</span><span class="n">double_value</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+      <span class="k">return</span> <span class="n">v</span><span class="o">.</span><span class="n">double_value</span>
+    <span class="k">elif</span> <span class="n">v</span><span class="o">.</span><span class="n">array_value</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+      <span class="k">return</span> <span class="n">from_json_value</span><span class="p">(</span><span class="n">v</span><span class="o">.</span><span class="n">array_value</span><span class="p">)</span>
+    <span class="k">elif</span> <span class="n">v</span><span class="o">.</span><span class="n">object_value</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+      <span class="k">return</span> <span class="n">from_json_value</span><span class="p">(</span><span class="n">v</span><span class="o">.</span><span class="n">object_value</span><span class="p">)</span>
+    <span class="k">elif</span> <span class="n">v</span><span class="o">.</span><span class="n">is_null</span><span class="p">:</span>
+      <span class="k">return</span> <span class="kc">None</span>
+  <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="n">extra_types</span><span class="o">.</span><span class="n">JsonArray</span><span class="p">):</span>
+    <span class="k">return</span> <span class="p">[</span><span class="n">from_json_value</span><span class="p">(</span><span class="n">e</span><span class="p">)</span> <span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="n">v</span><span class="o">.</span><span class="n">entries</span><span class="p">]</span>
+  <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="n">extra_types</span><span class="o">.</span><span class="n">JsonObject</span><span class="p">):</span>
+    <span class="k">return</span> <span class="p">{</span><span class="n">p</span><span class="o">.</span><span class="n">key</span><span class="p">:</span> <span class="n">from_json_value</span><span class="p">(</span><span class="n">p</span><span class="o">.</span><span class="n">value</span><span class="p">)</span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">v</span><span class="o">.</span><span class="n">properties</span><span class= [...]
+  <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s1">&#39;Cannot convert </span><span class="si">%s</span><span class="s1"> from a JSON value.&#39;</span> <span class="o">%</span> <span class="nb">repr</span><span class="p">(</span><span class="n">v</span><span class="p">))</span></div>
+</pre></div>
+
+           </div>
+           <div class="articleComments">
+            
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+        &copy; Copyright .
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../../../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file
diff --git a/pydoc/2.19.0/_modules/apache_beam/internal/http_client.html b/pydoc/2.19.0/_modules/apache_beam/internal/http_client.html
new file mode 100644
index 0000000..6c2315f
--- /dev/null
+++ b/pydoc/2.19.0/_modules/apache_beam/internal/http_client.html
@@ -0,0 +1,307 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>apache_beam.internal.http_client &mdash; Apache Beam  documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../search.html"/>
+    <link rel="top" title="Apache Beam  documentation" href="../../../index.html"/>
+        <link rel="up" title="Module code" href="../../index.html"/> 
+
+  
+  <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../index.html" class="icon icon-home"> Apache Beam
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.coders.html">apache_beam.coders package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.internal.html">apache_beam.internal package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.io.html">apache_beam.io package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.metrics.html">apache_beam.metrics package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.options.html">apache_beam.options package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.portability.html">apache_beam.portability package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.runners.html">apache_beam.runners package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.testing.html">apache_beam.testing package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.tools.html">apache_beam.tools package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.transforms.html">apache_beam.transforms package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.typehints.html">apache_beam.typehints package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.utils.html">apache_beam.utils package</a></li>
+</ul>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.error.html">apache_beam.error module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.pipeline.html">apache_beam.pipeline module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.pvalue.html">apache_beam.pvalue module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.version.html">apache_beam.version module</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../index.html">Apache Beam</a>
+        
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../index.html">Module code</a> &raquo;</li>
+        
+      <li>apache_beam.internal.http_client</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+            
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for apache_beam.internal.http_client</h1><div class="highlight"><pre>
+<span></span><span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one or more</span>
+<span class="c1"># contributor license agreements.  See the NOTICE file distributed with</span>
+<span class="c1"># this work for additional information regarding copyright ownership.</span>
+<span class="c1"># The ASF licenses this file to You under the Apache License, Version 2.0</span>
+<span class="c1"># (the &quot;License&quot;); you may not use this file except in compliance with</span>
+<span class="c1"># the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#    http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+<span class="c1">#</span>
+
+<span class="sd">&quot;&quot;&quot;Utility functions used for creating a common Http client from httplib2.</span>
+
+<span class="sd">For internal use only. No backwards compatibility guarantees.</span>
+<span class="sd">&quot;&quot;&quot;</span>
+<span class="c1"># pytype: skip-file</span>
+
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">absolute_import</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
+<span class="kn">import</span> <span class="nn">os</span>
+<span class="kn">import</span> <span class="nn">re</span>
+
+<span class="kn">import</span> <span class="nn">httplib2</span>
+
+<span class="c1"># This is the number of seconds the library will wait for GCS operations to</span>
+<span class="c1"># complete.</span>
+<span class="n">DEFAULT_HTTP_TIMEOUT_SECONDS</span> <span class="o">=</span> <span class="mi">60</span>
+
+
+<div class="viewcode-block" id="proxy_info_from_environment_var"><a class="viewcode-back" href="../../../apache_beam.internal.http_client.html#apache_beam.internal.http_client.proxy_info_from_environment_var">[docs]</a><span class="k">def</span> <span class="nf">proxy_info_from_environment_var</span><span class="p">(</span><span class="n">proxy_env_var</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;Reads proxy info from the environment and converts to httplib2.ProxyInfo.</span>
+
+<span class="sd">  Args:</span>
+<span class="sd">    proxy_env_var: environment variable string to read, http_proxy or</span>
+<span class="sd">       https_proxy (in lower case).</span>
+<span class="sd">       Example: http://myproxy.domain.com:8080</span>
+
+<span class="sd">  Returns:</span>
+<span class="sd">    httplib2.ProxyInfo constructed from the environment string.</span>
+<span class="sd">  &quot;&quot;&quot;</span>
+  <span class="n">proxy_url</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">proxy_env_var</span><span class="p">)</span>
+  <span class="k">if</span> <span class="ow">not</span> <span class="n">proxy_url</span><span class="p">:</span>
+    <span class="k">return</span> <span class="kc">None</span>
+  <span class="n">proxy_protocol</span> <span class="o">=</span> <span class="n">proxy_env_var</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39;_&#39;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
+  <span class="k">if</span> <span class="ow">not</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="s1">&#39;^https?://&#39;</span><span class="p">,</span> <span class="n">proxy_url</span><span class="p">,</span> <span class="n">flags</span><span class="o">=</span><span class="n">re</span><span class="o">.</span><span class="n">IGNORECASE</span><span class="p">):</span>
+    <span class="n">logging</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span><span class="s2">&quot;proxy_info_from_url requires a protocol, which is always &quot;</span>
+                    <span class="s2">&quot;http or https.&quot;</span><span class="p">)</span>
+    <span class="n">proxy_url</span> <span class="o">=</span> <span class="n">proxy_protocol</span> <span class="o">+</span> <span class="s1">&#39;://&#39;</span> <span class="o">+</span> <span class="n">proxy_url</span>
+  <span class="k">return</span> <span class="n">httplib2</span><span class="o">.</span><span class="n">proxy_info_from_url</span><span class="p">(</span><span class="n">proxy_url</span><span class="p">,</span> <span class="n">method</span><span class="o">=</span><span class="n">proxy_protocol</span><span class="p">)</span></div>
+
+
+<div class="viewcode-block" id="get_new_http"><a class="viewcode-back" href="../../../apache_beam.internal.http_client.html#apache_beam.internal.http_client.get_new_http">[docs]</a><span class="k">def</span> <span class="nf">get_new_http</span><span class="p">():</span>
+  <span class="sd">&quot;&quot;&quot;Creates and returns a new httplib2.Http instance.</span>
+
+<span class="sd">  Returns:</span>
+<span class="sd">    An initialized httplib2.Http instance.</span>
+<span class="sd">  &quot;&quot;&quot;</span>
+  <span class="n">proxy_info</span> <span class="o">=</span> <span class="kc">None</span>
+  <span class="k">for</span> <span class="n">proxy_env_var</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">&#39;http_proxy&#39;</span><span class="p">,</span> <span class="s1">&#39;https_proxy&#39;</span><span class="p">]:</span>
+    <span class="k">if</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="n">proxy_env_var</span><span class="p">):</span>
+      <span class="n">proxy_info</span> <span class="o">=</span> <span class="n">proxy_info_from_environment_var</span><span class="p">(</span><span class="n">proxy_env_var</span><span class="p">)</span>
+      <span class="k">break</span>
+  <span class="c1"># Use a non-infinite SSL timeout to avoid hangs during network flakiness.</span>
+  <span class="k">return</span> <span class="n">httplib2</span><span class="o">.</span><span class="n">Http</span><span class="p">(</span><span class="n">proxy_info</span><span class="o">=</span><span class="n">proxy_info</span><span class="p">,</span>
+                       <span class="n">timeout</span><span class="o">=</span><span class="n">DEFAULT_HTTP_TIMEOUT_SECONDS</span><span class="p">)</span></div>
+</pre></div>
+
+           </div>
+           <div class="articleComments">
+            
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+        &copy; Copyright .
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file
diff --git a/pydoc/2.19.0/_modules/apache_beam/internal/pickler.html b/pydoc/2.19.0/_modules/apache_beam/internal/pickler.html
new file mode 100644
index 0000000..99211f7
--- /dev/null
+++ b/pydoc/2.19.0/_modules/apache_beam/internal/pickler.html
@@ -0,0 +1,549 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>apache_beam.internal.pickler &mdash; Apache Beam  documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../search.html"/>
+    <link rel="top" title="Apache Beam  documentation" href="../../../index.html"/>
+        <link rel="up" title="Module code" href="../../index.html"/> 
+
+  
+  <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../index.html" class="icon icon-home"> Apache Beam
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.coders.html">apache_beam.coders package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.internal.html">apache_beam.internal package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.io.html">apache_beam.io package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.metrics.html">apache_beam.metrics package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.options.html">apache_beam.options package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.portability.html">apache_beam.portability package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.runners.html">apache_beam.runners package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.testing.html">apache_beam.testing package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.tools.html">apache_beam.tools package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.transforms.html">apache_beam.transforms package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.typehints.html">apache_beam.typehints package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.utils.html">apache_beam.utils package</a></li>
+</ul>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.error.html">apache_beam.error module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.pipeline.html">apache_beam.pipeline module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.pvalue.html">apache_beam.pvalue module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.version.html">apache_beam.version module</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../index.html">Apache Beam</a>
+        
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../index.html">Module code</a> &raquo;</li>
+        
+      <li>apache_beam.internal.pickler</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+            
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for apache_beam.internal.pickler</h1><div class="highlight"><pre>
+<span></span><span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one or more</span>
+<span class="c1"># contributor license agreements.  See the NOTICE file distributed with</span>
+<span class="c1"># this work for additional information regarding copyright ownership.</span>
+<span class="c1"># The ASF licenses this file to You under the Apache License, Version 2.0</span>
+<span class="c1"># (the &quot;License&quot;); you may not use this file except in compliance with</span>
+<span class="c1"># the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#    http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+<span class="c1">#</span>
+
+<span class="sd">&quot;&quot;&quot;Pickler for values, functions, and classes.</span>
+
+<span class="sd">For internal use only. No backwards compatibility guarantees.</span>
+
+<span class="sd">Pickles created by the pickling library contain non-ASCII characters, so</span>
+<span class="sd">we base64-encode the results so that we can put them in a JSON objects.</span>
+<span class="sd">The pickler is used to embed FlatMap callable objects into the workflow JSON</span>
+<span class="sd">description.</span>
+
+<span class="sd">The pickler module should be used to pickle functions and modules; for values,</span>
+<span class="sd">the coders.*PickleCoder classes should be used instead.</span>
+<span class="sd">&quot;&quot;&quot;</span>
+
+<span class="c1"># pytype: skip-file</span>
+
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">absolute_import</span>
+
+<span class="kn">import</span> <span class="nn">base64</span>
+<span class="kn">import</span> <span class="nn">logging</span>
+<span class="kn">import</span> <span class="nn">sys</span>
+<span class="kn">import</span> <span class="nn">threading</span>
+<span class="kn">import</span> <span class="nn">traceback</span>
+<span class="kn">import</span> <span class="nn">types</span>
+<span class="kn">import</span> <span class="nn">zlib</span>
+<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">Any</span>
+<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">Dict</span>
+<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">Tuple</span>
+
+<span class="kn">import</span> <span class="nn">dill</span>
+
+
+<span class="k">class</span> <span class="nc">_NoOpContextManager</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
+  <span class="k">def</span> <span class="fm">__enter__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">pass</span>
+
+  <span class="k">def</span> <span class="fm">__exit__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">unused_exc_info</span><span class="p">):</span>
+    <span class="k">pass</span>
+
+
+<span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">&gt;</span> <span class="mi">2</span><span class="p">:</span>
+  <span class="c1"># Pickling, especially unpickling, causes broken module imports on Python 3</span>
+  <span class="c1"># if executed concurrently, see: BEAM-8651, http://bugs.python.org/issue38884.</span>
+  <span class="n">pickle_lock_unless_py2</span> <span class="o">=</span> <span class="n">threading</span><span class="o">.</span><span class="n">RLock</span><span class="p">()</span>
+<span class="k">else</span><span class="p">:</span>
+  <span class="c1"># Avoid slow reentrant locks on Py2. See: https://bugs.python.org/issue3001.</span>
+  <span class="n">pickle_lock_unless_py2</span> <span class="o">=</span> <span class="n">_NoOpContextManager</span><span class="p">()</span>
+<span class="c1"># Dill 0.28.0 renamed dill.dill to dill._dill:</span>
+<span class="c1"># https://github.com/uqfoundation/dill/commit/f0972ecc7a41d0b8acada6042d557068cac69baa</span>
+<span class="c1"># TODO: Remove this once Beam depends on dill &gt;= 0.2.8</span>
+<span class="k">if</span> <span class="ow">not</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">dill</span><span class="p">,</span> <span class="s1">&#39;dill&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">):</span>
+  <span class="n">dill</span><span class="o">.</span><span class="n">dill</span> <span class="o">=</span> <span class="n">dill</span><span class="o">.</span><span class="n">_dill</span>
+  <span class="n">sys</span><span class="o">.</span><span class="n">modules</span><span class="p">[</span><span class="s1">&#39;dill.dill&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">dill</span><span class="o">.</span><span class="n">_dill</span>
+
+<span class="c1"># TODO: Remove once Dataflow has containers with a preinstalled dill &gt;= 0.2.8</span>
+<span class="k">if</span> <span class="ow">not</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">dill</span><span class="p">,</span> <span class="s1">&#39;_dill&#39;</span><span class="p">,</span> <span class="kc">None</span><span class="p">):</span>
+  <span class="n">dill</span><span class="o">.</span><span class="n">_dill</span> <span class="o">=</span> <span class="n">dill</span><span class="o">.</span><span class="n">dill</span>
+  <span class="n">sys</span><span class="o">.</span><span class="n">modules</span><span class="p">[</span><span class="s1">&#39;dill._dill&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">dill</span><span class="o">.</span><span class="n">dill</span>
+
+
+<span class="k">def</span> <span class="nf">_is_nested_class</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;Returns true if argument is a class object that appears to be nested.&quot;&quot;&quot;</span>
+  <span class="k">return</span> <span class="p">(</span><span class="nb">isinstance</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="nb">type</span><span class="p">)</span>
+          <span class="ow">and</span> <span class="bp">cls</span><span class="o">.</span><span class="vm">__module__</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
+          <span class="ow">and</span> <span class="bp">cls</span><span class="o">.</span><span class="vm">__module__</span> <span class="o">!=</span> <span class="s1">&#39;builtins&#39;</span>     <span class="c1"># Python 3</span>
+          <span class="ow">and</span> <span class="bp">cls</span><span class="o">.</span><span class="vm">__module__</span> <span class="o">!=</span> <span class="s1">&#39;__builtin__&#39;</span>  <span class="c1"># Python 2</span>
+          <span class="ow">and</span> <span class="bp">cls</span><span class="o">.</span><span class="vm">__name__</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">sys</span><span class="o">.</span><span class="n">modules</span><span class="p">[</span><span class="bp">cls</span><span class="o">.</span><span class="vm">__module__</span><span class="p">]</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">)</span>
+
+
+<span class="k">def</span> <span class="nf">_find_containing_class</span><span class="p">(</span><span class="n">nested_class</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;Finds containing class of a nested class passed as argument.&quot;&quot;&quot;</span>
+
+  <span class="n">seen</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
+
+  <span class="k">def</span> <span class="nf">_find_containing_class_inner</span><span class="p">(</span><span class="n">outer</span><span class="p">):</span>
+    <span class="k">if</span> <span class="n">outer</span> <span class="ow">in</span> <span class="n">seen</span><span class="p">:</span>
+      <span class="k">return</span> <span class="kc">None</span>
+    <span class="n">seen</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">outer</span><span class="p">)</span>
+    <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">outer</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
+      <span class="k">if</span> <span class="n">v</span> <span class="ow">is</span> <span class="n">nested_class</span><span class="p">:</span>
+        <span class="k">return</span> <span class="n">outer</span><span class="p">,</span> <span class="n">k</span>
+      <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="nb">type</span><span class="p">)</span> <span class="ow">and</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="s1">&#39;__dict__&#39;</span><span class="p">):</span>
+        <span class="n">res</span> <span class="o">=</span> <span class="n">_find_containing_class_inner</span><span class="p">(</span><span class="n">v</span><span class="p">)</span>
+        <span class="k">if</span> <span class="n">res</span><span class="p">:</span> <span class="k">return</span> <span class="n">res</span>
+
+  <span class="k">return</span> <span class="n">_find_containing_class_inner</span><span class="p">(</span><span class="n">sys</span><span class="o">.</span><span class="n">modules</span><span class="p">[</span><span class="n">nested_class</span><span class="o">.</span><span class="vm">__module__</span><span class="p">])</span>
+
+
+<span class="k">def</span> <span class="nf">_nested_type_wrapper</span><span class="p">(</span><span class="n">fun</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;A wrapper for the standard pickler handler for class objects.</span>
+
+<span class="sd">  Args:</span>
+<span class="sd">    fun: Original pickler handler for type objects.</span>
+
+<span class="sd">  Returns:</span>
+<span class="sd">    A wrapper for type objects that handles nested classes.</span>
+
+<span class="sd">  The wrapper detects if an object being pickled is a nested class object.</span>
+<span class="sd">  For nested class object only it will save the containing class object so</span>
+<span class="sd">  the nested structure is recreated during unpickle.</span>
+<span class="sd">  &quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="nf">wrapper</span><span class="p">(</span><span class="n">pickler</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span>
+    <span class="c1"># When the nested class is defined in the __main__ module we do not have to</span>
+    <span class="c1"># do anything special because the pickler itself will save the constituent</span>
+    <span class="c1"># parts of the type (i.e., name, base classes, dictionary) and then</span>
+    <span class="c1"># recreate it during unpickling.</span>
+    <span class="k">if</span> <span class="n">_is_nested_class</span><span class="p">(</span><span class="n">obj</span><span class="p">)</span> <span class="ow">and</span> <span class="n">obj</span><span class="o">.</span><span class="vm">__module__</span> <span class="o">!=</span> <span class="s1">&#39;__main__&#39;</span><span class="p">:</span>
+      <span class="n">containing_class_and_name</span> <span class="o">=</span> <span class="n">_find_containing_class</span><span class="p">(</span><span class="n">obj</span><span class="p">)</span>
+      <span class="k">if</span> <span class="n">containing_class_and_name</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+        <span class="k">return</span> <span class="n">pickler</span><span class="o">.</span><span class="n">save_reduce</span><span class="p">(</span>
+            <span class="nb">getattr</span><span class="p">,</span> <span class="n">containing_class_and_name</span><span class="p">,</span> <span class="n">obj</span><span class="o">=</span><span class="n">obj</span><span class="p">)</span>
+    <span class="k">try</span><span class="p">:</span>
+      <span class="k">return</span> <span class="n">fun</span><span class="p">(</span><span class="n">pickler</span><span class="p">,</span> <span class="n">obj</span><span class="p">)</span>
+    <span class="k">except</span> <span class="n">dill</span><span class="o">.</span><span class="n">dill</span><span class="o">.</span><span class="n">PicklingError</span><span class="p">:</span>
+      <span class="c1"># pylint: disable=protected-access</span>
+      <span class="k">return</span> <span class="n">pickler</span><span class="o">.</span><span class="n">save_reduce</span><span class="p">(</span>
+          <span class="n">dill</span><span class="o">.</span><span class="n">dill</span><span class="o">.</span><span class="n">_create_type</span><span class="p">,</span>
+          <span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="n">obj</span><span class="p">),</span> <span class="n">obj</span><span class="o">.</span><span class="vm">__name__</span><span class="p">,</span> <span class="n">obj</span><span class="o">.</span><span class="vm">__bases__</span><span class="p">,</span>
+           <span class="n">dill</span><span class="o">.</span><span class="n">dill</span><span class="o">.</span><span class="n">_dict_from_dictproxy</span><span class="p">(</span><span class="n">obj</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">)),</span>
+          <span class="n">obj</span><span class="o">=</span><span class="n">obj</span><span class="p">)</span>
+      <span class="c1"># pylint: enable=protected-access</span>
+
+  <span class="k">return</span> <span class="n">wrapper</span>
+
+
+<span class="c1"># Monkey patch the standard pickler dispatch table entry for type objects.</span>
+<span class="c1"># Dill, for certain types, defers to the standard pickler (including type</span>
+<span class="c1"># objects). We wrap the standard handler using type_wrapper() because</span>
+<span class="c1"># for nested class we want to pickle the actual enclosing class object so we</span>
+<span class="c1"># can recreate it during unpickling.</span>
+<span class="c1"># TODO(silviuc): Make sure we submit the fix upstream to GitHub dill project.</span>
+<span class="n">dill</span><span class="o">.</span><span class="n">dill</span><span class="o">.</span><span class="n">Pickler</span><span class="o">.</span><span class="n">dispatch</span><span class="p">[</span><span class="nb">type</span><span class="p">]</span> <span class="o">=</span> <span class="n">_nested_type_wrapper</span><span class="p">(</span>
+    <span class="n">dill</span><span class="o">.</span><span class="n">dill</span><span class="o">.</span><span class="n">Pickler</span><span class="o">.</span><span class="n">dispatch</span><span class="p">[</span><span class="nb">type</span><span class="p">])</span>
+
+
+<span class="c1"># Dill pickles generators objects without complaint, but unpickling produces</span>
+<span class="c1"># TypeError: object.__new__(generator) is not safe, use generator.__new__()</span>
+<span class="c1"># on some versions of Python.</span>
+<span class="k">def</span> <span class="nf">_reject_generators</span><span class="p">(</span><span class="n">unused_pickler</span><span class="p">,</span> <span class="n">unused_obj</span><span class="p">):</span>
+  <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">&quot;can&#39;t (safely) pickle generator objects&quot;</span><span class="p">)</span>
+
+
+<span class="n">dill</span><span class="o">.</span><span class="n">dill</span><span class="o">.</span><span class="n">Pickler</span><span class="o">.</span><span class="n">dispatch</span><span class="p">[</span><span class="n">types</span><span class="o">.</span><span class="n">GeneratorType</span><span class="p">]</span> <span class="o">=</span> <span class="n">_reject_generators</span>
+
+
+<span class="c1"># This if guards against dill not being full initialized when generating docs.</span>
+<span class="k">if</span> <span class="s1">&#39;save_module&#39;</span> <span class="ow">in</span> <span class="nb">dir</span><span class="p">(</span><span class="n">dill</span><span class="o">.</span><span class="n">dill</span><span class="p">):</span>
+
+  <span class="c1"># Always pickle non-main modules by name.</span>
+  <span class="n">old_save_module</span> <span class="o">=</span> <span class="n">dill</span><span class="o">.</span><span class="n">dill</span><span class="o">.</span><span class="n">save_module</span>
+
+<div class="viewcode-block" id="save_module"><a class="viewcode-back" href="../../../apache_beam.internal.pickler.html#apache_beam.internal.pickler.save_module">[docs]</a>  <span class="nd">@dill</span><span class="o">.</span><span class="n">dill</span><span class="o">.</span><span class="n">register</span><span class="p">(</span><span class="n">dill</span><span class="o">.</span><span class="n">dill</span><span class="o">.</span><span class="n">ModuleType</span><span class="p">)</span>
+  <span class="k">def</span> <span class="nf">save_module</span><span class="p">(</span><span class="n">pickler</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span>
+    <span class="k">if</span> <span class="n">dill</span><span class="o">.</span><span class="n">dill</span><span class="o">.</span><span class="n">is_dill</span><span class="p">(</span><span class="n">pickler</span><span class="p">)</span> <span class="ow">and</span> <span class="n">obj</span> <span class="ow">is</span> <span class="n">pickler</span><span class="o">.</span><span class="n">_main</span><span class="p">:</span>
+      <span class="k">return</span> <span class="n">old_save_module</span><span class="p">(</span><span class="n">pickler</span><span class="p">,</span> <span class="n">obj</span><span class="p">)</span>
+    <span class="k">else</span><span class="p">:</span>
+      <span class="n">dill</span><span class="o">.</span><span class="n">dill</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;M2: </span><span class="si">%s</span><span class="s1">&#39;</span> <span class="o">%</span> <span class="n">obj</span><span class="p">)</span>
+      <span class="c1"># pylint: disable=protected-access</span>
+      <span class="n">pickler</span><span class="o">.</span><span class="n">save_reduce</span><span class="p">(</span><span class="n">dill</span><span class="o">.</span><span class="n">dill</span><span class="o">.</span><span class="n">_import_module</span><span class="p">,</span> <span class="p">(</span><span class="n">obj</span><span class="o">.</span><span class="vm">__name__</span><span class="p">,),</span> <span class="n">obj</span><span class="o">=</span><span class="n">obj</span>< [...]
+      <span class="c1"># pylint: enable=protected-access</span>
+      <span class="n">dill</span><span class="o">.</span><span class="n">dill</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;# M2&#39;</span><span class="p">)</span></div>
+
+  <span class="c1"># Pickle module dictionaries (commonly found in lambda&#39;s globals)</span>
+  <span class="c1"># by referencing their module.</span>
+  <span class="n">old_save_module_dict</span> <span class="o">=</span> <span class="n">dill</span><span class="o">.</span><span class="n">dill</span><span class="o">.</span><span class="n">save_module_dict</span>
+  <span class="n">known_module_dicts</span> <span class="o">=</span> <span class="p">{}</span>  <span class="c1"># type: Dict[int, Tuple[types.ModuleType, Dict[str, Any]]]</span>
+
+<div class="viewcode-block" id="new_save_module_dict"><a class="viewcode-back" href="../../../apache_beam.internal.pickler.html#apache_beam.internal.pickler.new_save_module_dict">[docs]</a>  <span class="nd">@dill</span><span class="o">.</span><span class="n">dill</span><span class="o">.</span><span class="n">register</span><span class="p">(</span><span class="nb">dict</span><span class="p">)</span>
+  <span class="k">def</span> <span class="nf">new_save_module_dict</span><span class="p">(</span><span class="n">pickler</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span>
+    <span class="n">obj_id</span> <span class="o">=</span> <span class="nb">id</span><span class="p">(</span><span class="n">obj</span><span class="p">)</span>
+    <span class="k">if</span> <span class="ow">not</span> <span class="n">known_module_dicts</span> <span class="ow">or</span> <span class="s1">&#39;__file__&#39;</span> <span class="ow">in</span> <span class="n">obj</span> <span class="ow">or</span> <span class="s1">&#39;__package__&#39;</span> <span class="ow">in</span> <span class="n">obj</span><span class="p">:</span>
+      <span class="k">if</span> <span class="n">obj_id</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">known_module_dicts</span><span class="p">:</span>
+        <span class="c1"># Trigger loading of lazily loaded modules (such as pytest vendored</span>
+        <span class="c1"># modules).</span>
+        <span class="c1"># This first pass over sys.modules needs to iterate on a copy of</span>
+        <span class="c1"># sys.modules since lazy loading modifies the dictionary, hence the use</span>
+        <span class="c1"># of list().</span>
+        <span class="k">for</span> <span class="n">m</span> <span class="ow">in</span> <span class="nb">list</span><span class="p">(</span><span class="n">sys</span><span class="o">.</span><span class="n">modules</span><span class="o">.</span><span class="n">values</span><span class="p">()):</span>
+          <span class="k">try</span><span class="p">:</span>
+            <span class="n">_</span> <span class="o">=</span> <span class="n">m</span><span class="o">.</span><span class="vm">__dict__</span>
+          <span class="k">except</span> <span class="ne">AttributeError</span><span class="p">:</span>
+            <span class="k">pass</span>
+
+        <span class="k">for</span> <span class="n">m</span> <span class="ow">in</span> <span class="n">sys</span><span class="o">.</span><span class="n">modules</span><span class="o">.</span><span class="n">values</span><span class="p">():</span>
+          <span class="k">try</span><span class="p">:</span>
+            <span class="k">if</span> <span class="p">(</span><span class="n">m</span>
+                <span class="ow">and</span> <span class="n">m</span><span class="o">.</span><span class="vm">__name__</span> <span class="o">!=</span> <span class="s1">&#39;__main__&#39;</span>
+                <span class="ow">and</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">m</span><span class="p">,</span> <span class="n">dill</span><span class="o">.</span><span class="n">dill</span><span class="o">.</span><span class="n">ModuleType</span><span class="p">)):</span>
+              <span class="n">d</span> <span class="o">=</span> <span class="n">m</span><span class="o">.</span><span class="vm">__dict__</span>
+              <span class="n">known_module_dicts</span><span class="p">[</span><span class="nb">id</span><span class="p">(</span><span class="n">d</span><span class="p">)]</span> <span class="o">=</span> <span class="n">m</span><span class="p">,</span> <span class="n">d</span>
+          <span class="k">except</span> <span class="ne">AttributeError</span><span class="p">:</span>
+            <span class="c1"># Skip modules that do not have the __name__ attribute.</span>
+            <span class="k">pass</span>
+    <span class="k">if</span> <span class="n">obj_id</span> <span class="ow">in</span> <span class="n">known_module_dicts</span> <span class="ow">and</span> <span class="n">dill</span><span class="o">.</span><span class="n">dill</span><span class="o">.</span><span class="n">is_dill</span><span class="p">(</span><span class="n">pickler</span><span class="p">):</span>
+      <span class="n">m</span> <span class="o">=</span> <span class="n">known_module_dicts</span><span class="p">[</span><span class="n">obj_id</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span>
+      <span class="k">try</span><span class="p">:</span>
+        <span class="c1"># pylint: disable=protected-access</span>
+        <span class="n">dill</span><span class="o">.</span><span class="n">dill</span><span class="o">.</span><span class="n">_import_module</span><span class="p">(</span><span class="n">m</span><span class="o">.</span><span class="vm">__name__</span><span class="p">)</span>
+        <span class="k">return</span> <span class="n">pickler</span><span class="o">.</span><span class="n">save_reduce</span><span class="p">(</span>
+            <span class="nb">getattr</span><span class="p">,</span> <span class="p">(</span><span class="n">known_module_dicts</span><span class="p">[</span><span class="n">obj_id</span><span class="p">][</span><span class="mi">0</span><span class="p">],</span> <span class="s1">&#39;__dict__&#39;</span><span class="p">),</span> <span class="n">obj</span><span class="o">=</span><span class="n">obj</span><span class="p">)</span>
+      <span class="k">except</span> <span class="p">(</span><span class="ne">ImportError</span><span class="p">,</span> <span class="ne">AttributeError</span><span class="p">):</span>
+        <span class="k">return</span> <span class="n">old_save_module_dict</span><span class="p">(</span><span class="n">pickler</span><span class="p">,</span> <span class="n">obj</span><span class="p">)</span>
+    <span class="k">else</span><span class="p">:</span>
+      <span class="k">return</span> <span class="n">old_save_module_dict</span><span class="p">(</span><span class="n">pickler</span><span class="p">,</span> <span class="n">obj</span><span class="p">)</span></div>
+  <span class="n">dill</span><span class="o">.</span><span class="n">dill</span><span class="o">.</span><span class="n">save_module_dict</span> <span class="o">=</span> <span class="n">new_save_module_dict</span>
+
+  <span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">types</span><span class="p">,</span> <span class="s2">&quot;MappingProxyType&quot;</span><span class="p">):</span>
+    <span class="nd">@dill</span><span class="o">.</span><span class="n">dill</span><span class="o">.</span><span class="n">register</span><span class="p">(</span><span class="n">types</span><span class="o">.</span><span class="n">MappingProxyType</span><span class="p">)</span>
+    <span class="k">def</span> <span class="nf">save_mappingproxy</span><span class="p">(</span><span class="n">pickler</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span>
+      <span class="c1"># pickling mappingproxy AS IS, not as dict</span>
+      <span class="c1"># inspired from https://github.com/cloudpipe/cloudpickle/pull/245</span>
+      <span class="n">pickler</span><span class="o">.</span><span class="n">save_reduce</span><span class="p">(</span><span class="n">types</span><span class="o">.</span><span class="n">MappingProxyType</span><span class="p">,</span> <span class="p">(</span><span class="nb">dict</span><span class="p">(</span><span class="n">obj</span><span class="p">),),</span> <span class="n">obj</span><span class="o">=</span><span class="n">obj</span><span class="p">)</span>
+
+  <span class="k">def</span> <span class="nf">_nest_dill_logging</span><span class="p">():</span>
+    <span class="sd">&quot;&quot;&quot;Prefix all dill logging with its depth in the callstack.</span>
+
+<span class="sd">    Useful for debugging pickling of deeply nested structures.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">old_log_info</span> <span class="o">=</span> <span class="n">dill</span><span class="o">.</span><span class="n">dill</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span>
+
+    <span class="k">def</span> <span class="nf">new_log_info</span><span class="p">(</span><span class="n">msg</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+      <span class="n">old_log_info</span><span class="p">(</span>
+          <span class="p">(</span><span class="s1">&#39;1 2 3 4 5 6 7 8 9 0 &#39;</span> <span class="o">*</span> <span class="mi">10</span><span class="p">)[:</span><span class="nb">len</span><span class="p">(</span><span class="n">traceback</span><span class="o">.</span><span class="n">extract_stack</span><span class="p">())]</span> <span class="o">+</span> <span class="n">msg</span><span class="p">,</span>
+          <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+    <span class="n">dill</span><span class="o">.</span><span class="n">dill</span><span class="o">.</span><span class="n">log</span><span class="o">.</span><span class="n">info</span> <span class="o">=</span> <span class="n">new_log_info</span>
+
+
+<span class="c1"># Turn off verbose logging from the dill pickler.</span>
+<span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="s1">&#39;dill&#39;</span><span class="p">)</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">logging</span><span class="o">.</span><span class="n">WARN</span><span class="p">)</span>
+
+
+<span class="c1"># TODO(ccy): Currently, there are still instances of pickler.dumps() and</span>
+<span class="c1"># pickler.loads() being used for data, which results in an unnecessary base64</span>
+<span class="c1"># encoding.  This should be cleaned up.</span>
+<div class="viewcode-block" id="dumps"><a class="viewcode-back" href="../../../apache_beam.internal.pickler.html#apache_beam.internal.pickler.dumps">[docs]</a><span class="k">def</span> <span class="nf">dumps</span><span class="p">(</span><span class="n">o</span><span class="p">,</span> <span class="n">enable_trace</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
+  <span class="c1"># type: (...) -&gt; bytes</span>
+  <span class="sd">&quot;&quot;&quot;For internal use only; no backwards-compatibility guarantees.&quot;&quot;&quot;</span>
+  <span class="k">with</span> <span class="n">pickle_lock_unless_py2</span><span class="p">:</span>
+    <span class="k">try</span><span class="p">:</span>
+      <span class="n">s</span> <span class="o">=</span> <span class="n">dill</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">o</span><span class="p">)</span>
+    <span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>      <span class="c1"># pylint: disable=broad-except</span>
+      <span class="k">if</span> <span class="n">enable_trace</span><span class="p">:</span>
+        <span class="n">dill</span><span class="o">.</span><span class="n">dill</span><span class="o">.</span><span class="n">_trace</span><span class="p">(</span><span class="kc">True</span><span class="p">)</span>  <span class="c1"># pylint: disable=protected-access</span>
+        <span class="n">s</span> <span class="o">=</span> <span class="n">dill</span><span class="o">.</span><span class="n">dumps</span><span class="p">(</span><span class="n">o</span><span class="p">)</span>
+      <span class="k">else</span><span class="p">:</span>
+        <span class="k">raise</span>
+    <span class="k">finally</span><span class="p">:</span>
+      <span class="n">dill</span><span class="o">.</span><span class="n">dill</span><span class="o">.</span><span class="n">_trace</span><span class="p">(</span><span class="kc">False</span><span class="p">)</span>  <span class="c1"># pylint: disable=protected-access</span>
+
+  <span class="c1"># Compress as compactly as possible to decrease peak memory usage (of multiple</span>
+  <span class="c1"># in-memory copies) and free up some possibly large and no-longer-needed</span>
+  <span class="c1"># memory.</span>
+  <span class="n">c</span> <span class="o">=</span> <span class="n">zlib</span><span class="o">.</span><span class="n">compress</span><span class="p">(</span><span class="n">s</span><span class="p">,</span> <span class="mi">9</span><span class="p">)</span>
+  <span class="k">del</span> <span class="n">s</span>
+
+  <span class="k">return</span> <span class="n">base64</span><span class="o">.</span><span class="n">b64encode</span><span class="p">(</span><span class="n">c</span><span class="p">)</span></div>
+
+
+<div class="viewcode-block" id="loads"><a class="viewcode-back" href="../../../apache_beam.internal.pickler.html#apache_beam.internal.pickler.loads">[docs]</a><span class="k">def</span> <span class="nf">loads</span><span class="p">(</span><span class="n">encoded</span><span class="p">,</span> <span class="n">enable_trace</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;For internal use only; no backwards-compatibility guarantees.&quot;&quot;&quot;</span>
+
+  <span class="n">c</span> <span class="o">=</span> <span class="n">base64</span><span class="o">.</span><span class="n">b64decode</span><span class="p">(</span><span class="n">encoded</span><span class="p">)</span>
+
+  <span class="n">s</span> <span class="o">=</span> <span class="n">zlib</span><span class="o">.</span><span class="n">decompress</span><span class="p">(</span><span class="n">c</span><span class="p">)</span>
+  <span class="k">del</span> <span class="n">c</span>  <span class="c1"># Free up some possibly large and no-longer-needed memory.</span>
+
+  <span class="k">with</span> <span class="n">pickle_lock_unless_py2</span><span class="p">:</span>
+    <span class="k">try</span><span class="p">:</span>
+      <span class="k">return</span> <span class="n">dill</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">s</span><span class="p">)</span>
+    <span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>          <span class="c1"># pylint: disable=broad-except</span>
+      <span class="k">if</span> <span class="n">enable_trace</span><span class="p">:</span>
+        <span class="n">dill</span><span class="o">.</span><span class="n">dill</span><span class="o">.</span><span class="n">_trace</span><span class="p">(</span><span class="kc">True</span><span class="p">)</span>   <span class="c1"># pylint: disable=protected-access</span>
+        <span class="k">return</span> <span class="n">dill</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">s</span><span class="p">)</span>
+      <span class="k">else</span><span class="p">:</span>
+        <span class="k">raise</span>
+    <span class="k">finally</span><span class="p">:</span>
+      <span class="n">dill</span><span class="o">.</span><span class="n">dill</span><span class="o">.</span><span class="n">_trace</span><span class="p">(</span><span class="kc">False</span><span class="p">)</span>  <span class="c1"># pylint: disable=protected-access</span></div>
+
+
+<div class="viewcode-block" id="dump_session"><a class="viewcode-back" href="../../../apache_beam.internal.pickler.html#apache_beam.internal.pickler.dump_session">[docs]</a><span class="k">def</span> <span class="nf">dump_session</span><span class="p">(</span><span class="n">file_path</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;For internal use only; no backwards-compatibility guarantees.</span>
+
+<span class="sd">  Pickle the current python session to be used in the worker.</span>
+
+<span class="sd">  Note: Due to the inconsistency in the first dump of dill dump_session we</span>
+<span class="sd">  create and load the dump twice to have consistent results in the worker and</span>
+<span class="sd">  the running session. Check: https://github.com/uqfoundation/dill/issues/195</span>
+<span class="sd">  &quot;&quot;&quot;</span>
+  <span class="k">with</span> <span class="n">pickle_lock_unless_py2</span><span class="p">:</span>
+    <span class="n">dill</span><span class="o">.</span><span class="n">dump_session</span><span class="p">(</span><span class="n">file_path</span><span class="p">)</span>
+    <span class="n">dill</span><span class="o">.</span><span class="n">load_session</span><span class="p">(</span><span class="n">file_path</span><span class="p">)</span>
+    <span class="k">return</span> <span class="n">dill</span><span class="o">.</span><span class="n">dump_session</span><span class="p">(</span><span class="n">file_path</span><span class="p">)</span></div>
+
+
+<div class="viewcode-block" id="load_session"><a class="viewcode-back" href="../../../apache_beam.internal.pickler.html#apache_beam.internal.pickler.load_session">[docs]</a><span class="k">def</span> <span class="nf">load_session</span><span class="p">(</span><span class="n">file_path</span><span class="p">):</span>
+  <span class="k">with</span> <span class="n">pickle_lock_unless_py2</span><span class="p">:</span>
+    <span class="k">return</span> <span class="n">dill</span><span class="o">.</span><span class="n">load_session</span><span class="p">(</span><span class="n">file_path</span><span class="p">)</span></div>
+</pre></div>
+
+           </div>
+           <div class="articleComments">
+            
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+        &copy; Copyright .
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file
diff --git a/pydoc/2.19.0/_modules/apache_beam/internal/util.html b/pydoc/2.19.0/_modules/apache_beam/internal/util.html
new file mode 100644
index 0000000..e7b1066
--- /dev/null
+++ b/pydoc/2.19.0/_modules/apache_beam/internal/util.html
@@ -0,0 +1,398 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>apache_beam.internal.util &mdash; Apache Beam  documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../search.html"/>
+    <link rel="top" title="Apache Beam  documentation" href="../../../index.html"/>
+        <link rel="up" title="Module code" href="../../index.html"/> 
+
+  
+  <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../index.html" class="icon icon-home"> Apache Beam
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.coders.html">apache_beam.coders package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.internal.html">apache_beam.internal package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.io.html">apache_beam.io package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.metrics.html">apache_beam.metrics package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.options.html">apache_beam.options package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.portability.html">apache_beam.portability package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.runners.html">apache_beam.runners package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.testing.html">apache_beam.testing package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.tools.html">apache_beam.tools package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.transforms.html">apache_beam.transforms package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.typehints.html">apache_beam.typehints package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.utils.html">apache_beam.utils package</a></li>
+</ul>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.error.html">apache_beam.error module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.pipeline.html">apache_beam.pipeline module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.pvalue.html">apache_beam.pvalue module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.version.html">apache_beam.version module</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../index.html">Apache Beam</a>
+        
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../index.html">Module code</a> &raquo;</li>
+        
+      <li>apache_beam.internal.util</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+            
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for apache_beam.internal.util</h1><div class="highlight"><pre>
+<span></span><span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one or more</span>
+<span class="c1"># contributor license agreements.  See the NOTICE file distributed with</span>
+<span class="c1"># this work for additional information regarding copyright ownership.</span>
+<span class="c1"># The ASF licenses this file to You under the Apache License, Version 2.0</span>
+<span class="c1"># (the &quot;License&quot;); you may not use this file except in compliance with</span>
+<span class="c1"># the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#    http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+<span class="c1">#</span>
+
+<span class="sd">&quot;&quot;&quot;Utility functions used throughout the package.</span>
+
+<span class="sd">For internal use only. No backwards compatibility guarantees.</span>
+<span class="sd">&quot;&quot;&quot;</span>
+
+<span class="c1"># pytype: skip-file</span>
+
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">absolute_import</span>
+
+<span class="kn">import</span> <span class="nn">logging</span>
+<span class="kn">import</span> <span class="nn">threading</span>
+<span class="kn">import</span> <span class="nn">weakref</span>
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">object</span>
+<span class="kn">from</span> <span class="nn">multiprocessing.pool</span> <span class="kn">import</span> <span class="n">ThreadPool</span>
+<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">Any</span>
+<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">Dict</span>
+<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">Iterable</span>
+<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">List</span>
+<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">Tuple</span>
+<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">Type</span>
+<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">TypeVar</span>
+<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">Union</span>
+
+<span class="n">T</span> <span class="o">=</span> <span class="n">TypeVar</span><span class="p">(</span><span class="s1">&#39;T&#39;</span><span class="p">)</span>
+
+
+<div class="viewcode-block" id="ArgumentPlaceholder"><a class="viewcode-back" href="../../../apache_beam.internal.util.html#apache_beam.internal.util.ArgumentPlaceholder">[docs]</a><span class="k">class</span> <span class="nc">ArgumentPlaceholder</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;For internal use only; no backwards-compatibility guarantees.</span>
+
+<span class="sd">  A place holder object replacing PValues in argument lists.</span>
+
+<span class="sd">  A Fn object can take any number of &quot;side inputs&quot;, which are PValues that will</span>
+<span class="sd">  be evaluated during pipeline execution and will be provided to the function</span>
+<span class="sd">  at the moment of its execution as positional or keyword arguments.</span>
+
+<span class="sd">  This is used only internally and should never be used by user code. A custom</span>
+<span class="sd">  Fn object by the time it executes will have such values replaced with real</span>
+<span class="sd">  computed values.</span>
+<span class="sd">  &quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="fm">__eq__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;Tests for equality of two placeholder objects.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">      other: Another placeholder object to compare to.</span>
+
+<span class="sd">    This method is used only for test code. All placeholder objects are</span>
+<span class="sd">    equal to each other.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">return</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">other</span><span class="p">,</span> <span class="n">ArgumentPlaceholder</span><span class="p">)</span>
+
+  <span class="k">def</span> <span class="fm">__ne__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other</span><span class="p">):</span>
+    <span class="c1"># TODO(BEAM-5949): Needed for Python 2 compatibility.</span>
+    <span class="k">return</span> <span class="ow">not</span> <span class="bp">self</span> <span class="o">==</span> <span class="n">other</span>
+
+  <span class="k">def</span> <span class="fm">__hash__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="nb">hash</span><span class="p">(</span><span class="nb">type</span><span class="p">(</span><span class="bp">self</span><span class="p">))</span></div>
+
+
+<div class="viewcode-block" id="remove_objects_from_args"><a class="viewcode-back" href="../../../apache_beam.internal.util.html#apache_beam.internal.util.remove_objects_from_args">[docs]</a><span class="k">def</span> <span class="nf">remove_objects_from_args</span><span class="p">(</span><span class="n">args</span><span class="p">,</span>  <span class="c1"># type: Iterable[Any]</span>
+                             <span class="n">kwargs</span><span class="p">,</span>  <span class="c1"># type: Dict[str, Any]</span>
+                             <span class="n">pvalue_class</span>  <span class="c1"># type: Union[Type[T], Tuple[Type[T], ...]]</span>
+                            <span class="p">):</span>
+  <span class="c1"># type: (...) -&gt; Tuple[List[Any], Dict[str, Any], List[T]]</span>
+  <span class="sd">&quot;&quot;&quot;For internal use only; no backwards-compatibility guarantees.</span>
+
+<span class="sd">  Replaces all objects of a given type in args/kwargs with a placeholder.</span>
+
+<span class="sd">  Args:</span>
+<span class="sd">    args: A list of positional arguments.</span>
+<span class="sd">    kwargs: A dictionary of keyword arguments.</span>
+<span class="sd">    pvalue_class: A class object representing the types of arguments that must</span>
+<span class="sd">      be replaced with a placeholder value (instance of ArgumentPlaceholder).</span>
+
+<span class="sd">  Returns:</span>
+<span class="sd">    A 3-tuple containing a modified list of positional arguments, a modified</span>
+<span class="sd">    dictionary of keyword arguments, and a list of all objects replaced with</span>
+<span class="sd">    a placeholder value.</span>
+<span class="sd">  &quot;&quot;&quot;</span>
+  <span class="n">pvals</span> <span class="o">=</span> <span class="p">[]</span>
+
+  <span class="k">def</span> <span class="nf">swapper</span><span class="p">(</span><span class="n">value</span><span class="p">):</span>
+    <span class="n">pvals</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
+    <span class="k">return</span> <span class="n">ArgumentPlaceholder</span><span class="p">()</span>
+  <span class="n">new_args</span> <span class="o">=</span> <span class="p">[</span><span class="n">swapper</span><span class="p">(</span><span class="n">v</span><span class="p">)</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="n">pvalue_class</span><span class="p">)</span> <span class="k">else</span> <span class="n">v</span> <span class="k">for</span> <span class="n">v</span> <span cl [...]
+  <span class="c1"># Make sure the order in which we process the dictionary keys is predictable</span>
+  <span class="c1"># by sorting the entries first. This will be important when putting back</span>
+  <span class="c1"># PValues.</span>
+  <span class="n">new_kwargs</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">((</span><span class="n">k</span><span class="p">,</span> <span class="n">swapper</span><span class="p">(</span><span class="n">v</span><span class="p">))</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="n">pvalue_class</span><span class="p">)</span> <span class="k">else</span> <spa [...]
+                    <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">kwargs</span><span class="o">.</span><span class="n">items</span><span class="p">()))</span>
+  <span class="k">return</span> <span class="p">(</span><span class="n">new_args</span><span class="p">,</span> <span class="n">new_kwargs</span><span class="p">,</span> <span class="n">pvals</span><span class="p">)</span></div>
+
+
+<div class="viewcode-block" id="insert_values_in_args"><a class="viewcode-back" href="../../../apache_beam.internal.util.html#apache_beam.internal.util.insert_values_in_args">[docs]</a><span class="k">def</span> <span class="nf">insert_values_in_args</span><span class="p">(</span><span class="n">args</span><span class="p">,</span> <span class="n">kwargs</span><span class="p">,</span> <span class="n">values</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;For internal use only; no backwards-compatibility guarantees.</span>
+
+<span class="sd">  Replaces all placeholders in args/kwargs with actual values.</span>
+
+<span class="sd">  Args:</span>
+<span class="sd">    args: A list of positional arguments.</span>
+<span class="sd">    kwargs: A dictionary of keyword arguments.</span>
+<span class="sd">    values: A list of values that will be used to replace placeholder values.</span>
+
+<span class="sd">  Returns:</span>
+<span class="sd">    A 2-tuple containing a modified list of positional arguments, and a</span>
+<span class="sd">    modified dictionary of keyword arguments.</span>
+<span class="sd">  &quot;&quot;&quot;</span>
+  <span class="c1"># Use a local iterator so that we don&#39;t modify values.</span>
+  <span class="n">v_iter</span> <span class="o">=</span> <span class="nb">iter</span><span class="p">(</span><span class="n">values</span><span class="p">)</span>
+  <span class="n">new_args</span> <span class="o">=</span> <span class="p">[</span>
+      <span class="nb">next</span><span class="p">(</span><span class="n">v_iter</span><span class="p">)</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">arg</span><span class="p">,</span> <span class="n">ArgumentPlaceholder</span><span class="p">)</span> <span class="k">else</span> <span class="n">arg</span>
+      <span class="k">for</span> <span class="n">arg</span> <span class="ow">in</span> <span class="n">args</span><span class="p">]</span>
+  <span class="n">new_kwargs</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
+      <span class="p">(</span><span class="n">k</span><span class="p">,</span> <span class="nb">next</span><span class="p">(</span><span class="n">v_iter</span><span class="p">))</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">v</span><span class="p">,</span> <span class="n">ArgumentPlaceholder</span><span class="p">)</span> <span class="k">else</span> <span class="p">(</span><span class="n">k</span><span class="p">,</span> <span [...]
+      <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">kwargs</span><span class="o">.</span><span class="n">items</span><span class="p">()))</span>
+  <span class="k">return</span> <span class="p">(</span><span class="n">new_args</span><span class="p">,</span> <span class="n">new_kwargs</span><span class="p">)</span></div>
+
+
+<div class="viewcode-block" id="run_using_threadpool"><a class="viewcode-back" href="../../../apache_beam.internal.util.html#apache_beam.internal.util.run_using_threadpool">[docs]</a><span class="k">def</span> <span class="nf">run_using_threadpool</span><span class="p">(</span><span class="n">fn_to_execute</span><span class="p">,</span> <span class="n">inputs</span><span class="p">,</span> <span class="n">pool_size</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;For internal use only; no backwards-compatibility guarantees.</span>
+
+<span class="sd">  Runs the given function on given inputs using a thread pool.</span>
+
+<span class="sd">  Args:</span>
+<span class="sd">    fn_to_execute: Function to execute</span>
+<span class="sd">    inputs: Inputs on which given function will be executed in parallel.</span>
+<span class="sd">    pool_size: Size of thread pool.</span>
+<span class="sd">  Returns:</span>
+<span class="sd">    Results retrieved after executing the given function on given inputs.</span>
+<span class="sd">  &quot;&quot;&quot;</span>
+
+  <span class="c1"># ThreadPool crashes in old versions of Python (&lt; 2.7.5) if created</span>
+  <span class="c1"># from a child thread. (http://bugs.python.org/issue10015)</span>
+  <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="n">threading</span><span class="o">.</span><span class="n">current_thread</span><span class="p">(),</span> <span class="s1">&#39;_children&#39;</span><span class="p">):</span>
+    <span class="n">threading</span><span class="o">.</span><span class="n">current_thread</span><span class="p">()</span><span class="o">.</span><span class="n">_children</span> <span class="o">=</span> <span class="n">weakref</span><span class="o">.</span><span class="n">WeakKeyDictionary</span><span class="p">()</span>
+  <span class="n">pool</span> <span class="o">=</span> <span class="n">ThreadPool</span><span class="p">(</span><span class="nb">min</span><span class="p">(</span><span class="n">pool_size</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">inputs</span><span class="p">)))</span>
+  <span class="k">try</span><span class="p">:</span>
+    <span class="c1"># We record and reset logging level here since &#39;apitools&#39; library Beam</span>
+    <span class="c1"># depends on updates the logging level when used with a threadpool -</span>
+    <span class="c1"># https://github.com/google/apitools/issues/141</span>
+    <span class="c1"># TODO: Remove this once above issue in &#39;apitools&#39; is fixed.</span>
+    <span class="n">old_level</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">()</span><span class="o">.</span><span class="n">level</span>
+    <span class="k">return</span> <span class="n">pool</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="n">fn_to_execute</span><span class="p">,</span> <span class="n">inputs</span><span class="p">)</span>
+  <span class="k">finally</span><span class="p">:</span>
+    <span class="n">pool</span><span class="o">.</span><span class="n">terminate</span><span class="p">()</span>
+    <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">()</span><span class="o">.</span><span class="n">setLevel</span><span class="p">(</span><span class="n">old_level</span><span class="p">)</span></div>
+</pre></div>
+
+           </div>
+           <div class="articleComments">
+            
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+        &copy; Copyright .
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file
diff --git a/pydoc/2.19.0/_modules/apache_beam/io/avroio.html b/pydoc/2.19.0/_modules/apache_beam/io/avroio.html
new file mode 100644
index 0000000..8874f7e
--- /dev/null
+++ b/pydoc/2.19.0/_modules/apache_beam/io/avroio.html
@@ -0,0 +1,869 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>apache_beam.io.avroio &mdash; Apache Beam  documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../search.html"/>
+    <link rel="top" title="Apache Beam  documentation" href="../../../index.html"/>
+        <link rel="up" title="Module code" href="../../index.html"/> 
+
+  
+  <script src="../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../index.html" class="icon icon-home"> Apache Beam
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.coders.html">apache_beam.coders package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.internal.html">apache_beam.internal package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.io.html">apache_beam.io package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.metrics.html">apache_beam.metrics package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.options.html">apache_beam.options package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.portability.html">apache_beam.portability package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.runners.html">apache_beam.runners package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.testing.html">apache_beam.testing package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.tools.html">apache_beam.tools package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.transforms.html">apache_beam.transforms package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.typehints.html">apache_beam.typehints package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.utils.html">apache_beam.utils package</a></li>
+</ul>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.error.html">apache_beam.error module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.pipeline.html">apache_beam.pipeline module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.pvalue.html">apache_beam.pvalue module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../apache_beam.version.html">apache_beam.version module</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../index.html">Apache Beam</a>
+        
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../index.html">Module code</a> &raquo;</li>
+        
+      <li>apache_beam.io.avroio</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+            
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for apache_beam.io.avroio</h1><div class="highlight"><pre>
+<span></span><span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one or more</span>
+<span class="c1"># contributor license agreements.  See the NOTICE file distributed with</span>
+<span class="c1"># this work for additional information regarding copyright ownership.</span>
+<span class="c1"># The ASF licenses this file to You under the Apache License, Version 2.0</span>
+<span class="c1"># (the &quot;License&quot;); you may not use this file except in compliance with</span>
+<span class="c1"># the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#    http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+<span class="c1">#</span>
+<span class="sd">&quot;&quot;&quot;``PTransforms`` for reading from and writing to Avro files.</span>
+
+<span class="sd">Provides two read ``PTransform``s, ``ReadFromAvro`` and ``ReadAllFromAvro``,</span>
+<span class="sd">that produces a ``PCollection`` of records.</span>
+<span class="sd">Each record of this ``PCollection`` will contain a single record read from</span>
+<span class="sd">an Avro file. Records that are of simple types will be mapped into</span>
+<span class="sd">corresponding Python types. Records that are of Avro type &#39;RECORD&#39; will be</span>
+<span class="sd">mapped to Python dictionaries that comply with the schema contained in the</span>
+<span class="sd">Avro file that contains those records. In this case, keys of each dictionary</span>
+<span class="sd">will contain the corresponding field names and will be of type ``string``</span>
+<span class="sd">while the values of the dictionary will be of the type defined in the</span>
+<span class="sd">corresponding Avro schema.</span>
+
+<span class="sd">For example, if schema of the Avro file is the following.</span>
+<span class="sd">{&quot;namespace&quot;: &quot;example.avro&quot;,&quot;type&quot;: &quot;record&quot;,&quot;name&quot;: &quot;User&quot;,&quot;fields&quot;:</span>
+<span class="sd">[{&quot;name&quot;: &quot;name&quot;, &quot;type&quot;: &quot;string&quot;},</span>
+<span class="sd">{&quot;name&quot;: &quot;favorite_number&quot;,  &quot;type&quot;: [&quot;int&quot;, &quot;null&quot;]},</span>
+<span class="sd">{&quot;name&quot;: &quot;favorite_color&quot;, &quot;type&quot;: [&quot;string&quot;, &quot;null&quot;]}]}</span>
+
+<span class="sd">Then records generated by read transforms will be dictionaries of the</span>
+<span class="sd">following form.</span>
+<span class="sd">{u&#39;name&#39;: u&#39;Alyssa&#39;, u&#39;favorite_number&#39;: 256, u&#39;favorite_color&#39;: None}).</span>
+
+<span class="sd">Additionally, this module provides a write ``PTransform`` ``WriteToAvro``</span>
+<span class="sd">that can be used to write a given ``PCollection`` of Python objects to an</span>
+<span class="sd">Avro file.</span>
+<span class="sd">&quot;&quot;&quot;</span>
+<span class="c1"># pytype: skip-file</span>
+
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">absolute_import</span>
+
+<span class="kn">import</span> <span class="nn">io</span>
+<span class="kn">import</span> <span class="nn">os</span>
+<span class="kn">import</span> <span class="nn">sys</span>
+<span class="kn">import</span> <span class="nn">warnings</span>
+<span class="kn">import</span> <span class="nn">zlib</span>
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">object</span>
+<span class="kn">from</span> <span class="nn">functools</span> <span class="kn">import</span> <span class="n">partial</span>
+
+<span class="kn">import</span> <span class="nn">avro</span>
+<span class="kn">from</span> <span class="nn">avro</span> <span class="kn">import</span> <span class="n">io</span> <span class="k">as</span> <span class="n">avroio</span>
+<span class="kn">from</span> <span class="nn">avro</span> <span class="kn">import</span> <span class="n">datafile</span>
+<span class="kn">from</span> <span class="nn">fastavro.read</span> <span class="kn">import</span> <span class="n">block_reader</span>
+<span class="kn">from</span> <span class="nn">fastavro.write</span> <span class="kn">import</span> <span class="n">Writer</span>
+
+<span class="kn">import</span> <span class="nn">apache_beam</span> <span class="k">as</span> <span class="nn">beam</span>
+<span class="kn">from</span> <span class="nn">apache_beam.io</span> <span class="kn">import</span> <span class="n">filebasedsink</span>
+<span class="kn">from</span> <span class="nn">apache_beam.io</span> <span class="kn">import</span> <span class="n">filebasedsource</span>
+<span class="kn">from</span> <span class="nn">apache_beam.io</span> <span class="kn">import</span> <span class="n">iobase</span>
+<span class="kn">from</span> <span class="nn">apache_beam.io.filesystem</span> <span class="kn">import</span> <span class="n">CompressionTypes</span>
+<span class="kn">from</span> <span class="nn">apache_beam.io.iobase</span> <span class="kn">import</span> <span class="n">Read</span>
+<span class="kn">from</span> <span class="nn">apache_beam.transforms</span> <span class="kn">import</span> <span class="n">PTransform</span>
+
+<span class="c1"># pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports</span>
+<span class="k">try</span><span class="p">:</span>
+  <span class="kn">from</span> <span class="nn">avro.schema</span> <span class="kn">import</span> <span class="n">Parse</span> <span class="c1"># avro-python3 library for python3</span>
+<span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
+  <span class="kn">from</span> <span class="nn">avro.schema</span> <span class="kn">import</span> <span class="n">parse</span> <span class="k">as</span> <span class="n">Parse</span> <span class="c1"># avro library for python2</span>
+<span class="c1"># pylint: enable=wrong-import-order, wrong-import-position, ungrouped-imports</span>
+
+
+<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;ReadFromAvro&#39;</span><span class="p">,</span> <span class="s1">&#39;ReadAllFromAvro&#39;</span><span class="p">,</span> <span class="s1">&#39;WriteToAvro&#39;</span><span class="p">]</span>
+
+
+<span class="k">def</span> <span class="nf">_use_fastavro</span><span class="p">():</span>
+  <span class="k">return</span> <span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">&gt;=</span> <span class="mi">3</span>
+
+
+<div class="viewcode-block" id="ReadFromAvro"><a class="viewcode-back" href="../../../apache_beam.io.avroio.html#apache_beam.io.avroio.ReadFromAvro">[docs]</a><span class="k">class</span> <span class="nc">ReadFromAvro</span><span class="p">(</span><span class="n">PTransform</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;A :class:`~apache_beam.transforms.ptransform.PTransform` for reading avro</span>
+<span class="sd">  files.&quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">file_pattern</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">min_bundle_size</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">validate</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
+               <span class="n">use_fastavro</span><span class="o">=</span><span class="n">_use_fastavro</span><span class="p">()):</span>
+    <span class="sd">&quot;&quot;&quot;Initializes :class:`ReadFromAvro`.</span>
+
+<span class="sd">    Uses source :class:`~apache_beam.io._AvroSource` to read a set of Avro</span>
+<span class="sd">    files defined by a given file pattern.</span>
+
+<span class="sd">    If ``/mypath/myavrofiles*`` is a file-pattern that points to a set of Avro</span>
+<span class="sd">    files, a :class:`~apache_beam.pvalue.PCollection` for the records in</span>
+<span class="sd">    these Avro files can be created in the following manner.</span>
+
+<span class="sd">    .. testcode::</span>
+
+<span class="sd">      with beam.Pipeline() as p:</span>
+<span class="sd">        records = p | &#39;Read&#39; &gt;&gt; beam.io.ReadFromAvro(&#39;/mypath/myavrofiles*&#39;)</span>
+
+<span class="sd">    .. NOTE: We&#39;re not actually interested in this error; but if we get here,</span>
+<span class="sd">       it means that the way of calling this transform hasn&#39;t changed.</span>
+
+<span class="sd">    .. testoutput::</span>
+<span class="sd">      :hide:</span>
+
+<span class="sd">      Traceback (most recent call last):</span>
+<span class="sd">       ...</span>
+<span class="sd">      IOError: No files found based on the file pattern</span>
+
+<span class="sd">    Each record of this :class:`~apache_beam.pvalue.PCollection` will contain</span>
+<span class="sd">    a single record read from a source. Records that are of simple types will be</span>
+<span class="sd">    mapped into corresponding Python types. Records that are of Avro type</span>
+<span class="sd">    ``RECORD`` will be mapped to Python dictionaries that comply with the schema</span>
+<span class="sd">    contained in the Avro file that contains those records. In this case, keys</span>
+<span class="sd">    of each dictionary will contain the corresponding field names and will be of</span>
+<span class="sd">    type :class:`str` while the values of the dictionary will be of the type</span>
+<span class="sd">    defined in the corresponding Avro schema.</span>
+
+<span class="sd">    For example, if schema of the Avro file is the following. ::</span>
+
+<span class="sd">      {</span>
+<span class="sd">        &quot;namespace&quot;: &quot;example.avro&quot;,</span>
+<span class="sd">        &quot;type&quot;: &quot;record&quot;,</span>
+<span class="sd">        &quot;name&quot;: &quot;User&quot;,</span>
+<span class="sd">        &quot;fields&quot;: [</span>
+
+<span class="sd">          {&quot;name&quot;: &quot;name&quot;,</span>
+<span class="sd">           &quot;type&quot;: &quot;string&quot;},</span>
+
+<span class="sd">          {&quot;name&quot;: &quot;favorite_number&quot;,</span>
+<span class="sd">           &quot;type&quot;: [&quot;int&quot;, &quot;null&quot;]},</span>
+
+<span class="sd">          {&quot;name&quot;: &quot;favorite_color&quot;,</span>
+<span class="sd">           &quot;type&quot;: [&quot;string&quot;, &quot;null&quot;]}</span>
+
+<span class="sd">        ]</span>
+<span class="sd">      }</span>
+
+<span class="sd">    Then records generated by :class:`~apache_beam.io._AvroSource` will be</span>
+<span class="sd">    dictionaries of the following form. ::</span>
+
+<span class="sd">      {u&#39;name&#39;: u&#39;Alyssa&#39;, u&#39;favorite_number&#39;: 256, u&#39;favorite_color&#39;: None}).</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">      file_pattern (str): the file glob to read</span>
+<span class="sd">      min_bundle_size (int): the minimum size in bytes, to be considered when</span>
+<span class="sd">        splitting the input into bundles.</span>
+<span class="sd">      validate (bool): flag to verify that the files exist during the pipeline</span>
+<span class="sd">        creation time.</span>
+<span class="sd">      use_fastavro (bool); when set, use the `fastavro` library for IO, which</span>
+<span class="sd">        is significantly faster, and will likely become the default</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">&gt;=</span> <span class="mi">3</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">use_fastavro</span><span class="p">:</span>
+      <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">&quot;Due to a known issue in avro-python3 package, it is &quot;</span>
+                    <span class="s2">&quot;recommended to use fastavro with Beam Avro IO on &quot;</span>
+                    <span class="s2">&quot;Python 3 until BEAM-6522 is addressed.&quot;</span><span class="p">)</span>
+    <span class="nb">super</span><span class="p">(</span><span class="n">ReadFromAvro</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">_source</span> <span class="o">=</span> <span class="n">_create_avro_source</span><span class="p">(</span>
+        <span class="n">file_pattern</span><span class="p">,</span>
+        <span class="n">min_bundle_size</span><span class="p">,</span>
+        <span class="n">validate</span><span class="o">=</span><span class="n">validate</span><span class="p">,</span>
+        <span class="n">use_fastavro</span><span class="o">=</span><span class="n">use_fastavro</span>
+    <span class="p">)</span>
+
+<div class="viewcode-block" id="ReadFromAvro.expand"><a class="viewcode-back" href="../../../apache_beam.io.avroio.html#apache_beam.io.avroio.ReadFromAvro.expand">[docs]</a>  <span class="k">def</span> <span class="nf">expand</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">pvalue</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">pvalue</span><span class="o">.</span><span class="n">pipeline</span> <span class="o">|</span> <span class="n">Read</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_source</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="ReadFromAvro.display_data"><a class="viewcode-back" href="../../../apache_beam.io.avroio.html#apache_beam.io.avroio.ReadFromAvro.display_data">[docs]</a>  <span class="k">def</span> <span class="nf">display_data</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="p">{</span><span class="s1">&#39;source_dd&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_source</span><span class="p">}</span></div></div>
+
+
+<div class="viewcode-block" id="ReadAllFromAvro"><a class="viewcode-back" href="../../../apache_beam.io.avroio.html#apache_beam.io.avroio.ReadAllFromAvro">[docs]</a><span class="k">class</span> <span class="nc">ReadAllFromAvro</span><span class="p">(</span><span class="n">PTransform</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;A ``PTransform`` for reading ``PCollection`` of Avro files.</span>
+
+<span class="sd">   Uses source &#39;_AvroSource&#39; to read a ``PCollection`` of Avro files or</span>
+<span class="sd">   file patterns and produce a ``PCollection`` of Avro records.</span>
+<span class="sd">  &quot;&quot;&quot;</span>
+
+  <span class="n">DEFAULT_DESIRED_BUNDLE_SIZE</span> <span class="o">=</span> <span class="mi">64</span> <span class="o">*</span> <span class="mi">1024</span> <span class="o">*</span> <span class="mi">1024</span>  <span class="c1"># 64MB</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">min_bundle_size</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
+               <span class="n">desired_bundle_size</span><span class="o">=</span><span class="n">DEFAULT_DESIRED_BUNDLE_SIZE</span><span class="p">,</span>
+               <span class="n">use_fastavro</span><span class="o">=</span><span class="n">_use_fastavro</span><span class="p">(),</span>
+               <span class="n">label</span><span class="o">=</span><span class="s1">&#39;ReadAllFiles&#39;</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;Initializes ``ReadAllFromAvro``.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">      min_bundle_size: the minimum size in bytes, to be considered when</span>
+<span class="sd">                       splitting the input into bundles.</span>
+<span class="sd">      desired_bundle_size: the desired size in bytes, to be considered when</span>
+<span class="sd">                       splitting the input into bundles.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version_info</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">&gt;=</span> <span class="mi">3</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">use_fastavro</span><span class="p">:</span>
+      <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">&quot;Due to a known issue in avro-python3 package, it is &quot;</span>
+                    <span class="s2">&quot;recommended to use fastavro with Beam Avro IO on &quot;</span>
+                    <span class="s2">&quot;Python 3 until BEAM-6522 is addressed.&quot;</span><span class="p">)</span>
+    <span class="n">source_from_file</span> <span class="o">=</span> <span class="n">partial</span><span class="p">(</span>
+        <span class="n">_create_avro_source</span><span class="p">,</span>
+        <span class="n">min_bundle_size</span><span class="o">=</span><span class="n">min_bundle_size</span><span class="p">,</span>
+        <span class="n">use_fastavro</span><span class="o">=</span><span class="n">use_fastavro</span>
+    <span class="p">)</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">_read_all_files</span> <span class="o">=</span> <span class="n">filebasedsource</span><span class="o">.</span><span class="n">ReadAllFiles</span><span class="p">(</span>
+        <span class="kc">True</span><span class="p">,</span> <span class="n">CompressionTypes</span><span class="o">.</span><span class="n">AUTO</span><span class="p">,</span> <span class="n">desired_bundle_size</span><span class="p">,</span> <span class="n">min_bundle_size</span><span class="p">,</span>
+        <span class="n">source_from_file</span><span class="p">)</span>
+
+    <span class="bp">self</span><span class="o">.</span><span class="n">label</span> <span class="o">=</span> <span class="n">label</span>
+
+<div class="viewcode-block" id="ReadAllFromAvro.expand"><a class="viewcode-back" href="../../../apache_beam.io.avroio.html#apache_beam.io.avroio.ReadAllFromAvro.expand">[docs]</a>  <span class="k">def</span> <span class="nf">expand</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">pvalue</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">pvalue</span> <span class="o">|</span> <span class="bp">self</span><span class="o">.</span><span class="n">label</span> <span class="o">&gt;&gt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">_read_all_files</span></div></div>
+
+
+<span class="k">class</span> <span class="nc">_AvroUtils</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
+
+  <span class="nd">@staticmethod</span>
+  <span class="k">def</span> <span class="nf">read_meta_data_from_file</span><span class="p">(</span><span class="n">f</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;Reads metadata from a given Avro file.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">      f: Avro file to read.</span>
+<span class="sd">    Returns:</span>
+<span class="sd">      a tuple containing the codec, schema, and the sync marker of the Avro</span>
+<span class="sd">      file.</span>
+
+<span class="sd">    Raises:</span>
+<span class="sd">      ValueError: if the file does not start with the byte sequence defined in</span>
+<span class="sd">                  the specification.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">if</span> <span class="n">f</span><span class="o">.</span><span class="n">tell</span><span class="p">()</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
+      <span class="n">f</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
+    <span class="n">decoder</span> <span class="o">=</span> <span class="n">avroio</span><span class="o">.</span><span class="n">BinaryDecoder</span><span class="p">(</span><span class="n">f</span><span class="p">)</span>
+    <span class="n">header</span> <span class="o">=</span> <span class="n">avroio</span><span class="o">.</span><span class="n">DatumReader</span><span class="p">()</span><span class="o">.</span><span class="n">read_data</span><span class="p">(</span><span class="n">datafile</span><span class="o">.</span><span class="n">META_SCHEMA</span><span class="p">,</span>
+                                            <span class="n">datafile</span><span class="o">.</span><span class="n">META_SCHEMA</span><span class="p">,</span> <span class="n">decoder</span><span class="p">)</span>
+    <span class="k">if</span> <span class="n">header</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;magic&#39;</span><span class="p">)</span> <span class="o">!=</span> <span class="n">datafile</span><span class="o">.</span><span class="n">MAGIC</span><span class="p">:</span>
+      <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;Not an Avro file. File header should start with </span><span class="si">%s</span><span class="s1"> but&#39;</span>
+                       <span class="s1">&#39;started with </span><span class="si">%s</span><span class="s1"> instead.&#39;</span>
+                       <span class="o">%</span> <span class="p">(</span><span class="n">datafile</span><span class="o">.</span><span class="n">MAGIC</span><span class="p">,</span> <span class="n">header</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;magic&#39;</span><span class="p">)))</span>
+
+    <span class="n">meta</span> <span class="o">=</span> <span class="n">header</span><span class="p">[</span><span class="s1">&#39;meta&#39;</span><span class="p">]</span>
+
+    <span class="k">if</span> <span class="n">datafile</span><span class="o">.</span><span class="n">CODEC_KEY</span> <span class="ow">in</span> <span class="n">meta</span><span class="p">:</span>
+      <span class="n">codec</span> <span class="o">=</span> <span class="n">meta</span><span class="p">[</span><span class="n">datafile</span><span class="o">.</span><span class="n">CODEC_KEY</span><span class="p">]</span>
+    <span class="k">else</span><span class="p">:</span>
+      <span class="n">codec</span> <span class="o">=</span> <span class="sa">b</span><span class="s1">&#39;null&#39;</span>
+
+    <span class="n">schema_string</span> <span class="o">=</span> <span class="n">meta</span><span class="p">[</span><span class="n">datafile</span><span class="o">.</span><span class="n">SCHEMA_KEY</span><span class="p">]</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s1">&#39;utf-8&#39;</span><span class="p">)</span>
+    <span class="n">sync_marker</span> <span class="o">=</span> <span class="n">header</span><span class="p">[</span><span class="s1">&#39;sync&#39;</span><span class="p">]</span>
+
+    <span class="k">return</span> <span class="n">codec</span><span class="p">,</span> <span class="n">schema_string</span><span class="p">,</span> <span class="n">sync_marker</span>
+
+  <span class="nd">@staticmethod</span>
+  <span class="k">def</span> <span class="nf">read_block_from_file</span><span class="p">(</span><span class="n">f</span><span class="p">,</span> <span class="n">codec</span><span class="p">,</span> <span class="n">schema</span><span class="p">,</span> <span class="n">expected_sync_marker</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;Reads a block from a given Avro file.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">      f: Avro file to read.</span>
+<span class="sd">      codec: The codec to use for block-level decompression.</span>
+<span class="sd">        Supported codecs: &#39;null&#39;, &#39;deflate&#39;, &#39;snappy&#39;</span>
+<span class="sd">      schema: Avro Schema definition represented as JSON string.</span>
+<span class="sd">      expected_sync_marker: Avro synchronization marker. If the block&#39;s sync</span>
+<span class="sd">        marker does not match with this parameter then ValueError is thrown.</span>
+<span class="sd">    Returns:</span>
+<span class="sd">      A single _AvroBlock.</span>
+
+<span class="sd">    Raises:</span>
+<span class="sd">      ValueError: If the block cannot be read properly because the file doesn&#39;t</span>
+<span class="sd">        match the specification.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">offset</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">tell</span><span class="p">()</span>
+    <span class="n">decoder</span> <span class="o">=</span> <span class="n">avroio</span><span class="o">.</span><span class="n">BinaryDecoder</span><span class="p">(</span><span class="n">f</span><span class="p">)</span>
+    <span class="n">num_records</span> <span class="o">=</span> <span class="n">decoder</span><span class="o">.</span><span class="n">read_long</span><span class="p">()</span>
+    <span class="n">block_size</span> <span class="o">=</span> <span class="n">decoder</span><span class="o">.</span><span class="n">read_long</span><span class="p">()</span>
+    <span class="n">block_bytes</span> <span class="o">=</span> <span class="n">decoder</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="n">block_size</span><span class="p">)</span>
+    <span class="n">sync_marker</span> <span class="o">=</span> <span class="n">decoder</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">expected_sync_marker</span><span class="p">))</span>
+    <span class="k">if</span> <span class="n">sync_marker</span> <span class="o">!=</span> <span class="n">expected_sync_marker</span><span class="p">:</span>
+      <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;Unexpected sync marker (actual &quot;</span><span class="si">%s</span><span class="s1">&quot; vs expected &quot;</span><span class="si">%s</span><span class="s1">&quot;). &#39;</span>
+                       <span class="s1">&#39;Maybe the underlying avro file is corrupted?&#39;</span>
+                       <span class="o">%</span> <span class="p">(</span><span class="n">sync_marker</span><span class="p">,</span> <span class="n">expected_sync_marker</span><span class="p">))</span>
+    <span class="n">size</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">tell</span><span class="p">()</span> <span class="o">-</span> <span class="n">offset</span>
+    <span class="k">return</span> <span class="n">_AvroBlock</span><span class="p">(</span><span class="n">block_bytes</span><span class="p">,</span> <span class="n">num_records</span><span class="p">,</span> <span class="n">codec</span><span class="p">,</span> <span class="n">schema</span><span class="p">,</span> <span class="n">offset</span><span class="p">,</span> <span class="n">size</span><span class="p">)</span>
+
+  <span class="nd">@staticmethod</span>
+  <span class="k">def</span> <span class="nf">advance_file_past_next_sync_marker</span><span class="p">(</span><span class="n">f</span><span class="p">,</span> <span class="n">sync_marker</span><span class="p">):</span>
+    <span class="n">buf_size</span> <span class="o">=</span> <span class="mi">10000</span>
+
+    <span class="n">data</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="n">buf_size</span><span class="p">)</span>
+    <span class="k">while</span> <span class="n">data</span><span class="p">:</span>
+      <span class="n">pos</span> <span class="o">=</span> <span class="n">data</span><span class="o">.</span><span class="n">find</span><span class="p">(</span><span class="n">sync_marker</span><span class="p">)</span>
+      <span class="k">if</span> <span class="n">pos</span> <span class="o">&gt;=</span> <span class="mi">0</span><span class="p">:</span>
+        <span class="c1"># Adjusting the current position to the ending position of the sync</span>
+        <span class="c1"># marker.</span>
+        <span class="n">backtrack</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">data</span><span class="p">)</span> <span class="o">-</span> <span class="n">pos</span> <span class="o">-</span> <span class="nb">len</span><span class="p">(</span><span class="n">sync_marker</span><span class="p">)</span>
+        <span class="n">f</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span> <span class="o">*</span> <span class="n">backtrack</span><span class="p">,</span> <span class="n">os</span><span class="o">.</span><span class="n">SEEK_CUR</span><span class="p">)</span>
+        <span class="k">return</span> <span class="kc">True</span>
+      <span class="k">else</span><span class="p">:</span>
+        <span class="k">if</span> <span class="n">f</span><span class="o">.</span><span class="n">tell</span><span class="p">()</span> <span class="o">&gt;=</span> <span class="nb">len</span><span class="p">(</span><span class="n">sync_marker</span><span class="p">):</span>
+          <span class="c1"># Backtracking in case we partially read the sync marker during the</span>
+          <span class="c1"># previous read. We only have to backtrack if there are at least</span>
+          <span class="c1"># len(sync_marker) bytes before current position. We only have to</span>
+          <span class="c1"># backtrack (len(sync_marker) - 1) bytes.</span>
+          <span class="n">f</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span> <span class="o">*</span> <span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">sync_marker</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span><span class="p">),</span> <span class="n">os</span><span class="o">.</span><span class="n">SEEK_CUR</span><span class="p">)</span>
+        <span class="n">data</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="n">buf_size</span><span class="p">)</span>
+
+
+<span class="k">def</span> <span class="nf">_create_avro_source</span><span class="p">(</span><span class="n">file_pattern</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+                        <span class="n">min_bundle_size</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
+                        <span class="n">validate</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
+                        <span class="n">use_fastavro</span><span class="o">=</span><span class="n">_use_fastavro</span><span class="p">()):</span>
+  <span class="k">return</span> \
+      <span class="n">_FastAvroSource</span><span class="p">(</span>
+          <span class="n">file_pattern</span><span class="o">=</span><span class="n">file_pattern</span><span class="p">,</span>
+          <span class="n">min_bundle_size</span><span class="o">=</span><span class="n">min_bundle_size</span><span class="p">,</span>
+          <span class="n">validate</span><span class="o">=</span><span class="n">validate</span>
+      <span class="p">)</span> \
+      <span class="k">if</span> <span class="n">use_fastavro</span> \
+      <span class="k">else</span> \
+      <span class="n">_AvroSource</span><span class="p">(</span>
+          <span class="n">file_pattern</span><span class="o">=</span><span class="n">file_pattern</span><span class="p">,</span>
+          <span class="n">min_bundle_size</span><span class="o">=</span><span class="n">min_bundle_size</span><span class="p">,</span>
+          <span class="n">validate</span><span class="o">=</span><span class="n">validate</span>
+      <span class="p">)</span>
+
+
+<span class="k">class</span> <span class="nc">_AvroBlock</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;Represents a block of an Avro file.&quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">block_bytes</span><span class="p">,</span> <span class="n">num_records</span><span class="p">,</span> <span class="n">codec</span><span class="p">,</span> <span class="n">schema_string</span><span class="p">,</span>
+               <span class="n">offset</span><span class="p">,</span> <span class="n">size</span><span class="p">):</span>
+    <span class="c1"># Decompress data early on (if needed) and thus decrease the number of</span>
+    <span class="c1"># parallel copies of the data in memory at any given time during block</span>
+    <span class="c1"># iteration.</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">_decompressed_block_bytes</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_decompress_bytes</span><span class="p">(</span><span class="n">block_bytes</span><span class="p">,</span> <span class="n">codec</span><span class="p">)</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">_num_records</span> <span class="o">=</span> <span class="n">num_records</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">_schema</span> <span class="o">=</span> <span class="n">Parse</span><span class="p">(</span><span class="n">schema_string</span><span class="p">)</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">_offset</span> <span class="o">=</span> <span class="n">offset</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">_size</span> <span class="o">=</span> <span class="n">size</span>
+
+  <span class="k">def</span> <span class="nf">size</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_size</span>
+
+  <span class="k">def</span> <span class="nf">offset</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_offset</span>
+
+  <span class="nd">@staticmethod</span>
+  <span class="k">def</span> <span class="nf">_decompress_bytes</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">codec</span><span class="p">):</span>
+    <span class="k">if</span> <span class="n">codec</span> <span class="o">==</span> <span class="sa">b</span><span class="s1">&#39;null&#39;</span><span class="p">:</span>
+      <span class="k">return</span> <span class="n">data</span>
+    <span class="k">elif</span> <span class="n">codec</span> <span class="o">==</span> <span class="sa">b</span><span class="s1">&#39;deflate&#39;</span><span class="p">:</span>
+      <span class="c1"># zlib.MAX_WBITS is the window size. &#39;-&#39; sign indicates that this is</span>
+      <span class="c1"># raw data (without headers). See zlib and Avro documentations for more</span>
+      <span class="c1"># details.</span>
+      <span class="k">return</span> <span class="n">zlib</span><span class="o">.</span><span class="n">decompress</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="o">-</span><span class="n">zlib</span><span class="o">.</span><span class="n">MAX_WBITS</span><span class="p">)</span>
+    <span class="k">elif</span> <span class="n">codec</span> <span class="o">==</span> <span class="sa">b</span><span class="s1">&#39;snappy&#39;</span><span class="p">:</span>
+      <span class="c1"># Snappy is an optional avro codec.</span>
+      <span class="c1"># See Snappy and Avro documentation for more details.</span>
+      <span class="k">try</span><span class="p">:</span>
+        <span class="kn">import</span> <span class="nn">snappy</span>
+      <span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
+        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;python-snappy does not seem to be installed.&#39;</span><span class="p">)</span>
+
+      <span class="c1"># Compressed data includes a 4-byte CRC32 checksum which we verify.</span>
+      <span class="c1"># We take care to avoid extra copies of data while slicing large objects</span>
+      <span class="c1"># by use of a memoryview.</span>
+      <span class="n">result</span> <span class="o">=</span> <span class="n">snappy</span><span class="o">.</span><span class="n">decompress</span><span class="p">(</span><span class="nb">memoryview</span><span class="p">(</span><span class="n">data</span><span class="p">)[:</span><span class="o">-</span><span class="mi">4</span><span class="p">])</span>
+      <span class="n">avroio</span><span class="o">.</span><span class="n">BinaryDecoder</span><span class="p">(</span><span class="n">io</span><span class="o">.</span><span class="n">BytesIO</span><span class="p">(</span><span class="n">data</span><span class="p">[</span><span class="o">-</span><span class="mi">4</span><span class="p">:]))</span><span class="o">.</span><span class="n">check_crc32</span><span class="p">(</span><span class="n">result</span><span class="p">)</span>
+      <span class="k">return</span> <span class="n">result</span>
+    <span class="k">else</span><span class="p">:</span>
+      <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;Unknown codec: </span><span class="si">%r</span><span class="s1">&#39;</span> <span class="o">%</span> <span class="n">codec</span><span class="p">)</span>
+
+  <span class="k">def</span> <span class="nf">num_records</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_num_records</span>
+
+  <span class="k">def</span> <span class="nf">records</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="n">decoder</span> <span class="o">=</span> <span class="n">avroio</span><span class="o">.</span><span class="n">BinaryDecoder</span><span class="p">(</span>
+        <span class="n">io</span><span class="o">.</span><span class="n">BytesIO</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_decompressed_block_bytes</span><span class="p">))</span>
+
+    <span class="n">writer_schema</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_schema</span>
+    <span class="n">reader_schema</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_schema</span>
+    <span class="n">reader</span> <span class="o">=</span> <span class="n">avroio</span><span class="o">.</span><span class="n">DatumReader</span><span class="p">(</span><span class="n">writer_schema</span><span class="p">,</span> <span class="n">reader_schema</span><span class="p">)</span>
+
+    <span class="n">current_record</span> <span class="o">=</span> <span class="mi">0</span>
+    <span class="k">while</span> <span class="n">current_record</span> <span class="o">&lt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">_num_records</span><span class="p">:</span>
+      <span class="k">yield</span> <span class="n">reader</span><span class="o">.</span><span class="n">read</span><span class="p">(</span><span class="n">decoder</span><span class="p">)</span>
+      <span class="n">current_record</span> <span class="o">+=</span> <span class="mi">1</span>
+
+
+<span class="k">class</span> <span class="nc">_AvroSource</span><span class="p">(</span><span class="n">filebasedsource</span><span class="o">.</span><span class="n">FileBasedSource</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;A source for reading Avro files.</span>
+
+<span class="sd">  ``_AvroSource`` is implemented using the file-based source framework available</span>
+<span class="sd">  in module &#39;filebasedsource&#39;. Hence please refer to module &#39;filebasedsource&#39;</span>
+<span class="sd">  to fully understand how this source implements operations common to all</span>
+<span class="sd">  file-based sources such as file-pattern expansion and splitting into bundles</span>
+<span class="sd">  for parallel processing.</span>
+<span class="sd">  &quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="nf">read_records</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">file_name</span><span class="p">,</span> <span class="n">range_tracker</span><span class="p">):</span>
+    <span class="n">next_block_start</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span>
+
+    <span class="k">def</span> <span class="nf">split_points_unclaimed</span><span class="p">(</span><span class="n">stop_position</span><span class="p">):</span>
+      <span class="k">if</span> <span class="n">next_block_start</span> <span class="o">&gt;=</span> <span class="n">stop_position</span><span class="p">:</span>
+        <span class="c1"># Next block starts at or after the suggested stop position. Hence</span>
+        <span class="c1"># there will not be split points to be claimed for the range ending at</span>
+        <span class="c1"># suggested stop position.</span>
+        <span class="k">return</span> <span class="mi">0</span>
+
+      <span class="k">return</span> <span class="n">iobase</span><span class="o">.</span><span class="n">RangeTracker</span><span class="o">.</span><span class="n">SPLIT_POINTS_UNKNOWN</span>
+
+    <span class="n">range_tracker</span><span class="o">.</span><span class="n">set_split_points_unclaimed_callback</span><span class="p">(</span><span class="n">split_points_unclaimed</span><span class="p">)</span>
+
+    <span class="n">start_offset</span> <span class="o">=</span> <span class="n">range_tracker</span><span class="o">.</span><span class="n">start_position</span><span class="p">()</span>
+    <span class="k">if</span> <span class="n">start_offset</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+      <span class="n">start_offset</span> <span class="o">=</span> <span class="mi">0</span>
+
+    <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">open_file</span><span class="p">(</span><span class="n">file_name</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
+      <span class="n">codec</span><span class="p">,</span> <span class="n">schema_string</span><span class="p">,</span> <span class="n">sync_marker</span> <span class="o">=</span> \
+        <span class="n">_AvroUtils</span><span class="o">.</span><span class="n">read_meta_data_from_file</span><span class="p">(</span><span class="n">f</span><span class="p">)</span>
+
+      <span class="c1"># We have to start at current position if previous bundle ended at the</span>
+      <span class="c1"># end of a sync marker.</span>
+      <span class="n">start_offset</span> <span class="o">=</span> <span class="nb">max</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">start_offset</span> <span class="o">-</span> <span class="nb">len</span><span class="p">(</span><span class="n">sync_marker</span><span class="p">))</span>
+      <span class="n">f</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="n">start_offset</span><span class="p">)</span>
+      <span class="n">_AvroUtils</span><span class="o">.</span><span class="n">advance_file_past_next_sync_marker</span><span class="p">(</span><span class="n">f</span><span class="p">,</span> <span class="n">sync_marker</span><span class="p">)</span>
+
+      <span class="n">next_block_start</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">tell</span><span class="p">()</span>
+
+      <span class="k">while</span> <span class="n">range_tracker</span><span class="o">.</span><span class="n">try_claim</span><span class="p">(</span><span class="n">next_block_start</span><span class="p">):</span>
+        <span class="n">block</span> <span class="o">=</span> <span class="n">_AvroUtils</span><span class="o">.</span><span class="n">read_block_from_file</span><span class="p">(</span><span class="n">f</span><span class="p">,</span> <span class="n">codec</span><span class="p">,</span> <span class="n">schema_string</span><span class="p">,</span>
+                                                <span class="n">sync_marker</span><span class="p">)</span>
+        <span class="n">next_block_start</span> <span class="o">=</span> <span class="n">block</span><span class="o">.</span><span class="n">offset</span><span class="p">()</span> <span class="o">+</span> <span class="n">block</span><span class="o">.</span><span class="n">size</span><span class="p">()</span>
+        <span class="k">for</span> <span class="n">record</span> <span class="ow">in</span> <span class="n">block</span><span class="o">.</span><span class="n">records</span><span class="p">():</span>
+          <span class="k">yield</span> <span class="n">record</span>
+
+
+<span class="k">class</span> <span class="nc">_FastAvroSource</span><span class="p">(</span><span class="n">filebasedsource</span><span class="o">.</span><span class="n">FileBasedSource</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;A source for reading Avro files using the `fastavro` library.</span>
+
+<span class="sd">  ``_FastAvroSource`` is implemented using the file-based source framework</span>
+<span class="sd">  available in module &#39;filebasedsource&#39;. Hence please refer to module</span>
+<span class="sd">  &#39;filebasedsource&#39; to fully understand how this source implements operations</span>
+<span class="sd">  common to all file-based sources such as file-pattern expansion and splitting</span>
+<span class="sd">  into bundles for parallel processing.</span>
+
+<span class="sd">  TODO: remove ``_AvroSource`` in favor of using ``_FastAvroSource``</span>
+<span class="sd">  everywhere once it has been more widely tested</span>
+<span class="sd">  &quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="nf">read_records</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">file_name</span><span class="p">,</span> <span class="n">range_tracker</span><span class="p">):</span>
+    <span class="n">next_block_start</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span>
+
+    <span class="k">def</span> <span class="nf">split_points_unclaimed</span><span class="p">(</span><span class="n">stop_position</span><span class="p">):</span>
+      <span class="k">if</span> <span class="n">next_block_start</span> <span class="o">&gt;=</span> <span class="n">stop_position</span><span class="p">:</span>
+        <span class="c1"># Next block starts at or after the suggested stop position. Hence</span>
+        <span class="c1"># there will not be split points to be claimed for the range ending at</span>
+        <span class="c1"># suggested stop position.</span>
+        <span class="k">return</span> <span class="mi">0</span>
+
+      <span class="k">return</span> <span class="n">iobase</span><span class="o">.</span><span class="n">RangeTracker</span><span class="o">.</span><span class="n">SPLIT_POINTS_UNKNOWN</span>
+
+    <span class="n">range_tracker</span><span class="o">.</span><span class="n">set_split_points_unclaimed_callback</span><span class="p">(</span><span class="n">split_points_unclaimed</span><span class="p">)</span>
+
+    <span class="n">start_offset</span> <span class="o">=</span> <span class="n">range_tracker</span><span class="o">.</span><span class="n">start_position</span><span class="p">()</span>
+    <span class="k">if</span> <span class="n">start_offset</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
+      <span class="n">start_offset</span> <span class="o">=</span> <span class="mi">0</span>
+
+    <span class="k">with</span> <span class="bp">self</span><span class="o">.</span><span class="n">open_file</span><span class="p">(</span><span class="n">file_name</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
+      <span class="n">blocks</span> <span class="o">=</span> <span class="n">block_reader</span><span class="p">(</span><span class="n">f</span><span class="p">)</span>
+      <span class="n">sync_marker</span> <span class="o">=</span> <span class="n">blocks</span><span class="o">.</span><span class="n">_header</span><span class="p">[</span><span class="s1">&#39;sync&#39;</span><span class="p">]</span>
+
+      <span class="c1"># We have to start at current position if previous bundle ended at the</span>
+      <span class="c1"># end of a sync marker.</span>
+      <span class="n">start_offset</span> <span class="o">=</span> <span class="nb">max</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">start_offset</span> <span class="o">-</span> <span class="nb">len</span><span class="p">(</span><span class="n">sync_marker</span><span class="p">))</span>
+      <span class="n">f</span><span class="o">.</span><span class="n">seek</span><span class="p">(</span><span class="n">start_offset</span><span class="p">)</span>
+      <span class="n">_AvroUtils</span><span class="o">.</span><span class="n">advance_file_past_next_sync_marker</span><span class="p">(</span><span class="n">f</span><span class="p">,</span> <span class="n">sync_marker</span><span class="p">)</span>
+
+      <span class="n">next_block_start</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">tell</span><span class="p">()</span>
+
+      <span class="k">while</span> <span class="n">range_tracker</span><span class="o">.</span><span class="n">try_claim</span><span class="p">(</span><span class="n">next_block_start</span><span class="p">):</span>
+        <span class="n">block</span> <span class="o">=</span> <span class="nb">next</span><span class="p">(</span><span class="n">blocks</span><span class="p">)</span>
+        <span class="n">next_block_start</span> <span class="o">=</span> <span class="n">block</span><span class="o">.</span><span class="n">offset</span> <span class="o">+</span> <span class="n">block</span><span class="o">.</span><span class="n">size</span>
+        <span class="k">for</span> <span class="n">record</span> <span class="ow">in</span> <span class="n">block</span><span class="p">:</span>
+          <span class="k">yield</span> <span class="n">record</span>
+
+
+<div class="viewcode-block" id="WriteToAvro"><a class="viewcode-back" href="../../../apache_beam.io.avroio.html#apache_beam.io.avroio.WriteToAvro">[docs]</a><span class="k">class</span> <span class="nc">WriteToAvro</span><span class="p">(</span><span class="n">beam</span><span class="o">.</span><span class="n">transforms</span><span class="o">.</span><span class="n">PTransform</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;A ``PTransform`` for writing avro files.&quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+               <span class="n">file_path_prefix</span><span class="p">,</span>
+               <span class="n">schema</span><span class="p">,</span>
+               <span class="n">codec</span><span class="o">=</span><span class="s1">&#39;deflate&#39;</span><span class="p">,</span>
+               <span class="n">file_name_suffix</span><span class="o">=</span><span class="s1">&#39;&#39;</span><span class="p">,</span>
+               <span class="n">num_shards</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
+               <span class="n">shard_name_template</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+               <span class="n">mime_type</span><span class="o">=</span><span class="s1">&#39;application/x-avro&#39;</span><span class="p">,</span>
+               <span class="n">use_fastavro</span><span class="o">=</span><span class="n">_use_fastavro</span><span class="p">()):</span>
+    <span class="sd">&quot;&quot;&quot;Initialize a WriteToAvro transform.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">      file_path_prefix: The file path to write to. The files written will begin</span>
+<span class="sd">        with this prefix, followed by a shard identifier (see num_shards), and</span>
+<span class="sd">        end in a common extension, if given by file_name_suffix. In most cases,</span>
+<span class="sd">        only this argument is specified and num_shards, shard_name_template, and</span>
+<span class="sd">        file_name_suffix use default values.</span>
+<span class="sd">      schema: The schema to use, as returned by avro.schema.Parse</span>
+<span class="sd">      codec: The codec to use for block-level compression. Any string supported</span>
+<span class="sd">        by the Avro specification is accepted (for example &#39;null&#39;).</span>
+<span class="sd">      file_name_suffix: Suffix for the files written.</span>
+<span class="sd">      num_shards: The number of files (shards) used for output. If not set, the</span>
+<span class="sd">        service will decide on the optimal number of shards.</span>
+<span class="sd">        Constraining the number of shards is likely to reduce</span>
+<span class="sd">        the performance of a pipeline.  Setting this value is not recommended</span>
+<span class="sd">        unless you require a specific number of output files.</span>
+<span class="sd">      shard_name_template: A template string containing placeholders for</span>
+<span class="sd">        the shard number and shard count. When constructing a filename for a</span>
+<span class="sd">        particular shard number, the upper-case letters &#39;S&#39; and &#39;N&#39; are</span>
+<span class="sd">        replaced with the 0-padded shard number and shard count respectively.</span>
+<span class="sd">        This argument can be &#39;&#39; in which case it behaves as if num_shards was</span>
+<span class="sd">        set to 1 and only one file will be generated. The default pattern used</span>
+<span class="sd">        is &#39;-SSSSS-of-NNNNN&#39; if None is passed as the shard_name_template.</span>
+<span class="sd">      mime_type: The MIME type to use for the produced files, if the filesystem</span>
+<span class="sd">        supports specifying MIME types.</span>
+<span class="sd">      use_fastavro: when set, use the `fastavro` library for IO</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">      A WriteToAvro transform usable for writing.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">_sink</span> <span class="o">=</span> \
+      <span class="n">_create_avro_sink</span><span class="p">(</span>
+          <span class="n">file_path_prefix</span><span class="p">,</span>
+          <span class="n">schema</span><span class="p">,</span>
+          <span class="n">codec</span><span class="p">,</span>
+          <span class="n">file_name_suffix</span><span class="p">,</span>
+          <span class="n">num_shards</span><span class="p">,</span>
+          <span class="n">shard_name_template</span><span class="p">,</span>
+          <span class="n">mime_type</span><span class="p">,</span>
+          <span class="n">use_fastavro</span>
+      <span class="p">)</span>
+
+<div class="viewcode-block" id="WriteToAvro.expand"><a class="viewcode-back" href="../../../apache_beam.io.avroio.html#apache_beam.io.avroio.WriteToAvro.expand">[docs]</a>  <span class="k">def</span> <span class="nf">expand</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">pcoll</span><span class="p">):</span>
+    <span class="k">return</span> <span class="n">pcoll</span> <span class="o">|</span> <span class="n">beam</span><span class="o">.</span><span class="n">io</span><span class="o">.</span><span class="n">iobase</span><span class="o">.</span><span class="n">Write</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_sink</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="WriteToAvro.display_data"><a class="viewcode-back" href="../../../apache_beam.io.avroio.html#apache_beam.io.avroio.WriteToAvro.display_data">[docs]</a>  <span class="k">def</span> <span class="nf">display_data</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">return</span> <span class="p">{</span><span class="s1">&#39;sink_dd&#39;</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_sink</span><span class="p">}</span></div></div>
+
+
+<span class="k">def</span> <span class="nf">_create_avro_sink</span><span class="p">(</span><span class="n">file_path_prefix</span><span class="p">,</span>
+                      <span class="n">schema</span><span class="p">,</span>
+                      <span class="n">codec</span><span class="p">,</span>
+                      <span class="n">file_name_suffix</span><span class="p">,</span>
+                      <span class="n">num_shards</span><span class="p">,</span>
+                      <span class="n">shard_name_template</span><span class="p">,</span>
+                      <span class="n">mime_type</span><span class="p">,</span>
+                      <span class="n">use_fastavro</span><span class="p">):</span>
+  <span class="k">return</span> \
+      <span class="n">_FastAvroSink</span><span class="p">(</span>
+          <span class="n">file_path_prefix</span><span class="p">,</span>
+          <span class="n">schema</span><span class="p">,</span>
+          <span class="n">codec</span><span class="p">,</span>
+          <span class="n">file_name_suffix</span><span class="p">,</span>
+          <span class="n">num_shards</span><span class="p">,</span>
+          <span class="n">shard_name_template</span><span class="p">,</span>
+          <span class="n">mime_type</span>
+      <span class="p">)</span> \
+      <span class="k">if</span> <span class="n">use_fastavro</span> \
+      <span class="k">else</span> \
+      <span class="n">_AvroSink</span><span class="p">(</span>
+          <span class="n">file_path_prefix</span><span class="p">,</span>
+          <span class="n">schema</span><span class="p">,</span>
+          <span class="n">codec</span><span class="p">,</span>
+          <span class="n">file_name_suffix</span><span class="p">,</span>
+          <span class="n">num_shards</span><span class="p">,</span>
+          <span class="n">shard_name_template</span><span class="p">,</span>
+          <span class="n">mime_type</span>
+      <span class="p">)</span>
+
+
+<span class="k">class</span> <span class="nc">_BaseAvroSink</span><span class="p">(</span><span class="n">filebasedsink</span><span class="o">.</span><span class="n">FileBasedSink</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;A base for a sink for avro files. &quot;&quot;&quot;</span>
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+               <span class="n">file_path_prefix</span><span class="p">,</span>
+               <span class="n">schema</span><span class="p">,</span>
+               <span class="n">codec</span><span class="p">,</span>
+               <span class="n">file_name_suffix</span><span class="p">,</span>
+               <span class="n">num_shards</span><span class="p">,</span>
+               <span class="n">shard_name_template</span><span class="p">,</span>
+               <span class="n">mime_type</span><span class="p">):</span>
+    <span class="nb">super</span><span class="p">(</span><span class="n">_BaseAvroSink</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
+        <span class="n">file_path_prefix</span><span class="p">,</span>
+        <span class="n">file_name_suffix</span><span class="o">=</span><span class="n">file_name_suffix</span><span class="p">,</span>
+        <span class="n">num_shards</span><span class="o">=</span><span class="n">num_shards</span><span class="p">,</span>
+        <span class="n">shard_name_template</span><span class="o">=</span><span class="n">shard_name_template</span><span class="p">,</span>
+        <span class="n">coder</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
+        <span class="n">mime_type</span><span class="o">=</span><span class="n">mime_type</span><span class="p">,</span>
+        <span class="c1"># Compression happens at the block level using the supplied codec, and</span>
+        <span class="c1"># not at the file level.</span>
+        <span class="n">compression_type</span><span class="o">=</span><span class="n">CompressionTypes</span><span class="o">.</span><span class="n">UNCOMPRESSED</span><span class="p">)</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">_schema</span> <span class="o">=</span> <span class="n">schema</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">_codec</span> <span class="o">=</span> <span class="n">codec</span>
+
+  <span class="k">def</span> <span class="nf">display_data</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="n">res</span> <span class="o">=</span> <span class="nb">super</span><span class="p">(</span><span class="n">_BaseAvroSink</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">display_data</span><span class="p">()</span>
+    <span class="n">res</span><span class="p">[</span><span class="s1">&#39;codec&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_codec</span><span class="p">)</span>
+    <span class="n">res</span><span class="p">[</span><span class="s1">&#39;schema&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_schema</span><span class="p">)</span>
+    <span class="k">return</span> <span class="n">res</span>
+
+
+<span class="k">class</span> <span class="nc">_AvroSink</span><span class="p">(</span><span class="n">_BaseAvroSink</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;A sink for avro files using Avro. &quot;&quot;&quot;</span>
+  <span class="k">def</span> <span class="nf">open</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">temp_path</span><span class="p">):</span>
+    <span class="n">file_handle</span> <span class="o">=</span> <span class="nb">super</span><span class="p">(</span><span class="n">_AvroSink</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">temp_path</span><span class="p">)</span>
+    <span class="k">return</span> <span class="n">avro</span><span class="o">.</span><span class="n">datafile</span><span class="o">.</span><span class="n">DataFileWriter</span><span class="p">(</span>
+        <span class="n">file_handle</span><span class="p">,</span> <span class="n">avro</span><span class="o">.</span><span class="n">io</span><span class="o">.</span><span class="n">DatumWriter</span><span class="p">(),</span> <span class="bp">self</span><span class="o">.</span><span class="n">_schema</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_codec</span><span class="p">)</span>
+
+  <span class="k">def</span> <span class="nf">write_record</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">writer</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
+    <span class="n">writer</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
+
+
+<span class="k">class</span> <span class="nc">_FastAvroSink</span><span class="p">(</span><span class="n">_BaseAvroSink</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;A sink for avro files using FastAvro. &quot;&quot;&quot;</span>
+  <span class="k">def</span> <span class="nf">open</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">temp_path</span><span class="p">):</span>
+    <span class="n">file_handle</span> <span class="o">=</span> <span class="nb">super</span><span class="p">(</span><span class="n">_FastAvroSink</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">temp_path</span><span class="p">)</span>
+    <span class="k">return</span> <span class="n">Writer</span><span class="p">(</span><span class="n">file_handle</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_schema</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_codec</span><span class="p">)</span>
+
+  <span class="k">def</span> <span class="nf">write_record</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">writer</span><span class="p">,</span> <span class="n">value</span><span class="p">):</span>
+    <span class="n">writer</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">value</span><span class="p">)</span>
+
+  <span class="k">def</span> <span class="nf">close</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">writer</span><span class="p">):</span>
+    <span class="n">writer</span><span class="o">.</span><span class="n">flush</span><span class="p">()</span>
+    <span class="n">writer</span><span class="o">.</span><span class="n">fo</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
+</pre></div>
+
+           </div>
+           <div class="articleComments">
+            
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+        &copy; Copyright .
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../../../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file
diff --git a/pydoc/2.19.0/_modules/apache_beam/io/aws/clients/s3/boto3_client.html b/pydoc/2.19.0/_modules/apache_beam/io/aws/clients/s3/boto3_client.html
new file mode 100644
index 0000000..a6cb773
--- /dev/null
+++ b/pydoc/2.19.0/_modules/apache_beam/io/aws/clients/s3/boto3_client.html
@@ -0,0 +1,477 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>apache_beam.io.aws.clients.s3.boto3_client &mdash; Apache Beam  documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../../../../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="../../../../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../../../../search.html"/>
+    <link rel="top" title="Apache Beam  documentation" href="../../../../../../index.html"/>
+        <link rel="up" title="Module code" href="../../../../../index.html"/> 
+
+  
+  <script src="../../../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../../../index.html" class="icon icon-home"> Apache Beam
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.coders.html">apache_beam.coders package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.internal.html">apache_beam.internal package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.io.html">apache_beam.io package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.metrics.html">apache_beam.metrics package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.options.html">apache_beam.options package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.portability.html">apache_beam.portability package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.runners.html">apache_beam.runners package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.testing.html">apache_beam.testing package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.tools.html">apache_beam.tools package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.transforms.html">apache_beam.transforms package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.typehints.html">apache_beam.typehints package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.utils.html">apache_beam.utils package</a></li>
+</ul>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.error.html">apache_beam.error module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.pipeline.html">apache_beam.pipeline module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.pvalue.html">apache_beam.pvalue module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.version.html">apache_beam.version module</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../../../index.html">Apache Beam</a>
+        
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>apache_beam.io.aws.clients.s3.boto3_client</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+            
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for apache_beam.io.aws.clients.s3.boto3_client</h1><div class="highlight"><pre>
+<span></span><span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one or more</span>
+<span class="c1"># contributor license agreements.  See the NOTICE file distributed with</span>
+<span class="c1"># this work for additional information regarding copyright ownership.</span>
+<span class="c1"># The ASF licenses this file to You under the Apache License, Version 2.0</span>
+<span class="c1"># (the &quot;License&quot;); you may not use this file except in compliance with</span>
+<span class="c1"># the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#    http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+<span class="c1">#</span>
+
+<span class="c1"># pytype: skip-file</span>
+
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">absolute_import</span>
+
+<span class="kn">from</span> <span class="nn">apache_beam.io.aws.clients.s3</span> <span class="kn">import</span> <span class="n">messages</span>
+
+<span class="k">try</span><span class="p">:</span>
+  <span class="c1"># pylint: disable=wrong-import-order, wrong-import-position</span>
+  <span class="c1"># pylint: disable=ungrouped-imports</span>
+  <span class="kn">import</span> <span class="nn">boto3</span>
+
+<span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
+  <span class="n">boto3</span> <span class="o">=</span> <span class="kc">None</span>
+
+
+<div class="viewcode-block" id="Client"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.boto3_client.html#apache_beam.io.aws.clients.s3.boto3_client.Client">[docs]</a><span class="k">class</span> <span class="nc">Client</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">  Wrapper for boto3 library</span>
+<span class="sd">  &quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="k">assert</span> <span class="n">boto3</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">,</span> <span class="s1">&#39;Missing boto3 requirement&#39;</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">client</span> <span class="o">=</span> <span class="n">boto3</span><span class="o">.</span><span class="n">client</span><span class="p">(</span><span class="s1">&#39;s3&#39;</span><span class="p">)</span>
+
+<div class="viewcode-block" id="Client.get_object_metadata"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.boto3_client.html#apache_beam.io.aws.clients.s3.boto3_client.Client.get_object_metadata">[docs]</a>  <span class="k">def</span> <span class="nf">get_object_metadata</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">):</span>
+    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Retrieves an object&#39;s metadata.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">      request: (GetRequest) input message</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">      (Object) The response message.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">kwargs</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;Bucket&#39;</span><span class="p">:</span> <span class="n">request</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="s1">&#39;Key&#39;</span><span class="p">:</span> <span class="n">request</span><span class="o">.</span><span class="n">object</span><span class="p">}</span>
+
+    <span class="k">try</span><span class="p">:</span>
+      <span class="n">boto_response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">head_object</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+    <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+      <span class="n">message</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="n">response</span><span class="p">[</span><span class="s1">&#39;Error&#39;</span><span class="p">][</span><span class="s1">&#39;Message&#39;</span><span class="p">]</span>
+      <span class="n">code</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="n">response</span><span class="p">[</span><span class="s1">&#39;ResponseMetadata&#39;</span><span class="p">][</span><span class="s1">&#39;HTTPStatusCode&#39;</span><span class="p">]</span>
+      <span class="k">raise</span> <span class="n">messages</span><span class="o">.</span><span class="n">S3ClientError</span><span class="p">(</span><span class="n">message</span><span class="p">,</span> <span class="n">code</span><span class="p">)</span>
+
+    <span class="n">item</span> <span class="o">=</span> <span class="n">messages</span><span class="o">.</span><span class="n">Item</span><span class="p">(</span><span class="n">boto_response</span><span class="p">[</span><span class="s1">&#39;ETag&#39;</span><span class="p">],</span>
+                         <span class="n">request</span><span class="o">.</span><span class="n">object</span><span class="p">,</span>
+                         <span class="n">boto_response</span><span class="p">[</span><span class="s1">&#39;LastModified&#39;</span><span class="p">],</span>
+                         <span class="n">boto_response</span><span class="p">[</span><span class="s1">&#39;ContentLength&#39;</span><span class="p">],</span>
+                         <span class="n">boto_response</span><span class="p">[</span><span class="s1">&#39;ContentType&#39;</span><span class="p">])</span>
+
+    <span class="k">return</span> <span class="n">item</span></div>
+
+<div class="viewcode-block" id="Client.get_range"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.boto3_client.html#apache_beam.io.aws.clients.s3.boto3_client.Client.get_range">[docs]</a>  <span class="k">def</span> <span class="nf">get_range</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">,</span> <span class="n">start</span><span class="p">,</span> <span class="n">end</span>< [...]
+    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Retrieves an object&#39;s contents.</span>
+
+<span class="sd">      Args:</span>
+<span class="sd">        request: (GetRequest) request</span>
+<span class="sd">      Returns:</span>
+<span class="sd">        (bytes) The response message.</span>
+<span class="sd">      &quot;&quot;&quot;</span>
+    <span class="k">try</span><span class="p">:</span>
+      <span class="n">boto_response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">get_object</span><span class="p">(</span><span class="n">Bucket</span><span class="o">=</span><span class="n">request</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span>
+                                             <span class="n">Key</span><span class="o">=</span><span class="n">request</span><span class="o">.</span><span class="n">object</span><span class="p">,</span>
+                                             <span class="n">Range</span><span class="o">=</span><span class="s1">&#39;bytes=</span><span class="si">{}</span><span class="s1">-</span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+                                                 <span class="n">start</span><span class="p">,</span>
+                                                 <span class="n">end</span> <span class="o">-</span> <span class="mi">1</span><span class="p">))</span>
+    <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+      <span class="n">message</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="n">response</span><span class="p">[</span><span class="s1">&#39;Error&#39;</span><span class="p">][</span><span class="s1">&#39;Message&#39;</span><span class="p">]</span>
+      <span class="n">code</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="n">response</span><span class="p">[</span><span class="s1">&#39;ResponseMetadata&#39;</span><span class="p">][</span><span class="s1">&#39;HTTPStatusCode&#39;</span><span class="p">]</span>
+      <span class="k">raise</span> <span class="n">messages</span><span class="o">.</span><span class="n">S3ClientError</span><span class="p">(</span><span class="n">message</span><span class="p">,</span> <span class="n">code</span><span class="p">)</span>
+
+    <span class="k">return</span> <span class="n">boto_response</span><span class="p">[</span><span class="s1">&#39;Body&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">read</span><span class="p">()</span> <span class="c1"># A bytes object</span></div>
+
+<div class="viewcode-block" id="Client.list"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.boto3_client.html#apache_beam.io.aws.clients.s3.boto3_client.Client.list">[docs]</a>  <span class="k">def</span> <span class="nf">list</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">):</span>
+    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Retrieves a list of objects matching the criteria.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">      request: (ListRequest) input message</span>
+<span class="sd">    Returns:</span>
+<span class="sd">      (ListResponse) The response message.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">kwargs</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;Bucket&#39;</span><span class="p">:</span> <span class="n">request</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span>
+              <span class="s1">&#39;Prefix&#39;</span><span class="p">:</span> <span class="n">request</span><span class="o">.</span><span class="n">prefix</span><span class="p">}</span>
+
+    <span class="k">if</span> <span class="n">request</span><span class="o">.</span><span class="n">continuation_token</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+      <span class="n">kwargs</span><span class="p">[</span><span class="s1">&#39;ContinuationToken&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">request</span><span class="o">.</span><span class="n">continuation_token</span>
+
+    <span class="k">try</span><span class="p">:</span>
+      <span class="n">boto_response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">list_objects_v2</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
+    <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+      <span class="n">message</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="n">response</span><span class="p">[</span><span class="s1">&#39;Error&#39;</span><span class="p">][</span><span class="s1">&#39;Message&#39;</span><span class="p">]</span>
+      <span class="n">code</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="n">response</span><span class="p">[</span><span class="s1">&#39;ResponseMetadata&#39;</span><span class="p">][</span><span class="s1">&#39;HTTPStatusCode&#39;</span><span class="p">]</span>
+      <span class="k">raise</span> <span class="n">messages</span><span class="o">.</span><span class="n">S3ClientError</span><span class="p">(</span><span class="n">message</span><span class="p">,</span> <span class="n">code</span><span class="p">)</span>
+
+    <span class="k">if</span> <span class="n">boto_response</span><span class="p">[</span><span class="s1">&#39;KeyCount&#39;</span><span class="p">]</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+      <span class="n">message</span> <span class="o">=</span> <span class="s1">&#39;Tried to list nonexistent S3 path: s3://</span><span class="si">%s</span><span class="s1">/</span><span class="si">%s</span><span class="s1">&#39;</span> <span class="o">%</span> <span class="p">(</span>
+          <span class="n">request</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="n">request</span><span class="o">.</span><span class="n">prefix</span><span class="p">)</span>
+      <span class="k">raise</span> <span class="n">messages</span><span class="o">.</span><span class="n">S3ClientError</span><span class="p">(</span><span class="n">message</span><span class="p">,</span> <span class="mi">404</span><span class="p">)</span>
+
+    <span class="n">items</span> <span class="o">=</span> <span class="p">[</span><span class="n">messages</span><span class="o">.</span><span class="n">Item</span><span class="p">(</span><span class="n">etag</span><span class="o">=</span><span class="n">content</span><span class="p">[</span><span class="s1">&#39;ETag&#39;</span><span class="p">],</span>
+                           <span class="n">key</span><span class="o">=</span><span class="n">content</span><span class="p">[</span><span class="s1">&#39;Key&#39;</span><span class="p">],</span>
+                           <span class="n">last_modified</span><span class="o">=</span><span class="n">content</span><span class="p">[</span><span class="s1">&#39;LastModified&#39;</span><span class="p">],</span>
+                           <span class="n">size</span><span class="o">=</span><span class="n">content</span><span class="p">[</span><span class="s1">&#39;Size&#39;</span><span class="p">])</span>
+             <span class="k">for</span> <span class="n">content</span> <span class="ow">in</span> <span class="n">boto_response</span><span class="p">[</span><span class="s1">&#39;Contents&#39;</span><span class="p">]]</span>
+
+    <span class="k">try</span><span class="p">:</span>
+      <span class="n">next_token</span> <span class="o">=</span> <span class="n">boto_response</span><span class="p">[</span><span class="s1">&#39;NextContinuationToken&#39;</span><span class="p">]</span>
+    <span class="k">except</span> <span class="ne">KeyError</span><span class="p">:</span>
+      <span class="n">next_token</span> <span class="o">=</span> <span class="kc">None</span>
+
+    <span class="n">response</span> <span class="o">=</span> <span class="n">messages</span><span class="o">.</span><span class="n">ListResponse</span><span class="p">(</span><span class="n">items</span><span class="p">,</span> <span class="n">next_token</span><span class="p">)</span>
+    <span class="k">return</span> <span class="n">response</span></div>
+
+<div class="viewcode-block" id="Client.create_multipart_upload"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.boto3_client.html#apache_beam.io.aws.clients.s3.boto3_client.Client.create_multipart_upload">[docs]</a>  <span class="k">def</span> <span class="nf">create_multipart_upload</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">):</span>
+    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Initates a multipart upload to S3 for a given object</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">      request: (UploadRequest) input message</span>
+<span class="sd">    Returns:</span>
+<span class="sd">      (UploadResponse) The response message.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">try</span><span class="p">:</span>
+      <span class="n">boto_response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">create_multipart_upload</span><span class="p">(</span>
+          <span class="n">Bucket</span><span class="o">=</span><span class="n">request</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span>
+          <span class="n">Key</span><span class="o">=</span><span class="n">request</span><span class="o">.</span><span class="n">object</span><span class="p">,</span>
+          <span class="n">ContentType</span><span class="o">=</span><span class="n">request</span><span class="o">.</span><span class="n">mime_type</span>
+      <span class="p">)</span>
+      <span class="n">response</span> <span class="o">=</span> <span class="n">messages</span><span class="o">.</span><span class="n">UploadResponse</span><span class="p">(</span><span class="n">boto_response</span><span class="p">[</span><span class="s1">&#39;UploadId&#39;</span><span class="p">])</span>
+    <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+      <span class="n">message</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="n">response</span><span class="p">[</span><span class="s1">&#39;Error&#39;</span><span class="p">][</span><span class="s1">&#39;Message&#39;</span><span class="p">]</span>
+      <span class="n">code</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="n">response</span><span class="p">[</span><span class="s1">&#39;ResponseMetadata&#39;</span><span class="p">][</span><span class="s1">&#39;HTTPStatusCode&#39;</span><span class="p">]</span>
+      <span class="k">raise</span> <span class="n">messages</span><span class="o">.</span><span class="n">S3ClientError</span><span class="p">(</span><span class="n">message</span><span class="p">,</span> <span class="n">code</span><span class="p">)</span>
+    <span class="k">return</span> <span class="n">response</span></div>
+
+<div class="viewcode-block" id="Client.upload_part"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.boto3_client.html#apache_beam.io.aws.clients.s3.boto3_client.Client.upload_part">[docs]</a>  <span class="k">def</span> <span class="nf">upload_part</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">):</span>
+    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Uploads part of a file to S3 during a multipart upload</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">      request: (UploadPartRequest) input message</span>
+<span class="sd">    Returns:</span>
+<span class="sd">      (UploadPartResponse) The response message.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">try</span><span class="p">:</span>
+      <span class="n">boto_response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">upload_part</span><span class="p">(</span><span class="n">Body</span><span class="o">=</span><span class="n">request</span><span class="o">.</span><span class="n">bytes</span><span class="p">,</span>
+                                              <span class="n">Bucket</span><span class="o">=</span><span class="n">request</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span>
+                                              <span class="n">Key</span><span class="o">=</span><span class="n">request</span><span class="o">.</span><span class="n">object</span><span class="p">,</span>
+                                              <span class="n">PartNumber</span><span class="o">=</span><span class="n">request</span><span class="o">.</span><span class="n">part_number</span><span class="p">,</span>
+                                              <span class="n">UploadId</span><span class="o">=</span><span class="n">request</span><span class="o">.</span><span class="n">upload_id</span><span class="p">)</span>
+      <span class="n">response</span> <span class="o">=</span> <span class="n">messages</span><span class="o">.</span><span class="n">UploadPartResponse</span><span class="p">(</span><span class="n">boto_response</span><span class="p">[</span><span class="s1">&#39;ETag&#39;</span><span class="p">],</span>
+                                             <span class="n">request</span><span class="o">.</span><span class="n">part_number</span><span class="p">)</span>
+      <span class="k">return</span> <span class="n">response</span>
+    <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+      <span class="n">message</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="n">response</span><span class="p">[</span><span class="s1">&#39;Error&#39;</span><span class="p">][</span><span class="s1">&#39;Message&#39;</span><span class="p">]</span>
+      <span class="n">code</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="n">response</span><span class="p">[</span><span class="s1">&#39;ResponseMetadata&#39;</span><span class="p">][</span><span class="s1">&#39;HTTPStatusCode&#39;</span><span class="p">]</span>
+      <span class="k">raise</span> <span class="n">messages</span><span class="o">.</span><span class="n">S3ClientError</span><span class="p">(</span><span class="n">message</span><span class="p">,</span> <span class="n">code</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="Client.complete_multipart_upload"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.boto3_client.html#apache_beam.io.aws.clients.s3.boto3_client.Client.complete_multipart_upload">[docs]</a>  <span class="k">def</span> <span class="nf">complete_multipart_upload</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">):</span>
+    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Completes a multipart upload to S3</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">      request: (UploadPartRequest) input message</span>
+<span class="sd">    Returns:</span>
+<span class="sd">      (Void) The response message.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">parts</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;Parts&#39;</span><span class="p">:</span> <span class="n">request</span><span class="o">.</span><span class="n">parts</span><span class="p">}</span>
+    <span class="k">try</span><span class="p">:</span>
+      <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">complete_multipart_upload</span><span class="p">(</span><span class="n">Bucket</span><span class="o">=</span><span class="n">request</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span>
+                                            <span class="n">Key</span><span class="o">=</span><span class="n">request</span><span class="o">.</span><span class="n">object</span><span class="p">,</span>
+                                            <span class="n">UploadId</span><span class="o">=</span><span class="n">request</span><span class="o">.</span><span class="n">upload_id</span><span class="p">,</span>
+                                            <span class="n">MultipartUpload</span><span class="o">=</span><span class="n">parts</span><span class="p">)</span>
+    <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+      <span class="n">message</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="n">response</span><span class="p">[</span><span class="s1">&#39;Error&#39;</span><span class="p">][</span><span class="s1">&#39;Message&#39;</span><span class="p">]</span>
+      <span class="n">code</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="n">response</span><span class="p">[</span><span class="s1">&#39;ResponseMetadata&#39;</span><span class="p">][</span><span class="s1">&#39;HTTPStatusCode&#39;</span><span class="p">]</span>
+      <span class="k">raise</span> <span class="n">messages</span><span class="o">.</span><span class="n">S3ClientError</span><span class="p">(</span><span class="n">message</span><span class="p">,</span> <span class="n">code</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="Client.delete"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.boto3_client.html#apache_beam.io.aws.clients.s3.boto3_client.Client.delete">[docs]</a>  <span class="k">def</span> <span class="nf">delete</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">):</span>
+    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Deletes given object from bucket</span>
+<span class="sd">    Args:</span>
+<span class="sd">        request: (DeleteRequest) input message</span>
+<span class="sd">      Returns:</span>
+<span class="sd">        (void) Void, otherwise will raise if an error occurs</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">try</span><span class="p">:</span>
+      <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">delete_object</span><span class="p">(</span><span class="n">Bucket</span><span class="o">=</span><span class="n">request</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span>
+                                <span class="n">Key</span><span class="o">=</span><span class="n">request</span><span class="o">.</span><span class="n">object</span><span class="p">)</span>
+
+    <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+      <span class="n">message</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="n">response</span><span class="p">[</span><span class="s1">&#39;Error&#39;</span><span class="p">][</span><span class="s1">&#39;Message&#39;</span><span class="p">]</span>
+      <span class="n">code</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="n">response</span><span class="p">[</span><span class="s1">&#39;ResponseMetadata&#39;</span><span class="p">][</span><span class="s1">&#39;HTTPStatusCode&#39;</span><span class="p">]</span>
+      <span class="k">raise</span> <span class="n">messages</span><span class="o">.</span><span class="n">S3ClientError</span><span class="p">(</span><span class="n">message</span><span class="p">,</span> <span class="n">code</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="Client.delete_batch"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.boto3_client.html#apache_beam.io.aws.clients.s3.boto3_client.Client.delete_batch">[docs]</a>  <span class="k">def</span> <span class="nf">delete_batch</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">):</span>
+
+    <span class="n">aws_request</span> <span class="o">=</span> <span class="p">{</span>
+        <span class="s1">&#39;Bucket&#39;</span><span class="p">:</span> <span class="n">request</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span>
+        <span class="s1">&#39;Delete&#39;</span><span class="p">:</span> <span class="p">{</span>
+            <span class="s1">&#39;Objects&#39;</span><span class="p">:</span> <span class="p">[{</span><span class="s1">&#39;Key&#39;</span><span class="p">:</span> <span class="nb">object</span><span class="p">}</span> <span class="k">for</span> <span class="nb">object</span> <span class="ow">in</span> <span class="n">request</span><span class="o">.</span><span class="n">objects</span><span class="p">]</span>
+        <span class="p">}</span>
+    <span class="p">}</span>
+
+    <span class="k">try</span><span class="p">:</span>
+      <span class="n">aws_response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">delete_objects</span><span class="p">(</span><span class="o">**</span><span class="n">aws_request</span><span class="p">)</span>
+    <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+      <span class="n">message</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="n">response</span><span class="p">[</span><span class="s1">&#39;Error&#39;</span><span class="p">][</span><span class="s1">&#39;Message&#39;</span><span class="p">]</span>
+      <span class="n">code</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">e</span><span class="o">.</span><span class="n">response</span><span class="p">[</span><span class="s1">&#39;ResponseMetadata&#39;</span><span class="p">][</span><span class="s1">&#39;HTTPStatusCode&#39;</span><span class="p">])</span>
+      <span class="k">raise</span> <span class="n">messages</span><span class="o">.</span><span class="n">S3ClientError</span><span class="p">(</span><span class="n">message</span><span class="p">,</span> <span class="n">code</span><span class="p">)</span>
+
+    <span class="n">deleted</span> <span class="o">=</span> <span class="p">[</span><span class="n">obj</span><span class="p">[</span><span class="s1">&#39;Key&#39;</span><span class="p">]</span> <span class="k">for</span> <span class="n">obj</span> <span class="ow">in</span> <span class="n">aws_response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;Deleted&#39;</span><span class="p">,</span> <span class="p">[])]</span>
+
+    <span class="n">failed</span> <span class="o">=</span> <span class="p">[</span><span class="n">obj</span><span class="p">[</span><span class="s1">&#39;Key&#39;</span><span class="p">]</span> <span class="k">for</span> <span class="n">obj</span> <span class="ow">in</span> <span class="n">aws_response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;Errors&#39;</span><span class="p">,</span> <span class="p">[])]</span>
+
+    <span class="n">errors</span> <span class="o">=</span> <span class="p">[</span><span class="n">messages</span><span class="o">.</span><span class="n">S3ClientError</span><span class="p">(</span><span class="n">obj</span><span class="p">[</span><span class="s1">&#39;Message&#39;</span><span class="p">],</span> <span class="n">obj</span><span class="p">[</span><span class="s1">&#39;Code&#39;</span><span class="p">])</span>
+              <span class="k">for</span> <span class="n">obj</span> <span class="ow">in</span> <span class="n">aws_response</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s1">&#39;Errors&#39;</span><span class="p">,</span> <span class="p">[])]</span>
+
+    <span class="k">return</span> <span class="n">messages</span><span class="o">.</span><span class="n">DeleteBatchResponse</span><span class="p">(</span><span class="n">deleted</span><span class="p">,</span> <span class="n">failed</span><span class="p">,</span> <span class="n">errors</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="Client.copy"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.boto3_client.html#apache_beam.io.aws.clients.s3.boto3_client.Client.copy">[docs]</a>  <span class="k">def</span> <span class="nf">copy</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">):</span>
+    <span class="k">try</span><span class="p">:</span>
+      <span class="n">copy_src</span> <span class="o">=</span> <span class="p">{</span>
+          <span class="s1">&#39;Bucket&#39;</span><span class="p">:</span> <span class="n">request</span><span class="o">.</span><span class="n">src_bucket</span><span class="p">,</span>
+          <span class="s1">&#39;Key&#39;</span><span class="p">:</span> <span class="n">request</span><span class="o">.</span><span class="n">src_key</span>
+      <span class="p">}</span>
+      <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">copy</span><span class="p">(</span><span class="n">copy_src</span><span class="p">,</span> <span class="n">request</span><span class="o">.</span><span class="n">dest_bucket</span><span class="p">,</span> <span class="n">request</span><span class="o">.</span><span class="n">dest_key</span><span class="p">)</span>
+    <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+      <span class="n">message</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="n">response</span><span class="p">[</span><span class="s1">&#39;Error&#39;</span><span class="p">][</span><span class="s1">&#39;Message&#39;</span><span class="p">]</span>
+      <span class="n">code</span> <span class="o">=</span> <span class="n">e</span><span class="o">.</span><span class="n">response</span><span class="p">[</span><span class="s1">&#39;ResponseMetadata&#39;</span><span class="p">][</span><span class="s1">&#39;HTTPStatusCode&#39;</span><span class="p">]</span>
+      <span class="k">raise</span> <span class="n">messages</span><span class="o">.</span><span class="n">S3ClientError</span><span class="p">(</span><span class="n">message</span><span class="p">,</span> <span class="n">code</span><span class="p">)</span></div></div>
+</pre></div>
+
+           </div>
+           <div class="articleComments">
+            
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+        &copy; Copyright .
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../../../../../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="../../../../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../../../../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../../../../../../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file
diff --git a/pydoc/2.19.0/_modules/apache_beam/io/aws/clients/s3/fake_client.html b/pydoc/2.19.0/_modules/apache_beam/io/aws/clients/s3/fake_client.html
new file mode 100644
index 0000000..d382cc9
--- /dev/null
+++ b/pydoc/2.19.0/_modules/apache_beam/io/aws/clients/s3/fake_client.html
@@ -0,0 +1,473 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>apache_beam.io.aws.clients.s3.fake_client &mdash; Apache Beam  documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../../../../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="../../../../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../../../../search.html"/>
+    <link rel="top" title="Apache Beam  documentation" href="../../../../../../index.html"/>
+        <link rel="up" title="Module code" href="../../../../../index.html"/> 
+
+  
+  <script src="../../../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../../../index.html" class="icon icon-home"> Apache Beam
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.coders.html">apache_beam.coders package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.internal.html">apache_beam.internal package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.io.html">apache_beam.io package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.metrics.html">apache_beam.metrics package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.options.html">apache_beam.options package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.portability.html">apache_beam.portability package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.runners.html">apache_beam.runners package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.testing.html">apache_beam.testing package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.tools.html">apache_beam.tools package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.transforms.html">apache_beam.transforms package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.typehints.html">apache_beam.typehints package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.utils.html">apache_beam.utils package</a></li>
+</ul>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.error.html">apache_beam.error module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.pipeline.html">apache_beam.pipeline module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.pvalue.html">apache_beam.pvalue module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.version.html">apache_beam.version module</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../../../index.html">Apache Beam</a>
+        
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>apache_beam.io.aws.clients.s3.fake_client</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+            
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for apache_beam.io.aws.clients.s3.fake_client</h1><div class="highlight"><pre>
+<span></span><span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one or more</span>
+<span class="c1"># contributor license agreements.  See the NOTICE file distributed with</span>
+<span class="c1"># this work for additional information regarding copyright ownership.</span>
+<span class="c1"># The ASF licenses this file to You under the Apache License, Version 2.0</span>
+<span class="c1"># (the &quot;License&quot;); you may not use this file except in compliance with</span>
+<span class="c1"># the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#    http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+<span class="c1">#</span>
+
+<span class="c1"># pytype: skip-file</span>
+
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">absolute_import</span>
+
+<span class="kn">import</span> <span class="nn">datetime</span>
+<span class="kn">import</span> <span class="nn">time</span>
+
+<span class="kn">from</span> <span class="nn">apache_beam.io.aws.clients.s3</span> <span class="kn">import</span> <span class="n">messages</span>
+
+
+<div class="viewcode-block" id="FakeFile"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.fake_client.html#apache_beam.io.aws.clients.s3.fake_client.FakeFile">[docs]</a><span class="k">class</span> <span class="nc">FakeFile</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">contents</span><span class="p">,</span> <span class="n">etag</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span> <span class="o">=</span> <span class="n">bucket</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">key</span> <span class="o">=</span> <span class="n">key</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">contents</span> <span class="o">=</span> <span class="n">contents</span>
+
+    <span class="bp">self</span><span class="o">.</span><span class="n">last_modified</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>
+
+    <span class="k">if</span> <span class="ow">not</span> <span class="n">etag</span><span class="p">:</span>
+      <span class="bp">self</span><span class="o">.</span><span class="n">etag</span> <span class="o">=</span> <span class="s1">&#39;&quot;</span><span class="si">%s</span><span class="s1">-1&quot;&#39;</span> <span class="o">%</span> <span class="p">(</span><span class="s1">&#39;x&#39;</span> <span class="o">*</span> <span class="mi">32</span><span class="p">)</span>
+    <span class="k">else</span><span class="p">:</span>
+      <span class="bp">self</span><span class="o">.</span><span class="n">etag</span> <span class="o">=</span> <span class="n">etag</span>
+
+<div class="viewcode-block" id="FakeFile.get_metadata"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.fake_client.html#apache_beam.io.aws.clients.s3.fake_client.FakeFile.get_metadata">[docs]</a>  <span class="k">def</span> <span class="nf">get_metadata</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="n">last_modified_datetime</span> <span class="o">=</span> <span class="kc">None</span>
+    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">last_modified</span><span class="p">:</span>
+      <span class="n">last_modified_datetime</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">datetime</span><span class="o">.</span><span class="n">utcfromtimestamp</span><span class="p">(</span>
+          <span class="bp">self</span><span class="o">.</span><span class="n">last_modified</span><span class="p">)</span>
+
+    <span class="k">return</span> <span class="n">messages</span><span class="o">.</span><span class="n">Item</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">etag</span><span class="p">,</span>
+                         <span class="bp">self</span><span class="o">.</span><span class="n">key</span><span class="p">,</span>
+                         <span class="n">last_modified_datetime</span><span class="p">,</span>
+                         <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">contents</span><span class="p">),</span>
+                         <span class="n">mime_type</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span></div></div>
+
+
+<div class="viewcode-block" id="FakeS3Client"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.fake_client.html#apache_beam.io.aws.clients.s3.fake_client.FakeS3Client">[docs]</a><span class="k">class</span> <span class="nc">FakeS3Client</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">files</span> <span class="o">=</span> <span class="p">{}</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">list_continuation_tokens</span> <span class="o">=</span> <span class="p">{}</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">multipart_uploads</span> <span class="o">=</span> <span class="p">{}</span>
+
+    <span class="c1"># boto3 has different behavior when running some operations against a bucket</span>
+    <span class="c1"># that exists vs. against one that doesn&#39;t. To emulate that behavior, the</span>
+    <span class="c1"># mock client keeps a set of bucket names that it knows &quot;exist&quot;.</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">known_buckets</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
+
+<div class="viewcode-block" id="FakeS3Client.add_file"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.fake_client.html#apache_beam.io.aws.clients.s3.fake_client.FakeS3Client.add_file">[docs]</a>  <span class="k">def</span> <span class="nf">add_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">f</span><span class="p">):</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">files</span><span class="p">[(</span><span class="n">f</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="n">f</span><span class="o">.</span><span class="n">key</span><span class="p">)]</span> <span class="o">=</span> <span class="n">f</span>
+    <span class="k">if</span> <span class="n">f</span><span class="o">.</span><span class="n">bucket</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">known_buckets</span><span class="p">:</span>
+      <span class="bp">self</span><span class="o">.</span><span class="n">known_buckets</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">bucket</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="FakeS3Client.get_file"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.fake_client.html#apache_beam.io.aws.clients.s3.fake_client.FakeS3Client.get_file">[docs]</a>  <span class="k">def</span> <span class="nf">get_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span>
+    <span class="k">try</span><span class="p">:</span>
+      <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">files</span><span class="p">[</span><span class="n">bucket</span><span class="p">,</span> <span class="n">obj</span><span class="p">]</span>
+    <span class="k">except</span><span class="p">:</span>
+      <span class="k">raise</span> <span class="n">messages</span><span class="o">.</span><span class="n">S3ClientError</span><span class="p">(</span><span class="s1">&#39;Not Found&#39;</span><span class="p">,</span> <span class="mi">404</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="FakeS3Client.delete_file"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.fake_client.html#apache_beam.io.aws.clients.s3.fake_client.FakeS3Client.delete_file">[docs]</a>  <span class="k">def</span> <span class="nf">delete_file</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="n">obj</span><span class="p">):</span>
+    <span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">files</span><span class="p">[(</span><span class="n">bucket</span><span class="p">,</span> <span class="n">obj</span><span class="p">)]</span></div>
+
+<div class="viewcode-block" id="FakeS3Client.get_object_metadata"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.fake_client.html#apache_beam.io.aws.clients.s3.fake_client.FakeS3Client.get_object_metadata">[docs]</a>  <span class="k">def</span> <span class="nf">get_object_metadata</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">):</span>
+    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Retrieves an object&#39;s metadata.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">      request: (GetRequest) input message</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">      (Item) The response message.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="c1"># TODO: Do we want to mock out a lack of credentials?</span>
+    <span class="n">file_</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_file</span><span class="p">(</span><span class="n">request</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="n">request</span><span class="o">.</span><span class="n">object</span><span class="p">)</span>
+    <span class="k">return</span> <span class="n">file_</span><span class="o">.</span><span class="n">get_metadata</span><span class="p">()</span></div>
+
+<div class="viewcode-block" id="FakeS3Client.list"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.fake_client.html#apache_beam.io.aws.clients.s3.fake_client.FakeS3Client.list">[docs]</a>  <span class="k">def</span> <span class="nf">list</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">):</span>
+    <span class="n">bucket</span> <span class="o">=</span> <span class="n">request</span><span class="o">.</span><span class="n">bucket</span>
+    <span class="n">prefix</span> <span class="o">=</span> <span class="n">request</span><span class="o">.</span><span class="n">prefix</span> <span class="ow">or</span> <span class="s1">&#39;&#39;</span>
+    <span class="n">matching_files</span> <span class="o">=</span> <span class="p">[]</span>
+
+    <span class="k">for</span> <span class="n">file_bucket</span><span class="p">,</span> <span class="n">file_name</span> <span class="ow">in</span> <span class="nb">sorted</span><span class="p">(</span><span class="nb">iter</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">files</span><span class="p">)):</span>
+      <span class="k">if</span> <span class="n">bucket</span> <span class="o">==</span> <span class="n">file_bucket</span> <span class="ow">and</span> <span class="n">file_name</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">prefix</span><span class="p">):</span>
+        <span class="n">file_object</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_file</span><span class="p">(</span><span class="n">file_bucket</span><span class="p">,</span> <span class="n">file_name</span><span class="p">)</span><span class="o">.</span><span class="n">get_metadata</span><span class="p">()</span>
+        <span class="n">matching_files</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">file_object</span><span class="p">)</span>
+
+    <span class="k">if</span> <span class="ow">not</span> <span class="n">matching_files</span><span class="p">:</span>
+      <span class="n">message</span> <span class="o">=</span> <span class="s1">&#39;Tried to list nonexistent S3 path: s3://</span><span class="si">%s</span><span class="s1">/</span><span class="si">%s</span><span class="s1">&#39;</span> <span class="o">%</span> <span class="p">(</span>
+          <span class="n">bucket</span><span class="p">,</span> <span class="n">prefix</span><span class="p">)</span>
+      <span class="k">raise</span> <span class="n">messages</span><span class="o">.</span><span class="n">S3ClientError</span><span class="p">(</span><span class="n">message</span><span class="p">,</span> <span class="mi">404</span><span class="p">)</span>
+
+    <span class="c1"># Handle pagination.</span>
+    <span class="n">items_per_page</span> <span class="o">=</span> <span class="mi">5</span>
+    <span class="k">if</span> <span class="ow">not</span> <span class="n">request</span><span class="o">.</span><span class="n">continuation_token</span><span class="p">:</span>
+      <span class="n">range_start</span> <span class="o">=</span> <span class="mi">0</span>
+    <span class="k">else</span><span class="p">:</span>
+      <span class="k">if</span> <span class="n">request</span><span class="o">.</span><span class="n">continuation_token</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">list_continuation_tokens</span><span class="p">:</span>
+        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;Invalid page token.&#39;</span><span class="p">)</span>
+      <span class="n">range_start</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">list_continuation_tokens</span><span class="p">[</span><span class="n">request</span><span class="o">.</span><span class="n">continuation_token</span><span class="p">]</span>
+      <span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">list_continuation_tokens</span><span class="p">[</span><span class="n">request</span><span class="o">.</span><span class="n">continuation_token</span><span class="p">]</span>
+
+    <span class="n">result</span> <span class="o">=</span> <span class="n">messages</span><span class="o">.</span><span class="n">ListResponse</span><span class="p">(</span>
+        <span class="n">items</span><span class="o">=</span><span class="n">matching_files</span><span class="p">[</span><span class="n">range_start</span><span class="p">:</span><span class="n">range_start</span> <span class="o">+</span> <span class="n">items_per_page</span><span class="p">])</span>
+
+    <span class="k">if</span> <span class="n">range_start</span> <span class="o">+</span> <span class="n">items_per_page</span> <span class="o">&lt;</span> <span class="nb">len</span><span class="p">(</span><span class="n">matching_files</span><span class="p">):</span>
+      <span class="n">next_range_start</span> <span class="o">=</span> <span class="n">range_start</span> <span class="o">+</span> <span class="n">items_per_page</span>
+      <span class="n">next_continuation_token</span> <span class="o">=</span> <span class="s1">&#39;_page_token_</span><span class="si">%s</span><span class="s1">_</span><span class="si">%s</span><span class="s1">_</span><span class="si">%d</span><span class="s1">&#39;</span> <span class="o">%</span> <span class="p">(</span><span class="n">bucket</span><span class="p">,</span> <span class="n">prefix</span><span class="p">,</span>
+                                                          <span class="n">next_range_start</span><span class="p">)</span>
+      <span class="bp">self</span><span class="o">.</span><span class="n">list_continuation_tokens</span><span class="p">[</span><span class="n">next_continuation_token</span><span class="p">]</span> <span class="o">=</span> <span class="n">next_range_start</span>
+      <span class="n">result</span><span class="o">.</span><span class="n">next_token</span> <span class="o">=</span> <span class="n">next_continuation_token</span>
+
+    <span class="k">return</span> <span class="n">result</span></div>
+
+<div class="viewcode-block" id="FakeS3Client.get_range"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.fake_client.html#apache_beam.io.aws.clients.s3.fake_client.FakeS3Client.get_range">[docs]</a>  <span class="k">def</span> <span class="nf">get_range</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">,</span> <span class="n">start</span><span class="p">,</span> <span class="n">e [...]
+    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Retrieves an object.</span>
+
+<span class="sd">      Args:</span>
+<span class="sd">        request: (GetRequest) request</span>
+<span class="sd">      Returns:</span>
+<span class="sd">        (bytes) The response message.</span>
+<span class="sd">      &quot;&quot;&quot;</span>
+
+    <span class="n">file_</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_file</span><span class="p">(</span><span class="n">request</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="n">request</span><span class="o">.</span><span class="n">object</span><span class="p">)</span>
+
+    <span class="c1"># Replicates S3&#39;s behavior, per the spec here:</span>
+    <span class="c1"># https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35</span>
+    <span class="k">if</span> <span class="n">start</span> <span class="o">&lt;</span> <span class="mi">0</span> <span class="ow">or</span> <span class="n">end</span> <span class="o">&lt;=</span> <span class="n">start</span><span class="p">:</span>
+      <span class="k">return</span> <span class="n">file_</span><span class="o">.</span><span class="n">contents</span>
+
+    <span class="k">return</span> <span class="n">file_</span><span class="o">.</span><span class="n">contents</span><span class="p">[</span><span class="n">start</span><span class="p">:</span><span class="n">end</span><span class="p">]</span></div>
+
+<div class="viewcode-block" id="FakeS3Client.delete"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.fake_client.html#apache_beam.io.aws.clients.s3.fake_client.FakeS3Client.delete">[docs]</a>  <span class="k">def</span> <span class="nf">delete</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">):</span>
+    <span class="k">if</span> <span class="n">request</span><span class="o">.</span><span class="n">bucket</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">known_buckets</span><span class="p">:</span>
+      <span class="k">raise</span> <span class="n">messages</span><span class="o">.</span><span class="n">S3ClientError</span><span class="p">(</span><span class="s1">&#39;The specified bucket does not exist&#39;</span><span class="p">,</span> <span class="mi">404</span><span class="p">)</span>
+
+    <span class="k">if</span> <span class="p">(</span><span class="n">request</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="n">request</span><span class="o">.</span><span class="n">object</span><span class="p">)</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">files</span><span class="p">:</span>
+      <span class="bp">self</span><span class="o">.</span><span class="n">delete_file</span><span class="p">(</span><span class="n">request</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="n">request</span><span class="o">.</span><span class="n">object</span><span class="p">)</span>
+    <span class="k">else</span><span class="p">:</span>
+      <span class="c1"># S3 doesn&#39;t raise an error if you try to delete a nonexistent file from</span>
+      <span class="c1"># an extant bucket</span>
+      <span class="k">return</span></div>
+
+<div class="viewcode-block" id="FakeS3Client.delete_batch"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.fake_client.html#apache_beam.io.aws.clients.s3.fake_client.FakeS3Client.delete_batch">[docs]</a>  <span class="k">def</span> <span class="nf">delete_batch</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">):</span>
+
+    <span class="n">deleted</span><span class="p">,</span> <span class="n">failed</span><span class="p">,</span> <span class="n">errors</span> <span class="o">=</span> <span class="p">[],</span> <span class="p">[],</span> <span class="p">[]</span>
+    <span class="k">for</span> <span class="nb">object</span> <span class="ow">in</span> <span class="n">request</span><span class="o">.</span><span class="n">objects</span><span class="p">:</span>
+      <span class="k">try</span><span class="p">:</span>
+        <span class="n">delete_request</span> <span class="o">=</span> <span class="n">messages</span><span class="o">.</span><span class="n">DeleteRequest</span><span class="p">(</span><span class="n">request</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="nb">object</span><span class="p">)</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">delete</span><span class="p">(</span><span class="n">delete_request</span><span class="p">)</span>
+        <span class="n">deleted</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="nb">object</span><span class="p">)</span>
+      <span class="k">except</span> <span class="n">messages</span><span class="o">.</span><span class="n">S3ClientError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+        <span class="n">failed</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="nb">object</span><span class="p">)</span>
+        <span class="n">errors</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">e</span><span class="p">)</span>
+
+    <span class="k">return</span> <span class="n">messages</span><span class="o">.</span><span class="n">DeleteBatchResponse</span><span class="p">(</span><span class="n">deleted</span><span class="p">,</span> <span class="n">failed</span><span class="p">,</span> <span class="n">errors</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="FakeS3Client.copy"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.fake_client.html#apache_beam.io.aws.clients.s3.fake_client.FakeS3Client.copy">[docs]</a>  <span class="k">def</span> <span class="nf">copy</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">):</span>
+
+    <span class="n">src_file</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_file</span><span class="p">(</span><span class="n">request</span><span class="o">.</span><span class="n">src_bucket</span><span class="p">,</span> <span class="n">request</span><span class="o">.</span><span class="n">src_key</span><span class="p">)</span>
+    <span class="n">dest_file</span> <span class="o">=</span> <span class="n">FakeFile</span><span class="p">(</span><span class="n">request</span><span class="o">.</span><span class="n">dest_bucket</span><span class="p">,</span>
+                         <span class="n">request</span><span class="o">.</span><span class="n">dest_key</span><span class="p">,</span>
+                         <span class="n">src_file</span><span class="o">.</span><span class="n">contents</span><span class="p">)</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">add_file</span><span class="p">(</span><span class="n">dest_file</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="FakeS3Client.create_multipart_upload"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.fake_client.html#apache_beam.io.aws.clients.s3.fake_client.FakeS3Client.create_multipart_upload">[docs]</a>  <span class="k">def</span> <span class="nf">create_multipart_upload</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">):</span>
+    <span class="c1"># Create hash of bucket and key</span>
+    <span class="c1"># Store upload_id internally</span>
+    <span class="n">upload_id</span> <span class="o">=</span> <span class="n">request</span><span class="o">.</span><span class="n">bucket</span> <span class="o">+</span> <span class="n">request</span><span class="o">.</span><span class="n">object</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">multipart_uploads</span><span class="p">[</span><span class="n">upload_id</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>
+    <span class="k">return</span> <span class="n">messages</span><span class="o">.</span><span class="n">UploadResponse</span><span class="p">(</span><span class="n">upload_id</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="FakeS3Client.upload_part"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.fake_client.html#apache_beam.io.aws.clients.s3.fake_client.FakeS3Client.upload_part">[docs]</a>  <span class="k">def</span> <span class="nf">upload_part</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">):</span>
+    <span class="c1"># Save off bytes passed to internal data store</span>
+    <span class="n">upload_id</span><span class="p">,</span> <span class="n">part_number</span> <span class="o">=</span> <span class="n">request</span><span class="o">.</span><span class="n">upload_id</span><span class="p">,</span> <span class="n">request</span><span class="o">.</span><span class="n">part_number</span>
+
+    <span class="k">if</span> <span class="n">part_number</span> <span class="o">&lt;</span> <span class="mi">0</span> <span class="ow">or</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">part_number</span><span class="p">,</span> <span class="nb">int</span><span class="p">):</span>
+      <span class="k">raise</span> <span class="n">messages</span><span class="o">.</span><span class="n">S3ClientError</span><span class="p">(</span><span class="s1">&#39;Param validation failed on part number&#39;</span><span class="p">,</span>
+                                   <span class="mi">400</span><span class="p">)</span>
+
+    <span class="k">if</span> <span class="n">upload_id</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">multipart_uploads</span><span class="p">:</span>
+      <span class="k">raise</span> <span class="n">messages</span><span class="o">.</span><span class="n">S3ClientError</span><span class="p">(</span><span class="s1">&#39;The specified upload does not exist&#39;</span><span class="p">,</span> <span class="mi">404</span><span class="p">)</span>
+
+    <span class="bp">self</span><span class="o">.</span><span class="n">multipart_uploads</span><span class="p">[</span><span class="n">upload_id</span><span class="p">][</span><span class="n">part_number</span><span class="p">]</span> <span class="o">=</span> <span class="n">request</span><span class="o">.</span><span class="n">bytes</span>
+
+    <span class="n">etag</span> <span class="o">=</span> <span class="s1">&#39;&quot;</span><span class="si">%s</span><span class="s1">&quot;&#39;</span> <span class="o">%</span> <span class="p">(</span><span class="s1">&#39;x&#39;</span> <span class="o">*</span> <span class="mi">32</span><span class="p">)</span>
+    <span class="k">return</span> <span class="n">messages</span><span class="o">.</span><span class="n">UploadPartResponse</span><span class="p">(</span><span class="n">etag</span><span class="p">,</span> <span class="n">part_number</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="FakeS3Client.complete_multipart_upload"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.fake_client.html#apache_beam.io.aws.clients.s3.fake_client.FakeS3Client.complete_multipart_upload">[docs]</a>  <span class="k">def</span> <span class="nf">complete_multipart_upload</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">request</span><span class="p">):</span>
+    <span class="n">MIN_PART_SIZE</span> <span class="o">=</span> <span class="mi">5</span> <span class="o">*</span> <span class="mi">2</span><span class="o">**</span><span class="mi">10</span> <span class="c1"># 5 KiB</span>
+
+    <span class="n">parts_received</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">multipart_uploads</span><span class="p">[</span><span class="n">request</span><span class="o">.</span><span class="n">upload_id</span><span class="p">]</span>
+
+    <span class="c1"># Check that we got all the parts that they intended to send</span>
+    <span class="n">part_numbers_to_confirm</span> <span class="o">=</span> <span class="nb">set</span><span class="p">(</span><span class="n">part</span><span class="p">[</span><span class="s1">&#39;PartNumber&#39;</span><span class="p">]</span> <span class="k">for</span> <span class="n">part</span> <span class="ow">in</span> <span class="n">request</span><span class="o">.</span><span class="n">parts</span><span class="p">)</span>
+
+    <span class="c1"># Make sure all the expected parts are present</span>
+    <span class="k">if</span> <span class="n">part_numbers_to_confirm</span> <span class="o">!=</span> <span class="nb">set</span><span class="p">(</span><span class="n">parts_received</span><span class="o">.</span><span class="n">keys</span><span class="p">()):</span>
+      <span class="k">raise</span> <span class="n">messages</span><span class="o">.</span><span class="n">S3ClientError</span><span class="p">(</span>
+          <span class="s1">&#39;One or more of the specified parts could not be found&#39;</span><span class="p">,</span> <span class="mi">400</span><span class="p">)</span>
+
+    <span class="c1"># Sort by part number</span>
+    <span class="n">sorted_parts</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">parts_received</span><span class="o">.</span><span class="n">items</span><span class="p">(),</span> <span class="n">key</span><span class="o">=</span><span class="k">lambda</span> <span class="n">pair</span><span class="p">:</span> <span class="n">pair</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>
+    <span class="n">sorted_bytes</span> <span class="o">=</span> <span class="p">[</span><span class="n">bytes_</span> <span class="k">for</span> <span class="p">(</span><span class="n">_</span><span class="p">,</span> <span class="n">bytes_</span><span class="p">)</span> <span class="ow">in</span> <span class="n">sorted_parts</span><span class="p">]</span>
+
+    <span class="c1"># Make sure that the parts aren&#39;t too small (except the last part)</span>
+    <span class="n">part_sizes</span> <span class="o">=</span> <span class="p">[</span><span class="nb">len</span><span class="p">(</span><span class="n">bytes_</span><span class="p">)</span> <span class="k">for</span> <span class="n">bytes_</span> <span class="ow">in</span> <span class="n">sorted_bytes</span><span class="p">]</span>
+    <span class="k">if</span> <span class="nb">any</span><span class="p">(</span><span class="n">size</span> <span class="o">&lt;</span> <span class="n">MIN_PART_SIZE</span> <span class="k">for</span> <span class="n">size</span> <span class="ow">in</span> <span class="n">part_sizes</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">]):</span>
+      <span class="n">e_message</span> <span class="o">=</span> <span class="s2">&quot;&quot;&quot;</span>
+<span class="s2">      All parts but the last must be larger than </span><span class="si">%d</span><span class="s2"> bytes</span>
+<span class="s2">      &quot;&quot;&quot;</span> <span class="o">%</span> <span class="n">MIN_PART_SIZE</span>
+      <span class="k">raise</span> <span class="n">messages</span><span class="o">.</span><span class="n">S3ClientError</span><span class="p">(</span><span class="n">e_message</span><span class="p">,</span> <span class="mi">400</span><span class="p">)</span>
+
+    <span class="c1"># String together all bytes for the given upload</span>
+    <span class="n">final_contents</span> <span class="o">=</span> <span class="sa">b</span><span class="s1">&#39;&#39;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">sorted_bytes</span><span class="p">)</span>
+
+    <span class="c1"># Create FakeFile object</span>
+    <span class="n">num_parts</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">parts_received</span><span class="p">)</span>
+    <span class="n">etag</span> <span class="o">=</span> <span class="s1">&#39;&quot;</span><span class="si">%s</span><span class="s1">-</span><span class="si">%d</span><span class="s1">&quot;&#39;</span> <span class="o">%</span> <span class="p">(</span><span class="s1">&#39;x&#39;</span> <span class="o">*</span> <span class="mi">32</span><span class="p">,</span> <span class="n">num_parts</span><span class="p">)</span>
+    <span class="n">file_</span> <span class="o">=</span> <span class="n">FakeFile</span><span class="p">(</span><span class="n">request</span><span class="o">.</span><span class="n">bucket</span><span class="p">,</span> <span class="n">request</span><span class="o">.</span><span class="n">object</span><span class="p">,</span> <span class="n">final_contents</span><span class="p">,</span> <span class="n">etag</span><span class="o">=</span><span class="n">etag</span><span class="p">)</span>
+
+    <span class="c1"># Store FakeFile in self.files</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">add_file</span><span class="p">(</span><span class="n">file_</span><span class="p">)</span></div></div>
+</pre></div>
+
+           </div>
+           <div class="articleComments">
+            
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+        &copy; Copyright .
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../../../../../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="../../../../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../../../../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../../../../../../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file
diff --git a/pydoc/2.19.0/_modules/apache_beam/io/aws/clients/s3/messages.html b/pydoc/2.19.0/_modules/apache_beam/io/aws/clients/s3/messages.html
new file mode 100644
index 0000000..7e04f65
--- /dev/null
+++ b/pydoc/2.19.0/_modules/apache_beam/io/aws/clients/s3/messages.html
@@ -0,0 +1,404 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>apache_beam.io.aws.clients.s3.messages &mdash; Apache Beam  documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../../../../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="../../../../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../../../../search.html"/>
+    <link rel="top" title="Apache Beam  documentation" href="../../../../../../index.html"/>
+        <link rel="up" title="Module code" href="../../../../../index.html"/> 
+
+  
+  <script src="../../../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../../../index.html" class="icon icon-home"> Apache Beam
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.coders.html">apache_beam.coders package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.internal.html">apache_beam.internal package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.io.html">apache_beam.io package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.metrics.html">apache_beam.metrics package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.options.html">apache_beam.options package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.portability.html">apache_beam.portability package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.runners.html">apache_beam.runners package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.testing.html">apache_beam.testing package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.tools.html">apache_beam.tools package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.transforms.html">apache_beam.transforms package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.typehints.html">apache_beam.typehints package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.utils.html">apache_beam.utils package</a></li>
+</ul>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.error.html">apache_beam.error module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.pipeline.html">apache_beam.pipeline module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.pvalue.html">apache_beam.pvalue module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../../../apache_beam.version.html">apache_beam.version module</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../../../index.html">Apache Beam</a>
+        
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>apache_beam.io.aws.clients.s3.messages</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+            
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for apache_beam.io.aws.clients.s3.messages</h1><div class="highlight"><pre>
+<span></span><span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one or more</span>
+<span class="c1"># contributor license agreements.  See the NOTICE file distributed with</span>
+<span class="c1"># this work for additional information regarding copyright ownership.</span>
+<span class="c1"># The ASF licenses this file to You under the Apache License, Version 2.0</span>
+<span class="c1"># (the &quot;License&quot;); you may not use this file except in compliance with</span>
+<span class="c1"># the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#    http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+<span class="c1">#</span>
+
+<span class="c1"># pytype: skip-file</span>
+
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">absolute_import</span>
+
+
+<div class="viewcode-block" id="GetRequest"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.messages.html#apache_beam.io.aws.clients.s3.messages.GetRequest">[docs]</a><span class="k">class</span> <span class="nc">GetRequest</span><span class="p">():</span>
+  <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">  S3 request object for `Get` command</span>
+<span class="sd">  &quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="nb">object</span><span class="p">):</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span> <span class="o">=</span> <span class="n">bucket</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">object</span> <span class="o">=</span> <span class="nb">object</span></div>
+
+
+<div class="viewcode-block" id="UploadResponse"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.messages.html#apache_beam.io.aws.clients.s3.messages.UploadResponse">[docs]</a><span class="k">class</span> <span class="nc">UploadResponse</span><span class="p">():</span>
+  <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">  S3 response object for `StartUpload` command</span>
+<span class="sd">  &quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">upload_id</span><span class="p">):</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">upload_id</span> <span class="o">=</span> <span class="n">upload_id</span></div>
+
+
+<div class="viewcode-block" id="UploadRequest"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.messages.html#apache_beam.io.aws.clients.s3.messages.UploadRequest">[docs]</a><span class="k">class</span> <span class="nc">UploadRequest</span><span class="p">():</span>
+  <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">  S3 request object for `StartUpload` command</span>
+<span class="sd">  &quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="nb">object</span><span class="p">,</span> <span class="n">mime_type</span><span class="p">):</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span> <span class="o">=</span> <span class="n">bucket</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">object</span> <span class="o">=</span> <span class="nb">object</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">mime_type</span> <span class="o">=</span> <span class="n">mime_type</span></div>
+
+
+<div class="viewcode-block" id="UploadPartRequest"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.messages.html#apache_beam.io.aws.clients.s3.messages.UploadPartRequest">[docs]</a><span class="k">class</span> <span class="nc">UploadPartRequest</span><span class="p">():</span>
+  <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">  S3 request object for `UploadPart` command</span>
+<span class="sd">  &quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="nb">object</span><span class="p">,</span> <span class="n">upload_id</span><span class="p">,</span> <span class="n">part_number</span><span class="p">,</span> <span class="nb">bytes</span><span class="p">):</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span> <span class="o">=</span> <span class="n">bucket</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">object</span> <span class="o">=</span> <span class="nb">object</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">upload_id</span> <span class="o">=</span> <span class="n">upload_id</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">part_number</span> <span class="o">=</span> <span class="n">part_number</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">bytes</span> <span class="o">=</span> <span class="nb">bytes</span></div>
+    <span class="c1"># self.mime_type = mime_type</span>
+
+
+<div class="viewcode-block" id="UploadPartResponse"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.messages.html#apache_beam.io.aws.clients.s3.messages.UploadPartResponse">[docs]</a><span class="k">class</span> <span class="nc">UploadPartResponse</span><span class="p">():</span>
+  <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">  S3 response object for `UploadPart` command</span>
+<span class="sd">  &quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">etag</span><span class="p">,</span> <span class="n">part_number</span><span class="p">):</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">etag</span> <span class="o">=</span> <span class="n">etag</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">part_number</span> <span class="o">=</span> <span class="n">part_number</span></div>
+
+
+<div class="viewcode-block" id="CompleteMultipartUploadRequest"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.messages.html#apache_beam.io.aws.clients.s3.messages.CompleteMultipartUploadRequest">[docs]</a><span class="k">class</span> <span class="nc">CompleteMultipartUploadRequest</span><span class="p">():</span>
+  <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">  S3 request object for `UploadPart` command</span>
+<span class="sd">  &quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="nb">object</span><span class="p">,</span> <span class="n">upload_id</span><span class="p">,</span> <span class="n">parts</span><span class="p">):</span>
+    <span class="c1"># parts is a list of objects of the form</span>
+    <span class="c1"># {&#39;ETag&#39;: response.etag, &#39;PartNumber&#39;: response.part_number}</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span> <span class="o">=</span> <span class="n">bucket</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">object</span> <span class="o">=</span> <span class="nb">object</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">upload_id</span> <span class="o">=</span> <span class="n">upload_id</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">parts</span> <span class="o">=</span> <span class="n">parts</span></div>
+    <span class="c1"># self.mime_type = mime_type</span>
+
+
+<div class="viewcode-block" id="ListRequest"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.messages.html#apache_beam.io.aws.clients.s3.messages.ListRequest">[docs]</a><span class="k">class</span> <span class="nc">ListRequest</span><span class="p">():</span>
+  <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">  S3 request object for `List` command</span>
+<span class="sd">  &quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="n">prefix</span><span class="p">,</span> <span class="n">continuation_token</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span> <span class="o">=</span> <span class="n">bucket</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">prefix</span> <span class="o">=</span> <span class="n">prefix</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">continuation_token</span> <span class="o">=</span> <span class="n">continuation_token</span></div>
+
+
+<div class="viewcode-block" id="ListResponse"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.messages.html#apache_beam.io.aws.clients.s3.messages.ListResponse">[docs]</a><span class="k">class</span> <span class="nc">ListResponse</span><span class="p">():</span>
+  <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">  S3 response object for `List` command</span>
+<span class="sd">  &quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">items</span><span class="p">,</span> <span class="n">next_token</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">items</span> <span class="o">=</span> <span class="n">items</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">next_token</span> <span class="o">=</span> <span class="n">next_token</span></div>
+
+
+<div class="viewcode-block" id="Item"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.messages.html#apache_beam.io.aws.clients.s3.messages.Item">[docs]</a><span class="k">class</span> <span class="nc">Item</span><span class="p">():</span>
+  <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">  An item in S3</span>
+<span class="sd">  &quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">etag</span><span class="p">,</span> <span class="n">key</span><span class="p">,</span> <span class="n">last_modified</span><span class="p">,</span> <span class="n">size</span><span class="p">,</span> <span class="n">mime_type</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">etag</span> <span class="o">=</span> <span class="n">etag</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">key</span> <span class="o">=</span> <span class="n">key</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">last_modified</span> <span class="o">=</span> <span class="n">last_modified</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">size</span> <span class="o">=</span> <span class="n">size</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">mime_type</span> <span class="o">=</span> <span class="n">mime_type</span></div>
+
+
+<div class="viewcode-block" id="DeleteRequest"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.messages.html#apache_beam.io.aws.clients.s3.messages.DeleteRequest">[docs]</a><span class="k">class</span> <span class="nc">DeleteRequest</span><span class="p">():</span>
+  <span class="sd">&quot;&quot;&quot;</span>
+<span class="sd">  S3 request object for `Delete` command</span>
+<span class="sd">  &quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="nb">object</span><span class="p">):</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span> <span class="o">=</span> <span class="n">bucket</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">object</span> <span class="o">=</span> <span class="nb">object</span></div>
+
+
+<div class="viewcode-block" id="DeleteBatchRequest"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.messages.html#apache_beam.io.aws.clients.s3.messages.DeleteBatchRequest">[docs]</a><span class="k">class</span> <span class="nc">DeleteBatchRequest</span><span class="p">():</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">bucket</span><span class="p">,</span> <span class="n">objects</span><span class="p">):</span>
+    <span class="c1"># `objects` is a list of strings corresponding to the keys to be deleted</span>
+    <span class="c1"># in the bucket</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">bucket</span> <span class="o">=</span> <span class="n">bucket</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">objects</span> <span class="o">=</span> <span class="n">objects</span></div>
+
+
+<div class="viewcode-block" id="DeleteBatchResponse"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.messages.html#apache_beam.io.aws.clients.s3.messages.DeleteBatchResponse">[docs]</a><span class="k">class</span> <span class="nc">DeleteBatchResponse</span><span class="p">():</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">deleted</span><span class="p">,</span> <span class="n">failed</span><span class="p">,</span> <span class="n">errors</span><span class="p">):</span>
+    <span class="c1"># `deleted` is a list of strings corresponding to the keys that were deleted</span>
+    <span class="c1"># `failed` is a list of strings corresponding to the keys that caused errors</span>
+    <span class="c1"># `errors` is a list of S3ClientErrors, aligned with the order of `failed`</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">deleted</span> <span class="o">=</span> <span class="n">deleted</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">failed</span> <span class="o">=</span> <span class="n">failed</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">errors</span> <span class="o">=</span> <span class="n">errors</span></div>
+
+
+<div class="viewcode-block" id="CopyRequest"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.messages.html#apache_beam.io.aws.clients.s3.messages.CopyRequest">[docs]</a><span class="k">class</span> <span class="nc">CopyRequest</span><span class="p">():</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">src_bucket</span><span class="p">,</span> <span class="n">src_key</span><span class="p">,</span> <span class="n">dest_bucket</span><span class="p">,</span> <span class="n">dest_key</span><span class="p">):</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">src_bucket</span> <span class="o">=</span> <span class="n">src_bucket</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">src_key</span> <span class="o">=</span> <span class="n">src_key</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">dest_bucket</span> <span class="o">=</span> <span class="n">dest_bucket</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">dest_key</span> <span class="o">=</span> <span class="n">dest_key</span></div>
+
+
+<div class="viewcode-block" id="S3ClientError"><a class="viewcode-back" href="../../../../../../apache_beam.io.aws.clients.s3.messages.html#apache_beam.io.aws.clients.s3.messages.S3ClientError">[docs]</a><span class="k">class</span> <span class="nc">S3ClientError</span><span class="p">(</span><span class="ne">Exception</span><span class="p">):</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">message</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span> <span class="n">code</span> <span class="o">=</span> <span class="kc">None</span><span class="p">):</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">message</span> <span class="o">=</span> <span class="n">message</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">code</span> <span class="o">=</span> <span class="n">code</span></div>
+</pre></div>
+
+           </div>
+           <div class="articleComments">
+            
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+        &copy; Copyright .
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../../../../../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="../../../../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../../../../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../../../../../../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file
diff --git a/pydoc/2.19.0/_modules/apache_beam/io/aws/s3filesystem.html b/pydoc/2.19.0/_modules/apache_beam/io/aws/s3filesystem.html
new file mode 100644
index 0000000..a0f145f
--- /dev/null
+++ b/pydoc/2.19.0/_modules/apache_beam/io/aws/s3filesystem.html
@@ -0,0 +1,512 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>apache_beam.io.aws.s3filesystem &mdash; Apache Beam  documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="../../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../../search.html"/>
+    <link rel="top" title="Apache Beam  documentation" href="../../../../index.html"/>
+        <link rel="up" title="Module code" href="../../../index.html"/> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Apache Beam
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.coders.html">apache_beam.coders package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.internal.html">apache_beam.internal package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.io.html">apache_beam.io package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.metrics.html">apache_beam.metrics package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.options.html">apache_beam.options package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.portability.html">apache_beam.portability package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.runners.html">apache_beam.runners package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.testing.html">apache_beam.testing package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.tools.html">apache_beam.tools package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.transforms.html">apache_beam.transforms package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.typehints.html">apache_beam.typehints package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.utils.html">apache_beam.utils package</a></li>
+</ul>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.error.html">apache_beam.error module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.pipeline.html">apache_beam.pipeline module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.pvalue.html">apache_beam.pvalue module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.version.html">apache_beam.version module</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Apache Beam</a>
+        
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>apache_beam.io.aws.s3filesystem</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+            
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for apache_beam.io.aws.s3filesystem</h1><div class="highlight"><pre>
+<span></span><span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one or more</span>
+<span class="c1"># contributor license agreements.  See the NOTICE file distributed with</span>
+<span class="c1"># this work for additional information regarding copyright ownership.</span>
+<span class="c1"># The ASF licenses this file to You under the Apache License, Version 2.0</span>
+<span class="c1"># (the &quot;License&quot;); you may not use this file except in compliance with</span>
+<span class="c1"># the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#    http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+<span class="c1">#</span>
+<span class="sd">&quot;&quot;&quot;S3 file system implementation for accessing files on AWS S3.&quot;&quot;&quot;</span>
+
+<span class="c1"># pytype: skip-file</span>
+
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">absolute_import</span>
+
+<span class="kn">from</span> <span class="nn">future.utils</span> <span class="kn">import</span> <span class="n">iteritems</span>
+
+<span class="kn">from</span> <span class="nn">apache_beam.io.aws</span> <span class="kn">import</span> <span class="n">s3io</span>
+<span class="kn">from</span> <span class="nn">apache_beam.io.filesystem</span> <span class="kn">import</span> <span class="n">BeamIOError</span>
+<span class="kn">from</span> <span class="nn">apache_beam.io.filesystem</span> <span class="kn">import</span> <span class="n">CompressedFile</span>
+<span class="kn">from</span> <span class="nn">apache_beam.io.filesystem</span> <span class="kn">import</span> <span class="n">CompressionTypes</span>
+<span class="kn">from</span> <span class="nn">apache_beam.io.filesystem</span> <span class="kn">import</span> <span class="n">FileMetadata</span>
+<span class="kn">from</span> <span class="nn">apache_beam.io.filesystem</span> <span class="kn">import</span> <span class="n">FileSystem</span>
+
+<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;S3FileSystem&#39;</span><span class="p">]</span>
+
+
+<div class="viewcode-block" id="S3FileSystem"><a class="viewcode-back" href="../../../../apache_beam.io.aws.s3filesystem.html#apache_beam.io.aws.s3filesystem.S3FileSystem">[docs]</a><span class="k">class</span> <span class="nc">S3FileSystem</span><span class="p">(</span><span class="n">FileSystem</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;An S3 `FileSystem` implementation for accessing files on AWS S3</span>
+<span class="sd">  &quot;&quot;&quot;</span>
+
+  <span class="n">CHUNK_SIZE</span> <span class="o">=</span> <span class="n">s3io</span><span class="o">.</span><span class="n">MAX_BATCH_OPERATION_SIZE</span>
+  <span class="n">S3_PREFIX</span> <span class="o">=</span> <span class="s1">&#39;s3://&#39;</span>
+
+<div class="viewcode-block" id="S3FileSystem.scheme"><a class="viewcode-back" href="../../../../apache_beam.io.aws.s3filesystem.html#apache_beam.io.aws.s3filesystem.S3FileSystem.scheme">[docs]</a>  <span class="nd">@classmethod</span>
+  <span class="k">def</span> <span class="nf">scheme</span><span class="p">(</span><span class="bp">cls</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;URI scheme for the FileSystem</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">return</span> <span class="s1">&#39;s3&#39;</span></div>
+
+<div class="viewcode-block" id="S3FileSystem.join"><a class="viewcode-back" href="../../../../apache_beam.io.aws.s3filesystem.html#apache_beam.io.aws.s3filesystem.S3FileSystem.join">[docs]</a>  <span class="k">def</span> <span class="nf">join</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">basepath</span><span class="p">,</span> <span class="o">*</span><span class="n">paths</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;Join two or more pathname components for the filesystem</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">      basepath: string path of the first component of the path</span>
+<span class="sd">      paths: path components to be added</span>
+
+<span class="sd">    Returns: full path after combining all of the return nulled components</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">if</span> <span class="ow">not</span> <span class="n">basepath</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">S3FileSystem</span><span class="o">.</span><span class="n">S3_PREFIX</span><span class="p">):</span>
+      <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;Basepath </span><span class="si">%r</span><span class="s1"> must be S3 path.&#39;</span> <span class="o">%</span> <span class="n">basepath</span><span class="p">)</span>
+
+    <span class="n">path</span> <span class="o">=</span> <span class="n">basepath</span>
+    <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">paths</span><span class="p">:</span>
+      <span class="n">path</span> <span class="o">=</span> <span class="n">path</span><span class="o">.</span><span class="n">rstrip</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">)</span> <span class="o">+</span> <span class="s1">&#39;/&#39;</span> <span class="o">+</span> <span class="n">p</span><span class="o">.</span><span class="n">lstrip</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">)</span>
+    <span class="k">return</span> <span class="n">path</span></div>
+
+<div class="viewcode-block" id="S3FileSystem.split"><a class="viewcode-back" href="../../../../apache_beam.io.aws.s3filesystem.html#apache_beam.io.aws.s3filesystem.S3FileSystem.split">[docs]</a>  <span class="k">def</span> <span class="nf">split</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;Splits the given path into two parts.</span>
+
+<span class="sd">    Splits the path into a pair (head, tail) such that tail contains the last</span>
+<span class="sd">    component of the path and head contains everything up to that.</span>
+
+<span class="sd">    Head will include the S3 prefix (&#39;s3://&#39;).</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">      path: path as a string</span>
+<span class="sd">    Returns:</span>
+<span class="sd">      a pair of path components as strings.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">path</span> <span class="o">=</span> <span class="n">path</span><span class="o">.</span><span class="n">strip</span><span class="p">()</span>
+    <span class="k">if</span> <span class="ow">not</span> <span class="n">path</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="n">S3FileSystem</span><span class="o">.</span><span class="n">S3_PREFIX</span><span class="p">):</span>
+      <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;Path </span><span class="si">%r</span><span class="s1"> must be S3 path.&#39;</span> <span class="o">%</span> <span class="n">path</span><span class="p">)</span>
+
+    <span class="n">prefix_len</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">S3FileSystem</span><span class="o">.</span><span class="n">S3_PREFIX</span><span class="p">)</span>
+    <span class="n">last_sep</span> <span class="o">=</span> <span class="n">path</span><span class="p">[</span><span class="n">prefix_len</span><span class="p">:]</span><span class="o">.</span><span class="n">rfind</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">)</span>
+    <span class="k">if</span> <span class="n">last_sep</span> <span class="o">&gt;=</span> <span class="mi">0</span><span class="p">:</span>
+      <span class="n">last_sep</span> <span class="o">+=</span> <span class="n">prefix_len</span>
+
+    <span class="k">if</span> <span class="n">last_sep</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
+      <span class="k">return</span> <span class="p">(</span><span class="n">path</span><span class="p">[:</span><span class="n">last_sep</span><span class="p">],</span> <span class="n">path</span><span class="p">[</span><span class="n">last_sep</span> <span class="o">+</span> <span class="mi">1</span><span class="p">:])</span>
+    <span class="k">elif</span> <span class="n">last_sep</span> <span class="o">&lt;</span> <span class="mi">0</span><span class="p">:</span>
+      <span class="k">return</span> <span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="s1">&#39;&#39;</span><span class="p">)</span>
+    <span class="k">else</span><span class="p">:</span>
+      <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;Invalid path: </span><span class="si">%s</span><span class="s1">&#39;</span> <span class="o">%</span> <span class="n">path</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="S3FileSystem.mkdirs"><a class="viewcode-back" href="../../../../apache_beam.io.aws.s3filesystem.html#apache_beam.io.aws.s3filesystem.S3FileSystem.mkdirs">[docs]</a>  <span class="k">def</span> <span class="nf">mkdirs</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;Recursively create directories for the provided path.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">      path: string path of the directory structure that should be created</span>
+
+<span class="sd">    Raises:</span>
+<span class="sd">      IOError if leaf directory already exists.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">pass</span></div>
+
+<div class="viewcode-block" id="S3FileSystem.has_dirs"><a class="viewcode-back" href="../../../../apache_beam.io.aws.s3filesystem.html#apache_beam.io.aws.s3filesystem.S3FileSystem.has_dirs">[docs]</a>  <span class="k">def</span> <span class="nf">has_dirs</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;Whether this FileSystem supports directories.&quot;&quot;&quot;</span>
+    <span class="k">return</span> <span class="kc">False</span></div>
+
+  <span class="k">def</span> <span class="nf">_list</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">dir_or_prefix</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;List files in a location.</span>
+
+<span class="sd">    Listing is non-recursive, for filesystems that support directories.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">      dir_or_prefix: (string) A directory or location prefix (for filesystems</span>
+<span class="sd">        that don&#39;t have directories).</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">      Generator of ``FileMetadata`` objects.</span>
+
+<span class="sd">    Raises:</span>
+<span class="sd">      ``BeamIOError`` if listing fails, but not if no files were found.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">try</span><span class="p">:</span>
+      <span class="k">for</span> <span class="n">path</span><span class="p">,</span> <span class="n">size</span> <span class="ow">in</span> <span class="n">iteritems</span><span class="p">(</span><span class="n">s3io</span><span class="o">.</span><span class="n">S3IO</span><span class="p">()</span><span class="o">.</span><span class="n">list_prefix</span><span class="p">(</span><span class="n">dir_or_prefix</span><span class="p">)):</span>
+        <span class="k">yield</span> <span class="n">FileMetadata</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">size</span><span class="p">)</span>
+    <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>  <span class="c1"># pylint: disable=broad-except</span>
+      <span class="k">raise</span> <span class="n">BeamIOError</span><span class="p">(</span><span class="s2">&quot;List operation failed&quot;</span><span class="p">,</span> <span class="p">{</span><span class="n">dir_or_prefix</span><span class="p">:</span> <span class="n">e</span><span class="p">})</span>
+
+  <span class="k">def</span> <span class="nf">_path_open</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">mode</span><span class="p">,</span> <span class="n">mime_type</span><span class="o">=</span><span class="s1">&#39;application/octet-stream&#39;</span><span class="p">,</span>
+                 <span class="n">compression_type</span><span class="o">=</span><span class="n">CompressionTypes</span><span class="o">.</span><span class="n">AUTO</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;Helper functions to open a file in the provided mode.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">compression_type</span> <span class="o">=</span> <span class="n">FileSystem</span><span class="o">.</span><span class="n">_get_compression_type</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">compression_type</span><span class="p">)</span>
+    <span class="n">mime_type</span> <span class="o">=</span> <span class="n">CompressionTypes</span><span class="o">.</span><span class="n">mime_type</span><span class="p">(</span><span class="n">compression_type</span><span class="p">,</span> <span class="n">mime_type</span><span class="p">)</span>
+    <span class="n">raw_file</span> <span class="o">=</span> <span class="n">s3io</span><span class="o">.</span><span class="n">S3IO</span><span class="p">()</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">mode</span><span class="p">,</span> <span class="n">mime_type</span><span class="o">=</span><span class="n">mime_type</span><span class="p">)</span>
+    <span class="k">if</span> <span class="n">compression_type</span> <span class="o">==</span> <span class="n">CompressionTypes</span><span class="o">.</span><span class="n">UNCOMPRESSED</span><span class="p">:</span>
+      <span class="k">return</span> <span class="n">raw_file</span>
+    <span class="k">return</span> <span class="n">CompressedFile</span><span class="p">(</span><span class="n">raw_file</span><span class="p">,</span> <span class="n">compression_type</span><span class="o">=</span><span class="n">compression_type</span><span class="p">)</span>
+
+<div class="viewcode-block" id="S3FileSystem.create"><a class="viewcode-back" href="../../../../apache_beam.io.aws.s3filesystem.html#apache_beam.io.aws.s3filesystem.S3FileSystem.create">[docs]</a>  <span class="k">def</span> <span class="nf">create</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">mime_type</span><span class="o">=</span><span class="s1">&#39;application/octet-stream&#39;< [...]
+             <span class="n">compression_type</span><span class="o">=</span><span class="n">CompressionTypes</span><span class="o">.</span><span class="n">AUTO</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;Returns a write channel for the given file path.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">      path: string path of the file object to be written to the system</span>
+<span class="sd">      mime_type: MIME type to specify the type of content in the file object</span>
+<span class="sd">      compression_type: Type of compression to be used for this object</span>
+
+<span class="sd">    Returns: file handle with a close function for the user to use</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_path_open</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="s1">&#39;wb&#39;</span><span class="p">,</span> <span class="n">mime_type</span><span class="p">,</span> <span class="n">compression_type</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="S3FileSystem.open"><a class="viewcode-back" href="../../../../apache_beam.io.aws.s3filesystem.html#apache_beam.io.aws.s3filesystem.S3FileSystem.open">[docs]</a>  <span class="k">def</span> <span class="nf">open</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">,</span> <span class="n">mime_type</span><span class="o">=</span><span class="s1">&#39;application/octet-stream&#39;</span> [...]
+           <span class="n">compression_type</span><span class="o">=</span><span class="n">CompressionTypes</span><span class="o">.</span><span class="n">AUTO</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;Returns a read channel for the given file path.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">      path: string path of the file object to be written to the system</span>
+<span class="sd">      mime_type: MIME type to specify the type of content in the file object</span>
+<span class="sd">      compression_type: Type of compression to be used for this object</span>
+
+<span class="sd">    Returns: file handle with a close function for the user to use</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_path_open</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="s1">&#39;rb&#39;</span><span class="p">,</span> <span class="n">mime_type</span><span class="p">,</span> <span class="n">compression_type</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="S3FileSystem.copy"><a class="viewcode-back" href="../../../../apache_beam.io.aws.s3filesystem.html#apache_beam.io.aws.s3filesystem.S3FileSystem.copy">[docs]</a>  <span class="k">def</span> <span class="nf">copy</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">source_file_names</span><span class="p">,</span> <span class="n">destination_file_names</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;Recursively copy the file tree from the source to the destination</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">      source_file_names: list of source file objects that needs to be copied</span>
+<span class="sd">      destination_file_names: list of destination of the new object</span>
+
+<span class="sd">    Raises:</span>
+<span class="sd">      ``BeamIOError`` if any of the copy operations fail</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">if</span> <span class="ow">not</span> <span class="nb">len</span><span class="p">(</span><span class="n">source_file_names</span><span class="p">)</span> <span class="o">==</span> <span class="nb">len</span><span class="p">(</span><span class="n">destination_file_names</span><span class="p">):</span>
+      <span class="n">message</span> <span class="o">=</span> <span class="s1">&#39;Unable to copy unequal number of sources and destinations&#39;</span>
+      <span class="k">raise</span> <span class="n">BeamIOError</span><span class="p">(</span><span class="n">message</span><span class="p">)</span>
+    <span class="n">src_dest_pairs</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">zip</span><span class="p">(</span><span class="n">source_file_names</span><span class="p">,</span> <span class="n">destination_file_names</span><span class="p">))</span>
+    <span class="k">return</span> <span class="n">s3io</span><span class="o">.</span><span class="n">S3IO</span><span class="p">()</span><span class="o">.</span><span class="n">copy_paths</span><span class="p">(</span><span class="n">src_dest_pairs</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="S3FileSystem.rename"><a class="viewcode-back" href="../../../../apache_beam.io.aws.s3filesystem.html#apache_beam.io.aws.s3filesystem.S3FileSystem.rename">[docs]</a>  <span class="k">def</span> <span class="nf">rename</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">source_file_names</span><span class="p">,</span> <span class="n">destination_file_names</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;Rename the files at the source list to the destination list.</span>
+<span class="sd">    Source and destination lists should be of the same size.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">      source_file_names: List of file paths that need to be moved</span>
+<span class="sd">      destination_file_names: List of destination_file_names for the files</span>
+
+<span class="sd">    Raises:</span>
+<span class="sd">      ``BeamIOError`` if any of the rename operations fail</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">if</span> <span class="ow">not</span> <span class="nb">len</span><span class="p">(</span><span class="n">source_file_names</span><span class="p">)</span> <span class="o">==</span> <span class="nb">len</span><span class="p">(</span><span class="n">destination_file_names</span><span class="p">):</span>
+      <span class="n">message</span> <span class="o">=</span> <span class="s1">&#39;Unable to rename unequal number of sources and destinations&#39;</span>
+      <span class="k">raise</span> <span class="n">BeamIOError</span><span class="p">(</span><span class="n">message</span><span class="p">)</span>
+    <span class="n">src_dest_pairs</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">zip</span><span class="p">(</span><span class="n">source_file_names</span><span class="p">,</span> <span class="n">destination_file_names</span><span class="p">))</span>
+    <span class="n">results</span> <span class="o">=</span> <span class="n">s3io</span><span class="o">.</span><span class="n">S3IO</span><span class="p">()</span><span class="o">.</span><span class="n">rename_files</span><span class="p">(</span><span class="n">src_dest_pairs</span><span class="p">)</span>
+    <span class="n">exceptions</span> <span class="o">=</span> <span class="p">{(</span><span class="n">src</span><span class="p">,</span> <span class="n">dest</span><span class="p">):</span> <span class="n">error</span> <span class="k">for</span> <span class="p">(</span><span class="n">src</span><span class="p">,</span> <span class="n">dest</span><span class="p">,</span> <span class="n">error</span><span class="p">)</span> <span class="ow">in</span> <span class="n">results</span>
+                  <span class="k">if</span> <span class="n">error</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">}</span>
+    <span class="k">if</span> <span class="n">exceptions</span><span class="p">:</span>
+      <span class="k">raise</span> <span class="n">BeamIOError</span><span class="p">(</span><span class="s2">&quot;Rename operation failed&quot;</span><span class="p">,</span> <span class="n">exceptions</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="S3FileSystem.exists"><a class="viewcode-back" href="../../../../apache_beam.io.aws.s3filesystem.html#apache_beam.io.aws.s3filesystem.S3FileSystem.exists">[docs]</a>  <span class="k">def</span> <span class="nf">exists</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;Check if the provided path exists on the FileSystem.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">      path: string path that needs to be checked.</span>
+
+<span class="sd">    Returns: boolean flag indicating if path exists</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">try</span><span class="p">:</span>
+      <span class="k">return</span> <span class="n">s3io</span><span class="o">.</span><span class="n">S3IO</span><span class="p">()</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
+    <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>  <span class="c1"># pylint: disable=broad-except</span>
+      <span class="k">raise</span> <span class="n">BeamIOError</span><span class="p">(</span><span class="s2">&quot;exists() operation failed&quot;</span><span class="p">,</span> <span class="p">{</span><span class="n">path</span><span class="p">:</span> <span class="n">e</span><span class="p">})</span></div>
+
+<div class="viewcode-block" id="S3FileSystem.size"><a class="viewcode-back" href="../../../../apache_beam.io.aws.s3filesystem.html#apache_beam.io.aws.s3filesystem.S3FileSystem.size">[docs]</a>  <span class="k">def</span> <span class="nf">size</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;Get size of path on the FileSystem.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">      path: string path in question.</span>
+
+<span class="sd">    Returns: int size of path according to the FileSystem.</span>
+
+<span class="sd">    Raises:</span>
+<span class="sd">      ``BeamIOError`` if path doesn&#39;t exist.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">try</span><span class="p">:</span>
+      <span class="k">return</span> <span class="n">s3io</span><span class="o">.</span><span class="n">S3IO</span><span class="p">()</span><span class="o">.</span><span class="n">size</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
+    <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>  <span class="c1"># pylint: disable=broad-except</span>
+      <span class="k">raise</span> <span class="n">BeamIOError</span><span class="p">(</span><span class="s2">&quot;size() operation failed&quot;</span><span class="p">,</span> <span class="p">{</span><span class="n">path</span><span class="p">:</span> <span class="n">e</span><span class="p">})</span></div>
+
+<div class="viewcode-block" id="S3FileSystem.last_updated"><a class="viewcode-back" href="../../../../apache_beam.io.aws.s3filesystem.html#apache_beam.io.aws.s3filesystem.S3FileSystem.last_updated">[docs]</a>  <span class="k">def</span> <span class="nf">last_updated</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;Get UNIX Epoch time in seconds on the FileSystem.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">      path: string path of file.</span>
+
+<span class="sd">    Returns: float UNIX Epoch time</span>
+
+<span class="sd">    Raises:</span>
+<span class="sd">      ``BeamIOError`` if path doesn&#39;t exist.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">try</span><span class="p">:</span>
+      <span class="k">return</span> <span class="n">s3io</span><span class="o">.</span><span class="n">S3IO</span><span class="p">()</span><span class="o">.</span><span class="n">last_updated</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
+    <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>  <span class="c1"># pylint: disable=broad-except</span>
+      <span class="k">raise</span> <span class="n">BeamIOError</span><span class="p">(</span><span class="s2">&quot;last_updated operation failed&quot;</span><span class="p">,</span> <span class="p">{</span><span class="n">path</span><span class="p">:</span> <span class="n">e</span><span class="p">})</span></div>
+
+<div class="viewcode-block" id="S3FileSystem.checksum"><a class="viewcode-back" href="../../../../apache_beam.io.aws.s3filesystem.html#apache_beam.io.aws.s3filesystem.S3FileSystem.checksum">[docs]</a>  <span class="k">def</span> <span class="nf">checksum</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;Fetch checksum metadata of a file on the</span>
+<span class="sd">    :class:`~apache_beam.io.filesystem.FileSystem`.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">      path: string path of a file.</span>
+
+<span class="sd">    Returns: string containing checksum</span>
+
+<span class="sd">    Raises:</span>
+<span class="sd">      ``BeamIOError`` if path isn&#39;t a file or doesn&#39;t exist.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">try</span><span class="p">:</span>
+      <span class="k">return</span> <span class="n">s3io</span><span class="o">.</span><span class="n">S3IO</span><span class="p">()</span><span class="o">.</span><span class="n">checksum</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
+    <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>  <span class="c1"># pylint: disable=broad-except</span>
+      <span class="k">raise</span> <span class="n">BeamIOError</span><span class="p">(</span><span class="s2">&quot;Checksum operation failed&quot;</span><span class="p">,</span> <span class="p">{</span><span class="n">path</span><span class="p">:</span> <span class="n">e</span><span class="p">})</span></div>
+
+<div class="viewcode-block" id="S3FileSystem.delete"><a class="viewcode-back" href="../../../../apache_beam.io.aws.s3filesystem.html#apache_beam.io.aws.s3filesystem.S3FileSystem.delete">[docs]</a>  <span class="k">def</span> <span class="nf">delete</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">paths</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;Deletes files or directories at the provided paths.</span>
+<span class="sd">    Directories will be deleted recursively.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">      paths: list of paths that give the file objects to be deleted</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">results</span> <span class="o">=</span> <span class="n">s3io</span><span class="o">.</span><span class="n">S3IO</span><span class="p">()</span><span class="o">.</span><span class="n">delete_paths</span><span class="p">(</span><span class="n">paths</span><span class="p">)</span>
+    <span class="n">exceptions</span> <span class="o">=</span> <span class="p">{</span><span class="n">path</span><span class="p">:</span> <span class="n">error</span> <span class="k">for</span> <span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">error</span><span class="p">)</span> <span class="ow">in</span> <span class="n">results</span>
+                  <span class="k">if</span> <span class="n">error</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">}</span>
+    <span class="k">if</span> <span class="n">exceptions</span><span class="p">:</span>
+      <span class="k">raise</span> <span class="n">BeamIOError</span><span class="p">(</span><span class="s2">&quot;Delete operation failed&quot;</span><span class="p">,</span> <span class="n">exceptions</span><span class="p">)</span></div></div>
+</pre></div>
+
+           </div>
+           <div class="articleComments">
+            
+           </div>
+          </div>
+          <footer>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+        &copy; Copyright .
+
+    </p>
+  </div>
+  Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. 
+
+</footer>
+
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+  
+
+
+  
+
+    <script type="text/javascript">
+        var DOCUMENTATION_OPTIONS = {
+            URL_ROOT:'../../../../',
+            VERSION:'',
+            COLLAPSE_INDEX:false,
+            FILE_SUFFIX:'.html',
+            HAS_SOURCE:  true,
+            SOURCELINK_SUFFIX: '.txt'
+        };
+    </script>
+      <script type="text/javascript" src="../../../../_static/jquery.js"></script>
+      <script type="text/javascript" src="../../../../_static/underscore.js"></script>
+      <script type="text/javascript" src="../../../../_static/doctools.js"></script>
+
+  
+
+  
+  
+    <script type="text/javascript" src="../../../../_static/js/theme.js"></script>
+  
+
+  
+  
+  <script type="text/javascript">
+      jQuery(function () {
+          SphinxRtdTheme.StickyNav.enable();
+      });
+  </script>
+   
+
+</body>
+</html>
\ No newline at end of file
diff --git a/pydoc/2.19.0/_modules/apache_beam/io/aws/s3io.html b/pydoc/2.19.0/_modules/apache_beam/io/aws/s3io.html
new file mode 100644
index 0000000..4f2c6fa
--- /dev/null
+++ b/pydoc/2.19.0/_modules/apache_beam/io/aws/s3io.html
@@ -0,0 +1,843 @@
+
+
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  <title>apache_beam.io.aws.s3io &mdash; Apache Beam  documentation</title>
+  
+
+  
+  
+  
+  
+
+  
+
+  
+  
+    
+
+  
+
+  
+  
+    <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
+  
+
+  
+
+  
+        <link rel="index" title="Index"
+              href="../../../../genindex.html"/>
+        <link rel="search" title="Search" href="../../../../search.html"/>
+    <link rel="top" title="Apache Beam  documentation" href="../../../../index.html"/>
+        <link rel="up" title="Module code" href="../../../index.html"/> 
+
+  
+  <script src="../../../../_static/js/modernizr.min.js"></script>
+
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+   
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
+      <div class="wy-side-scroll">
+        <div class="wy-side-nav-search">
+          
+
+          
+            <a href="../../../../index.html" class="icon icon-home"> Apache Beam
+          
+
+          
+          </a>
+
+          
+            
+            
+          
+
+          
+<div role="search">
+  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" />
+    <input type="hidden" name="check_keywords" value="yes" />
+    <input type="hidden" name="area" value="default" />
+  </form>
+</div>
+
+          
+        </div>
+
+        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+          
+            
+            
+              
+            
+            
+              <ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.coders.html">apache_beam.coders package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.internal.html">apache_beam.internal package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.io.html">apache_beam.io package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.metrics.html">apache_beam.metrics package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.options.html">apache_beam.options package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.portability.html">apache_beam.portability package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.runners.html">apache_beam.runners package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.testing.html">apache_beam.testing package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.tools.html">apache_beam.tools package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.transforms.html">apache_beam.transforms package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.typehints.html">apache_beam.typehints package</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.utils.html">apache_beam.utils package</a></li>
+</ul>
+<ul>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.error.html">apache_beam.error module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.pipeline.html">apache_beam.pipeline module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.pvalue.html">apache_beam.pvalue module</a></li>
+<li class="toctree-l1"><a class="reference internal" href="../../../../apache_beam.version.html">apache_beam.version module</a></li>
+</ul>
+
+            
+          
+        </div>
+      </div>
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        
+          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+          <a href="../../../../index.html">Apache Beam</a>
+        
+      </nav>
+
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    
+      <li><a href="../../../../index.html">Docs</a> &raquo;</li>
+        
+          <li><a href="../../../index.html">Module code</a> &raquo;</li>
+        
+      <li>apache_beam.io.aws.s3io</li>
+    
+    
+      <li class="wy-breadcrumbs-aside">
+        
+            
+        
+      </li>
+    
+  </ul>
+
+  
+  <hr/>
+</div>
+          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
+           <div itemprop="articleBody">
+            
+  <h1>Source code for apache_beam.io.aws.s3io</h1><div class="highlight"><pre>
+<span></span><span class="c1">#</span>
+<span class="c1"># Licensed to the Apache Software Foundation (ASF) under one or more</span>
+<span class="c1"># contributor license agreements.  See the NOTICE file distributed with</span>
+<span class="c1"># this work for additional information regarding copyright ownership.</span>
+<span class="c1"># The ASF licenses this file to You under the Apache License, Version 2.0</span>
+<span class="c1"># (the &quot;License&quot;); you may not use this file except in compliance with</span>
+<span class="c1"># the License.  You may obtain a copy of the License at</span>
+<span class="c1">#</span>
+<span class="c1">#    http://www.apache.org/licenses/LICENSE-2.0</span>
+<span class="c1">#</span>
+<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
+<span class="c1"># distributed under the License is distributed on an &quot;AS IS&quot; BASIS,</span>
+<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
+<span class="c1"># See the License for the specific language governing permissions and</span>
+<span class="c1"># limitations under the License.</span>
+<span class="c1">#</span>
+<span class="sd">&quot;&quot;&quot;AWS S3 client</span>
+<span class="sd">&quot;&quot;&quot;</span>
+
+<span class="c1"># pytype: skip-file</span>
+
+<span class="kn">from</span> <span class="nn">__future__</span> <span class="kn">import</span> <span class="n">absolute_import</span>
+
+<span class="kn">import</span> <span class="nn">errno</span>
+<span class="kn">import</span> <span class="nn">io</span>
+<span class="kn">import</span> <span class="nn">logging</span>
+<span class="kn">import</span> <span class="nn">re</span>
+<span class="kn">import</span> <span class="nn">time</span>
+<span class="kn">import</span> <span class="nn">traceback</span>
+<span class="kn">from</span> <span class="nn">builtins</span> <span class="kn">import</span> <span class="nb">object</span>
+
+<span class="kn">from</span> <span class="nn">apache_beam.io.aws.clients.s3</span> <span class="kn">import</span> <span class="n">messages</span>
+<span class="kn">from</span> <span class="nn">apache_beam.io.filesystemio</span> <span class="kn">import</span> <span class="n">Downloader</span>
+<span class="kn">from</span> <span class="nn">apache_beam.io.filesystemio</span> <span class="kn">import</span> <span class="n">DownloaderStream</span>
+<span class="kn">from</span> <span class="nn">apache_beam.io.filesystemio</span> <span class="kn">import</span> <span class="n">Uploader</span>
+<span class="kn">from</span> <span class="nn">apache_beam.io.filesystemio</span> <span class="kn">import</span> <span class="n">UploaderStream</span>
+<span class="kn">from</span> <span class="nn">apache_beam.utils</span> <span class="kn">import</span> <span class="n">retry</span>
+
+<span class="k">try</span><span class="p">:</span>
+  <span class="c1"># pylint: disable=wrong-import-order, wrong-import-position</span>
+  <span class="c1"># pylint: disable=ungrouped-imports</span>
+  <span class="kn">from</span> <span class="nn">apache_beam.io.aws.clients.s3</span> <span class="kn">import</span> <span class="n">boto3_client</span>
+  <span class="n">BOTO3_INSTALLED</span> <span class="o">=</span> <span class="kc">True</span>
+<span class="k">except</span> <span class="ne">ImportError</span><span class="p">:</span>
+  <span class="n">BOTO3_INSTALLED</span> <span class="o">=</span> <span class="kc">False</span>
+
+<span class="n">MAX_BATCH_OPERATION_SIZE</span> <span class="o">=</span> <span class="mi">100</span>
+
+
+<div class="viewcode-block" id="parse_s3_path"><a class="viewcode-back" href="../../../../apache_beam.io.aws.s3io.html#apache_beam.io.aws.s3io.parse_s3_path">[docs]</a><span class="k">def</span> <span class="nf">parse_s3_path</span><span class="p">(</span><span class="n">s3_path</span><span class="p">,</span> <span class="n">object_optional</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;Return the bucket and object names of the given s3:// path.&quot;&quot;&quot;</span>
+  <span class="n">match</span> <span class="o">=</span> <span class="n">re</span><span class="o">.</span><span class="n">match</span><span class="p">(</span><span class="s1">&#39;^s3://([^/]+)/(.*)$&#39;</span><span class="p">,</span> <span class="n">s3_path</span><span class="p">)</span>
+  <span class="k">if</span> <span class="n">match</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">or</span> <span class="p">(</span><span class="n">match</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span> <span class="o">==</span> <span class="s1">&#39;&#39;</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">object_optional</span><span class="p">):</span>
+    <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;S3 path must be in the form s3://&lt;bucket&gt;/&lt;object&gt;.&#39;</span><span class="p">)</span>
+  <span class="k">return</span> <span class="n">match</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">1</span><span class="p">),</span> <span class="n">match</span><span class="o">.</span><span class="n">group</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span></div>
+
+
+<div class="viewcode-block" id="S3IO"><a class="viewcode-back" href="../../../../apache_beam.io.aws.s3io.html#apache_beam.io.aws.s3io.S3IO">[docs]</a><span class="k">class</span> <span class="nc">S3IO</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
+  <span class="sd">&quot;&quot;&quot;S3 I/O client.&quot;&quot;&quot;</span>
+
+  <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">client</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
+    <span class="k">if</span> <span class="n">client</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
+      <span class="bp">self</span><span class="o">.</span><span class="n">client</span> <span class="o">=</span> <span class="n">client</span>
+    <span class="k">elif</span> <span class="n">BOTO3_INSTALLED</span><span class="p">:</span>
+      <span class="bp">self</span><span class="o">.</span><span class="n">client</span> <span class="o">=</span> <span class="n">boto3_client</span><span class="o">.</span><span class="n">Client</span><span class="p">()</span>
+    <span class="k">else</span><span class="p">:</span>
+      <span class="n">message</span> <span class="o">=</span> <span class="s1">&#39;AWS dependencies are not installed, and no alternative &#39;</span> \
+      <span class="s1">&#39;client was provided to S3IO.&#39;</span>
+      <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="n">message</span><span class="p">)</span>
+
+<div class="viewcode-block" id="S3IO.open"><a class="viewcode-back" href="../../../../apache_beam.io.aws.s3io.html#apache_beam.io.aws.s3io.S3IO.open">[docs]</a>  <span class="k">def</span> <span class="nf">open</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
+           <span class="n">filename</span><span class="p">,</span>
+           <span class="n">mode</span><span class="o">=</span><span class="s1">&#39;r&#39;</span><span class="p">,</span>
+           <span class="n">read_buffer_size</span><span class="o">=</span><span class="mi">16</span><span class="o">*</span><span class="mi">1024</span><span class="o">*</span><span class="mi">1024</span><span class="p">,</span>
+           <span class="n">mime_type</span><span class="o">=</span><span class="s1">&#39;application/octet-stream&#39;</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;Open an S3 file path for reading or writing.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">      filename (str): S3 file path in the form ``s3://&lt;bucket&gt;/&lt;object&gt;``.</span>
+<span class="sd">      mode (str): ``&#39;r&#39;`` for reading or ``&#39;w&#39;`` for writing.</span>
+<span class="sd">      read_buffer_size (int): Buffer size to use during read operations.</span>
+<span class="sd">      mime_type (str): Mime type to set for write operations.</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">      S3 file object.</span>
+
+<span class="sd">    Raises:</span>
+<span class="sd">      ~exceptions.ValueError: Invalid open file mode.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">if</span> <span class="n">mode</span> <span class="o">==</span> <span class="s1">&#39;r&#39;</span> <span class="ow">or</span> <span class="n">mode</span> <span class="o">==</span> <span class="s1">&#39;rb&#39;</span><span class="p">:</span>
+      <span class="n">downloader</span> <span class="o">=</span> <span class="n">S3Downloader</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="p">,</span> <span class="n">filename</span><span class="p">,</span>
+                                <span class="n">buffer_size</span><span class="o">=</span><span class="n">read_buffer_size</span><span class="p">)</span>
+      <span class="k">return</span> <span class="n">io</span><span class="o">.</span><span class="n">BufferedReader</span><span class="p">(</span><span class="n">DownloaderStream</span><span class="p">(</span><span class="n">downloader</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="n">mode</span><span class="p">),</span>
+                               <span class="n">buffer_size</span><span class="o">=</span><span class="n">read_buffer_size</span><span class="p">)</span>
+    <span class="k">elif</span> <span class="n">mode</span> <span class="o">==</span> <span class="s1">&#39;w&#39;</span> <span class="ow">or</span> <span class="n">mode</span> <span class="o">==</span> <span class="s1">&#39;wb&#39;</span><span class="p">:</span>
+      <span class="n">uploader</span> <span class="o">=</span> <span class="n">S3Uploader</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="p">,</span> <span class="n">filename</span><span class="p">,</span> <span class="n">mime_type</span><span class="p">)</span>
+      <span class="k">return</span> <span class="n">io</span><span class="o">.</span><span class="n">BufferedWriter</span><span class="p">(</span><span class="n">UploaderStream</span><span class="p">(</span><span class="n">uploader</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="n">mode</span><span class="p">),</span>
+                               <span class="n">buffer_size</span><span class="o">=</span><span class="mi">128</span> <span class="o">*</span> <span class="mi">1024</span><span class="p">)</span>
+    <span class="k">else</span><span class="p">:</span>
+      <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;Invalid file open mode: </span><span class="si">%s</span><span class="s1">.&#39;</span> <span class="o">%</span> <span class="n">mode</span><span class="p">)</span></div>
+
+<div class="viewcode-block" id="S3IO.list_prefix"><a class="viewcode-back" href="../../../../apache_beam.io.aws.s3io.html#apache_beam.io.aws.s3io.S3IO.list_prefix">[docs]</a>  <span class="nd">@retry</span><span class="o">.</span><span class="n">with_exponential_backoff</span><span class="p">(</span>
+      <span class="n">retry_filter</span><span class="o">=</span><span class="n">retry</span><span class="o">.</span><span class="n">retry_on_server_errors_and_timeout_filter</span><span class="p">)</span>
+  <span class="k">def</span> <span class="nf">list_prefix</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;Lists files matching the prefix.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">      path: S3 file path pattern in the form s3://&lt;bucket&gt;/[name].</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">      Dictionary of file name -&gt; size.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">bucket</span><span class="p">,</span> <span class="n">prefix</span> <span class="o">=</span> <span class="n">parse_s3_path</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="n">object_optional</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
+    <span class="n">request</span> <span class="o">=</span> <span class="n">messages</span><span class="o">.</span><span class="n">ListRequest</span><span class="p">(</span><span class="n">bucket</span><span class="o">=</span><span class="n">bucket</span><span class="p">,</span> <span class="n">prefix</span><span class="o">=</span><span class="n">prefix</span><span class="p">)</span>
+
+    <span class="n">file_sizes</span> <span class="o">=</span> <span class="p">{}</span>
+    <span class="n">counter</span> <span class="o">=</span> <span class="mi">0</span>
+    <span class="n">start_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>
+
+    <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Starting the size estimation of the input&quot;</span><span class="p">)</span>
+
+    <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
+      <span class="n">response</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">list</span><span class="p">(</span><span class="n">request</span><span class="p">)</span>
+      <span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">response</span><span class="o">.</span><span class="n">items</span><span class="p">:</span>
+        <span class="n">file_name</span> <span class="o">=</span> <span class="s1">&#39;s3://</span><span class="si">%s</span><span class="s1">/</span><span class="si">%s</span><span class="s1">&#39;</span> <span class="o">%</span> <span class="p">(</span><span class="n">bucket</span><span class="p">,</span> <span class="n">item</span><span class="o">.</span><span class="n">key</span><span class="p">)</span>
+        <span class="n">file_sizes</span><span class="p">[</span><span class="n">file_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">item</span><span class="o">.</span><span class="n">size</span>
+        <span class="n">counter</span> <span class="o">+=</span> <span class="mi">1</span>
+        <span class="k">if</span> <span class="n">counter</span> <span class="o">%</span> <span class="mi">10000</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
+          <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Finished computing size of: </span><span class="si">%s</span><span class="s2"> files&quot;</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">file_sizes</span><span class="p">))</span>
+      <span class="k">if</span> <span class="n">response</span><span class="o">.</span><span class="n">next_token</span><span class="p">:</span>
+        <span class="n">request</span><span class="o">.</span><span class="n">continuation_token</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">next_token</span>
+      <span class="k">else</span><span class="p">:</span>
+        <span class="k">break</span>
+
+    <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s2">&quot;Finished listing </span><span class="si">%s</span><span class="s2"> files in </span><span class="si">%s</span><span class="s2"> seconds.&quot;</span><span class="p">,</span>
+                 <span class="n">counter</span><span class="p">,</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span> <span class="o">-</span> <span class="n">start_time</span><span class="p">)</span>
+
+    <span class="k">return</span> <span class="n">file_sizes</span></div>
+
+<div class="viewcode-block" id="S3IO.checksum"><a class="viewcode-back" href="../../../../apache_beam.io.aws.s3io.html#apache_beam.io.aws.s3io.S3IO.checksum">[docs]</a>  <span class="nd">@retry</span><span class="o">.</span><span class="n">with_exponential_backoff</span><span class="p">(</span>
+      <span class="n">retry_filter</span><span class="o">=</span><span class="n">retry</span><span class="o">.</span><span class="n">retry_on_server_errors_and_timeout_filter</span><span class="p">)</span>
+  <span class="k">def</span> <span class="nf">checksum</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;Looks up the checksum of an S3 object.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">      path: S3 file path pattern in the form s3://&lt;bucket&gt;/&lt;name&gt;.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">bucket</span><span class="p">,</span> <span class="n">object_path</span> <span class="o">=</span> <span class="n">parse_s3_path</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
+    <span class="n">request</span> <span class="o">=</span> <span class="n">messages</span><span class="o">.</span><span class="n">GetRequest</span><span class="p">(</span><span class="n">bucket</span><span class="p">,</span> <span class="n">object_path</span><span class="p">)</span>
+    <span class="n">item</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">get_object_metadata</span><span class="p">(</span><span class="n">request</span><span class="p">)</span>
+    <span class="k">return</span> <span class="n">item</span><span class="o">.</span><span class="n">etag</span></div>
+
+<div class="viewcode-block" id="S3IO.copy"><a class="viewcode-back" href="../../../../apache_beam.io.aws.s3io.html#apache_beam.io.aws.s3io.S3IO.copy">[docs]</a>  <span class="nd">@retry</span><span class="o">.</span><span class="n">with_exponential_backoff</span><span class="p">(</span>
+      <span class="n">retry_filter</span><span class="o">=</span><span class="n">retry</span><span class="o">.</span><span class="n">retry_on_server_errors_and_timeout_filter</span><span class="p">)</span>
+  <span class="k">def</span> <span class="nf">copy</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">src</span><span class="p">,</span> <span class="n">dest</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;Copies a single S3 file object from src to dest.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">      src: S3 file path pattern in the form s3://&lt;bucket&gt;/&lt;name&gt;.</span>
+<span class="sd">      dest: S3 file path pattern in the form s3://&lt;bucket&gt;/&lt;name&gt;.</span>
+
+<span class="sd">    Raises:</span>
+<span class="sd">      TimeoutError on timeout.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="n">src_bucket</span><span class="p">,</span> <span class="n">src_key</span> <span class="o">=</span> <span class="n">parse_s3_path</span><span class="p">(</span><span class="n">src</span><span class="p">)</span>
+    <span class="n">dest_bucket</span><span class="p">,</span> <span class="n">dest_key</span> <span class="o">=</span> <span class="n">parse_s3_path</span><span class="p">(</span><span class="n">dest</span><span class="p">)</span>
+    <span class="n">request</span> <span class="o">=</span> <span class="n">messages</span><span class="o">.</span><span class="n">CopyRequest</span><span class="p">(</span><span class="n">src_bucket</span><span class="p">,</span> <span class="n">src_key</span><span class="p">,</span> <span class="n">dest_bucket</span><span class="p">,</span> <span class="n">dest_key</span><span class="p">)</span>
+    <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">copy</span><span class="p">(</span><span class="n">request</span><span class="p">)</span></div>
+
+  <span class="c1"># We intentionally do not decorate this method with a retry, since the</span>
+  <span class="c1"># underlying copy and delete operations are already idempotent operations</span>
+  <span class="c1"># protected by retry decorators.</span>
+<div class="viewcode-block" id="S3IO.copy_paths"><a class="viewcode-back" href="../../../../apache_beam.io.aws.s3io.html#apache_beam.io.aws.s3io.S3IO.copy_paths">[docs]</a>  <span class="k">def</span> <span class="nf">copy_paths</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">src_dest_pairs</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;Copies the given S3 objects from src to dest. This can handle directory</span>
+<span class="sd">    or file paths.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">      src_dest_pairs: list of (src, dest) tuples of s3://&lt;bucket&gt;/&lt;name&gt; file</span>
+<span class="sd">                      paths to copy from src to dest</span>
+<span class="sd">    Returns: List of tuples of (src, dest, exception) in the same order as the</span>
+<span class="sd">            src_dest_pairs argument, where exception is None if the operation</span>
+<span class="sd">            succeeded or the relevant exception if the operation failed.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">if</span> <span class="ow">not</span> <span class="n">src_dest_pairs</span><span class="p">:</span> <span class="k">return</span> <span class="p">[]</span>
+
+    <span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
+
+    <span class="k">for</span> <span class="n">src_path</span><span class="p">,</span> <span class="n">dest_path</span> <span class="ow">in</span> <span class="n">src_dest_pairs</span><span class="p">:</span>
+
+      <span class="c1"># Copy a directory with self.copy_tree</span>
+      <span class="k">if</span> <span class="n">src_path</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">)</span> <span class="ow">and</span> <span class="n">dest_path</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">):</span>
+        <span class="k">try</span><span class="p">:</span>
+          <span class="n">results</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">copy_tree</span><span class="p">(</span><span class="n">src_path</span><span class="p">,</span> <span class="n">dest_path</span><span class="p">)</span>
+        <span class="k">except</span> <span class="n">messages</span><span class="o">.</span><span class="n">S3ClientError</span> <span class="k">as</span> <span class="n">err</span><span class="p">:</span>
+          <span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">src_path</span><span class="p">,</span> <span class="n">dest_path</span><span class="p">,</span> <span class="n">err</span><span class="p">))</span>
+
+      <span class="c1"># Copy individual files with self.copy</span>
+      <span class="k">elif</span> <span class="ow">not</span> <span class="n">src_path</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">)</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">dest_path</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">):</span>
+        <span class="n">src_bucket</span><span class="p">,</span> <span class="n">src_key</span> <span class="o">=</span> <span class="n">parse_s3_path</span><span class="p">(</span><span class="n">src_path</span><span class="p">)</span>
+        <span class="n">dest_bucket</span><span class="p">,</span> <span class="n">dest_key</span> <span class="o">=</span> <span class="n">parse_s3_path</span><span class="p">(</span><span class="n">dest_path</span><span class="p">)</span>
+        <span class="n">request</span> <span class="o">=</span> <span class="n">messages</span><span class="o">.</span><span class="n">CopyRequest</span><span class="p">(</span><span class="n">src_bucket</span><span class="p">,</span>
+                                       <span class="n">src_key</span><span class="p">,</span>
+                                       <span class="n">dest_bucket</span><span class="p">,</span>
+                                       <span class="n">dest_key</span><span class="p">)</span>
+
+        <span class="k">try</span><span class="p">:</span>
+          <span class="bp">self</span><span class="o">.</span><span class="n">client</span><span class="o">.</span><span class="n">copy</span><span class="p">(</span><span class="n">request</span><span class="p">)</span>
+          <span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">src_path</span><span class="p">,</span> <span class="n">dest_path</span><span class="p">,</span> <span class="kc">None</span><span class="p">))</span>
+        <span class="k">except</span> <span class="n">messages</span><span class="o">.</span><span class="n">S3ClientError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+          <span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">src_path</span><span class="p">,</span> <span class="n">dest_path</span><span class="p">,</span> <span class="n">e</span><span class="p">))</span>
+
+      <span class="c1"># Mismatched paths (one directory, one non-directory) get an error result</span>
+      <span class="k">else</span><span class="p">:</span>
+        <span class="n">err</span> <span class="o">=</span> <span class="n">messages</span><span class="o">.</span><span class="n">S3ClientError</span><span class="p">(</span>
+            <span class="s2">&quot;Can&#39;t copy mismatched paths (one directory, one non-directory):&quot;</span> <span class="o">+</span>
+            <span class="s1">&#39; </span><span class="si">%s</span><span class="s1">, </span><span class="si">%s</span><span class="s1">&#39;</span> <span class="o">%</span> <span class="p">(</span><span class="n">src_path</span><span class="p">,</span> <span class="n">dest_path</span><span class="p">),</span>
+            <span class="mi">400</span><span class="p">)</span>
+        <span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">src_path</span><span class="p">,</span> <span class="n">dest_path</span><span class="p">,</span> <span class="n">err</span><span class="p">))</span>
+
+    <span class="k">return</span> <span class="n">results</span></div>
+
+  <span class="c1"># We intentionally do not decorate this method with a retry, since the</span>
+  <span class="c1"># underlying copy and delete operations are already idempotent operations</span>
+  <span class="c1"># protected by retry decorators.</span>
+<div class="viewcode-block" id="S3IO.copy_tree"><a class="viewcode-back" href="../../../../apache_beam.io.aws.s3io.html#apache_beam.io.aws.s3io.S3IO.copy_tree">[docs]</a>  <span class="k">def</span> <span class="nf">copy_tree</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">src</span><span class="p">,</span> <span class="n">dest</span><span class="p">):</span>
+    <span class="sd">&quot;&quot;&quot;Renames the given S3 directory and it&#39;s contents recursively</span>
+<span class="sd">    from src to dest.</span>
+
+<span class="sd">    Args:</span>
+<span class="sd">      src: S3 file path pattern in the form s3://&lt;bucket&gt;/&lt;name&gt;/.</span>
+<span class="sd">      dest: S3 file path pattern in the form s3://&lt;bucket&gt;/&lt;name&gt;/.</span>
+
+<span class="sd">    Returns:</span>
+<span class="sd">      List of tuples of (src, dest, exception) where exception is None if the</span>
+<span class="sd">      operation succeeded or the relevant exception if the operation failed.</span>
+<span class="sd">    &quot;&quot;&quot;</span>
+    <span class="k">assert</span> <span class="n">src</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">)</span>
+    <span class="k">assert</span> <span class="n">dest</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s1">&#39;/&#39;</span><span class="p">)</span>
+
+    <span class="n">results</span> <span class="o">=</span> <span class="p">[]</span>
+    <span class="k">for</span> <span class="n">entry</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">list_prefix</span><span class="p">(</span><span class="n">src</span><span class="p">):</span>
+      <span class="n">rel_path</span> <span class="o">=</span> <span class="n">entry</span><span class="p">[</span><span class="nb">len</span><span class="p">(</span><span class="n">src</span><span class="p">):]</span>
+      <span class="k">try</span><span class="p">:</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">copy</span><span class="p">(</span><span class="n">entry</span><span class="p">,</span> <span class="n">dest</span> <span class="o">+</span> <span class="n">rel_path</span><span class="p">)</span>
+        <span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">entry</span><span class="p">,</span> <span class="n">dest</span> <span class="o">+</span> <span class="n">rel_path</span><span class="p">,</span> <span class="kc">None</span><span class="p">))</span>
+      <span class="k">except</span> <span class="n">messages</span><span class="o">.</span><span class="n">S3ClientError</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
+        <span class="n">results</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">entry</span><span class="p">,</span> <span class="n">dest</span> <span class="o">+</span> <span class="n">rel_path</span><span class="p">,</span> <span class="n">e</span><span class="p">))</span>
+
... 208067 lines suppressed ...


Mime
View raw message