beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From al...@apache.org
Subject [1/2] beam git commit: Updating Dataflow API protos and client
Date Wed, 03 May 2017 01:01:26 GMT
Repository: beam
Updated Branches:
  refs/heads/master d7151fb2b -> 2d22485c1


Updating Dataflow API protos and client


Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/7a48ddda
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/7a48ddda
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/7a48ddda

Branch: refs/heads/master
Commit: 7a48ddda873eb5c2b3d616a3d8bc225a576cd4cb
Parents: d7151fb
Author: Pablo <pabloem@google.com>
Authored: Tue May 2 14:23:44 2017 -0700
Committer: Ahmet Altay <altay@altay-macbookpro2.roam.corp.google.com>
Committed: Tue May 2 18:01:08 2017 -0700

----------------------------------------------------------------------
 .../clients/dataflow/dataflow_v1b3_messages.py  | 564 +++++++++++++++++--
 1 file changed, 528 insertions(+), 36 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/beam/blob/7a48ddda/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py
b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py
index 4f1ccdb..eb88bce 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py
@@ -101,6 +101,60 @@ class ApproximateSplitRequest(_messages.Message):
   position = _messages.MessageField('Position', 2)
 
 
+class AutoscalingEvent(_messages.Message):
+  """A structured message reporting an autoscaling decision made by the
+  Dataflow service.
+
+  Enums:
+    EventTypeValueValuesEnum: The type of autoscaling event to report.
+
+  Fields:
+    currentNumWorkers: The current number of workers the job has.
+    description: A message describing why the system decided to adjust the
+      current number of workers, why it failed, or why the system decided to
+      not make any changes to the number of workers.
+    eventType: The type of autoscaling event to report.
+    targetNumWorkers: The target number of workers the worker pool wants to
+      resize to use.
+    time: The time this event was emitted to indicate a new target or current
+      num_workers value.
+  """
+
+  class EventTypeValueValuesEnum(_messages.Enum):
+    """The type of autoscaling event to report.
+
+    Values:
+      TYPE_UNKNOWN: Default type for the enum.  Value should never be
+        returned.
+      TARGET_NUM_WORKERS_CHANGED: The TARGET_NUM_WORKERS_CHANGED type should
+        be used when the target worker pool size has changed at the start of
+        an actuation. An event should always be specified as
+        TARGET_NUM_WORKERS_CHANGED if it reflects a change in the
+        target_num_workers.
+      CURRENT_NUM_WORKERS_CHANGED: The CURRENT_NUM_WORKERS_CHANGED type should
+        be used when actual worker pool size has been changed, but the
+        target_num_workers has not changed.
+      ACTUATION_FAILURE: The ACTUATION_FAILURE type should be used when we
+        want to report an error to the user indicating why the current number
+        of workers in the pool could not be changed. Displayed in the current
+        status and history widgets.
+      NO_CHANGE: Used when we want to report to the user a reason why we are
+        not currently adjusting the number of workers. Should specify both
+        target_num_workers, current_num_workers and a decision_message.
+    """
+    TYPE_UNKNOWN = 0
+    TARGET_NUM_WORKERS_CHANGED = 1
+    CURRENT_NUM_WORKERS_CHANGED = 2
+    ACTUATION_FAILURE = 3
+    NO_CHANGE = 4
+
+  currentNumWorkers = _messages.IntegerField(1)
+  description = _messages.MessageField('StructuredMessage', 2)
+  eventType = _messages.EnumField('EventTypeValueValuesEnum', 3)
+  targetNumWorkers = _messages.IntegerField(4)
+  time = _messages.StringField(5)
+
+
 class AutoscalingSettings(_messages.Message):
   """Settings for WorkerPool autoscaling.
 
@@ -406,6 +460,7 @@ class CreateJobFromTemplateRequest(_messages.Message):
       create the job. Must be a valid Cloud Storage URL, beginning with
       `gs://`.
     jobName: Required. The job name to use for the created job.
+    location: The location to which to direct the request.
     parameters: The runtime parameters to pass to the job.
   """
 
@@ -436,7 +491,8 @@ class CreateJobFromTemplateRequest(_messages.Message):
   environment = _messages.MessageField('RuntimeEnvironment', 1)
   gcsPath = _messages.StringField(2)
   jobName = _messages.StringField(3)
-  parameters = _messages.MessageField('ParametersValue', 4)
+  location = _messages.StringField(4)
+  parameters = _messages.MessageField('ParametersValue', 5)
 
 
 class CustomSourceLocation(_messages.Message):
@@ -773,6 +829,40 @@ class DataflowProjectsLocationsJobsCreateRequest(_messages.Message):
   view = _messages.EnumField('ViewValueValuesEnum', 5)
 
 
+class DataflowProjectsLocationsJobsDebugGetConfigRequest(_messages.Message):
+  """A DataflowProjectsLocationsJobsDebugGetConfigRequest object.
+
+  Fields:
+    getDebugConfigRequest: A GetDebugConfigRequest resource to be passed as
+      the request body.
+    jobId: The job id.
+    location: The location which contains the job specified by job_id.
+    projectId: The project id.
+  """
+
+  getDebugConfigRequest = _messages.MessageField('GetDebugConfigRequest', 1)
+  jobId = _messages.StringField(2, required=True)
+  location = _messages.StringField(3, required=True)
+  projectId = _messages.StringField(4, required=True)
+
+
+class DataflowProjectsLocationsJobsDebugSendCaptureRequest(_messages.Message):
+  """A DataflowProjectsLocationsJobsDebugSendCaptureRequest object.
+
+  Fields:
+    jobId: The job id.
+    location: The location which contains the job specified by job_id.
+    projectId: The project id.
+    sendDebugCaptureRequest: A SendDebugCaptureRequest resource to be passed
+      as the request body.
+  """
+
+  jobId = _messages.StringField(1, required=True)
+  location = _messages.StringField(2, required=True)
+  projectId = _messages.StringField(3, required=True)
+  sendDebugCaptureRequest = _messages.MessageField('SendDebugCaptureRequest', 4)
+
+
 class DataflowProjectsLocationsJobsGetMetricsRequest(_messages.Message):
   """A DataflowProjectsLocationsJobsGetMetricsRequest object.
 
@@ -984,6 +1074,89 @@ class DataflowProjectsLocationsJobsWorkItemsReportStatusRequest(_messages.Messag
   reportWorkItemStatusRequest = _messages.MessageField('ReportWorkItemStatusRequest', 4)
 
 
+class DataflowProjectsLocationsTemplatesCreateRequest(_messages.Message):
+  """A DataflowProjectsLocationsTemplatesCreateRequest object.
+
+  Fields:
+    createJobFromTemplateRequest: A CreateJobFromTemplateRequest resource to
+      be passed as the request body.
+    location: The location to which to direct the request.
+    projectId: Required. The ID of the Cloud Platform project that the job
+      belongs to.
+  """
+
+  createJobFromTemplateRequest = _messages.MessageField('CreateJobFromTemplateRequest', 1)
+  location = _messages.StringField(2, required=True)
+  projectId = _messages.StringField(3, required=True)
+
+
+class DataflowProjectsLocationsTemplatesGetRequest(_messages.Message):
+  """A DataflowProjectsLocationsTemplatesGetRequest object.
+
+  Enums:
+    ViewValueValuesEnum: The view to retrieve. Defaults to METADATA_ONLY.
+
+  Fields:
+    gcsPath: Required. A Cloud Storage path to the template from which to
+      create the job. Must be a valid Cloud Storage URL, beginning with
+      `gs://`.
+    location: The location to which to direct the request.
+    projectId: Required. The ID of the Cloud Platform project that the job
+      belongs to.
+    view: The view to retrieve. Defaults to METADATA_ONLY.
+  """
+
+  class ViewValueValuesEnum(_messages.Enum):
+    """The view to retrieve. Defaults to METADATA_ONLY.
+
+    Values:
+      METADATA_ONLY: <no description>
+    """
+    METADATA_ONLY = 0
+
+  gcsPath = _messages.StringField(1)
+  location = _messages.StringField(2, required=True)
+  projectId = _messages.StringField(3, required=True)
+  view = _messages.EnumField('ViewValueValuesEnum', 4)
+
+
+class DataflowProjectsLocationsTemplatesLaunchRequest(_messages.Message):
+  """A DataflowProjectsLocationsTemplatesLaunchRequest object.
+
+  Fields:
+    gcsPath: Required. A Cloud Storage path to the template from which to
+      create the job. Must be valid Cloud Storage URL, beginning with 'gs://'.
+    launchTemplateParameters: A LaunchTemplateParameters resource to be passed
+      as the request body.
+    location: The location to which to direct the request.
+    projectId: Required. The ID of the Cloud Platform project that the job
+      belongs to.
+    validateOnly: If true, the request is validated but not actually executed.
+      Defaults to false.
+  """
+
+  gcsPath = _messages.StringField(1)
+  launchTemplateParameters = _messages.MessageField('LaunchTemplateParameters', 2)
+  location = _messages.StringField(3, required=True)
+  projectId = _messages.StringField(4, required=True)
+  validateOnly = _messages.BooleanField(5)
+
+
+class DataflowProjectsLocationsWorkerMessagesRequest(_messages.Message):
+  """A DataflowProjectsLocationsWorkerMessagesRequest object.
+
+  Fields:
+    location: The location which contains the job
+    projectId: The project to send the WorkerMessages to.
+    sendWorkerMessagesRequest: A SendWorkerMessagesRequest resource to be
+      passed as the request body.
+  """
+
+  location = _messages.StringField(1, required=True)
+  projectId = _messages.StringField(2, required=True)
+  sendWorkerMessagesRequest = _messages.MessageField('SendWorkerMessagesRequest', 3)
+
+
 class DataflowProjectsTemplatesCreateRequest(_messages.Message):
   """A DataflowProjectsTemplatesCreateRequest object.
 
@@ -998,6 +1171,58 @@ class DataflowProjectsTemplatesCreateRequest(_messages.Message):
   projectId = _messages.StringField(2, required=True)
 
 
+class DataflowProjectsTemplatesGetRequest(_messages.Message):
+  """A DataflowProjectsTemplatesGetRequest object.
+
+  Enums:
+    ViewValueValuesEnum: The view to retrieve. Defaults to METADATA_ONLY.
+
+  Fields:
+    gcsPath: Required. A Cloud Storage path to the template from which to
+      create the job. Must be a valid Cloud Storage URL, beginning with
+      `gs://`.
+    location: The location to which to direct the request.
+    projectId: Required. The ID of the Cloud Platform project that the job
+      belongs to.
+    view: The view to retrieve. Defaults to METADATA_ONLY.
+  """
+
+  class ViewValueValuesEnum(_messages.Enum):
+    """The view to retrieve. Defaults to METADATA_ONLY.
+
+    Values:
+      METADATA_ONLY: <no description>
+    """
+    METADATA_ONLY = 0
+
+  gcsPath = _messages.StringField(1)
+  location = _messages.StringField(2)
+  projectId = _messages.StringField(3, required=True)
+  view = _messages.EnumField('ViewValueValuesEnum', 4)
+
+
+class DataflowProjectsTemplatesLaunchRequest(_messages.Message):
+  """A DataflowProjectsTemplatesLaunchRequest object.
+
+  Fields:
+    gcsPath: Required. A Cloud Storage path to the template from which to
+      create the job. Must be valid Cloud Storage URL, beginning with 'gs://'.
+    launchTemplateParameters: A LaunchTemplateParameters resource to be passed
+      as the request body.
+    location: The location to which to direct the request.
+    projectId: Required. The ID of the Cloud Platform project that the job
+      belongs to.
+    validateOnly: If true, the request is validated but not actually executed.
+      Defaults to false.
+  """
+
+  gcsPath = _messages.StringField(1)
+  launchTemplateParameters = _messages.MessageField('LaunchTemplateParameters', 2)
+  location = _messages.StringField(3)
+  projectId = _messages.StringField(4, required=True)
+  validateOnly = _messages.BooleanField(5)
+
+
 class DataflowProjectsWorkerMessagesRequest(_messages.Message):
   """A DataflowProjectsWorkerMessagesRequest object.
 
@@ -1121,6 +1346,8 @@ class DistributionUpdate(_messages.Message):
 
   Fields:
     count: The count of the number of elements present in the distribution.
+    logBuckets: (Optional) Logarithmic histogram of values. Each log may be in
+      no more than one bucket. Order does not matter.
     max: The maximum value present in the distribution.
     min: The minimum value present in the distribution.
     sum: Use an int64 since we'd prefer the added precision. If overflow is a
@@ -1130,10 +1357,11 @@ class DistributionUpdate(_messages.Message):
   """
 
   count = _messages.MessageField('SplitInt64', 1)
-  max = _messages.MessageField('SplitInt64', 2)
-  min = _messages.MessageField('SplitInt64', 3)
-  sum = _messages.MessageField('SplitInt64', 4)
-  sumOfSquares = _messages.FloatField(5)
+  logBuckets = _messages.MessageField('LogBucket', 2, repeated=True)
+  max = _messages.MessageField('SplitInt64', 3)
+  min = _messages.MessageField('SplitInt64', 4)
+  sum = _messages.MessageField('SplitInt64', 5)
+  sumOfSquares = _messages.FloatField(6)
 
 
 class DynamicSourceSplit(_messages.Message):
@@ -1312,6 +1540,84 @@ class Environment(_messages.Message):
   workerPools = _messages.MessageField('WorkerPool', 10, repeated=True)
 
 
+class ExecutionStageState(_messages.Message):
+  """A message describing the state of a particular execution stage.
+
+  Enums:
+    ExecutionStageStateValueValuesEnum: Executions stage states allow the same
+      set of values as JobState.
+
+  Fields:
+    currentStateTime: The time at which the stage transitioned to this state.
+    executionStageName: The name of the execution stage.
+    executionStageState: Executions stage states allow the same set of values
+      as JobState.
+  """
+
+  class ExecutionStageStateValueValuesEnum(_messages.Enum):
+    """Executions stage states allow the same set of values as JobState.
+
+    Values:
+      JOB_STATE_UNKNOWN: The job's run state isn't specified.
+      JOB_STATE_STOPPED: `JOB_STATE_STOPPED` indicates that the job has not
+        yet started to run.
+      JOB_STATE_RUNNING: `JOB_STATE_RUNNING` indicates that the job is
+        currently running.
+      JOB_STATE_DONE: `JOB_STATE_DONE` indicates that the job has successfully
+        completed. This is a terminal job state.  This state may be set by the
+        Cloud Dataflow service, as a transition from `JOB_STATE_RUNNING`. It
+        may also be set via a Cloud Dataflow `UpdateJob` call, if the job has
+        not yet reached a terminal state.
+      JOB_STATE_FAILED: `JOB_STATE_FAILED` indicates that the job has failed.
+        This is a terminal job state.  This state may only be set by the Cloud
+        Dataflow service, and only as a transition from `JOB_STATE_RUNNING`.
+      JOB_STATE_CANCELLED: `JOB_STATE_CANCELLED` indicates that the job has
+        been explicitly cancelled. This is a terminal job state. This state
+        may only be set via a Cloud Dataflow `UpdateJob` call, and only if the
+        job has not yet reached another terminal state.
+      JOB_STATE_UPDATED: `JOB_STATE_UPDATED` indicates that the job was
+        successfully updated, meaning that this job was stopped and another
+        job was started, inheriting state from this one. This is a terminal
+        job state. This state may only be set by the Cloud Dataflow service,
+        and only as a transition from `JOB_STATE_RUNNING`.
+      JOB_STATE_DRAINING: `JOB_STATE_DRAINING` indicates that the job is in
+        the process of draining. A draining job has stopped pulling from its
+        input sources and is processing any data that remains in-flight. This
+        state may be set via a Cloud Dataflow `UpdateJob` call, but only as a
+        transition from `JOB_STATE_RUNNING`. Jobs that are draining may only
+        transition to `JOB_STATE_DRAINED`, `JOB_STATE_CANCELLED`, or
+        `JOB_STATE_FAILED`.
+      JOB_STATE_DRAINED: `JOB_STATE_DRAINED` indicates that the job has been
+        drained. A drained job terminated by stopping pulling from its input
+        sources and processing any data that remained in-flight when draining
+        was requested. This state is a terminal state, may only be set by the
+        Cloud Dataflow service, and only as a transition from
+        `JOB_STATE_DRAINING`.
+      JOB_STATE_PENDING: 'JOB_STATE_PENDING' indicates that the job has been
+        created but is not yet running.  Jobs that are pending may only
+        transition to `JOB_STATE_RUNNING`, or `JOB_STATE_FAILED`.
+      JOB_STATE_CANCELLING: 'JOB_STATE_CANCELLING' indicates that the job has
+        been explicitly cancelled and is in the process of stopping.  Jobs
+        that are cancelling may only transition to 'JOB_STATE_CANCELLED' or
+        'JOB_STATE_FAILED'.
+    """
+    JOB_STATE_UNKNOWN = 0
+    JOB_STATE_STOPPED = 1
+    JOB_STATE_RUNNING = 2
+    JOB_STATE_DONE = 3
+    JOB_STATE_FAILED = 4
+    JOB_STATE_CANCELLED = 5
+    JOB_STATE_UPDATED = 6
+    JOB_STATE_DRAINING = 7
+    JOB_STATE_DRAINED = 8
+    JOB_STATE_PENDING = 9
+    JOB_STATE_CANCELLING = 10
+
+  currentStateTime = _messages.StringField(1)
+  executionStageName = _messages.StringField(2)
+  executionStageState = _messages.EnumField('ExecutionStageStateValueValuesEnum', 3)
+
+
 class ExecutionStageSummary(_messages.Message):
   """Description of the composing transforms, names/ids, and input/outputs of
   a stage of execution.  Some composing transforms and sources may have been
@@ -1358,7 +1664,7 @@ class ExecutionStageSummary(_messages.Message):
 
   componentSource = _messages.MessageField('ComponentSource', 1, repeated=True)
   componentTransform = _messages.MessageField('ComponentTransform', 2, repeated=True)
-  id = _messages.IntegerField(3)
+  id = _messages.StringField(3)
   inputSource = _messages.MessageField('StageSource', 4, repeated=True)
   kind = _messages.EnumField('KindValueValuesEnum', 5)
   name = _messages.StringField(6)
@@ -1414,11 +1720,13 @@ class GetDebugConfigRequest(_messages.Message):
   Fields:
     componentId: The internal component id for which debug configuration is
       requested.
+    location: The location which contains the job specified by job_id.
     workerId: The worker id, i.e., VM hostname.
   """
 
   componentId = _messages.StringField(1)
-  workerId = _messages.StringField(2)
+  location = _messages.StringField(2)
+  workerId = _messages.StringField(3)
 
 
 class GetDebugConfigResponse(_messages.Message):
@@ -1431,6 +1739,20 @@ class GetDebugConfigResponse(_messages.Message):
   config = _messages.StringField(1)
 
 
+class GetTemplateResponse(_messages.Message):
+  """The response to a GetTemplate request.
+
+  Fields:
+    metadata: The template metadata describing the template name, available
+      parameters, etc.
+    status: The status of the get template request. Any problems with the
+      request will be indicated in the error_details.
+  """
+
+  metadata = _messages.MessageField('TemplateMetadata', 1)
+  status = _messages.MessageField('Status', 2)
+
+
 class InstructionInput(_messages.Message):
   """An input of an instruction, as a reference to an output of a producer
   instruction.
@@ -1600,6 +1922,8 @@ class Job(_messages.Message):
       set a job's requested state to `JOB_STATE_CANCELLED` or
       `JOB_STATE_DONE`, irrevocably terminating the job if it has not already
       reached a terminal state.
+    stageStates: This field may be mutated by the Cloud Dataflow service;
+      callers cannot mutate it.
     steps: The top-level steps that constitute the entire job.
     tempFiles: A set of files the system should be aware of that are used for
       temporary storage. These temporary files will be removed on job
@@ -1656,6 +1980,13 @@ class Job(_messages.Message):
         was requested. This state is a terminal state, may only be set by the
         Cloud Dataflow service, and only as a transition from
         `JOB_STATE_DRAINING`.
+      JOB_STATE_PENDING: 'JOB_STATE_PENDING' indicates that the job has been
+        created but is not yet running.  Jobs that are pending may only
+        transition to `JOB_STATE_RUNNING`, or `JOB_STATE_FAILED`.
+      JOB_STATE_CANCELLING: 'JOB_STATE_CANCELLING' indicates that the job has
+        been explicitly cancelled and is in the process of stopping.  Jobs
+        that are cancelling may only transition to 'JOB_STATE_CANCELLED' or
+        'JOB_STATE_FAILED'.
     """
     JOB_STATE_UNKNOWN = 0
     JOB_STATE_STOPPED = 1
@@ -1666,6 +1997,8 @@ class Job(_messages.Message):
     JOB_STATE_UPDATED = 6
     JOB_STATE_DRAINING = 7
     JOB_STATE_DRAINED = 8
+    JOB_STATE_PENDING = 9
+    JOB_STATE_CANCELLING = 10
 
   class RequestedStateValueValuesEnum(_messages.Enum):
     """The job's requested state.  `UpdateJob` may be used to switch between
@@ -1710,6 +2043,13 @@ class Job(_messages.Message):
         was requested. This state is a terminal state, may only be set by the
         Cloud Dataflow service, and only as a transition from
         `JOB_STATE_DRAINING`.
+      JOB_STATE_PENDING: 'JOB_STATE_PENDING' indicates that the job has been
+        created but is not yet running.  Jobs that are pending may only
+        transition to `JOB_STATE_RUNNING`, or `JOB_STATE_FAILED`.
+      JOB_STATE_CANCELLING: 'JOB_STATE_CANCELLING' indicates that the job has
+        been explicitly cancelled and is in the process of stopping.  Jobs
+        that are cancelling may only transition to 'JOB_STATE_CANCELLED' or
+        'JOB_STATE_FAILED'.
     """
     JOB_STATE_UNKNOWN = 0
     JOB_STATE_STOPPED = 1
@@ -1720,6 +2060,8 @@ class Job(_messages.Message):
     JOB_STATE_UPDATED = 6
     JOB_STATE_DRAINING = 7
     JOB_STATE_DRAINED = 8
+    JOB_STATE_PENDING = 9
+    JOB_STATE_CANCELLING = 10
 
   class TypeValueValuesEnum(_messages.Enum):
     """The type of Cloud Dataflow job.
@@ -1806,10 +2148,11 @@ class Job(_messages.Message):
   replaceJobId = _messages.StringField(13)
   replacedByJobId = _messages.StringField(14)
   requestedState = _messages.EnumField('RequestedStateValueValuesEnum', 15)
-  steps = _messages.MessageField('Step', 16, repeated=True)
-  tempFiles = _messages.StringField(17, repeated=True)
-  transformNameMapping = _messages.MessageField('TransformNameMappingValue', 18)
-  type = _messages.EnumField('TypeValueValuesEnum', 19)
+  stageStates = _messages.MessageField('ExecutionStageState', 16, repeated=True)
+  steps = _messages.MessageField('Step', 17, repeated=True)
+  tempFiles = _messages.StringField(18, repeated=True)
+  transformNameMapping = _messages.MessageField('TransformNameMappingValue', 19)
+  type = _messages.EnumField('TypeValueValuesEnum', 20)
 
 
 class JobExecutionInfo(_messages.Message):
@@ -1871,8 +2214,7 @@ class JobMessage(_messages.Message):
     MessageImportanceValueValuesEnum: Importance level of the message.
 
   Fields:
-    id: Identifies the message.  This is automatically generated by the
-      service; the caller should treat it as an opaque string.
+    id: Deprecated.
     messageImportance: Importance level of the message.
     messageText: The text of the message.
     time: The timestamp of the message.
@@ -1967,19 +2309,72 @@ class KeyRangeLocation(_messages.Message):
       "myproject-1014-104817-4c2-harness-0-disk-1".
     deliveryEndpoint: The physical location of this range assignment to be
       used for streaming computation cross-worker message delivery.
+    deprecatedPersistentDirectory: DEPRECATED. The location of the persistent
+      state for this range, as a persistent directory in the worker local
+      filesystem.
     end: The end (exclusive) of the key range.
-    persistentDirectory: The location of the persistent state for this range,
-      as a persistent directory in the worker local filesystem.
     start: The start (inclusive) of the key range.
   """
 
   dataDisk = _messages.StringField(1)
   deliveryEndpoint = _messages.StringField(2)
-  end = _messages.StringField(3)
-  persistentDirectory = _messages.StringField(4)
+  deprecatedPersistentDirectory = _messages.StringField(3)
+  end = _messages.StringField(4)
   start = _messages.StringField(5)
 
 
+class LaunchTemplateParameters(_messages.Message):
+  """Parameters to provide to the template being launched.
+
+  Messages:
+    ParametersValue: The runtime parameters to pass to the job.
+
+  Fields:
+    environment: The runtime environment for the job.
+    jobName: Required. The job name to use for the created job.
+    parameters: The runtime parameters to pass to the job.
+  """
+
+  @encoding.MapUnrecognizedFields('additionalProperties')
+  class ParametersValue(_messages.Message):
+    """The runtime parameters to pass to the job.
+
+    Messages:
+      AdditionalProperty: An additional property for a ParametersValue object.
+
+    Fields:
+      additionalProperties: Additional properties of type ParametersValue
+    """
+
+    class AdditionalProperty(_messages.Message):
+      """An additional property for a ParametersValue object.
+
+      Fields:
+        key: Name of the additional property.
+        value: A string attribute.
+      """
+
+      key = _messages.StringField(1)
+      value = _messages.StringField(2)
+
+    additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
+
+  environment = _messages.MessageField('RuntimeEnvironment', 1)
+  jobName = _messages.StringField(2)
+  parameters = _messages.MessageField('ParametersValue', 3)
+
+
+class LaunchTemplateResponse(_messages.Message):
+  """Response to the request to launch a template.
+
+  Fields:
+    job: The job that was launched, if the request was not a dry run and the
+      job was successfully launched.
+  """
+
+  job = _messages.MessageField('Job', 1)
+
+
 class LeaseWorkItemRequest(_messages.Message):
   """Request to lease WorkItems.
 
@@ -2016,13 +2411,15 @@ class ListJobMessagesResponse(_messages.Message):
   """Response to a request to list job messages.
 
   Fields:
+    autoscalingEvents: Autoscaling events in ascending timestamp order.
     jobMessages: Messages in ascending timestamp order.
     nextPageToken: The token to obtain the next page of results if there are
       more.
   """
 
-  jobMessages = _messages.MessageField('JobMessage', 1, repeated=True)
-  nextPageToken = _messages.StringField(2)
+  autoscalingEvents = _messages.MessageField('AutoscalingEvent', 1, repeated=True)
+  jobMessages = _messages.MessageField('JobMessage', 2, repeated=True)
+  nextPageToken = _messages.StringField(3)
 
 
 class ListJobsResponse(_messages.Message):
@@ -2041,6 +2438,20 @@ class ListJobsResponse(_messages.Message):
   nextPageToken = _messages.StringField(3)
 
 
+class LogBucket(_messages.Message):
+  """Bucket of values for Distribution's logarithmic histogram.
+
+  Fields:
+    count: Number of values in this bucket.
+    log: floor(log2(value)); defined to be zero for nonpositive values.
+      log(-1) = 0   log(0) = 0   log(1) = 0   log(2) = 1   log(3) = 1   log(4)
+      = 2   log(5) = 2
+  """
+
+  count = _messages.IntegerField(1)
+  log = _messages.IntegerField(2, variant=_messages.Variant.INT32)
+
+
 class MapTask(_messages.Message):
   """MapTask consists of an ordered set of instructions, each of which
   describes one particular low-level operation for the worker to perform in
@@ -2139,12 +2550,14 @@ class MetricUpdate(_messages.Message):
       aggregate value accumulated since the worker started working on this
       WorkItem. By default this is false, indicating that this metric is
       reported as a delta that is not associated with any WorkItem.
+    distribution: A struct value describing properties of a distribution of
+      numeric values.
     internal: Worker-computed aggregate value for internal use by the Dataflow
       service.
     kind: Metric aggregation kind.  The possible metric aggregation kinds are
-      "Sum", "Max", "Min", "Mean", "Set", "And", and "Or". The specified
-      aggregation kind is case-insensitive.  If omitted, this is not an
-      aggregated value but instead a single metric sample value.
+      "Sum", "Max", "Min", "Mean", "Set", "And", "Or", and "Distribution". The
+      specified aggregation kind is case-insensitive.  If omitted, this is not
+      an aggregated value but instead a single metric sample value.
     meanCount: Worker-computed aggregate value for the "Mean" aggregation
       kind. This holds the count of the aggregated values and is used in
       combination with mean_sum above to obtain the actual mean aggregate
@@ -2167,14 +2580,15 @@ class MetricUpdate(_messages.Message):
   """
 
   cumulative = _messages.BooleanField(1)
-  internal = _messages.MessageField('extra_types.JsonValue', 2)
-  kind = _messages.StringField(3)
-  meanCount = _messages.MessageField('extra_types.JsonValue', 4)
-  meanSum = _messages.MessageField('extra_types.JsonValue', 5)
-  name = _messages.MessageField('MetricStructuredName', 6)
-  scalar = _messages.MessageField('extra_types.JsonValue', 7)
-  set = _messages.MessageField('extra_types.JsonValue', 8)
-  updateTime = _messages.StringField(9)
+  distribution = _messages.MessageField('extra_types.JsonValue', 2)
+  internal = _messages.MessageField('extra_types.JsonValue', 3)
+  kind = _messages.StringField(4)
+  meanCount = _messages.MessageField('extra_types.JsonValue', 5)
+  meanSum = _messages.MessageField('extra_types.JsonValue', 6)
+  name = _messages.MessageField('MetricStructuredName', 7)
+  scalar = _messages.MessageField('extra_types.JsonValue', 8)
+  set = _messages.MessageField('extra_types.JsonValue', 9)
+  updateTime = _messages.StringField(10)
 
 
 class MountedDataDisk(_messages.Message):
@@ -2337,6 +2751,37 @@ class ParallelInstruction(_messages.Message):
   write = _messages.MessageField('WriteInstruction', 9)
 
 
+class Parameter(_messages.Message):
+  """Structured data associated with this message.
+
+  Fields:
+    key: Key or name for this parameter.
+    value: Value for this parameter.
+  """
+
+  key = _messages.StringField(1)
+  value = _messages.MessageField('extra_types.JsonValue', 2)
+
+
+class ParameterMetadata(_messages.Message):
+  """Metadata for a specific parameter.
+
+  Fields:
+    helpText: Required. The help text to display for the parameter.
+    isOptional: Optional. Whether the parameter is optional. Defaults to
+      false.
+    label: Required. The label to display for the parameter.
+    name: Required. The name of the parameter.
+    regexes: Optional. Regexes that the parameter must match.
+  """
+
+  helpText = _messages.StringField(1)
+  isOptional = _messages.BooleanField(2)
+  label = _messages.StringField(3)
+  name = _messages.StringField(4)
+  regexes = _messages.StringField(5, repeated=True)
+
+
 class PartialGroupByKeyInstruction(_messages.Message):
   """An instruction that does a partial group-by-key. One input and one
   output.
@@ -2600,12 +3045,14 @@ class SendDebugCaptureRequest(_messages.Message):
     componentId: The internal component id for which debug information is
       sent.
     data: The encoded debug information.
+    location: The location which contains the job specified by job_id.
     workerId: The worker id, i.e., VM hostname.
   """
 
   componentId = _messages.StringField(1)
   data = _messages.StringField(2)
-  workerId = _messages.StringField(3)
+  location = _messages.StringField(3)
+  workerId = _messages.StringField(4)
 
 
 class SendDebugCaptureResponse(_messages.Message):
@@ -2617,10 +3064,12 @@ class SendWorkerMessagesRequest(_messages.Message):
   """A request for sending worker messages to the service.
 
   Fields:
+    location: The location which contains the job
     workerMessages: The WorkerMessages to send.
   """
 
-  workerMessages = _messages.MessageField('WorkerMessage', 1, repeated=True)
+  location = _messages.StringField(1)
+  workerMessages = _messages.MessageField('WorkerMessage', 2, repeated=True)
 
 
 class SendWorkerMessagesResponse(_messages.Message):
@@ -3148,16 +3597,15 @@ class StageSource(_messages.Message):
 
   Fields:
     name: Dataflow service generated name for this source.
-    originalUserTransformOrCollection: User name for the original user
-      transform or collection with which this source is most closely
-      associated.
+    originalTransformOrCollection: User name for the original user transform
+      or collection with which this source is most closely associated.
     sizeBytes: Size of the source, if measurable.
     userName: Human-readable name for this source; may be user or system
       generated.
   """
 
   name = _messages.StringField(1)
-  originalUserTransformOrCollection = _messages.StringField(2)
+  originalTransformOrCollection = _messages.StringField(2)
   sizeBytes = _messages.IntegerField(3)
   userName = _messages.StringField(4)
 
@@ -3469,6 +3917,12 @@ class StreamingConfigTask(_messages.Message):
   Fields:
     streamingComputationConfigs: Set of computation configuration information.
     userStepToStateFamilyNameMap: Map from user step names to state families.
+    windmillServiceEndpoint: If present, the worker must use this endpoint to
+      communicate with Windmill Service dispatchers, otherwise the worker must
+      continue to use whatever endpoint it had been using.
+    windmillServicePort: If present, the worker must use this port to
+      communicate with Windmill Service dispatchers. Only applicable when
+      windmill_service_endpoint is specified.
   """
 
   @encoding.MapUnrecognizedFields('additionalProperties')
@@ -3500,6 +3954,8 @@ class StreamingConfigTask(_messages.Message):
 
   streamingComputationConfigs = _messages.MessageField('StreamingComputationConfig', 1, repeated=True)
   userStepToStateFamilyNameMap = _messages.MessageField('UserStepToStateFamilyNameMapValue',
2)
+  windmillServiceEndpoint = _messages.StringField(3)
+  windmillServicePort = _messages.IntegerField(4)
 
 
 class StreamingSetupTask(_messages.Message):
@@ -3556,6 +4012,23 @@ class StringList(_messages.Message):
   elements = _messages.StringField(1, repeated=True)
 
 
+class StructuredMessage(_messages.Message):
+  """A rich message format, including a human readable string, a key for
+  identifying the message, and structured data associated with the message for
+  programmatic consumption.
+
+  Fields:
+    messageKey: Idenfier for this message type.  Used by external systems to
+      internationalize or personalize message.
+    messageText: Human-readable version of message.
+    parameters: The structured data associated with this message.
+  """
+
+  messageKey = _messages.StringField(1)
+  messageText = _messages.StringField(2)
+  parameters = _messages.MessageField('Parameter', 3, repeated=True)
+
+
 class TaskRunnerSettings(_messages.Message):
   """Taskrunner configuration settings.
 
@@ -3619,6 +4092,25 @@ class TaskRunnerSettings(_messages.Message):
   workflowFileName = _messages.StringField(19)
 
 
+class TemplateMetadata(_messages.Message):
+  """Metadata describing a template.
+
+  Fields:
+    bypassTempDirValidation: If true, will bypass the validation that the temp
+      directory is writable. This should only be used with templates for
+      pipelines that are guaranteed not to need to write to the temp
+      directory, which is subject to change based on the optimizer.
+    description: Optional. A description of the template.
+    name: Required. The name of the template.
+    parameters: The parameters for the template.
+  """
+
+  bypassTempDirValidation = _messages.BooleanField(1)
+  description = _messages.StringField(2)
+  name = _messages.StringField(3)
+  parameters = _messages.MessageField('ParameterMetadata', 4, repeated=True)
+
+
 class TopologyConfig(_messages.Message):
   """Global topology of the streaming Dataflow job, including all computations
   and their sharded locations.


Mime
View raw message