beam-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From k...@apache.org
Subject [beam] branch master updated: Updates python SDK GCP client API in sync with the current Google internal definition. (#8554)
Date Fri, 17 May 2019 22:40:53 GMT
This is an automated email from the ASF dual-hosted git repository.

kenn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new a1cb61a  Updates python SDK GCP client API in sync with the current Google internal
definition. (#8554)
a1cb61a is described below

commit a1cb61ab63acff2c3d93f166d59f4789d45ecc11
Author: slavachernyak <chernyak@google.com>
AuthorDate: Fri May 17 15:40:41 2019 -0700

    Updates python SDK GCP client API in sync with the current Google internal definition.
(#8554)
---
 .../clients/dataflow/dataflow_v1b3_client.py       | 218 +++++++++++++++++++++
 .../clients/dataflow/dataflow_v1b3_messages.py     | 182 ++++++++++++++++-
 2 files changed, 391 insertions(+), 9 deletions(-)

diff --git a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py
b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py
index 9a2d4d5..061a60b 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_client.py
@@ -64,8 +64,11 @@ class DataflowV1b3(base_api.BaseApiClient):
     self.projects_locations_jobs_messages = self.ProjectsLocationsJobsMessagesService(self)
     self.projects_locations_jobs_workItems = self.ProjectsLocationsJobsWorkItemsService(self)
     self.projects_locations_jobs = self.ProjectsLocationsJobsService(self)
+    self.projects_locations_snapshots = self.ProjectsLocationsSnapshotsService(self)
+    self.projects_locations_sql = self.ProjectsLocationsSqlService(self)
     self.projects_locations_templates = self.ProjectsLocationsTemplatesService(self)
     self.projects_locations = self.ProjectsLocationsService(self)
+    self.projects_snapshots = self.ProjectsSnapshotsService(self)
     self.projects_templates = self.ProjectsTemplatesService(self)
     self.projects = self.ProjectsService(self)
 
@@ -821,6 +824,133 @@ of jobs that are running in `us-central1`.
         supports_download=False,
     )
 
+  class ProjectsLocationsSnapshotsService(base_api.BaseApiService):
+    """Service class for the projects_locations_snapshots resource."""
+
+    _NAME = u'projects_locations_snapshots'
+
+    def __init__(self, client):
+      super(DataflowV1b3.ProjectsLocationsSnapshotsService, self).__init__(client)
+      self._upload_configs = {
+          }
+
+    def Delete(self, request, global_params=None):
+      r"""Deletes a snapshot.
+
+      Args:
+        request: (DataflowProjectsLocationsSnapshotsDeleteRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (DeleteSnapshotResponse) The response message.
+      """
+      config = self.GetMethodConfig('Delete')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    Delete.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'DELETE',
+        method_id=u'dataflow.projects.locations.snapshots.delete',
+        ordered_params=[u'projectId', u'location', u'snapshotId'],
+        path_params=[u'location', u'projectId', u'snapshotId'],
+        query_params=[],
+        relative_path=u'v1b3/projects/{projectId}/locations/{location}/snapshots/{snapshotId}',
+        request_field='',
+        request_type_name=u'DataflowProjectsLocationsSnapshotsDeleteRequest',
+        response_type_name=u'DeleteSnapshotResponse',
+        supports_download=False,
+    )
+
+    def Get(self, request, global_params=None):
+      r"""Gets information about a snapshot.
+
+      Args:
+        request: (DataflowProjectsLocationsSnapshotsGetRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (Snapshot) The response message.
+      """
+      config = self.GetMethodConfig('Get')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    Get.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'GET',
+        method_id=u'dataflow.projects.locations.snapshots.get',
+        ordered_params=[u'projectId', u'location', u'snapshotId'],
+        path_params=[u'location', u'projectId', u'snapshotId'],
+        query_params=[],
+        relative_path=u'v1b3/projects/{projectId}/locations/{location}/snapshots/{snapshotId}',
+        request_field='',
+        request_type_name=u'DataflowProjectsLocationsSnapshotsGetRequest',
+        response_type_name=u'Snapshot',
+        supports_download=False,
+    )
+
+    def List(self, request, global_params=None):
+      r"""Lists snapshots.
+
+      Args:
+        request: (DataflowProjectsLocationsSnapshotsListRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (ListSnapshotsResponse) The response message.
+      """
+      config = self.GetMethodConfig('List')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    List.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'GET',
+        method_id=u'dataflow.projects.locations.snapshots.list',
+        ordered_params=[u'projectId', u'location'],
+        path_params=[u'location', u'projectId'],
+        query_params=[],
+        relative_path=u'v1b3/projects/{projectId}/locations/{location}/snapshots',
+        request_field='',
+        request_type_name=u'DataflowProjectsLocationsSnapshotsListRequest',
+        response_type_name=u'ListSnapshotsResponse',
+        supports_download=False,
+    )
+
+  class ProjectsLocationsSqlService(base_api.BaseApiService):
+    """Service class for the projects_locations_sql resource."""
+
+    _NAME = u'projects_locations_sql'
+
+    def __init__(self, client):
+      super(DataflowV1b3.ProjectsLocationsSqlService, self).__init__(client)
+      self._upload_configs = {
+          }
+
+    def Validate(self, request, global_params=None):
+      r"""Validates a GoogleSQL query for Cloud Dataflow syntax. Will always.
+confirm the given query parses correctly, and if able to look up
+schema information from DataCatalog, will validate that the query
+analyzes properly as well.
+
+      Args:
+        request: (DataflowProjectsLocationsSqlValidateRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (ValidateResponse) The response message.
+      """
+      config = self.GetMethodConfig('Validate')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    Validate.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'GET',
+        method_id=u'dataflow.projects.locations.sql.validate',
+        ordered_params=[u'projectId', u'location'],
+        path_params=[u'location', u'projectId'],
+        query_params=[u'query'],
+        relative_path=u'v1b3/projects/{projectId}/locations/{location}/sql:validate',
+        request_field='',
+        request_type_name=u'DataflowProjectsLocationsSqlValidateRequest',
+        response_type_name=u'ValidateResponse',
+        supports_download=False,
+    )
+
   class ProjectsLocationsTemplatesService(base_api.BaseApiService):
     """Service class for the projects_locations_templates resource."""
 
@@ -945,6 +1075,68 @@ of jobs that are running in `us-central1`.
         supports_download=False,
     )
 
+  class ProjectsSnapshotsService(base_api.BaseApiService):
+    """Service class for the projects_snapshots resource."""
+
+    _NAME = u'projects_snapshots'
+
+    def __init__(self, client):
+      super(DataflowV1b3.ProjectsSnapshotsService, self).__init__(client)
+      self._upload_configs = {
+          }
+
+    def Get(self, request, global_params=None):
+      r"""Gets information about a snapshot.
+
+      Args:
+        request: (DataflowProjectsSnapshotsGetRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (Snapshot) The response message.
+      """
+      config = self.GetMethodConfig('Get')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    Get.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'GET',
+        method_id=u'dataflow.projects.snapshots.get',
+        ordered_params=[u'projectId', u'snapshotId'],
+        path_params=[u'projectId', u'snapshotId'],
+        query_params=[u'location'],
+        relative_path=u'v1b3/projects/{projectId}/snapshots/{snapshotId}',
+        request_field='',
+        request_type_name=u'DataflowProjectsSnapshotsGetRequest',
+        response_type_name=u'Snapshot',
+        supports_download=False,
+    )
+
+    def List(self, request, global_params=None):
+      r"""Lists snapshots.
+
+      Args:
+        request: (DataflowProjectsSnapshotsListRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (ListSnapshotsResponse) The response message.
+      """
+      config = self.GetMethodConfig('List')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    List.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'GET',
+        method_id=u'dataflow.projects.snapshots.list',
+        ordered_params=[u'projectId'],
+        path_params=[u'projectId'],
+        query_params=[u'location'],
+        relative_path=u'v1b3/projects/{projectId}/snapshots',
+        request_field='',
+        request_type_name=u'DataflowProjectsSnapshotsListRequest',
+        response_type_name=u'ListSnapshotsResponse',
+        supports_download=False,
+    )
+
   class ProjectsTemplatesService(base_api.BaseApiService):
     """Service class for the projects_templates resource."""
 
@@ -1043,6 +1235,32 @@ of jobs that are running in `us-central1`.
       self._upload_configs = {
           }
 
+    def DeleteSnapshots(self, request, global_params=None):
+      r"""Deletes a snapshot.
+
+      Args:
+        request: (DataflowProjectsDeleteSnapshotsRequest) input message
+        global_params: (StandardQueryParameters, default: None) global arguments
+      Returns:
+        (DeleteSnapshotResponse) The response message.
+      """
+      config = self.GetMethodConfig('DeleteSnapshots')
+      return self._RunMethod(
+          config, request, global_params=global_params)
+
+    DeleteSnapshots.method_config = lambda: base_api.ApiMethodInfo(
+        http_method=u'DELETE',
+        method_id=u'dataflow.projects.deleteSnapshots',
+        ordered_params=[u'projectId'],
+        path_params=[u'projectId'],
+        query_params=[u'location', u'snapshotId'],
+        relative_path=u'v1b3/projects/{projectId}/snapshots',
+        request_field='',
+        request_type_name=u'DataflowProjectsDeleteSnapshotsRequest',
+        response_type_name=u'DeleteSnapshotResponse',
+        supports_download=False,
+    )
+
     def WorkerMessages(self, request, global_params=None):
       r"""Send a worker_message to the service.
 
diff --git a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py
b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py
index 5d8fdae..021f394 100644
--- a/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py
+++ b/sdks/python/apache_beam/runners/dataflow/internal/clients/dataflow/dataflow_v1b3_messages.py
@@ -575,6 +575,21 @@ class DataDiskAssignment(_messages.Message):
   vmInstance = _messages.StringField(2)
 
 
+class DataflowProjectsDeleteSnapshotsRequest(_messages.Message):
+  r"""A DataflowProjectsDeleteSnapshotsRequest object.
+
+  Fields:
+    location: The location that contains this snapshot.
+    projectId: The ID of the Cloud Platform project that the snapshot belongs
+      to.
+    snapshotId: The ID of the snapshot.
+  """
+
+  location = _messages.StringField(1)
+  projectId = _messages.StringField(2, required=True)
+  snapshotId = _messages.StringField(3)
+
+
 class DataflowProjectsJobsAggregatedRequest(_messages.Message):
   r"""A DataflowProjectsJobsAggregatedRequest object.
 
@@ -1251,6 +1266,65 @@ class DataflowProjectsLocationsJobsWorkItemsReportStatusRequest(_messages.Messag
   reportWorkItemStatusRequest = _messages.MessageField('ReportWorkItemStatusRequest', 4)
 
 
+class DataflowProjectsLocationsSnapshotsDeleteRequest(_messages.Message):
+  r"""A DataflowProjectsLocationsSnapshotsDeleteRequest object.
+
+  Fields:
+    location: The location that contains this snapshot.
+    projectId: The ID of the Cloud Platform project that the snapshot belongs
+      to.
+    snapshotId: The ID of the snapshot.
+  """
+
+  location = _messages.StringField(1, required=True)
+  projectId = _messages.StringField(2, required=True)
+  snapshotId = _messages.StringField(3, required=True)
+
+
+class DataflowProjectsLocationsSnapshotsGetRequest(_messages.Message):
+  r"""A DataflowProjectsLocationsSnapshotsGetRequest object.
+
+  Fields:
+    location: The location that contains this snapshot.
+    projectId: The ID of the Cloud Platform project that the snapshot belongs
+      to.
+    snapshotId: The ID of the snapshot.
+  """
+
+  location = _messages.StringField(1, required=True)
+  projectId = _messages.StringField(2, required=True)
+  snapshotId = _messages.StringField(3, required=True)
+
+
+class DataflowProjectsLocationsSnapshotsListRequest(_messages.Message):
+  r"""A DataflowProjectsLocationsSnapshotsListRequest object.
+
+  Fields:
+    location: The location to list snapshots in.
+    projectId: The project ID to list snapshots for.
+  """
+
+  location = _messages.StringField(1, required=True)
+  projectId = _messages.StringField(2, required=True)
+
+
+class DataflowProjectsLocationsSqlValidateRequest(_messages.Message):
+  r"""A DataflowProjectsLocationsSqlValidateRequest object.
+
+  Fields:
+    location: The [regional endpoint]
+      (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) to
+      which to direct the request.
+    projectId: Required. The ID of the Cloud Platform project that the job
+      belongs to.
+    query: The sql query to validate.
+  """
+
+  location = _messages.StringField(1, required=True)
+  projectId = _messages.StringField(2, required=True)
+  query = _messages.StringField(3)
+
+
 class DataflowProjectsLocationsTemplatesCreateRequest(_messages.Message):
   r"""A DataflowProjectsLocationsTemplatesCreateRequest object.
 
@@ -1347,6 +1421,33 @@ class DataflowProjectsLocationsWorkerMessagesRequest(_messages.Message):
   sendWorkerMessagesRequest = _messages.MessageField('SendWorkerMessagesRequest', 3)
 
 
+class DataflowProjectsSnapshotsGetRequest(_messages.Message):
+  r"""A DataflowProjectsSnapshotsGetRequest object.
+
+  Fields:
+    location: The location that contains this snapshot.
+    projectId: The ID of the Cloud Platform project that the snapshot belongs
+      to.
+    snapshotId: The ID of the snapshot.
+  """
+
+  location = _messages.StringField(1)
+  projectId = _messages.StringField(2, required=True)
+  snapshotId = _messages.StringField(3, required=True)
+
+
+class DataflowProjectsSnapshotsListRequest(_messages.Message):
+  r"""A DataflowProjectsSnapshotsListRequest object.
+
+  Fields:
+    location: The location to list snapshots in.
+    projectId: The project ID to list snapshots for.
+  """
+
+  location = _messages.StringField(1)
+  projectId = _messages.StringField(2, required=True)
+
+
 class DataflowProjectsTemplatesCreateRequest(_messages.Message):
   r"""A DataflowProjectsTemplatesCreateRequest object.
 
@@ -1447,6 +1548,10 @@ class DatastoreIODetails(_messages.Message):
   projectId = _messages.StringField(2)
 
 
+class DeleteSnapshotResponse(_messages.Message):
+  r"""Response from deleting a snapshot."""
+
+
 class DerivedSource(_messages.Message):
   r"""Specification of one of the bundles produced as a result of splitting a
   Source (e.g. when executing a SourceSplitRequest, or when splitting an
@@ -2849,6 +2954,16 @@ class ListJobsResponse(_messages.Message):
   nextPageToken = _messages.StringField(3)
 
 
+class ListSnapshotsResponse(_messages.Message):
+  r"""List of snapshots.
+
+  Fields:
+    snapshots: Returned snapshots.
+  """
+
+  snapshots = _messages.MessageField('Snapshot', 1, repeated=True)
+
+
 class MapTask(_messages.Message):
   r"""MapTask consists of an ordered set of instructions, each of which
   describes one particular low-level operation for the worker to perform in
@@ -3507,11 +3622,17 @@ class RuntimeEnvironment(_messages.Message):
   r"""The environment values to set at runtime.
 
   Messages:
-    AdditionalUserLabelsValue: Additional user labels attached to the job.
+    AdditionalUserLabelsValue: Additional user labels to be specified for the
+      job. Keys and values should follow the restrictions specified in the
+      [labeling restrictions](https://cloud.google.com/compute/docs/labeling-
+      resources#restrictions) page.
 
   Fields:
     additionalExperiments: Additional experiment flags for the job.
-    additionalUserLabels: Additional user labels attached to the job.
+    additionalUserLabels: Additional user labels to be specified for the job.
+      Keys and values should follow the restrictions specified in the
+      [labeling restrictions](https://cloud.google.com/compute/docs/labeling-
+      resources#restrictions) page.
     bypassTempDirValidation: Whether to bypass the safety checks for the job's
       temporary directory. Use with caution.
     machineType: The machine type to use for the job. Defaults to the value
@@ -3535,7 +3656,10 @@ class RuntimeEnvironment(_messages.Message):
 
   @encoding.MapUnrecognizedFields('additionalProperties')
   class AdditionalUserLabelsValue(_messages.Message):
-    r"""Additional user labels attached to the job.
+    r"""Additional user labels to be specified for the job. Keys and values
+    should follow the restrictions specified in the [labeling
+    restrictions](https://cloud.google.com/compute/docs/labeling-
+    resources#restrictions) page.
 
     Messages:
       AdditionalProperty: An additional property for a
@@ -3841,19 +3965,43 @@ class Sink(_messages.Message):
 class Snapshot(_messages.Message):
   r"""Represents a snapshot of a job.
 
+  Enums:
+    StateValueValuesEnum: State of the snapshot.
+
   Fields:
     creationTime: The time this snapshot was created.
     id: The unique ID of this snapshot.
     projectId: The project this snapshot belongs to.
     sourceJobId: The job this snapshot was created from.
+    state: State of the snapshot.
     ttl: The time after which this snapshot will be automatically deleted.
   """
 
+  class StateValueValuesEnum(_messages.Enum):
+    r"""State of the snapshot.
+
+    Values:
+      UNKNOWN_SNAPSHOT_STATE: Unknown state.
+      PENDING: Snapshot intent to create has been persisted, snapshotting of
+        state has not yet started.
+      RUNNING: Snapshotting is being performed.
+      READY: Snapshot has been created and is ready to be used.
+      FAILED: Snapshot failed to be created.
+      DELETED: Snapshot has been deleted.
+    """
+    UNKNOWN_SNAPSHOT_STATE = 0
+    PENDING = 1
+    RUNNING = 2
+    READY = 3
+    FAILED = 4
+    DELETED = 5
+
   creationTime = _messages.StringField(1)
   id = _messages.StringField(2)
   projectId = _messages.StringField(3)
   sourceJobId = _messages.StringField(4)
-  ttl = _messages.StringField(5)
+  state = _messages.EnumField('StateValueValuesEnum', 5)
+  ttl = _messages.StringField(6)
 
 
 class SnapshotJobRequest(_messages.Message):
@@ -4888,6 +5036,16 @@ class TransformSummary(_messages.Message):
   outputCollectionName = _messages.StringField(6, repeated=True)
 
 
+class ValidateResponse(_messages.Message):
+  r"""Response to the validation request.
+
+  Fields:
+    errorMessage: Will be empty if validation succeeds.
+  """
+
+  errorMessage = _messages.StringField(1)
+
+
 class WorkItem(_messages.Message):
   r"""WorkItem represents basic information about a WorkItem to be executed in
   the cloud.
@@ -5079,6 +5237,7 @@ class WorkerHealthReport(_messages.Message):
     PodsValueListEntry: A PodsValueListEntry object.
 
   Fields:
+    msg: A message describing any unusual health reports.
     pods: The pods running on the worker. See: http://kubernetes.io/v1.1/docs
       /api-reference/v1/definitions.html#_v1_pod  This field is used by the
       worker to send the status of the indvidual containers running on each
@@ -5086,7 +5245,10 @@ class WorkerHealthReport(_messages.Message):
     reportInterval: The interval at which the worker is sending health
       reports. The default value of 0 should be interpreted as the field is
       not being explicitly set by the worker.
-    vmIsHealthy: Whether the VM is healthy.
+    vmIsBroken: Whether the VM is in a permanently broken state. Broken VMs
+      should be abandoned or deleted ASAP to avoid assigning or completing any
+      work.
+    vmIsHealthy: Whether the VM is currently healthy.
     vmStartupTime: The time the VM was booted.
   """
 
@@ -5115,10 +5277,12 @@ class WorkerHealthReport(_messages.Message):
 
     additionalProperties = _messages.MessageField('AdditionalProperty', 1, repeated=True)
 
-  pods = _messages.MessageField('PodsValueListEntry', 1, repeated=True)
-  reportInterval = _messages.StringField(2)
-  vmIsHealthy = _messages.BooleanField(3)
-  vmStartupTime = _messages.StringField(4)
+  msg = _messages.StringField(1)
+  pods = _messages.MessageField('PodsValueListEntry', 2, repeated=True)
+  reportInterval = _messages.StringField(3)
+  vmIsBroken = _messages.BooleanField(4)
+  vmIsHealthy = _messages.BooleanField(5)
+  vmStartupTime = _messages.StringField(6)
 
 
 class WorkerHealthReportResponse(_messages.Message):


Mime
View raw message