diff --git a/google/cloud/dataproc/dataproc.yaml b/google/cloud/dataproc/dataproc.yaml index 023448c3..86831d63 100644 --- a/google/cloud/dataproc/dataproc.yaml +++ b/google/cloud/dataproc/dataproc.yaml @@ -4,14 +4,18 @@ name: dataproc.googleapis.com title: Cloud Dataproc API apis: +- name: google.cloud.dataproc.v1.ClusterController +- name: google.cloud.dataproc.v1.JobController +- name: google.cloud.dataproc.v1.WorkflowTemplateService - name: google.cloud.dataproc.v1beta2.AutoscalingPolicyService - name: google.cloud.dataproc.v1beta2.ClusterController - name: google.cloud.dataproc.v1beta2.JobController - name: google.cloud.dataproc.v1beta2.WorkflowTemplateService types: -- name: google.cloud.dataproc.v1.DiagnoseClusterResults - name: google.cloud.dataproc.v1.ClusterOperationMetadata +- name: google.cloud.dataproc.v1.DiagnoseClusterResults +- name: google.cloud.dataproc.v1.WorkflowMetadata - name: google.cloud.dataproc.v1beta2.DiagnoseClusterResults - name: google.cloud.dataproc.v1beta2.ClusterOperationMetadata - name: google.cloud.dataproc.v1beta2.WorkflowMetadata @@ -157,19 +161,15 @@ http: authentication: rules: - - selector: 'google.cloud.dataproc.v1beta2.AutoscalingPolicyService.*' + - selector: 'google.cloud.dataproc.v1.ClusterController.*' oauth: canonical_scopes: |- https://www.googleapis.com/auth/cloud-platform - - selector: 'google.cloud.dataproc.v1beta2.ClusterController.*' + - selector: 'google.cloud.dataproc.v1.JobController.*' oauth: canonical_scopes: |- https://www.googleapis.com/auth/cloud-platform - - selector: 'google.cloud.dataproc.v1beta2.JobController.*' - oauth: - canonical_scopes: |- - https://www.googleapis.com/auth/cloud-platform - - selector: 'google.cloud.dataproc.v1beta2.WorkflowTemplateService.*' + - selector: 'google.cloud.dataproc.v1.WorkflowTemplateService.*' oauth: canonical_scopes: |- https://www.googleapis.com/auth/cloud-platform diff --git a/google/cloud/dataproc/v1/clusters.proto b/google/cloud/dataproc/v1/clusters.proto index 31873aad..638b72ac 100644 --- a/google/cloud/dataproc/v1/clusters.proto +++ b/google/cloud/dataproc/v1/clusters.proto @@ -1,4 +1,4 @@ -// Copyright 2018 Google LLC. +// Copyright 2019 Google LLC. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,7 +18,7 @@ syntax = "proto3"; package google.cloud.dataproc.v1; import "google/api/annotations.proto"; -import "google/cloud/dataproc/v1/operations.proto"; +import "google/cloud/dataproc/v1/shared.proto"; import "google/longrunning/operations.proto"; import "google/protobuf/duration.proto"; import "google/protobuf/field_mask.proto"; @@ -33,8 +33,7 @@ option java_package = "com.google.cloud.dataproc.v1"; // of Compute Engine instances. service ClusterController { // Creates a cluster in a project. - rpc CreateCluster(CreateClusterRequest) - returns (google.longrunning.Operation) { + rpc CreateCluster(CreateClusterRequest) returns (google.longrunning.Operation) { option (google.api.http) = { post: "/v1/projects/{project_id}/regions/{region}/clusters" body: "cluster" @@ -42,8 +41,7 @@ service ClusterController { } // Updates a cluster in a project. - rpc UpdateCluster(UpdateClusterRequest) - returns (google.longrunning.Operation) { + rpc UpdateCluster(UpdateClusterRequest) returns (google.longrunning.Operation) { option (google.api.http) = { patch: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}" body: "cluster" @@ -51,8 +49,7 @@ service ClusterController { } // Deletes a cluster in a project. - rpc DeleteCluster(DeleteClusterRequest) - returns (google.longrunning.Operation) { + rpc DeleteCluster(DeleteClusterRequest) returns (google.longrunning.Operation) { option (google.api.http) = { delete: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}" }; @@ -75,8 +72,7 @@ service ClusterController { // Gets cluster diagnostic information. // After the operation completes, the Operation.response field // contains `DiagnoseClusterOutputLocation`. - rpc DiagnoseCluster(DiagnoseClusterRequest) - returns (google.longrunning.Operation) { + rpc DiagnoseCluster(DiagnoseClusterRequest) returns (google.longrunning.Operation) { option (google.api.http) = { post: "/v1/projects/{project_id}/regions/{region}/clusters/{cluster_name}:diagnose" body: "*" @@ -126,15 +122,18 @@ message Cluster { // The cluster config. message ClusterConfig { - // Optional. A Cloud Storage staging bucket used for sharing generated - // SSH keys and config. If you do not specify a staging bucket, Cloud - // Dataproc will determine an appropriate Cloud Storage location (US, + // Optional. A Google Cloud Storage bucket used to stage job + // dependencies, config files, and job driver console output. + // If you do not specify a staging bucket, Cloud + // Dataproc will determine a Cloud Storage location (US, // ASIA, or EU) for your cluster's staging bucket according to the Google - // Compute Engine zone where your cluster is deployed, and then it will create - // and manage this project-level, per-location bucket for you. + // Compute Engine zone where your cluster is deployed, and then create + // and manage this project-level, per-location bucket (see + // [Cloud Dataproc staging + // bucket](/dataproc/docs/concepts/configuring-clusters/staging-bucket)). string config_bucket = 1; - // Required. The shared Compute Engine config settings for + // Optional. The shared Compute Engine config settings for // all instances in a cluster. GceClusterConfig gce_cluster_config = 8; @@ -213,8 +212,8 @@ message GceClusterConfig { // // A full URL, partial URI, or short name are valid. Examples: // - // * `https://www.googleapis.com/compute/v1/projects/[project_id]/regions/us-east1/sub0` - // * `projects/[project_id]/regions/us-east1/sub0` + // * `https://www.googleapis.com/compute/v1/projects/[project_id]/regions/us-east1/subnetworks/sub0` + // * `projects/[project_id]/regions/us-east1/subnetworks/sub0` // * `sub0` string subnetwork_uri = 6; @@ -447,13 +446,13 @@ message SoftwareConfig { // such as "1.2" (including a subminor version, such as "1.2.29"), or the // ["preview" // version](/dataproc/docs/concepts/versioning/dataproc-versions#other_versions). - // If unspecified, it defaults to the latest version. + // If unspecified, it defaults to the latest Debian version. string image_version = 1; // Optional. The properties to set on daemon config files. // - // Property keys are specified in `prefix:property` format, such as - // `core:fs.defaultFS`. The following are supported prefixes + // Property keys are specified in `prefix:property` format, for example + // `core:hadoop.tmp.dir`. The following are supported prefixes // and their mappings: // // * capacity-scheduler: `capacity-scheduler.xml` @@ -469,6 +468,9 @@ message SoftwareConfig { // For more information, see // [Cluster properties](/dataproc/docs/concepts/cluster-properties). map properties = 2; + + // The set of optional components to activate on the cluster. + repeated Component optional_components = 3; } // Contains cluster daemon metrics, such as HDFS and YARN stats. @@ -496,11 +498,10 @@ message CreateClusterRequest { Cluster cluster = 2; // Optional. A unique id used to identify the request. If the server - // receives two - // [CreateClusterRequest][google.cloud.dataproc.v1.CreateClusterRequest] - // requests with the same id, then the second request will be ignored and the - // first [google.longrunning.Operation][google.longrunning.Operation] created - // and stored in the backend is returned. + // receives two [CreateClusterRequest][google.cloud.dataproc.v1.CreateClusterRequest] requests with the same + // id, then the second request will be ignored and the + // first [google.longrunning.Operation][google.longrunning.Operation] created and stored in the backend + // is returned. // // It is recommended to always set this value to a // [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier). @@ -585,11 +586,10 @@ message UpdateClusterRequest { google.protobuf.FieldMask update_mask = 4; // Optional. A unique id used to identify the request. If the server - // receives two - // [UpdateClusterRequest][google.cloud.dataproc.v1.UpdateClusterRequest] - // requests with the same id, then the second request will be ignored and the - // first [google.longrunning.Operation][google.longrunning.Operation] created - // and stored in the backend is returned. + // receives two [UpdateClusterRequest][google.cloud.dataproc.v1.UpdateClusterRequest] requests with the same + // id, then the second request will be ignored and the + // first [google.longrunning.Operation][google.longrunning.Operation] created and stored in the + // backend is returned. // // It is recommended to always set this value to a // [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier). @@ -616,11 +616,10 @@ message DeleteClusterRequest { string cluster_uuid = 4; // Optional. A unique id used to identify the request. If the server - // receives two - // [DeleteClusterRequest][google.cloud.dataproc.v1.DeleteClusterRequest] - // requests with the same id, then the second request will be ignored and the - // first [google.longrunning.Operation][google.longrunning.Operation] created - // and stored in the backend is returned. + // receives two [DeleteClusterRequest][google.cloud.dataproc.v1.DeleteClusterRequest] requests with the same + // id, then the second request will be ignored and the + // first [google.longrunning.Operation][google.longrunning.Operation] created and stored in the + // backend is returned. // // It is recommended to always set this value to a // [UUID](https://en.wikipedia.org/wiki/Universally_unique_identifier). diff --git a/google/cloud/dataproc/v1/dataproc_gapic.yaml b/google/cloud/dataproc/v1/dataproc_gapic.yaml index ab7e2bd3..87097283 100644 --- a/google/cloud/dataproc/v1/dataproc_gapic.yaml +++ b/google/cloud/dataproc/v1/dataproc_gapic.yaml @@ -19,7 +19,7 @@ language_settings: domain_layer_location: google-cloud # A list of API interface configurations. interfaces: - # The fully qualified name of the API interface. +# The fully qualified name of the API interface. - name: google.cloud.dataproc.v1.ClusterController # A list of resource collection configurations. # Consists of a name_pattern and an entity_name. @@ -74,15 +74,6 @@ interfaces: # required_fields - Fields that are always required for a request to be # valid. # - # resource_name_treatment - An enum that specifies how to treat the resource - # name formats defined in the field_name_patterns and - # response_field_name_patterns fields. - # UNSET: default value - # NONE: the collection configs will not be used by the generated code. - # VALIDATE: string fields will be validated by the client against the - # specified resource name formats. - # STATIC_TYPES: the client will use generated types for resource names. - # # page_streaming - Specifies the configuration for paging. # Describes information for generating a method which transforms a paging # list RPC into a stream of resources. @@ -295,15 +286,6 @@ interfaces: # required_fields - Fields that are always required for a request to be # valid. # - # resource_name_treatment - An enum that specifies how to treat the resource - # name formats defined in the field_name_patterns and - # response_field_name_patterns fields. - # UNSET: default value - # NONE: the collection configs will not be used by the generated code. - # VALIDATE: string fields will be validated by the client against the - # specified resource name formats. - # STATIC_TYPES: the client will use generated types for resource names. - # # page_streaming - Specifies the configuration for paging. # Describes information for generating a method which transforms a paging # list RPC into a stream of resources. @@ -476,15 +458,6 @@ interfaces: # required_fields - Fields that are always required for a request to be # valid. # - # resource_name_treatment - An enum that specifies how to treat the resource - # name formats defined in the field_name_patterns and - # response_field_name_patterns fields. - # UNSET: default value - # NONE: the collection configs will not be used by the generated code. - # VALIDATE: string fields will be validated by the client against the - # specified resource name formats. - # STATIC_TYPES: the client will use generated types for resource names. - # # page_streaming - Specifies the configuration for paging. # Describes information for generating a method which transforms a paging # list RPC into a stream of resources. @@ -526,7 +499,6 @@ interfaces: - template retry_codes_name: non_idempotent retry_params_name: default - resource_name_treatment: STATIC_TYPES field_name_patterns: parent: region timeout_millis: 30000 @@ -540,7 +512,6 @@ interfaces: - name retry_codes_name: idempotent retry_params_name: default - resource_name_treatment: STATIC_TYPES field_name_patterns: name: workflow_template timeout_millis: 60000 @@ -556,7 +527,6 @@ interfaces: - name retry_codes_name: non_idempotent retry_params_name: default - resource_name_treatment: STATIC_TYPES field_name_patterns: name: workflow_template timeout_millis: 60000 @@ -620,7 +590,6 @@ interfaces: resources_field: templates retry_codes_name: idempotent retry_params_name: default - resource_name_treatment: STATIC_TYPES field_name_patterns: parent: region timeout_millis: 60000 @@ -633,7 +602,6 @@ interfaces: - name retry_codes_name: non_idempotent retry_params_name: default - resource_name_treatment: STATIC_TYPES field_name_patterns: name: workflow_template timeout_millis: 60000 @@ -650,6 +618,9 @@ resource_name_generation: - message_name: InstantiateWorkflowTemplateRequest field_entity_map: name: workflow_template +- message_name: InstantiateInlineWorkflowTemplateRequest + field_entity_map: + parent: region - message_name: ListWorkflowTemplatesRequest field_entity_map: parent: region diff --git a/google/cloud/dataproc/v1/jobs.proto b/google/cloud/dataproc/v1/jobs.proto index c331ee23..7ead7bb9 100644 --- a/google/cloud/dataproc/v1/jobs.proto +++ b/google/cloud/dataproc/v1/jobs.proto @@ -1,4 +1,4 @@ -// Copyright 2018 Google LLC. +// Copyright 2019 Google LLC. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -473,11 +473,12 @@ message JobReference { // belongs to. string project_id = 1; - // Optional. The job ID, which must be unique within the project. The job ID - // is generated by the server upon job submission or provided by the user as a - // means to perform retries without creating duplicate jobs. The ID must - // contain only letters (a-z, A-Z), numbers (0-9), underscores (_), or - // hyphens (-). The maximum length is 100 characters. + // Optional. The job ID, which must be unique within the project. + // + // The ID must contain only letters (a-z, A-Z), numbers (0-9), + // underscores (_), or hyphens (-). The maximum length is 100 characters. + // + // If not specified by the caller, the job ID will be provided by the server. string job_id = 2; } @@ -634,8 +635,8 @@ message SubmitJobRequest { Job job = 2; // Optional. A unique id used to identify the request. If the server - // receives two [SubmitJobRequest][google.cloud.dataproc.v1.SubmitJobRequest] - // requests with the same id, then the second request will be ignored and the + // receives two [SubmitJobRequest][google.cloud.dataproc.v1.SubmitJobRequest] requests with the same + // id, then the second request will be ignored and the // first [Job][google.cloud.dataproc.v1.Job] created and stored in the backend // is returned. // diff --git a/google/cloud/dataproc/v1/operations.proto b/google/cloud/dataproc/v1/operations.proto index 7c058766..c820cd8e 100644 --- a/google/cloud/dataproc/v1/operations.proto +++ b/google/cloud/dataproc/v1/operations.proto @@ -1,4 +1,4 @@ -// Copyright 2018 Google LLC. +// Copyright 2019 Google LLC. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/google/cloud/dataproc/v1/shared.proto b/google/cloud/dataproc/v1/shared.proto new file mode 100644 index 00000000..74bd56a8 --- /dev/null +++ b/google/cloud/dataproc/v1/shared.proto @@ -0,0 +1,43 @@ +// Copyright 2019 Google LLC. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +syntax = "proto3"; + +package google.cloud.dataproc.v1; + +import "google/api/annotations.proto"; + +option go_package = "google.golang.org/genproto/googleapis/cloud/dataproc/v1;dataproc"; +option java_multiple_files = true; +option java_outer_classname = "SharedProto"; +option java_package = "com.google.cloud.dataproc.v1"; + +// Cluster components that can be activated. +enum Component { + // Unspecified component. + COMPONENT_UNSPECIFIED = 0; + + // The Anaconda python distribution. + ANACONDA = 5; + + // The Hive Web HCatalog (the REST service for accessing HCatalog). + HIVE_WEBHCAT = 3; + + // The Jupyter Notebook. + JUPYTER = 1; + + // The Zeppelin notebook. + ZEPPELIN = 4; +} diff --git a/google/cloud/dataproc/v1/workflow_templates.proto b/google/cloud/dataproc/v1/workflow_templates.proto index a536698b..61295a55 100644 --- a/google/cloud/dataproc/v1/workflow_templates.proto +++ b/google/cloud/dataproc/v1/workflow_templates.proto @@ -1,4 +1,4 @@ -// Copyright 2018 Google LLC. +// Copyright 2019 Google LLC. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -33,8 +33,7 @@ option java_package = "com.google.cloud.dataproc.v1"; // Cloud Dataproc API. service WorkflowTemplateService { // Creates new workflow template. - rpc CreateWorkflowTemplate(CreateWorkflowTemplateRequest) - returns (WorkflowTemplate) { + rpc CreateWorkflowTemplate(CreateWorkflowTemplateRequest) returns (WorkflowTemplate) { option (google.api.http) = { post: "/v1/{parent=projects/*/locations/*}/workflowTemplates" body: "template" @@ -49,8 +48,7 @@ service WorkflowTemplateService { // // Can retrieve previously instantiated template by specifying optional // version parameter. - rpc GetWorkflowTemplate(GetWorkflowTemplateRequest) - returns (WorkflowTemplate) { + rpc GetWorkflowTemplate(GetWorkflowTemplateRequest) returns (WorkflowTemplate) { option (google.api.http) = { get: "/v1/{name=projects/*/locations/*/workflowTemplates/*}" additional_bindings { @@ -77,8 +75,7 @@ service WorkflowTemplateService { // On successful completion, // [Operation.response][google.longrunning.Operation.response] will be // [Empty][google.protobuf.Empty]. - rpc InstantiateWorkflowTemplate(InstantiateWorkflowTemplateRequest) - returns (google.longrunning.Operation) { + rpc InstantiateWorkflowTemplate(InstantiateWorkflowTemplateRequest) returns (google.longrunning.Operation) { option (google.api.http) = { post: "/v1/{name=projects/*/locations/*/workflowTemplates/*}:instantiate" body: "*" @@ -92,8 +89,7 @@ service WorkflowTemplateService { // Instantiates a template and begins execution. // // This method is equivalent to executing the sequence - // [CreateWorkflowTemplate][google.cloud.dataproc.v1.WorkflowTemplateService.CreateWorkflowTemplate], - // [InstantiateWorkflowTemplate][google.cloud.dataproc.v1.WorkflowTemplateService.InstantiateWorkflowTemplate], + // [CreateWorkflowTemplate][google.cloud.dataproc.v1.WorkflowTemplateService.CreateWorkflowTemplate], [InstantiateWorkflowTemplate][google.cloud.dataproc.v1.WorkflowTemplateService.InstantiateWorkflowTemplate], // [DeleteWorkflowTemplate][google.cloud.dataproc.v1.WorkflowTemplateService.DeleteWorkflowTemplate]. // // The returned Operation can be used to track execution of @@ -112,9 +108,7 @@ service WorkflowTemplateService { // On successful completion, // [Operation.response][google.longrunning.Operation.response] will be // [Empty][google.protobuf.Empty]. - rpc InstantiateInlineWorkflowTemplate( - InstantiateInlineWorkflowTemplateRequest) - returns (google.longrunning.Operation) { + rpc InstantiateInlineWorkflowTemplate(InstantiateInlineWorkflowTemplateRequest) returns (google.longrunning.Operation) { option (google.api.http) = { post: "/v1/{parent=projects/*/locations/*}/workflowTemplates:instantiateInline" body: "template" @@ -127,8 +121,7 @@ service WorkflowTemplateService { // Updates (replaces) workflow template. The updated template // must contain version that matches the current server version. - rpc UpdateWorkflowTemplate(UpdateWorkflowTemplateRequest) - returns (WorkflowTemplate) { + rpc UpdateWorkflowTemplate(UpdateWorkflowTemplateRequest) returns (WorkflowTemplate) { option (google.api.http) = { put: "/v1/{template.name=projects/*/locations/*/workflowTemplates/*}" body: "template" @@ -140,8 +133,7 @@ service WorkflowTemplateService { } // Lists workflows that match the specified filter in the request. - rpc ListWorkflowTemplates(ListWorkflowTemplatesRequest) - returns (ListWorkflowTemplatesResponse) { + rpc ListWorkflowTemplates(ListWorkflowTemplatesRequest) returns (ListWorkflowTemplatesResponse) { option (google.api.http) = { get: "/v1/{parent=projects/*/locations/*}/workflowTemplates" additional_bindings { @@ -151,8 +143,7 @@ service WorkflowTemplateService { } // Deletes a workflow template. It does not cancel in-progress workflows. - rpc DeleteWorkflowTemplate(DeleteWorkflowTemplateRequest) - returns (google.protobuf.Empty) { + rpc DeleteWorkflowTemplate(DeleteWorkflowTemplateRequest) returns (google.protobuf.Empty) { option (google.api.http) = { delete: "/v1/{name=projects/*/locations/*/workflowTemplates/*}" additional_bindings { @@ -284,8 +275,8 @@ message OrderedJob { // // The step id is used as prefix for job id, as job // `goog-dataproc-workflow-step-id` label, and in - // [prerequisiteStepIds][google.cloud.dataproc.v1.OrderedJob.prerequisite_step_ids] - // field from other steps. + // [prerequisiteStepIds][google.cloud.dataproc.v1.OrderedJob.prerequisite_step_ids] field from other + // steps. // // The id must contain only letters (a-z, A-Z), numbers (0-9), // underscores (_), and hyphens (-). Cannot begin or end with underscore @@ -356,10 +347,10 @@ message TemplateParameter { // A field is allowed to appear in at most one parameter's list of field // paths. // - // A field path is similar in syntax to a - // [google.protobuf.FieldMask][google.protobuf.FieldMask]. For example, a - // field path that references the zone field of a workflow template's cluster - // selector would be specified as `placement.clusterSelector.zone`. + // A field path is similar in syntax to a [google.protobuf.FieldMask][google.protobuf.FieldMask]. + // For example, a field path that references the zone field of a workflow + // template's cluster selector would be specified as + // `placement.clusterSelector.zone`. // // Also, field paths can reference fields using the following syntax: // diff --git a/google/cloud/dataproc/v1beta2/dataproc_gapic.yaml b/google/cloud/dataproc/v1beta2/dataproc_gapic.yaml index 67940f45..576bfc8e 100644 --- a/google/cloud/dataproc/v1beta2/dataproc_gapic.yaml +++ b/google/cloud/dataproc/v1beta2/dataproc_gapic.yaml @@ -670,6 +670,7 @@ interfaces: - parameters: - name - parameters: + - name - parameters required_fields: - name