diff --git a/google/privacy/dlp/v2/dlp.proto b/google/privacy/dlp/v2/dlp.proto index 0dc4119b..6dcf88ca 100644 --- a/google/privacy/dlp/v2/dlp.proto +++ b/google/privacy/dlp/v2/dlp.proto @@ -1,4 +1,4 @@ -// Copyright 2019 Google LLC. +// Copyright 2020 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,7 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -// syntax = "proto3"; @@ -49,7 +48,8 @@ option php_namespace = "Google\\Cloud\\Dlp\\V2"; // https://cloud.google.com/dlp/docs/. service DlpService { option (google.api.default_host) = "dlp.googleapis.com"; - option (google.api.oauth_scopes) = "https://www.googleapis.com/auth/cloud-platform"; + option (google.api.oauth_scopes) = + "https://www.googleapis.com/auth/cloud-platform"; // Finds potentially sensitive info in content. // This method has limits on input size, processing time, and output size. @@ -98,7 +98,8 @@ service DlpService { // When no InfoTypes or CustomInfoTypes are specified in this request, the // system will automatically choose what detectors to run. By default this may // be all types, but may change over time as detectors are updated. - rpc DeidentifyContent(DeidentifyContentRequest) returns (DeidentifyContentResponse) { + rpc DeidentifyContent(DeidentifyContentRequest) + returns (DeidentifyContentResponse) { option (google.api.http) = { post: "/v2/{parent=projects/*}/content:deidentify" body: "*" @@ -113,7 +114,8 @@ service DlpService { // See // https://cloud.google.com/dlp/docs/pseudonymization#re-identification_in_free_text_code_example // to learn more. - rpc ReidentifyContent(ReidentifyContentRequest) returns (ReidentifyContentResponse) { + rpc ReidentifyContent(ReidentifyContentRequest) + returns (ReidentifyContentResponse) { option (google.api.http) = { post: "/v2/{parent=projects/*}/content:reidentify" body: "*" @@ -130,9 +132,7 @@ service DlpService { rpc ListInfoTypes(ListInfoTypesRequest) returns (ListInfoTypesResponse) { option (google.api.http) = { get: "/v2/infoTypes" - additional_bindings { - get: "/v2/locations/{location_id}/infoTypes" - } + additional_bindings { get: "/v2/locations/{location_id}/infoTypes" } }; option (google.api.method_signature) = "location_id"; } @@ -140,7 +140,8 @@ service DlpService { // Creates an InspectTemplate for re-using frequently used configuration // for inspecting content, images, and storage. // See https://cloud.google.com/dlp/docs/creating-templates to learn more. - rpc CreateInspectTemplate(CreateInspectTemplateRequest) returns (InspectTemplate) { + rpc CreateInspectTemplate(CreateInspectTemplateRequest) + returns (InspectTemplate) { option (google.api.http) = { post: "/v2/{parent=organizations/*}/inspectTemplates" body: "*" @@ -158,12 +159,14 @@ service DlpService { } }; option (google.api.method_signature) = "parent,inspect_template"; - option (google.api.method_signature) = "parent,inspect_template,location_id"; + option (google.api.method_signature) = + "parent,inspect_template,location_id"; } // Updates the InspectTemplate. // See https://cloud.google.com/dlp/docs/creating-templates to learn more. - rpc UpdateInspectTemplate(UpdateInspectTemplateRequest) returns (InspectTemplate) { + rpc UpdateInspectTemplate(UpdateInspectTemplateRequest) + returns (InspectTemplate) { option (google.api.http) = { patch: "/v2/{name=organizations/*/inspectTemplates/*}" body: "*" @@ -191,9 +194,7 @@ service DlpService { additional_bindings { get: "/v2/{name=organizations/*/locations/*/inspectTemplates/*}" } - additional_bindings { - get: "/v2/{name=projects/*/inspectTemplates/*}" - } + additional_bindings { get: "/v2/{name=projects/*/inspectTemplates/*}" } additional_bindings { get: "/v2/{name=projects/*/locations/*/inspectTemplates/*}" } @@ -203,15 +204,14 @@ service DlpService { // Lists InspectTemplates. // See https://cloud.google.com/dlp/docs/creating-templates to learn more. - rpc ListInspectTemplates(ListInspectTemplatesRequest) returns (ListInspectTemplatesResponse) { + rpc ListInspectTemplates(ListInspectTemplatesRequest) + returns (ListInspectTemplatesResponse) { option (google.api.http) = { get: "/v2/{parent=organizations/*}/inspectTemplates" additional_bindings { get: "/v2/{parent=organizations/*}/locations/{location_id}/inspectTemplates" } - additional_bindings { - get: "/v2/{parent=projects/*}/inspectTemplates" - } + additional_bindings { get: "/v2/{parent=projects/*}/inspectTemplates" } additional_bindings { get: "/v2/{parent=projects/*}/locations/{location_id}/inspectTemplates" } @@ -221,15 +221,14 @@ service DlpService { // Deletes an InspectTemplate. // See https://cloud.google.com/dlp/docs/creating-templates to learn more. - rpc DeleteInspectTemplate(DeleteInspectTemplateRequest) returns (google.protobuf.Empty) { + rpc DeleteInspectTemplate(DeleteInspectTemplateRequest) + returns (google.protobuf.Empty) { option (google.api.http) = { delete: "/v2/{name=organizations/*/inspectTemplates/*}" additional_bindings { delete: "/v2/{name=organizations/*/locations/*/inspectTemplates/*}" } - additional_bindings { - delete: "/v2/{name=projects/*/inspectTemplates/*}" - } + additional_bindings { delete: "/v2/{name=projects/*/inspectTemplates/*}" } additional_bindings { delete: "/v2/{name=projects/*/locations/*/inspectTemplates/*}" } @@ -241,7 +240,8 @@ service DlpService { // for de-identifying content, images, and storage. // See https://cloud.google.com/dlp/docs/creating-templates-deid to learn // more. - rpc CreateDeidentifyTemplate(CreateDeidentifyTemplateRequest) returns (DeidentifyTemplate) { + rpc CreateDeidentifyTemplate(CreateDeidentifyTemplateRequest) + returns (DeidentifyTemplate) { option (google.api.http) = { post: "/v2/{parent=organizations/*}/deidentifyTemplates" body: "*" @@ -259,13 +259,15 @@ service DlpService { } }; option (google.api.method_signature) = "parent,deidentify_template"; - option (google.api.method_signature) = "parent,deidentify_template,location_id"; + option (google.api.method_signature) = + "parent,deidentify_template,location_id"; } // Updates the DeidentifyTemplate. // See https://cloud.google.com/dlp/docs/creating-templates-deid to learn // more. - rpc UpdateDeidentifyTemplate(UpdateDeidentifyTemplateRequest) returns (DeidentifyTemplate) { + rpc UpdateDeidentifyTemplate(UpdateDeidentifyTemplateRequest) + returns (DeidentifyTemplate) { option (google.api.http) = { patch: "/v2/{name=organizations/*/deidentifyTemplates/*}" body: "*" @@ -282,21 +284,21 @@ service DlpService { body: "*" } }; - option (google.api.method_signature) = "name,deidentify_template,update_mask"; + option (google.api.method_signature) = + "name,deidentify_template,update_mask"; } // Gets a DeidentifyTemplate. // See https://cloud.google.com/dlp/docs/creating-templates-deid to learn // more. - rpc GetDeidentifyTemplate(GetDeidentifyTemplateRequest) returns (DeidentifyTemplate) { + rpc GetDeidentifyTemplate(GetDeidentifyTemplateRequest) + returns (DeidentifyTemplate) { option (google.api.http) = { get: "/v2/{name=organizations/*/deidentifyTemplates/*}" additional_bindings { get: "/v2/{name=organizations/*/locations/*/deidentifyTemplates/*}" } - additional_bindings { - get: "/v2/{name=projects/*/deidentifyTemplates/*}" - } + additional_bindings { get: "/v2/{name=projects/*/deidentifyTemplates/*}" } additional_bindings { get: "/v2/{name=projects/*/locations/*/deidentifyTemplates/*}" } @@ -307,15 +309,14 @@ service DlpService { // Lists DeidentifyTemplates. // See https://cloud.google.com/dlp/docs/creating-templates-deid to learn // more. - rpc ListDeidentifyTemplates(ListDeidentifyTemplatesRequest) returns (ListDeidentifyTemplatesResponse) { + rpc ListDeidentifyTemplates(ListDeidentifyTemplatesRequest) + returns (ListDeidentifyTemplatesResponse) { option (google.api.http) = { get: "/v2/{parent=organizations/*}/deidentifyTemplates" additional_bindings { get: "/v2/{parent=organizations/*}/locations/{location_id}/deidentifyTemplates" } - additional_bindings { - get: "/v2/{parent=projects/*}/deidentifyTemplates" - } + additional_bindings { get: "/v2/{parent=projects/*}/deidentifyTemplates" } additional_bindings { get: "/v2/{parent=projects/*}/locations/{location_id}/deidentifyTemplates" } @@ -326,7 +327,8 @@ service DlpService { // Deletes a DeidentifyTemplate. // See https://cloud.google.com/dlp/docs/creating-templates-deid to learn // more. - rpc DeleteDeidentifyTemplate(DeleteDeidentifyTemplateRequest) returns (google.protobuf.Empty) { + rpc DeleteDeidentifyTemplate(DeleteDeidentifyTemplateRequest) + returns (google.protobuf.Empty) { option (google.api.http) = { delete: "/v2/{name=organizations/*/deidentifyTemplates/*}" additional_bindings { @@ -372,6 +374,21 @@ service DlpService { option (google.api.method_signature) = "name,job_trigger,update_mask"; } + // Inspect hybrid content and store findings to a trigger. The inspection + // will be processed asynchronously. To review the findings monitor the + // jobs within the trigger. + // Early access feature is in a pre-release state and might change or have + // limited support. For more information, see + // https://cloud.google.com/products#product-launch-stages. + rpc HybridInspectJobTrigger(HybridInspectJobTriggerRequest) + returns (HybridInspectResponse) { + option (google.api.http) = { + post: "/v2/{name=projects/*/locations/*/jobTriggers/*}:hybridInspect" + body: "*" + }; + option (google.api.method_signature) = "name"; + } + // Gets a job trigger. // See https://cloud.google.com/dlp/docs/creating-job-triggers to learn more. rpc GetJobTrigger(GetJobTriggerRequest) returns (JobTrigger) { @@ -386,7 +403,8 @@ service DlpService { // Lists job triggers. // See https://cloud.google.com/dlp/docs/creating-job-triggers to learn more. - rpc ListJobTriggers(ListJobTriggersRequest) returns (ListJobTriggersResponse) { + rpc ListJobTriggers(ListJobTriggersRequest) + returns (ListJobTriggersResponse) { option (google.api.http) = { get: "/v2/{parent=projects/*}/jobTriggers" additional_bindings { @@ -398,7 +416,8 @@ service DlpService { // Deletes a job trigger. // See https://cloud.google.com/dlp/docs/creating-job-triggers to learn more. - rpc DeleteJobTrigger(DeleteJobTriggerRequest) returns (google.protobuf.Empty) { + rpc DeleteJobTrigger(DeleteJobTriggerRequest) + returns (google.protobuf.Empty) { option (google.api.http) = { delete: "/v2/{name=projects/*/jobTriggers/*}" additional_bindings { @@ -462,9 +481,7 @@ service DlpService { rpc GetDlpJob(GetDlpJobRequest) returns (DlpJob) { option (google.api.http) = { get: "/v2/{name=projects/*/dlpJobs/*}" - additional_bindings { - get: "/v2/{name=projects/*/locations/*/dlpJobs/*}" - } + additional_bindings { get: "/v2/{name=projects/*/locations/*/dlpJobs/*}" } }; option (google.api.method_signature) = "name"; } @@ -503,7 +520,8 @@ service DlpService { // Creates a pre-built stored infoType to be used for inspection. // See https://cloud.google.com/dlp/docs/creating-stored-infotypes to // learn more. - rpc CreateStoredInfoType(CreateStoredInfoTypeRequest) returns (StoredInfoType) { + rpc CreateStoredInfoType(CreateStoredInfoTypeRequest) + returns (StoredInfoType) { option (google.api.http) = { post: "/v2/{parent=organizations/*}/storedInfoTypes" body: "*" @@ -528,7 +546,8 @@ service DlpService { // will continue to be used until the new version is ready. // See https://cloud.google.com/dlp/docs/creating-stored-infotypes to // learn more. - rpc UpdateStoredInfoType(UpdateStoredInfoTypeRequest) returns (StoredInfoType) { + rpc UpdateStoredInfoType(UpdateStoredInfoTypeRequest) + returns (StoredInfoType) { option (google.api.http) = { patch: "/v2/{name=organizations/*/storedInfoTypes/*}" body: "*" @@ -557,9 +576,7 @@ service DlpService { additional_bindings { get: "/v2/{name=organizations/*/locations/*/storedInfoTypes/*}" } - additional_bindings { - get: "/v2/{name=projects/*/storedInfoTypes/*}" - } + additional_bindings { get: "/v2/{name=projects/*/storedInfoTypes/*}" } additional_bindings { get: "/v2/{name=projects/*/locations/*/storedInfoTypes/*}" } @@ -570,15 +587,14 @@ service DlpService { // Lists stored infoTypes. // See https://cloud.google.com/dlp/docs/creating-stored-infotypes to // learn more. - rpc ListStoredInfoTypes(ListStoredInfoTypesRequest) returns (ListStoredInfoTypesResponse) { + rpc ListStoredInfoTypes(ListStoredInfoTypesRequest) + returns (ListStoredInfoTypesResponse) { option (google.api.http) = { get: "/v2/{parent=organizations/*}/storedInfoTypes" additional_bindings { get: "/v2/{parent=organizations/*}/locations/{location_id}/storedInfoTypes" } - additional_bindings { - get: "/v2/{parent=projects/*}/storedInfoTypes" - } + additional_bindings { get: "/v2/{parent=projects/*}/storedInfoTypes" } additional_bindings { get: "/v2/{parent=projects/*}/locations/{location_id}/storedInfoTypes" } @@ -589,21 +605,47 @@ service DlpService { // Deletes a stored infoType. // See https://cloud.google.com/dlp/docs/creating-stored-infotypes to // learn more. - rpc DeleteStoredInfoType(DeleteStoredInfoTypeRequest) returns (google.protobuf.Empty) { + rpc DeleteStoredInfoType(DeleteStoredInfoTypeRequest) + returns (google.protobuf.Empty) { option (google.api.http) = { delete: "/v2/{name=organizations/*/storedInfoTypes/*}" additional_bindings { delete: "/v2/{name=organizations/*/locations/*/storedInfoTypes/*}" } - additional_bindings { - delete: "/v2/{name=projects/*/storedInfoTypes/*}" - } + additional_bindings { delete: "/v2/{name=projects/*/storedInfoTypes/*}" } additional_bindings { delete: "/v2/{name=projects/*/locations/*/storedInfoTypes/*}" } }; option (google.api.method_signature) = "name"; } + + // Inspect hybrid content and store findings to a job. + // To review the findings inspect the job. Inspection will occur + // asynchronously. + // Early access feature is in a pre-release state and might change or have + // limited support. For more information, see + // https://cloud.google.com/products#product-launch-stages. + rpc HybridInspectDlpJob(HybridInspectDlpJobRequest) + returns (HybridInspectResponse) { + option (google.api.http) = { + post: "/v2/{name=projects/*/locations/*/dlpJobs/*}:hybridInspect" + body: "*" + }; + option (google.api.method_signature) = "name"; + } + + // Finish a running hybrid DlpJob. Triggers the finalization steps and running + // of any enabled actions that have not yet run. + // Early access feature is in a pre-release state and might change or have + // limited support. For more information, see + // https://cloud.google.com/products#product-launch-stages. + rpc FinishDlpJob(FinishDlpJobRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { + post: "/v2/{name=projects/*/locations/*/dlpJobs/*}:finish" + body: "*" + }; + } } // List of exclude infoTypes. @@ -681,7 +723,7 @@ message InspectConfig { } // Max number of findings that will be returned for each item scanned. - // When set within `InspectDataSourceRequest`, + // When set within `InspectJobConfig`, // the maximum returned is 2000 regardless if this is set higher. // When set within `InspectContentRequest`, this field is ignored. int32 max_findings_per_item = 1; @@ -823,6 +865,16 @@ message InspectResult { // Represents a piece of potentially sensitive content. message Finding { + option (google.api.resource) = { + type: "dlp.googleapis.com/InspectFinding" + pattern: "projects/{project}/locations/{location}/findings/{finding}" + }; + + // Resource name in format + // projects/{project}/locations/{location}/findings/{finding} + // Populated only when viewing persisted findings. + string name = 14; + // The content that was found. Even if the content is not textual, it // may be converted to a textual representation here. // Provided if `include_quote` is true and the finding is @@ -847,6 +899,44 @@ message Finding { // to true and a supported infoType was requested. Currently supported // infoTypes: DATE, DATE_OF_BIRTH and TIME. QuoteInfo quote_info = 7; + + // The job that stored the finding. + string resource_name = 8 + [(google.api.resource_reference) = { type: "dlp.googleapis.com/DlpJob" }]; + + // Job trigger name, if applicable, for this finding. + // (-- api-linter: core::0122::name-suffix=disabled + // aip.dev/not-precedent: AIP-122 discourages _name suffixes for + // resource names, but this has existed as part of the bigquery schema + // before this rule existed. --) + string trigger_name = 9 [ + (google.api.resource_reference) = { type: "dlp.googleapis.com/JobTrigger" } + ]; + + // The labels associated with this `InspectFinding`. + // + // Label keys must be between 1 and 63 characters long and must conform + // to the following regular expression: \[a-z\](\[-a-z0-9\]*\[a-z0-9\])?. + // + // Label values must be between 0 and 63 characters long and must conform + // to the regular expression (\[a-z\](\[-a-z0-9\]*\[a-z0-9\])?)?. + // + // No more than 10 labels can be associated with a given finding. + // + // Example: "environment" : "production" + // Example: "pipeline" : "etl" + map labels = 10; + + // Time the job started that produced this finding. + google.protobuf.Timestamp job_create_time = 11; + + // The job that stored the finding. + // (-- api-linter: core::0122::name-suffix=disabled + // aip.dev/not-precedent: AIP-122 discourages _name suffixes for + // resource names, but this has existed as part of the bigquery schema + // before this rule existed. --) + string job_name = 13 + [(google.api.resource_reference) = { type: "dlp.googleapis.com/DlpJob" }]; } // Specifies the location of the finding. @@ -866,9 +956,13 @@ message Location { // List of nested objects pointing to the precise location of the finding // within the file or record. repeated ContentLocation content_locations = 7; + + // Information about the container where this finding occurred, if available. + Container container = 8; } -// Findings container location data. +// Precise location of the finding within a document, record, image, or metadata +// container. message ContentLocation { // Name of the container where the finding is located. // The top level name is the source file name or table name. Names of some @@ -930,6 +1024,49 @@ message TableLocation { int64 row_index = 1; } +// Represents a container that may contain DLP findings. +// Examples of a container include a file, table, or database record. +message Container { + // Container type, for example BigQuery or Google Cloud Storage. + string type = 1; + + // Project where the finding was found. + // Can be different from the project that owns the finding. + string project_id = 2; + + // A string representation of the full container name. + // Examples: + // - BigQuery: 'Project:DataSetId.TableId' + // - Google Cloud Storage: 'gs://Bucket/folders/filename.txt' + string full_path = 3; + + // The root of the container. + // Examples: + // - For BigQuery table `project_id:dataset_id.table_id`, the root is + // `dataset_id` + // - For Google Cloud Storage file `gs://bucket/folder/filename.txt`, the root + // is `gs://bucket` + string root_path = 4; + + // The rest of the path after the root. + // Examples: + // - For BigQuery table `project_id:dataset_id.table_id`, the relative path is + // `table_id` + // - Google Cloud Storage file `gs://bucket/folder/filename.txt`, the relative + // path is `folder/filename.txt` + string relative_path = 5; + + // Findings container modification timestamp, if applicable. + // For Google Cloud Storage contains last file modification timestamp. + // For BigQuery table contains last_modified_time property. + // For Datastore - not populated. + google.protobuf.Timestamp update_time = 6; + + // Findings container version, if available + // ("generation" for Google Cloud Storage). + string version = 7; +} + // Generic half-open interval [start, end) message Range { // Index of the first character of the range (inclusive). @@ -985,8 +1122,8 @@ message RedactImageRequest { // The parent resource name, for example projects/my-project-id. string parent = 1 [(google.api.resource_reference) = { - type: "cloudresourcemanager.googleapis.com/Project" - }]; + type: "cloudresourcemanager.googleapis.com/Project" + }]; // The geographic location to process the request. Reserved for future // extensions. @@ -1036,8 +1173,8 @@ message RedactImageResponse { message DeidentifyContentRequest { // The parent resource name, for example projects/my-project-id. string parent = 1 [(google.api.resource_reference) = { - type: "cloudresourcemanager.googleapis.com/Project" - }]; + type: "cloudresourcemanager.googleapis.com/Project" + }]; // Configuration for the de-identification of the content item. // Items specified here will override the template referenced by the @@ -1141,8 +1278,8 @@ message ReidentifyContentResponse { message InspectContentRequest { // The parent resource name, for example projects/my-project-id. string parent = 1 [(google.api.resource_reference) = { - type: "cloudresourcemanager.googleapis.com/Project" - }]; + type: "cloudresourcemanager.googleapis.com/Project" + }]; // Configuration for the inspector. What specified here will override // the template referenced by the inspect_template_name argument. @@ -1172,6 +1309,7 @@ message InspectContentResponse { // Cloud repository for storing output. message OutputStorageConfig { // Predefined schemas for storing findings. + // Only for use with external storage. enum OutputSchema { // Unused. OUTPUT_SCHEMA_UNSPECIFIED = 0; @@ -1221,6 +1359,7 @@ message OutputStorageConfig { // If unspecified, then all available columns will be used for a new table or // an (existing) table with no schema, and no changes will be made to an // existing table that has a schema. + // Only for use with external storage. OutputSchema output_schema = 3; } @@ -1256,6 +1395,12 @@ message InspectDataSourceDetails { // Statistics of how many instances of each info type were found during // inspect job. repeated InfoTypeStats info_type_stats = 3; + + // Statistics related to the processing of hybrid inspect. + // Early access feature is in a pre-release state and might change or have + // limited support. For more information, see + // https://cloud.google.com/products#product-launch-stages. + HybridInspectStatistics hybrid_stats = 7; } // The configuration used for this job. @@ -1265,6 +1410,23 @@ message InspectDataSourceDetails { Result result = 3; } +// Statistics related to processing hybrid inspect requests.s +message HybridInspectStatistics { + // The number of hybrid inspection requests processed within this job. + int64 processed_count = 1; + + // The number of hybrid inspection requests aborted because the job ran + // out of quota or was ended before they could be processed. + int64 aborted_count = 2; + + // The number of hybrid requests currently being processed. Only populated + // when called via method `getDlpJob`. + // A burst of traffic may cause hybrid inspect requests to be enqueued. + // Processing will take place as quickly as possible, but resource limitations + // may impact how long a request is enqueued for. + int64 pending_count = 3; +} + // InfoType description. message InfoTypeDescription { // Internal name of the infoType. @@ -1368,10 +1530,11 @@ message StatisticalTable { BigQueryTable table = 3 [(google.api.field_behavior) = REQUIRED]; // Required. Quasi-identifier columns. - repeated QuasiIdentifierField quasi_ids = 1 [(google.api.field_behavior) = REQUIRED]; + repeated QuasiIdentifierField quasi_ids = 1 + [(google.api.field_behavior) = REQUIRED]; - // Required. The relative frequency column must contain a floating-point number - // between 0 and 1 (inclusive). Null values are assumed to be zero. + // Required. The relative frequency column must contain a floating-point + // number between 0 and 1 (inclusive). Null values are assumed to be zero. FieldId relative_frequency = 2 [(google.api.field_behavior) = REQUIRED]; } @@ -1484,15 +1647,16 @@ message PrivacyMetric { BigQueryTable table = 3 [(google.api.field_behavior) = REQUIRED]; // Required. Quasi-identifier columns. - repeated QuasiIdField quasi_ids = 1 [(google.api.field_behavior) = REQUIRED]; + repeated QuasiIdField quasi_ids = 1 + [(google.api.field_behavior) = REQUIRED]; - // Required. The relative frequency column must contain a floating-point number - // between 0 and 1 (inclusive). Null values are assumed to be zero. + // Required. The relative frequency column must contain a floating-point + // number between 0 and 1 (inclusive). Null values are assumed to be zero. FieldId relative_frequency = 2 [(google.api.field_behavior) = REQUIRED]; } - // Required. Fields considered to be quasi-identifiers. No two columns can have the - // same tag. + // Required. Fields considered to be quasi-identifiers. No two columns can + // have the same tag. repeated TaggedField quasi_ids = 1 [(google.api.field_behavior) = REQUIRED]; // ISO 3166-1 alpha-2 region code to use in the statistical modeling. @@ -1511,8 +1675,8 @@ message PrivacyMetric { // Similarly to the k-map metric, we cannot compute δ-presence exactly without // knowing the attack dataset, so we use a statistical model instead. message DeltaPresenceEstimationConfig { - // Required. Fields considered to be quasi-identifiers. No two fields can have the - // same tag. + // Required. Fields considered to be quasi-identifiers. No two fields can + // have the same tag. repeated QuasiId quasi_ids = 1 [(google.api.field_behavior) = REQUIRED]; // ISO 3166-1 alpha-2 region code to use in the statistical modeling. @@ -1585,7 +1749,8 @@ message AnalyzeDataSourceRiskDetails { } // Histogram of value frequencies in the column. - repeated CategoricalStatsHistogramBucket value_frequency_histogram_buckets = 5; + repeated CategoricalStatsHistogramBucket value_frequency_histogram_buckets = + 5; } // Result of the k-anonymity computation. @@ -1665,7 +1830,8 @@ message AnalyzeDataSourceRiskDetails { } // Histogram of l-diversity equivalence class sensitive value frequencies. - repeated LDiversityHistogramBucket sensitive_value_frequency_histogram_buckets = 5; + repeated LDiversityHistogramBucket + sensitive_value_frequency_histogram_buckets = 5; } // Result of the reidentifiability analysis. Note that these results are an @@ -1771,7 +1937,8 @@ message AnalyzeDataSourceRiskDetails { // {min_probability: 0.3, max_probability: 0.4, frequency: 99} // mean that there are no record with an estimated probability in [0.1, 0.2) // nor larger or equal to 0.4. - repeated DeltaPresenceEstimationHistogramBucket delta_presence_estimation_histogram = 1; + repeated DeltaPresenceEstimationHistogramBucket + delta_presence_estimation_histogram = 1; } // Privacy metric to compute. @@ -1891,6 +2058,38 @@ message DeidentifyConfig { // a column within a table. RecordTransformations record_transformations = 2; } + + // Mode for handling transformation errors. If left unspecified, the default + // mode is `TransformationErrorHandling.ThrowError`. + TransformationErrorHandling transformation_error_handling = 3; +} + +// How to handle transformation errors during de-identification. A +// transformation error occurs when the requested transformation is incompatible +// with the data. For example, trying to de-identify an IP address using a +// `DateShift` transformation would result in a transformation error, since date +// info cannot be extracted from an IP address. +// Information about any incompatible transformations, and how they were +// handled, is returned in the response as part of the +// `TransformationOverviews`. +message TransformationErrorHandling { + // Throw an error and fail the request when a transformation error occurs. + message ThrowError {} + + // Skips the data without modifying it if the requested transformation would + // cause an error. For example, if a `DateShift` transformation were applied + // an an IP address, this mode would leave the IP address unchanged in the + // response. + message LeaveUntransformed {} + + // How transformation errors should be handled. + oneof mode { + // Throw an error + ThrowError throw_error = 1; + + // Ignore errors + LeaveUntransformed leave_untransformed = 2; + } } // A rule for transforming a value. @@ -2043,16 +2242,12 @@ message ReplaceValueConfig { } // Replace each matching finding with the name of the info_type. -message ReplaceWithInfoTypeConfig { - -} +message ReplaceWithInfoTypeConfig {} // Redact a given value. For example, if used with an `InfoTypeTransformation` // transforming PHONE_NUMBER, and input 'My phone number is 206-555-0123', the // output would be 'My phone number is '. -message RedactConfig { - -} +message RedactConfig {} // Characters to skip when doing deidentification of a value. These will be left // alone and skipped. @@ -2136,18 +2331,18 @@ message CharacterMaskConfig { // // See https://cloud.google.com/dlp/docs/concepts-bucketing to learn more. message FixedSizeBucketingConfig { - // Required. Lower bound value of buckets. All values less than `lower_bound` are - // grouped together into a single bucket; for example if `lower_bound` = 10, - // then all values less than 10 are replaced with the value “-10”. + // Required. Lower bound value of buckets. All values less than `lower_bound` + // are grouped together into a single bucket; for example if `lower_bound` = + // 10, then all values less than 10 are replaced with the value “-10”. Value lower_bound = 1 [(google.api.field_behavior) = REQUIRED]; - // Required. Upper bound value of buckets. All values greater than upper_bound are - // grouped together into a single bucket; for example if `upper_bound` = 89, - // then all values greater than 89 are replaced with the value “89+”. + // Required. Upper bound value of buckets. All values greater than upper_bound + // are grouped together into a single bucket; for example if `upper_bound` = + // 89, then all values greater than 89 are replaced with the value “89+”. Value upper_bound = 2 [(google.api.field_behavior) = REQUIRED]; - // Required. Size of each bucket (except for minimum and maximum buckets). So if - // `lower_bound` = 10, `upper_bound` = 89, and `bucket_size` = 10, then the + // Required. Size of each bucket (except for minimum and maximum buckets). So + // if `lower_bound` = 10, `upper_bound` = 89, and `bucket_size` = 10, then the // following buckets would be used: -10, 10-20, 20-30, 30-40, 40-50, 50-60, // 60-70, 70-80, 80-89, 89+. Precision up to 2 decimals works. double bucket_size = 3 [(google.api.field_behavior) = REQUIRED]; @@ -2341,14 +2536,15 @@ message KmsWrappedCryptoKey { // same context. See https://cloud.google.com/dlp/docs/concepts-date-shifting // to learn more. message DateShiftConfig { - // Required. Range of shift in days. Actual shift will be selected at random within this - // range (inclusive ends). Negative means shift to earlier in time. Must not - // be more than 365250 days (1000 years) each direction. + // Required. Range of shift in days. Actual shift will be selected at random + // within this range (inclusive ends). Negative means shift to earlier in + // time. Must not be more than 365250 days (1000 years) each direction. // // For example, 3 means shift date to at most 3 days into the future. int32 upper_bound_days = 1 [(google.api.field_behavior) = REQUIRED]; - // Required. For example, -5 means shift date to at most 5 days back in the past. + // Required. For example, -5 means shift date to at most 5 days back in the + // past. int32 lower_bound_days = 2 [(google.api.field_behavior) = REQUIRED]; // Points to the field that contains the context, for example, an entity id. @@ -2380,12 +2576,14 @@ message InfoTypeTransformations { repeated InfoType info_types = 1; // Required. Primitive transformation to apply to the infoType. - PrimitiveTransformation primitive_transformation = 2 [(google.api.field_behavior) = REQUIRED]; + PrimitiveTransformation primitive_transformation = 2 + [(google.api.field_behavior) = REQUIRED]; } // Required. Transformation for each infoType. Cannot specify more than one // for a given infoType. - repeated InfoTypeTransformation transformations = 1 [(google.api.field_behavior) = REQUIRED]; + repeated InfoTypeTransformation transformations = 1 + [(google.api.field_behavior) = REQUIRED]; } // The transformation to apply to the field. @@ -2577,6 +2775,10 @@ message Schedule { } } +// Job trigger option for hybrid jobs. Jobs must be manually created +// and finished. +message Manual {} + // The inspectTemplate contains a configuration (set of types of sensitive data // to be detected) to be used anywhere you otherwise would normally specify // InspectConfig. See https://cloud.google.com/dlp/docs/concepts-templates @@ -2586,6 +2788,8 @@ message InspectTemplate { type: "dlp.googleapis.com/InspectTemplate" pattern: "organizations/{organization}/inspectTemplates/{inspect_template}" pattern: "projects/{project}/inspectTemplates/{inspect_template}" + pattern: "organizations/{organization}/locations/{location}/inspectTemplates/{inspect_template}" + pattern: "projects/{project}/locations/{location}/inspectTemplates/{inspect_template}" }; // Output only. The template name. @@ -2602,10 +2806,12 @@ message InspectTemplate { string description = 3; // Output only. The creation timestamp of an inspectTemplate. - google.protobuf.Timestamp create_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; + google.protobuf.Timestamp create_time = 4 + [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The last update timestamp of an inspectTemplate. - google.protobuf.Timestamp update_time = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; + google.protobuf.Timestamp update_time = 5 + [(google.api.field_behavior) = OUTPUT_ONLY]; // The core content of the template. Configuration of the scanning process. InspectConfig inspect_config = 6; @@ -2618,6 +2824,8 @@ message DeidentifyTemplate { type: "dlp.googleapis.com/DeidentifyTemplate" pattern: "organizations/{organization}/deidentifyTemplates/{deidentify_template}" pattern: "projects/{project}/deidentifyTemplates/{deidentify_template}" + pattern: "organizations/{organization}/locations/{location}/deidentifyTemplates/{deidentify_template}" + pattern: "projects/{project}/locations/{location}/deidentifyTemplates/{deidentify_template}" }; // Output only. The template name. @@ -2634,10 +2842,12 @@ message DeidentifyTemplate { string description = 3; // Output only. The creation timestamp of an inspectTemplate. - google.protobuf.Timestamp create_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY]; + google.protobuf.Timestamp create_time = 4 + [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The last update timestamp of an inspectTemplate. - google.protobuf.Timestamp update_time = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; + google.protobuf.Timestamp update_time = 5 + [(google.api.field_behavior) = OUTPUT_ONLY]; // ///////////// // The core content of the template // /////////////// DeidentifyConfig deidentify_config = 6; @@ -2659,6 +2869,7 @@ message JobTrigger { option (google.api.resource) = { type: "dlp.googleapis.com/JobTrigger" pattern: "projects/{project}/jobTriggers/{job_trigger}" + pattern: "projects/{project}/locations/{location}/jobTriggers/{job_trigger}" }; // What event needs to occur for a new job to be started. @@ -2666,6 +2877,12 @@ message JobTrigger { oneof trigger { // Create a job on a repeating basis based on the elapse of time. Schedule schedule = 1; + + // For use with hybrid jobs. Jobs must be manually created and finished. + // Early access feature is in a pre-release state and might change or have + // limited support. For more information, see + // https://cloud.google.com/products#product-launch-stages. + Manual manual = 2; } } @@ -2709,20 +2926,23 @@ message JobTrigger { // a single Schedule trigger and must have at least one object. repeated Trigger triggers = 5; - // Output only. A stream of errors encountered when the trigger was activated. Repeated - // errors may result in the JobTrigger automatically being paused. + // Output only. A stream of errors encountered when the trigger was activated. + // Repeated errors may result in the JobTrigger automatically being paused. // Will return the last 100 errors. Whenever the JobTrigger is modified // this list will be cleared. repeated Error errors = 6 [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The creation timestamp of a triggeredJob. - google.protobuf.Timestamp create_time = 7 [(google.api.field_behavior) = OUTPUT_ONLY]; + google.protobuf.Timestamp create_time = 7 + [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The last update timestamp of a triggeredJob. - google.protobuf.Timestamp update_time = 8 [(google.api.field_behavior) = OUTPUT_ONLY]; + google.protobuf.Timestamp update_time = 8 + [(google.api.field_behavior) = OUTPUT_ONLY]; // Output only. The timestamp of the last time this trigger executed. - google.protobuf.Timestamp last_run_time = 9 [(google.api.field_behavior) = OUTPUT_ONLY]; + google.protobuf.Timestamp last_run_time = 9 + [(google.api.field_behavior) = OUTPUT_ONLY]; // Required. A status for this trigger. Status status = 10 [(google.api.field_behavior) = REQUIRED]; @@ -2763,9 +2983,7 @@ message Action { // service-specific policy, see https://cloud.google.com/terms/service-terms // Only a single instance of this action can be specified. // Compatible with: Inspect - message PublishSummaryToCscc { - - } + message PublishSummaryToCscc {} // Publish findings of a DlpJob to Cloud Data Catalog. Labels summarizing the // results of the DlpJob will be applied to the entry for the resource scanned @@ -2777,23 +2995,17 @@ message Action { // Only a single instance of this action can be specified and only allowed if // all resources being scanned are BigQuery tables. // Compatible with: Inspect - message PublishFindingsToCloudDataCatalog { - - } + message PublishFindingsToCloudDataCatalog {} // Enable email notification to project owners and editors on jobs's // completion/failure. - message JobNotificationEmails { - - } + message JobNotificationEmails {} // Enable Stackdriver metric dlp.googleapis.com/finding_count. This // will publish a metric to stack driver on each infotype requested and // how many findings were found for it. CustomDetectors will be bucketed // as 'Custom' under the Stackdriver label 'info_type'. - message PublishToStackdriver { - - } + message PublishToStackdriver {} oneof action { // Save resulting findings in a provided location. @@ -2806,7 +3018,8 @@ message Action { PublishSummaryToCscc publish_summary_to_cscc = 3; // Publish findings to Cloud Datahub. - PublishFindingsToCloudDataCatalog publish_findings_to_cloud_data_catalog = 5; + PublishFindingsToCloudDataCatalog publish_findings_to_cloud_data_catalog = + 5; // Enable email notification for project owners and editors on job's // completion/failure. @@ -2844,8 +3057,8 @@ message CreateInspectTemplateRequest { // Request message for UpdateInspectTemplate. message UpdateInspectTemplateRequest { - // Required. Resource name of organization and inspectTemplate to be updated, for - // example `organizations/433245324/inspectTemplates/432452342` or + // Required. Resource name of organization and inspectTemplate to be updated, + // for example `organizations/433245324/inspectTemplates/432452342` or // projects/project-id/inspectTemplates/432452342. string name = 1 [ (google.api.field_behavior) = REQUIRED, @@ -2863,8 +3076,8 @@ message UpdateInspectTemplateRequest { // Request message for GetInspectTemplate. message GetInspectTemplateRequest { - // Required. Resource name of the organization and inspectTemplate to be read, for - // example `organizations/433245324/inspectTemplates/432452342` or + // Required. Resource name of the organization and inspectTemplate to be read, + // for example `organizations/433245324/inspectTemplates/432452342` or // projects/project-id/inspectTemplates/432452342. string name = 1 [ (google.api.field_behavior) = REQUIRED, @@ -2925,9 +3138,9 @@ message ListInspectTemplatesResponse { // Request message for DeleteInspectTemplate. message DeleteInspectTemplateRequest { - // Required. Resource name of the organization and inspectTemplate to be deleted, for - // example `organizations/433245324/inspectTemplates/432452342` or - // projects/project-id/inspectTemplates/432452342. + // Required. Resource name of the organization and inspectTemplate to be + // deleted, for example `organizations/433245324/inspectTemplates/432452342` + // or projects/project-id/inspectTemplates/432452342. string name = 1 [ (google.api.field_behavior) = REQUIRED, (google.api.resource_reference) = { @@ -2966,9 +3179,7 @@ message ActivateJobTriggerRequest { // `projects/dlp-test-project/jobTriggers/53234423`. string name = 1 [ (google.api.field_behavior) = REQUIRED, - (google.api.resource_reference) = { - type: "dlp.googleapis.com/JobTrigger" - } + (google.api.resource_reference) = { type: "dlp.googleapis.com/JobTrigger" } ]; } @@ -2978,9 +3189,7 @@ message UpdateJobTriggerRequest { // `projects/dlp-test-project/jobTriggers/53234423`. string name = 1 [ (google.api.field_behavior) = REQUIRED, - (google.api.resource_reference) = { - type: "dlp.googleapis.com/JobTrigger" - } + (google.api.resource_reference) = { type: "dlp.googleapis.com/JobTrigger" } ]; // New JobTrigger value. @@ -2996,9 +3205,7 @@ message GetJobTriggerRequest { // `projects/dlp-test-project/jobTriggers/53234423`. string name = 1 [ (google.api.field_behavior) = REQUIRED, - (google.api.resource_reference) = { - type: "dlp.googleapis.com/JobTrigger" - } + (google.api.resource_reference) = { type: "dlp.googleapis.com/JobTrigger" } ]; } @@ -3116,9 +3323,7 @@ message DeleteJobTriggerRequest { // `projects/dlp-test-project/jobTriggers/53234423`. string name = 1 [ (google.api.field_behavior) = REQUIRED, - (google.api.resource_reference) = { - type: "dlp.googleapis.com/JobTrigger" - } + (google.api.resource_reference) = { type: "dlp.googleapis.com/JobTrigger" } ]; } @@ -3144,9 +3349,10 @@ message DlpJob { option (google.api.resource) = { type: "dlp.googleapis.com/DlpJob" pattern: "projects/{project}/dlpJobs/{dlp_job}" + pattern: "projects/{project}/locations/{location}/dlpJobs/{dlp_job}" }; - // Possible states of a job. + // Possible states of a job. New items may be added. enum JobState { // Unused. JOB_STATE_UNSPECIFIED = 0; @@ -3154,7 +3360,8 @@ message DlpJob { // The job has not yet started. PENDING = 1; - // The job is currently running. + // The job is currently running. Once a job has finished it will transition + // to FAILED or DONE. RUNNING = 2; // The job is no longer running. @@ -3165,6 +3372,12 @@ message DlpJob { // The job had an error and did not complete. FAILED = 5; + + // The job is currently accepting findings via hybridInspect. + // A hybrid job in ACTIVE state may continue to have findings added to it + // through calling of hybridInspect. After the job has finished no more + // calls to hybridInspect may be made. ACTIVE jobs can transition to DONE. + ACTIVE = 6; } // The server-assigned name. @@ -3206,9 +3419,7 @@ message GetDlpJobRequest { // Required. The name of the DlpJob resource. string name = 1 [ (google.api.field_behavior) = REQUIRED, - (google.api.resource_reference) = { - type: "dlp.googleapis.com/DlpJob" - } + (google.api.resource_reference) = { type: "dlp.googleapis.com/DlpJob" } ]; } @@ -3295,9 +3506,16 @@ message CancelDlpJobRequest { // Required. The name of the DlpJob resource to be cancelled. string name = 1 [ (google.api.field_behavior) = REQUIRED, - (google.api.resource_reference) = { - type: "dlp.googleapis.com/DlpJob" - } + (google.api.resource_reference) = { type: "dlp.googleapis.com/DlpJob" } + ]; +} + +// The request message for finishing a DLP hybrid job. +message FinishDlpJobRequest { + // Required. The name of the DlpJob resource to be cancelled. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { type: "dlp.googleapis.com/DlpJob" } ]; } @@ -3306,9 +3524,7 @@ message DeleteDlpJobRequest { // Required. The name of the DlpJob resource to be deleted. string name = 1 [ (google.api.field_behavior) = REQUIRED, - (google.api.resource_reference) = { - type: "dlp.googleapis.com/DlpJob" - } + (google.api.resource_reference) = { type: "dlp.googleapis.com/DlpJob" } ]; } @@ -3324,7 +3540,8 @@ message CreateDeidentifyTemplateRequest { ]; // Required. The DeidentifyTemplate to create. - DeidentifyTemplate deidentify_template = 2 [(google.api.field_behavior) = REQUIRED]; + DeidentifyTemplate deidentify_template = 2 + [(google.api.field_behavior) = REQUIRED]; // The template id can contain uppercase and lowercase letters, // numbers, and hyphens; that is, it must match the regular @@ -3339,8 +3556,9 @@ message CreateDeidentifyTemplateRequest { // Request message for UpdateDeidentifyTemplate. message UpdateDeidentifyTemplateRequest { - // Required. Resource name of organization and deidentify template to be updated, for - // example `organizations/433245324/deidentifyTemplates/432452342` or + // Required. Resource name of organization and deidentify template to be + // updated, for example + // `organizations/433245324/deidentifyTemplates/432452342` or // projects/project-id/deidentifyTemplates/432452342. string name = 1 [ (google.api.field_behavior) = REQUIRED, @@ -3358,9 +3576,9 @@ message UpdateDeidentifyTemplateRequest { // Request message for GetDeidentifyTemplate. message GetDeidentifyTemplateRequest { - // Required. Resource name of the organization and deidentify template to be read, for - // example `organizations/433245324/deidentifyTemplates/432452342` or - // projects/project-id/deidentifyTemplates/432452342. + // Required. Resource name of the organization and deidentify template to be + // read, for example `organizations/433245324/deidentifyTemplates/432452342` + // or projects/project-id/deidentifyTemplates/432452342. string name = 1 [ (google.api.field_behavior) = REQUIRED, (google.api.resource_reference) = { @@ -3421,8 +3639,9 @@ message ListDeidentifyTemplatesResponse { // Request message for DeleteDeidentifyTemplate. message DeleteDeidentifyTemplateRequest { - // Required. Resource name of the organization and deidentify template to be deleted, - // for example `organizations/433245324/deidentifyTemplates/432452342` or + // Required. Resource name of the organization and deidentify template to be + // deleted, for example + // `organizations/433245324/deidentifyTemplates/432452342` or // projects/project-id/deidentifyTemplates/432452342. string name = 1 [ (google.api.field_behavior) = REQUIRED, @@ -3474,6 +3693,12 @@ message StoredInfoTypeConfig { oneof type { // StoredInfoType where findings are defined by a dictionary of phrases. LargeCustomDictionaryConfig large_custom_dictionary = 3; + + // Store dictionary-based CustomInfoType. + CustomInfoType.Dictionary dictionary = 4; + + // Store regular expression-based StoredInfoType. + CustomInfoType.Regex regex = 5; } } @@ -3526,6 +3751,8 @@ message StoredInfoType { type: "dlp.googleapis.com/StoredInfoType" pattern: "organizations/{organization}/storedInfoTypes/{stored_info_type}" pattern: "projects/{project}/storedInfoTypes/{stored_info_type}" + pattern: "organizations/{organization}/locations/{location}/storedInfoTypes/{stored_info_type}" + pattern: "projects/{project}/locations/{location}/storedInfoTypes/{stored_info_type}" }; // Resource name. @@ -3566,8 +3793,8 @@ message CreateStoredInfoTypeRequest { // Request message for UpdateStoredInfoType. message UpdateStoredInfoTypeRequest { - // Required. Resource name of organization and storedInfoType to be updated, for - // example `organizations/433245324/storedInfoTypes/432452342` or + // Required. Resource name of organization and storedInfoType to be updated, + // for example `organizations/433245324/storedInfoTypes/432452342` or // projects/project-id/storedInfoTypes/432452342. string name = 1 [ (google.api.field_behavior) = REQUIRED, @@ -3587,8 +3814,8 @@ message UpdateStoredInfoTypeRequest { // Request message for GetStoredInfoType. message GetStoredInfoTypeRequest { - // Required. Resource name of the organization and storedInfoType to be read, for - // example `organizations/433245324/storedInfoTypes/432452342` or + // Required. Resource name of the organization and storedInfoType to be read, + // for example `organizations/433245324/storedInfoTypes/432452342` or // projects/project-id/storedInfoTypes/432452342. string name = 1 [ (google.api.field_behavior) = REQUIRED, @@ -3650,8 +3877,8 @@ message ListStoredInfoTypesResponse { // Request message for DeleteStoredInfoType. message DeleteStoredInfoTypeRequest { - // Required. Resource name of the organization and storedInfoType to be deleted, for - // example `organizations/433245324/storedInfoTypes/432452342` or + // Required. Resource name of the organization and storedInfoType to be + // deleted, for example `organizations/433245324/storedInfoTypes/432452342` or // projects/project-id/storedInfoTypes/432452342. string name = 1 [ (google.api.field_behavior) = REQUIRED, @@ -3661,6 +3888,86 @@ message DeleteStoredInfoTypeRequest { ]; } +// Request to search for potentially sensitive info in a custom location. +message HybridInspectJobTriggerRequest { + // Required. Resource name of the trigger to execute a hybrid inspect on, for + // example `projects/dlp-test-project/jobTriggers/53234423`. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { type: "dlp.googleapis.com/JobTrigger" } + ]; + + // The item to inspect. + HybridContentItem hybrid_item = 3; +} + +// Request to search for potentially sensitive info in a custom location. +message HybridInspectDlpJobRequest { + // Required. Resource name of the job to execute a hybrid inspect on, for + // example `projects/dlp-test-project/dlpJob/53234423`. + string name = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { type: "dlp.googleapis.com/DlpJob" } + ]; + + // The item to inspect. + HybridContentItem hybrid_item = 3; +} + +// An individual hybrid item to inspect. Will be stored temporarily during +// processing. +message HybridContentItem { + // The item to inspect. + ContentItem item = 1; + + // Supplementary information that will be added to each finding. + HybridFindingDetails finding_details = 2; +} + +// Populate to associate additional data with each finding. +message HybridFindingDetails { + // Details about the container where the content being inspected is from. + Container container_details = 1; + + // Offset in bytes of the line, from the beginning of the file, where the + // finding is located. Populate if the item being scanned is only part of a + // bigger item, such as a shard of a file and you want to track the absolute + // position of the finding. + int64 file_offset = 2; + + // Offset of the row for tables. Populate if the row(s) being scanned are + // part of a bigger dataset and you want to keep track of their absolute + // position. + int64 row_offset = 3; + + // If the container is a table, additional information to make findings + // meaningful such as the columns that are primary keys. If not known ahead + // of time, can also be set within each inspect hybrid call and the two + // will be merged. Note that identifying_fields will only be stored to + // BigQuery, and only if the BigQuery action has been included. + TableOptions table_options = 4; + + // Labels to represent user provided metadata about the data being inspected. + // If configured by the job, some key values may be required. + // The labels associated with `Finding`'s produced by hybrid + // inspection. + // + // Label keys must be between 1 and 63 characters long and must conform + // to the following regular expression: \[a-z\](\[-a-z0-9\]*\[a-z0-9\])?. + // + // Label values must be between 0 and 63 characters long and must conform + // to the regular expression (\[a-z\](\[-a-z0-9\]*\[a-z0-9\])?)?. + // + // No more than 10 labels can be associated with a given finding. + // + // Example: "environment" : "production" + // Example: "pipeline" : "etl" + map labels = 5; +} + +// Quota exceeded errors will be thrown once quota has been met. +message HybridInspectResponse {} + // Operators available for comparing the value of fields. enum RelationalOperator { // Unused diff --git a/google/privacy/dlp/v2/dlp_gapic.yaml b/google/privacy/dlp/v2/dlp_gapic.yaml index 12b51484..9357709f 100644 --- a/google/privacy/dlp/v2/dlp_gapic.yaml +++ b/google/privacy/dlp/v2/dlp_gapic.yaml @@ -337,6 +337,21 @@ interfaces: field_name_patterns: name: dlp_job timeout_millis: 300000 + - name: FinishDlpJob + required_fields: + - name + resource_name_treatment: STATIC_TYPES + retry_codes_name: non_idempotent + retry_params_name: default + field_name_patterns: + name: dlp_job + timeout_millis: 300000 + - name: HybridInspectDlpJob + required_fields: + - name + retry_codes_name: non_idempotent + retry_params_name: default + timeout_millis: 300000 - name: ListJobTriggers required_fields: - parent @@ -380,6 +395,12 @@ interfaces: - python - php visibility: DISABLED + - name: HybridInspectJobTrigger + required_fields: + - name + retry_codes_name: non_idempotent + retry_params_name: default + timeout_millis: 300000 - name: UpdateJobTrigger required_fields: - name @@ -525,6 +546,9 @@ resource_name_generation: - message_name: CancelDlpJobRequest field_entity_map: name: dlp_job +- message_name: FinishDlpJobRequest + field_entity_map: + name: dlp_job - message_name: CreateStoredInfoTypeRequest field_entity_map: parent: organization diff --git a/google/privacy/dlp/v2/dlp_grpc_service_config.json b/google/privacy/dlp/v2/dlp_grpc_service_config.json index cd346fe0..e2017e24 100755 --- a/google/privacy/dlp/v2/dlp_grpc_service_config.json +++ b/google/privacy/dlp/v2/dlp_grpc_service_config.json @@ -81,6 +81,18 @@ { "service": "google.privacy.dlp.v2.DlpService", "method": "DeleteStoredInfoType" + }, + { + "service": "google.privacy.dlp.v2.DlpService", + "method": "FinishDlpJob" + }, + { + "service": "google.privacy.dlp.v2.DlpService", + "method": "HybridInspectDlpJob" + }, + { + "service": "google.privacy.dlp.v2.DlpService", + "method": "HybridInspectJobTrigger" } ], "timeout": "300s", diff --git a/google/privacy/dlp/v2/storage.proto b/google/privacy/dlp/v2/storage.proto index aebc873a..585d777d 100644 --- a/google/privacy/dlp/v2/storage.proto +++ b/google/privacy/dlp/v2/storage.proto @@ -1,4 +1,4 @@ -// Copyright 2019 Google LLC. +// Copyright 2020 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,7 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -// syntax = "proto3"; @@ -134,9 +133,7 @@ message CustomInfoType { // output. This should be used in conjunction with a field on the // transformation such as `surrogate_info_type`. This CustomInfoType does // not support the use of `detection_rules`. - message SurrogateType { - - } + message SurrogateType {} // Deprecated; use `InspectionRuleSet` instead. Rule for modifying a // `CustomInfoType` to alter behavior under certain circumstances, depending @@ -435,8 +432,11 @@ message BigQueryOptions { // Complete BigQuery table reference. BigQueryTable table_reference = 1; - // References to fields uniquely identifying rows within the table. - // Nested fields in the format, like `person.birthdate.year`, are allowed. + // Table fields that may uniquely identify a row within the table. When + // `actions.saveFindings.outputConfig.table` is specified, the values of + // columns specified here are available in the output table under + // `location.content_locations.record_location.record_key.id_values`. Nested + // fields such as `person.birthdate.year` are allowed. repeated FieldId identifying_fields = 2; // Max number of rows to scan. If the table has more rows than this value, the @@ -495,14 +495,20 @@ message StorageConfig { } oneof type { - // Google Cloud Datastore options specification. + // Google Cloud Datastore options. DatastoreOptions datastore_options = 2; - // Google Cloud Storage options specification. + // Google Cloud Storage options. CloudStorageOptions cloud_storage_options = 3; - // BigQuery options specification. + // BigQuery options. BigQueryOptions big_query_options = 4; + + // Hybrid inspection options. + // Early access feature is in a pre-release state and might change or have + // limited support. For more information, see + // https://cloud.google.com/products#product-launch-stages. + HybridOptions hybrid_options = 9; } TimespanConfig timespan_config = 6; @@ -534,13 +540,52 @@ enum FileType { AVRO = 7; } +// Configuration to control jobs where the content being inspected is outside +// of Google Cloud Platform. +message HybridOptions { + // A short description of where the data is coming from. Will be stored once + // in the job. 256 max length. + string description = 1; + + // These are labels that each inspection request must include within their + // 'finding_labels' map. Request may contain others, but any missing one of + // these will be rejected. + // + // Label keys must be between 1 and 63 characters long and must conform + // to the following regular expression: \[a-z\](\[-a-z0-9\]*\[a-z0-9\])?. + // + // No more than 10 keys can be required. + repeated string required_finding_label_keys = 2; + + // To organize findings, these labels will be added to each finding. + // + // Label keys must be between 1 and 63 characters long and must conform + // to the following regular expression: \[a-z\](\[-a-z0-9\]*\[a-z0-9\])?. + // + // Label values must be between 0 and 63 characters long and must conform + // to the regular expression (\[a-z\](\[-a-z0-9\]*\[a-z0-9\])?)?. + // + // No more than 10 labels can be associated with a given finding. + // + // Example: "environment" : "production" + // Example: "pipeline" : "etl" + map labels = 3; + + // If the container is a table, additional information to make findings + // meaningful such as the columns that are primary keys. + TableOptions table_options = 4; +} + // Row key for identifying a record in BigQuery table. message BigQueryKey { // Complete BigQuery table reference. BigQueryTable table_reference = 1; - // Absolute number of the row from the beginning of the table at the time - // of scanning. + // Row number inferred at the time the table was scanned. This value is + // nondeterministic, cannot be queried, and may be null for inspection + // jobs. To locate findings within a table, specify + // `inspect_job.storage_config.big_query_options.identifying_fields` in + // `CreateDlpJobRequest`. int64 row_number = 2; } @@ -607,7 +652,7 @@ message RecordKey { } // Values of identifying columns in the given row. Order of values matches - // the order of field identifiers specified in the scanning request. + // the order of `identifying_fields` specified in the scanning request. repeated string id_values = 5; } @@ -646,3 +691,12 @@ message EntityId { // Composite key indicating which field contains the entity identifier. FieldId field = 1; } + +// Instructions regarding the table content being inspected. +message TableOptions { + // The columns that are the primary keys for table objects included in + // ContentItem. A copy of this cell's value will stored alongside alongside + // each finding so that the finding can be traced to the specific row it came + // from. No more than 3 may be provided. + repeated FieldId identifying_fields = 1; +}