From 5e53d6b6dde0e72fa9510ec1d796176d128afa40 Mon Sep 17 00:00:00 2001 From: Google APIs Date: Tue, 15 Sep 2020 20:04:02 -0700 Subject: [PATCH] feat: added support for per type and partition export for Cloud Asset API Clients can now specify two more args when export assets to bigquery PiperOrigin-RevId: 331912851 --- google/cloud/asset/v1/asset_service.proto | 77 +++++++++++++++++++ .../v1/cloudasset_grpc_service_config.json | 21 +++++ 2 files changed, 98 insertions(+) diff --git a/google/cloud/asset/v1/asset_service.proto b/google/cloud/asset/v1/asset_service.proto index 4aa3ee74..6b66d444 100644 --- a/google/cloud/asset/v1/asset_service.proto +++ b/google/cloud/asset/v1/asset_service.proto @@ -393,6 +393,83 @@ message BigQueryDestination { // is `FALSE` or unset and the destination table already exists, the export // call returns an INVALID_ARGUMEMT error. bool force = 3; + + // [partition_spec] determines whether to export to partitioned table(s) and + // how to partition the data. + // + // If [partition_spec] is unset or [partition_spec.partion_key] is unset or + // `PARTITION_KEY_UNSPECIFIED`, the snapshot results will be exported to + // non-partitioned table(s). [force] will decide whether to overwrite existing + // table(s). + // + // If [partition_spec] is specified. First, the snapshot results will be + // written to partitioned table(s) with two additional timestamp columns, + // readTime and requestTime, one of which will be the partition key. Secondly, + // in the case when any destination table already exists, it will first try to + // update existing table's schema as necessary by appending additional + // columns. Then, if [force] is `TRUE`, the corresponding partition will be + // overwritten by the snapshot results (data in different partitions will + // remain intact); if [force] is unset or `FALSE`, it will append the data. An + // error will be returned if the schema update or data appension fails. + PartitionSpec partition_spec = 4; + + // If this flag is `TRUE`, the snapshot results will be written to one or + // multiple tables, each of which contains results of one asset type. The + // [force] and [partition_spec] fields will apply to each of them. + // + // Field [table] will be concatenated with "_" and the asset type names (see + // https://cloud.google.com/asset-inventory/docs/supported-asset-types for + // supported asset types) to construct per-asset-type table names, in which + // all non-alphanumeric characters like "." and "/" will be substituted by + // "_". Example: if field [table] is "mytable" and snapshot results + // contain "storage.googleapis.com/Bucket" assets, the corresponding table + // name will be "mytable_storage_googleapis_com_Bucket". If any of these + // tables does not exist, a new table with the concatenated name will be + // created. + // + // When [content_type] in the ExportAssetsRequest is `RESOURCE`, the schema of + // each table will include RECORD-type columns mapped to the nested fields in + // the Asset.resource.data field of that asset type (up to the 15 nested level + // BigQuery supports + // (https://cloud.google.com/bigquery/docs/nested-repeated#limitations)). The + // fields in >15 nested levels will be stored in JSON format string as a child + // column of its parent RECORD column. + // + // If error occurs when exporting to any table, the whole export call will + // return an error but the export results that already succeed will persist. + // Example: if exporting to table_type_A succeeds when exporting to + // table_type_B fails during one export call, the results in table_type_A will + // persist and there will not be partial results persisting in a table. + bool separate_tables_per_asset_type = 5; +} + +// Specifications of BigQuery partitioned table as export destination. +message PartitionSpec { + // This enum is used to determine the partition key column when exporting + // assets to BigQuery partitioned table(s). Note that, if the partition key is + // a timestamp column, the actual partition is based on its date value + // (expressed in UTC. see details in + // https://cloud.google.com/bigquery/docs/partitioned-tables#date_timestamp_partitioned_tables). + enum PartitionKey { + // Unspecified partition key. If used, it means using non-partitioned table. + PARTITION_KEY_UNSPECIFIED = 0; + + // The time when the snapshot is taken. If specified as partition key, the + // result table(s) is partitoned by the additional timestamp column, + // readTime. If [read_time] in ExportAssetsRequest is specified, the + // readTime column's value will be the same as it. Otherwise, its value will + // be the current time that is used to take the snapshot. + READ_TIME = 1; + + // The time when the request is received and started to be processed. If + // specified as partition key, the result table(s) is partitoned by the + // requestTime column, an additional timestamp column representing when the + // request was received. + REQUEST_TIME = 2; + } + + // The partition key for BigQuery partitioned table. + PartitionKey partition_key = 1; } // A Pub/Sub destination. diff --git a/google/cloud/asset/v1/cloudasset_grpc_service_config.json b/google/cloud/asset/v1/cloudasset_grpc_service_config.json index 770c5fd3..64cb6d21 100755 --- a/google/cloud/asset/v1/cloudasset_grpc_service_config.json +++ b/google/cloud/asset/v1/cloudasset_grpc_service_config.json @@ -13,6 +13,10 @@ { "service": "google.cloud.asset.v1.AssetService", "method": "UpdateFeed" + }, + { + "service": "google.cloud.asset.v1.AssetService", + "method": "ExportIamPolicyAnalysis" } ], "timeout": "60s" @@ -69,6 +73,23 @@ "UNAVAILABLE" ] } + }, + { + "name": [ + { + "service": "google.cloud.asset.v1.AssetService", + "method": "AnalyzeIamPolicy" + } + ], + "timeout": "300s", + "retryPolicy": { + "initialBackoff": "0.100s", + "maxBackoff": "60s", + "backoffMultiplier": 1.3, + "retryableStatusCodes": [ + "UNAVAILABLE" + ] + } } ] }