From bf17ae5fd93929beb44ac4c6b04f5088c3ee4a02 Mon Sep 17 00:00:00 2001 From: Google APIs Date: Tue, 12 May 2020 13:14:37 -0700 Subject: [PATCH] feat: Add a Flush API to enable finer grained data commit needs for dataflow. fix: add resource definition for Table/ReadStream/WriteStream message fix: add proper resource_reference for messages chore: update copyright committer: @xiaozhenliugg PiperOrigin-RevId: 311188524 --- .../bigquery/storage/v1alpha2/storage.proto | 185 +++++++++++------- 1 file changed, 109 insertions(+), 76 deletions(-) diff --git a/google/cloud/bigquery/storage/v1alpha2/storage.proto b/google/cloud/bigquery/storage/v1alpha2/storage.proto index fd13ba3a..e9a893b9 100644 --- a/google/cloud/bigquery/storage/v1alpha2/storage.proto +++ b/google/cloud/bigquery/storage/v1alpha2/storage.proto @@ -46,6 +46,97 @@ message CreateWriteStreamRequest { WriteStream write_stream = 2 [(google.api.field_behavior) = REQUIRED]; } +// BigQuery Write API. +// +// The Write API can be used to write data to BigQuery. +service BigQueryWrite { + option (google.api.default_host) = "bigquerystorage.googleapis.com"; + option (google.api.oauth_scopes) = + "https://www.googleapis.com/auth/bigquery," + "https://www.googleapis.com/auth/bigquery.insertdata," + "https://www.googleapis.com/auth/cloud-platform"; + + // Creates a write stream to the given table. + rpc CreateWriteStream(CreateWriteStreamRequest) returns (WriteStream) { + option (google.api.http) = { + post: "/v1alpha2/{parent=projects/*/datasets/*/tables/*}" + body: "write_stream" + }; + option (google.api.method_signature) = "parent,write_stream"; + } + + // Appends data to the given stream. + // + // If `offset` is specified, the `offset` is checked against the end of + // stream. The server returns `OUT_OF_RANGE` in `AppendRowsResponse` if an + // attempt is made to append to an offset beyond the current end of the stream + // or `ALREADY_EXISTS` if user provids an `offset` that has already been + // written to. User can retry with adjusted offset within the same RPC + // stream. If `offset` is not specified, append happens at the end of the + // stream. + // + // The response contains the offset at which the append happened. Responses + // are received in the same order in which requests are sent. There will be + // one response for each successful request. If the `offset` is not set in + // response, it means append didn't happen due to some errors. If one request + // fails, all the subsequent requests will also fail until a success request + // is made again. + // + // If the stream is of `PENDING` type, data will only be available for read + // operations after the stream is committed. + rpc AppendRows(stream AppendRowsRequest) returns (stream AppendRowsResponse) { + option (google.api.http) = { + post: "/v1alpha2/{write_stream=projects/*/datasets/*/tables/*/streams/*}" + body: "*" + }; + option (google.api.method_signature) = "write_stream"; + } + + // Gets a write stream. + rpc GetWriteStream(GetWriteStreamRequest) returns (WriteStream) { + option (google.api.http) = { + post: "/v1alpha2/{name=projects/*/datasets/*/tables/*/streams/*}" + body: "*" + }; + option (google.api.method_signature) = "name"; + } + + // Finalize a write stream so that no new data can be appended to the + // stream. + rpc FinalizeWriteStream(FinalizeWriteStreamRequest) returns (FinalizeWriteStreamResponse) { + option (google.api.http) = { + post: "/v1alpha2/{name=projects/*/datasets/*/tables/*/streams/*}" + body: "*" + }; + option (google.api.method_signature) = "name"; + } + + // Atomically commits a group of `PENDING` streams that belong to the same + // `parent` table. + // Streams must be finalized before commit and cannot be committed multiple + // times. Once a stream is committed, data in the stream becomes available + // for read operations. + rpc BatchCommitWriteStreams(BatchCommitWriteStreamsRequest) returns (BatchCommitWriteStreamsResponse) { + option (google.api.http) = { + get: "/v1alpha2/{parent=projects/*/datasets/*/tables/*}" + }; + option (google.api.method_signature) = "parent"; + } + + // Flushes rows to a BUFFERED stream. + // If users are appending rows to BUFFERED stream, flush operation is + // required in order for the rows to become available for reading. A + // Flush operation flushes up to any previously flushed offset in a BUFFERED + // stream, to the offset specified in the request. + rpc FlushRows(FlushRowsRequest) returns (FlushRowsResponse) { + option (google.api.http) = { + post: "/v1alpha2/{write_stream=projects/*/datasets/*/tables/*/streams/*}" + body: "*" + }; + option (google.api.method_signature) = "write_stream"; + } +} + // Request message for `AppendRows`. message AppendRowsRequest { message ProtoData { @@ -143,81 +234,23 @@ message FinalizeWriteStreamResponse { int64 row_count = 1; } -// BigQuery Write API. -// -// The Write API can be used to write data to BigQuery. -service BigQueryWrite { - option (google.api.default_host) = "bigquerystorage.googleapis.com"; - option (google.api.oauth_scopes) = - "https://www.googleapis.com/auth/bigquery," - "https://www.googleapis.com/auth/bigquery.insertdata," - "https://www.googleapis.com/auth/cloud-platform"; - - // Creates a write stream to the given table. - rpc CreateWriteStream(CreateWriteStreamRequest) returns (WriteStream) { - option (google.api.http) = { - post: "/v1alpha2/{parent=projects/*/datasets/*/tables/*}" - body: "write_stream" - }; - option (google.api.method_signature) = "parent,write_stream"; - } - - // Appends data to the given stream. - // - // If `offset` is specified, the `offset` is checked against the end of - // stream. The server returns `OUT_OF_RANGE` in `AppendRowsResponse` if an - // attempt is made to append to an offset beyond the current end of the stream - // or `ALREADY_EXISTS` if user provids an `offset` that has already been - // written to. User can retry with adjusted offset within the same RPC - // stream. If `offset` is not specified, append happens at the end of the - // stream. - // - // The response contains the offset at which the append happened. Responses - // are received in the same order in which requests are sent. There will be - // one response for each successful request. If the `offset` is not set in - // response, it means append didn't happen due to some errors. If one request - // fails, all the subsequent requests will also fail until a success request - // is made again. - // - // If the stream is of `PENDING` type, data will only be available for read - // operations after the stream is committed. - rpc AppendRows(stream AppendRowsRequest) returns (stream AppendRowsResponse) { - option (google.api.http) = { - post: "/v1alpha2/{write_stream=projects/*/datasets/*/tables/*/streams/*}" - body: "*" - }; - option (google.api.method_signature) = "write_stream"; - } - - // Gets a write stream. - rpc GetWriteStream(GetWriteStreamRequest) returns (WriteStream) { - option (google.api.http) = { - post: "/v1alpha2/{name=projects/*/datasets/*/tables/*/streams/*}" - body: "*" - }; - option (google.api.method_signature) = "name"; - } - - // Finalize a write stream so that no new data can be appended to the - // stream. - rpc FinalizeWriteStream(FinalizeWriteStreamRequest) returns (FinalizeWriteStreamResponse) { - option (google.api.http) = { - post: "/v1alpha2/{name=projects/*/datasets/*/tables/*/streams/*}" - body: "*" - }; - option (google.api.method_signature) = "name"; - } - - // Atomically commits a group of `PENDING` streams that belong to the same - // `parent` table. - // Streams must be finalized before commit and cannot be committed multiple - // times. Once a stream is committed, data in the stream becomes available - // for read operations. - rpc BatchCommitWriteStreams(BatchCommitWriteStreamsRequest) returns (BatchCommitWriteStreamsResponse) { - option (google.api.http) = { - get: "/v1alpha2/{parent=projects/*/datasets/*/tables/*}" - }; - option (google.api.method_signature) = "parent"; - } +// Request message for `FlushRows`. +message FlushRowsRequest { + // Required. The stream that is the target of the flush operation. + string write_stream = 1 [ + (google.api.field_behavior) = REQUIRED, + (google.api.resource_reference) = { + type: "bigquerystorage.googleapis.com/WriteStream" + } + ]; + // Ending offset of the flush operation. Rows before this offset(including + // this offset) will be flushed. + int64 offset = 2; +} + +// Respond message for `FlushRows`. +message FlushRowsResponse { + // The rows before this offset (including this offset) are flushed. + int64 offset = 1; }