From 149dcc1f21c2de95492af221987f46ea8fc67359 Mon Sep 17 00:00:00 2001 From: Sharon Lynn Date: Fri, 8 May 2026 05:20:55 +0000 Subject: [PATCH 1/3] feat(storage): add feature flag and models for appendable upload Introduces the `google_cloud_unstable_storage_appendable_upload` feature flag to gate the upcoming GCS Appendable Uploads functionality. --- .gcb/builds/triggers/main.tf | 1 + .github/workflows/sdk.yaml | 2 +- Cargo.toml | 1 + src/storage/src/model_ext.rs | 26 ++++++++++++++++++++++++ src/storage/src/storage.rs | 2 ++ src/storage/src/storage/append_object.rs | 15 ++++++++++++++ 6 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 src/storage/src/storage/append_object.rs diff --git a/.gcb/builds/triggers/main.tf b/.gcb/builds/triggers/main.tf index 0a40d9c964..68b1eeb5e8 100644 --- a/.gcb/builds/triggers/main.tf +++ b/.gcb/builds/triggers/main.tf @@ -35,6 +35,7 @@ locals { gcb_secret_name = "projects/${var.project}/secrets/github-github-oauthtoken-319d75/versions/latest" unstable_flags = join(" ", [ + "--cfg google_cloud_unstable_storage_appendable_upload", "--cfg google_cloud_unstable_trust_boundaries", "--cfg google_cloud_unstable_storage_bidi", "--cfg google_cloud_unstable_grpc_server_streaming" diff --git a/.github/workflows/sdk.yaml b/.github/workflows/sdk.yaml index df2e13d69b..3d0a8d8106 100644 --- a/.github/workflows/sdk.yaml +++ b/.github/workflows/sdk.yaml @@ -25,7 +25,7 @@ env: GHA_RUST_VERSIONS: '{ "rust:current": "1.95" }' GHA_GO_VERSIONS: '{ "go:current": "1.25.6" }' # Define the unstable flags once - UNSTABLE_CFGS: &unstable_cfgs '--cfg google_cloud_unstable_trust_boundaries --cfg google_cloud_unstable_storage_bidi --cfg google_cloud_unstable_grpc_server_streaming' + UNSTABLE_CFGS: &unstable_cfgs '--cfg google_cloud_unstable_storage_appendable_upload --cfg google_cloud_unstable_trust_boundaries --cfg google_cloud_unstable_storage_bidi --cfg google_cloud_unstable_grpc_server_streaming' jobs: build: diff --git a/Cargo.toml b/Cargo.toml index 4e4a9d459d..74047eca9d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -536,6 +536,7 @@ storage-grpc-mock = { path = "src/storage/grpc-mock" } [workspace.lints.rust] unexpected_cfgs = { level = "deny", check-cfg = [ + 'cfg(google_cloud_unstable_storage_appendable_upload)', 'cfg(google_cloud_unstable_grpc_server_streaming)', 'cfg(google_cloud_unstable_storage_bidi)', 'cfg(google_cloud_unstable_tracing)', diff --git a/src/storage/src/model_ext.rs b/src/storage/src/model_ext.rs index 7a336aae74..7d3d17701c 100644 --- a/src/storage/src/model_ext.rs +++ b/src/storage/src/model_ext.rs @@ -310,6 +310,32 @@ pub struct WriteObjectRequest { pub params: Option, } +#[cfg(google_cloud_unstable_storage_appendable_upload)] +/// The first message sent in an appendable upload stream. +#[derive(Debug, PartialEq)] +#[non_exhaustive] +pub enum AppendObjectFirstMessage { + /// Create a new appendable object. + Create(Box), + /// Take over an existing appendable object. + Takeover(crate::google::storage::v2::AppendObjectSpec), +} + +#[cfg(google_cloud_unstable_storage_appendable_upload)] +/// Represents the parameters of an [AppendObject] request. +/// +/// This type is only used in mocks of the `Storage` client. +/// +/// [AppendObject]: crate::builder::storage::AppendObject +#[derive(Debug, PartialEq)] +#[non_exhaustive] +pub struct AppendObjectRequest { + /// The first message sent in the stream, which determines whether this is a create or takeover operation. + pub first_message: AppendObjectFirstMessage, + /// Additional request parameters that are not part of the object attributes. + pub params: Option, +} + #[cfg(test)] pub(crate) mod tests { use super::*; diff --git a/src/storage/src/storage.rs b/src/storage/src/storage.rs index accfcabf9d..02d27f0211 100644 --- a/src/storage/src/storage.rs +++ b/src/storage/src/storage.rs @@ -16,6 +16,8 @@ pub(crate) mod bidi; pub(crate) mod checksum; pub(crate) mod client; pub(crate) mod common_options; +#[cfg(google_cloud_unstable_storage_appendable_upload)] +pub mod append_object; pub(crate) mod open_object; pub(crate) mod perform_upload; pub(crate) mod read_object; diff --git a/src/storage/src/storage/append_object.rs b/src/storage/src/storage/append_object.rs new file mode 100644 index 0000000000..0b69b45ddf --- /dev/null +++ b/src/storage/src/storage/append_object.rs @@ -0,0 +1,15 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Appendable object uploads (unstable). From 790d68fd519883c56439edf14f9d1720ec1c3ddc Mon Sep 17 00:00:00 2001 From: Sharon Lynn Date: Mon, 11 May 2026 06:02:29 +0000 Subject: [PATCH 2/3] Incorporate auto code review suggestions and fix failing tests --- Cargo.toml | 2 +- src/storage/src/model_ext.rs | 49 +++++++++++++++++++++++++++++++----- src/storage/src/storage.rs | 4 +-- 3 files changed, 46 insertions(+), 9 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 74047eca9d..6c6c757fa3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -536,8 +536,8 @@ storage-grpc-mock = { path = "src/storage/grpc-mock" } [workspace.lints.rust] unexpected_cfgs = { level = "deny", check-cfg = [ - 'cfg(google_cloud_unstable_storage_appendable_upload)', 'cfg(google_cloud_unstable_grpc_server_streaming)', + 'cfg(google_cloud_unstable_storage_appendable_upload)', 'cfg(google_cloud_unstable_storage_bidi)', 'cfg(google_cloud_unstable_tracing)', 'cfg(google_cloud_unstable_trust_boundaries)', diff --git a/src/storage/src/model_ext.rs b/src/storage/src/model_ext.rs index 7d3d17701c..5a8038b80a 100644 --- a/src/storage/src/model_ext.rs +++ b/src/storage/src/model_ext.rs @@ -311,24 +311,23 @@ pub struct WriteObjectRequest { } #[cfg(google_cloud_unstable_storage_appendable_upload)] -/// The first message sent in an appendable upload stream. +/// The first message sent in an appendable upload bidi stream. #[derive(Debug, PartialEq)] #[non_exhaustive] pub enum AppendObjectFirstMessage { /// Create a new appendable object. Create(Box), /// Take over an existing appendable object. - Takeover(crate::google::storage::v2::AppendObjectSpec), + Takeover(AppendObjectSpec), } #[cfg(google_cloud_unstable_storage_appendable_upload)] -/// Represents the parameters of an [AppendObject] request. +/// The request type for an appendable upload bidi streaming RPC. /// -/// This type is only used in mocks of the `Storage` client. -/// -/// [AppendObject]: crate::builder::storage::AppendObject +/// This type might be used in mocks of the `Storage` client. #[derive(Debug, PartialEq)] #[non_exhaustive] +#[allow(dead_code)] pub struct AppendObjectRequest { /// The first message sent in the stream, which determines whether this is a create or takeover operation. pub first_message: AppendObjectFirstMessage, @@ -336,6 +335,44 @@ pub struct AppendObjectRequest { pub params: Option, } +#[cfg(google_cloud_unstable_storage_appendable_upload)] +/// Describes an existing target object to be appended to. +#[derive(Clone, Debug, Default, PartialEq)] +#[non_exhaustive] +pub struct AppendObjectSpec { + /// The bucket containing the target object. + pub bucket: String, + /// The target object name. + pub object: String, + /// The target object generation to append to. + pub generation: i64, + /// If set, return an error if the current metageneration does not match the value. + pub if_metageneration_match: Option, + /// If set, return an error if the current metageneration matches the value. + pub if_metageneration_not_match: Option, + /// Optional. A routing token for the request. + pub routing_token: Option, + /// Optional. A write handle from a previous `AppendObject` operation. + pub write_handle: Option, +} + +#[cfg(google_cloud_unstable_storage_appendable_upload)] +impl From for crate::google::storage::v2::AppendObjectSpec { + fn from(value: AppendObjectSpec) -> Self { + Self { + bucket: value.bucket, + object: value.object, + generation: value.generation, + if_metageneration_match: value.if_metageneration_match, + if_metageneration_not_match: value.if_metageneration_not_match, + routing_token: value.routing_token, + write_handle: value + .write_handle + .map(|h| crate::google::storage::v2::BidiWriteHandle { handle: h }), + } + } +} + #[cfg(test)] pub(crate) mod tests { use super::*; diff --git a/src/storage/src/storage.rs b/src/storage/src/storage.rs index 02d27f0211..6d67c26261 100644 --- a/src/storage/src/storage.rs +++ b/src/storage/src/storage.rs @@ -12,12 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +#[cfg(google_cloud_unstable_storage_appendable_upload)] +pub(crate) mod append_object; pub(crate) mod bidi; pub(crate) mod checksum; pub(crate) mod client; pub(crate) mod common_options; -#[cfg(google_cloud_unstable_storage_appendable_upload)] -pub mod append_object; pub(crate) mod open_object; pub(crate) mod perform_upload; pub(crate) mod read_object; From 45c1729c61ac13af1f0e4ea351492d6d9b481804 Mon Sep 17 00:00:00 2001 From: Sharon Lynn Date: Wed, 13 May 2026 08:49:44 +0000 Subject: [PATCH 3/3] Use existing cfg flag and change API Use existing `google_cloud_unstable_storage_bidi`, because currently it's not being used for anything. Separate the API into two, instead of one. --- .gcb/builds/triggers/main.tf | 1 - .github/workflows/sdk.yaml | 2 +- Cargo.toml | 1 - src/storage/src/model_ext.rs | 86 +++++++++++++++++++++++++----------- src/storage/src/storage.rs | 2 +- 5 files changed, 61 insertions(+), 31 deletions(-) diff --git a/.gcb/builds/triggers/main.tf b/.gcb/builds/triggers/main.tf index 68b1eeb5e8..0a40d9c964 100644 --- a/.gcb/builds/triggers/main.tf +++ b/.gcb/builds/triggers/main.tf @@ -35,7 +35,6 @@ locals { gcb_secret_name = "projects/${var.project}/secrets/github-github-oauthtoken-319d75/versions/latest" unstable_flags = join(" ", [ - "--cfg google_cloud_unstable_storage_appendable_upload", "--cfg google_cloud_unstable_trust_boundaries", "--cfg google_cloud_unstable_storage_bidi", "--cfg google_cloud_unstable_grpc_server_streaming" diff --git a/.github/workflows/sdk.yaml b/.github/workflows/sdk.yaml index 3d0a8d8106..df2e13d69b 100644 --- a/.github/workflows/sdk.yaml +++ b/.github/workflows/sdk.yaml @@ -25,7 +25,7 @@ env: GHA_RUST_VERSIONS: '{ "rust:current": "1.95" }' GHA_GO_VERSIONS: '{ "go:current": "1.25.6" }' # Define the unstable flags once - UNSTABLE_CFGS: &unstable_cfgs '--cfg google_cloud_unstable_storage_appendable_upload --cfg google_cloud_unstable_trust_boundaries --cfg google_cloud_unstable_storage_bidi --cfg google_cloud_unstable_grpc_server_streaming' + UNSTABLE_CFGS: &unstable_cfgs '--cfg google_cloud_unstable_trust_boundaries --cfg google_cloud_unstable_storage_bidi --cfg google_cloud_unstable_grpc_server_streaming' jobs: build: diff --git a/Cargo.toml b/Cargo.toml index 6c6c757fa3..4e4a9d459d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -537,7 +537,6 @@ storage-grpc-mock = { path = "src/storage/grpc-mock" } [workspace.lints.rust] unexpected_cfgs = { level = "deny", check-cfg = [ 'cfg(google_cloud_unstable_grpc_server_streaming)', - 'cfg(google_cloud_unstable_storage_appendable_upload)', 'cfg(google_cloud_unstable_storage_bidi)', 'cfg(google_cloud_unstable_tracing)', 'cfg(google_cloud_unstable_trust_boundaries)', diff --git a/src/storage/src/model_ext.rs b/src/storage/src/model_ext.rs index 5a8038b80a..d3c24d1a62 100644 --- a/src/storage/src/model_ext.rs +++ b/src/storage/src/model_ext.rs @@ -310,36 +310,26 @@ pub struct WriteObjectRequest { pub params: Option, } -#[cfg(google_cloud_unstable_storage_appendable_upload)] -/// The first message sent in an appendable upload bidi stream. -#[derive(Debug, PartialEq)] -#[non_exhaustive] -pub enum AppendObjectFirstMessage { - /// Create a new appendable object. - Create(Box), - /// Take over an existing appendable object. - Takeover(AppendObjectSpec), -} - -#[cfg(google_cloud_unstable_storage_appendable_upload)] -/// The request type for an appendable upload bidi streaming RPC. +#[cfg(google_cloud_unstable_storage_bidi)] +/// Represents the parameters of a request to open a new object for exclusive appends. /// /// This type might be used in mocks of the `Storage` client. #[derive(Debug, PartialEq)] #[non_exhaustive] -#[allow(dead_code)] -pub struct AppendObjectRequest { - /// The first message sent in the stream, which determines whether this is a create or takeover operation. - pub first_message: AppendObjectFirstMessage, - /// Additional request parameters that are not part of the object attributes. +pub struct OpenAppendableObjectRequest { + /// The object attributes and pre-conditions for the open operation. + pub spec: Box, + /// Additional request parameters. pub params: Option, } -#[cfg(google_cloud_unstable_storage_appendable_upload)] -/// Describes an existing target object to be appended to. -#[derive(Clone, Debug, Default, PartialEq)] +#[cfg(google_cloud_unstable_storage_bidi)] +/// Represents the parameters of a request to reopen an existing object for appends. +/// +/// This type might be used in mocks of the `Storage` client. +#[derive(Debug, PartialEq)] #[non_exhaustive] -pub struct AppendObjectSpec { +pub struct ReopenAppendableObjectRequest { /// The bucket containing the target object. pub bucket: String, /// The target object name. @@ -350,15 +340,17 @@ pub struct AppendObjectSpec { pub if_metageneration_match: Option, /// If set, return an error if the current metageneration matches the value. pub if_metageneration_not_match: Option, - /// Optional. A routing token for the request. + /// A routing token from a previous operation. pub routing_token: Option, - /// Optional. A write handle from a previous `AppendObject` operation. + /// A write handle from a previous operation. pub write_handle: Option, + /// Additional request parameters. + pub params: Option, } -#[cfg(google_cloud_unstable_storage_appendable_upload)] -impl From for crate::google::storage::v2::AppendObjectSpec { - fn from(value: AppendObjectSpec) -> Self { +#[cfg(google_cloud_unstable_storage_bidi)] +impl From for crate::google::storage::v2::AppendObjectSpec { + fn from(value: ReopenAppendableObjectRequest) -> Self { Self { bucket: value.bucket, object: value.object, @@ -492,4 +484,44 @@ pub(crate) mod tests { assert_eq!(key_aes_256.to_string(), key_base64); Ok(()) } + + #[cfg(google_cloud_unstable_storage_bidi)] + #[test] + fn test_open_appendable_object_request() { + let req = OpenAppendableObjectRequest { + spec: Box::new(crate::model::WriteObjectSpec::default()), + params: None, + }; + assert_eq!(req.spec.resource, None); + assert_eq!(req.params, None); + } + + #[cfg(google_cloud_unstable_storage_bidi)] + #[test] + fn test_reopen_appendable_object_request_from() { + let req = ReopenAppendableObjectRequest { + bucket: "my-bucket".into(), + object: "my-object".into(), + generation: 42, + if_metageneration_match: Some(1), + if_metageneration_not_match: Some(2), + routing_token: Some("token".into()), + write_handle: Some(bytes::Bytes::from("handle")), + params: None, + }; + + let spec = crate::google::storage::v2::AppendObjectSpec::from(req); + assert_eq!(spec.bucket, "my-bucket"); + assert_eq!(spec.object, "my-object"); + assert_eq!(spec.generation, 42); + assert_eq!(spec.if_metageneration_match, Some(1)); + assert_eq!(spec.if_metageneration_not_match, Some(2)); + assert_eq!(spec.routing_token, Some("token".into())); + assert_eq!( + spec.write_handle, + Some(crate::google::storage::v2::BidiWriteHandle { + handle: bytes::Bytes::from("handle") + }) + ); + } } diff --git a/src/storage/src/storage.rs b/src/storage/src/storage.rs index 6d67c26261..6dbbb5cfc2 100644 --- a/src/storage/src/storage.rs +++ b/src/storage/src/storage.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#[cfg(google_cloud_unstable_storage_appendable_upload)] +#[cfg(google_cloud_unstable_storage_bidi)] pub(crate) mod append_object; pub(crate) mod bidi; pub(crate) mod checksum;