From e2a595a9be207b9e6da232844e0bb5d5372bc948 Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Thu, 18 Dec 2025 08:40:45 +0000 Subject: [PATCH 01/21] feat: compute chunk wise checksum for bidi_writes --- .../asyncio/async_appendable_object_writer.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py b/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py index 27c4b4f19..ac522793d 100644 --- a/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py +++ b/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py @@ -22,6 +22,8 @@ """ from typing import Optional, Union +from google.api_core import exceptions +from google_crc32c import Checksum, implementation as crc32c_impl from google.cloud import _storage_v2 from google.cloud.storage._experimental.asyncio.async_grpc_client import ( AsyncGrpcClient, @@ -100,6 +102,14 @@ def __init__( :param write_handle: (Optional) An existing handle for writing the object. If provided, opening the bidi-gRPC connection will be faster. """ + # Verify that the fast, C-accelerated version of crc32c is available. + # If not, raise an error to prevent silent performance degradation. + if crc32c_impl != "c": + raise exceptions.NotFound( + "The google-crc32c package is not installed with C support. " + "Bidi reads require the C extension for data integrity checks." + "For more information, see https://github.com/googleapis/python-crc32c." + ) self.client = client self.bucket_name = bucket_name self.object_name = object_name @@ -191,11 +201,13 @@ async def append(self, data: bytes) -> None: bytes_to_flush = 0 while start_idx < total_bytes: end_idx = min(start_idx + _MAX_CHUNK_SIZE_BYTES, total_bytes) + data_chunk = data[start_idx:end_idx] await self.write_obj_stream.send( _storage_v2.BidiWriteObjectRequest( write_offset=self.offset, checksummed_data=_storage_v2.ChecksummedData( - content=data[start_idx:end_idx] + content=data_chunk, + crc32c=int.from_bytes(Checksum(data_chunk).digest(), "big"), ), ) ) From 4f3d18eab45d446f3035fa8d237735973c63f9ad Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Thu, 18 Dec 2025 08:54:53 +0000 Subject: [PATCH 02/21] move common code to utils --- .../storage/_experimental/asyncio/_utils.py | 20 +++++++++++++++++++ .../asyncio/async_appendable_object_writer.py | 15 ++++++-------- .../asyncio/async_multi_range_downloader.py | 15 ++++---------- 3 files changed, 30 insertions(+), 20 deletions(-) create mode 100644 google/cloud/storage/_experimental/asyncio/_utils.py diff --git a/google/cloud/storage/_experimental/asyncio/_utils.py b/google/cloud/storage/_experimental/asyncio/_utils.py new file mode 100644 index 000000000..b386af2eb --- /dev/null +++ b/google/cloud/storage/_experimental/asyncio/_utils.py @@ -0,0 +1,20 @@ +import google_crc32c + +from google.cloud.storage import exceptions + +def raise_if_no_fast_crc32c(): + """Check if the C-accelerated version of google-crc32c is available. + + If not, raise an error to prevent silent performance degradation. + + raises google.api_core.exceptions.NotFound: If the C extension is not available. + returns: True if the C extension is available. + rtype: bool + + """ + if google_crc32c.implementation != "c": + raise exceptions.NotFound( + "The google-crc32c package is not installed with C support. " + "Bidi reads require the C extension for data integrity checks." + "For more information, see https://github.com/googleapis/python-crc32c." + ) \ No newline at end of file diff --git a/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py b/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py index ac522793d..d9095003e 100644 --- a/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py +++ b/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py @@ -21,9 +21,13 @@ if you want to use these Rapid Storage APIs. """ +import time from typing import Optional, Union + from google.api_core import exceptions -from google_crc32c import Checksum, implementation as crc32c_impl +from google_crc32c import Checksum + +from ._utils import raise_if_no_fast_crc32c from google.cloud import _storage_v2 from google.cloud.storage._experimental.asyncio.async_grpc_client import ( AsyncGrpcClient, @@ -102,14 +106,7 @@ def __init__( :param write_handle: (Optional) An existing handle for writing the object. If provided, opening the bidi-gRPC connection will be faster. """ - # Verify that the fast, C-accelerated version of crc32c is available. - # If not, raise an error to prevent silent performance degradation. - if crc32c_impl != "c": - raise exceptions.NotFound( - "The google-crc32c package is not installed with C support. " - "Bidi reads require the C extension for data integrity checks." - "For more information, see https://github.com/googleapis/python-crc32c." - ) + raise_if_no_fast_crc32c() self.client = client self.bucket_name = bucket_name self.object_name = object_name diff --git a/google/cloud/storage/_experimental/asyncio/async_multi_range_downloader.py b/google/cloud/storage/_experimental/asyncio/async_multi_range_downloader.py index fecd685d4..5c3f9512e 100644 --- a/google/cloud/storage/_experimental/asyncio/async_multi_range_downloader.py +++ b/google/cloud/storage/_experimental/asyncio/async_multi_range_downloader.py @@ -14,12 +14,12 @@ from __future__ import annotations import asyncio -import google_crc32c +from typing import List, Optional, Tuple + from google.api_core import exceptions from google_crc32c import Checksum -from typing import List, Optional, Tuple - +from ._utils import raise_if_no_fast_crc32c from google.cloud.storage._experimental.asyncio.async_read_object_stream import ( _AsyncReadObjectStream, ) @@ -160,14 +160,7 @@ def __init__( :param read_handle: (Optional) An existing read handle. """ - # Verify that the fast, C-accelerated version of crc32c is available. - # If not, raise an error to prevent silent performance degradation. - if google_crc32c.implementation != "c": - raise exceptions.NotFound( - "The google-crc32c package is not installed with C support. " - "Bidi reads require the C extension for data integrity checks." - "For more information, see https://github.com/googleapis/python-crc32c." - ) + raise_if_no_fast_crc32c() self.client = client self.bucket_name = bucket_name From 62a2c89ac4183ef398b2a2c2ccab97eec4fbf47a Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Thu, 18 Dec 2025 09:19:55 +0000 Subject: [PATCH 03/21] add and fix failing unit tests --- .../storage/_experimental/asyncio/_utils.py | 4 ++-- .../test_async_appendable_object_writer.py | 18 ++++++++++++++++++ .../test_async_multi_range_downloader.py | 4 +--- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/google/cloud/storage/_experimental/asyncio/_utils.py b/google/cloud/storage/_experimental/asyncio/_utils.py index b386af2eb..93e12e223 100644 --- a/google/cloud/storage/_experimental/asyncio/_utils.py +++ b/google/cloud/storage/_experimental/asyncio/_utils.py @@ -1,6 +1,6 @@ import google_crc32c -from google.cloud.storage import exceptions +from google.api_core import exceptions def raise_if_no_fast_crc32c(): """Check if the C-accelerated version of google-crc32c is available. @@ -17,4 +17,4 @@ def raise_if_no_fast_crc32c(): "The google-crc32c package is not installed with C support. " "Bidi reads require the C extension for data integrity checks." "For more information, see https://github.com/googleapis/python-crc32c." - ) \ No newline at end of file + ) diff --git a/tests/unit/asyncio/test_async_appendable_object_writer.py b/tests/unit/asyncio/test_async_appendable_object_writer.py index 089c3d88f..7eceda360 100644 --- a/tests/unit/asyncio/test_async_appendable_object_writer.py +++ b/tests/unit/asyncio/test_async_appendable_object_writer.py @@ -15,6 +15,7 @@ import pytest from unittest import mock +from google.api_core import exceptions from google.cloud.storage._experimental.asyncio.async_appendable_object_writer import ( AsyncAppendableObjectWriter, ) @@ -85,6 +86,23 @@ def test_init_with_optional_args(mock_write_object_stream, mock_client): ) +@mock.patch("google.cloud.storage._experimental.asyncio._utils.google_crc32c") +@mock.patch( + "google.cloud.storage._experimental.asyncio.async_grpc_client.AsyncGrpcClient.grpc_client" +) +def test_init_raises_if_crc32c_c_extension_is_missing( + mock_grpc_client, mock_google_crc32c +): + mock_google_crc32c.implementation = "python" + + with pytest.raises(exceptions.NotFound) as exc_info: + AsyncAppendableObjectWriter(mock_grpc_client, "bucket", "object") + + assert "The google-crc32c package is not installed with C support" in str( + exc_info.value + ) + + @pytest.mark.asyncio @mock.patch( "google.cloud.storage._experimental.asyncio.async_appendable_object_writer._AsyncWriteObjectStream" diff --git a/tests/unit/asyncio/test_async_multi_range_downloader.py b/tests/unit/asyncio/test_async_multi_range_downloader.py index 1460e4df8..92bee84dd 100644 --- a/tests/unit/asyncio/test_async_multi_range_downloader.py +++ b/tests/unit/asyncio/test_async_multi_range_downloader.py @@ -349,9 +349,7 @@ async def test_downloading_without_opening_should_throw_error( assert str(exc.value) == "Underlying bidi-gRPC stream is not open" assert not mrd.is_stream_open - @mock.patch( - "google.cloud.storage._experimental.asyncio.async_multi_range_downloader.google_crc32c" - ) + @mock.patch("google.cloud.storage._experimental.asyncio._utils.google_crc32c") @mock.patch( "google.cloud.storage._experimental.asyncio.async_grpc_client.AsyncGrpcClient.grpc_client" ) From e89e2050acecd86d319025bcb6dd751f82283fbc Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Thu, 18 Dec 2025 12:03:44 +0000 Subject: [PATCH 04/21] add license info in _utils file --- .../cloud/storage/_experimental/asyncio/_utils.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/google/cloud/storage/_experimental/asyncio/_utils.py b/google/cloud/storage/_experimental/asyncio/_utils.py index 93e12e223..a0ab78999 100644 --- a/google/cloud/storage/_experimental/asyncio/_utils.py +++ b/google/cloud/storage/_experimental/asyncio/_utils.py @@ -1,3 +1,17 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import google_crc32c from google.api_core import exceptions From 9353e58808c8e4e9bf9e8836e07ccca9683acdb5 Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Thu, 18 Dec 2025 12:08:07 +0000 Subject: [PATCH 05/21] use FailedPreCondition instead of NotFound --- google/cloud/storage/_experimental/asyncio/_utils.py | 8 ++++---- tests/unit/asyncio/test_async_appendable_object_writer.py | 2 +- tests/unit/asyncio/test_async_multi_range_downloader.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/google/cloud/storage/_experimental/asyncio/_utils.py b/google/cloud/storage/_experimental/asyncio/_utils.py index a0ab78999..38eb659fc 100644 --- a/google/cloud/storage/_experimental/asyncio/_utils.py +++ b/google/cloud/storage/_experimental/asyncio/_utils.py @@ -20,15 +20,15 @@ def raise_if_no_fast_crc32c(): """Check if the C-accelerated version of google-crc32c is available. If not, raise an error to prevent silent performance degradation. - - raises google.api_core.exceptions.NotFound: If the C extension is not available. + + raises google.api_core.exceptions.FailedPrecondition: If the C extension is not available. returns: True if the C extension is available. rtype: bool """ if google_crc32c.implementation != "c": - raise exceptions.NotFound( + raise exceptions.FailedPrecondition( "The google-crc32c package is not installed with C support. " - "Bidi reads require the C extension for data integrity checks." + "C extension is required for faster data integrity checks." "For more information, see https://github.com/googleapis/python-crc32c." ) diff --git a/tests/unit/asyncio/test_async_appendable_object_writer.py b/tests/unit/asyncio/test_async_appendable_object_writer.py index 7eceda360..c2d125011 100644 --- a/tests/unit/asyncio/test_async_appendable_object_writer.py +++ b/tests/unit/asyncio/test_async_appendable_object_writer.py @@ -95,7 +95,7 @@ def test_init_raises_if_crc32c_c_extension_is_missing( ): mock_google_crc32c.implementation = "python" - with pytest.raises(exceptions.NotFound) as exc_info: + with pytest.raises(exceptions.FailedPrecondition) as exc_info: AsyncAppendableObjectWriter(mock_grpc_client, "bucket", "object") assert "The google-crc32c package is not installed with C support" in str( diff --git a/tests/unit/asyncio/test_async_multi_range_downloader.py b/tests/unit/asyncio/test_async_multi_range_downloader.py index 92bee84dd..8afef104b 100644 --- a/tests/unit/asyncio/test_async_multi_range_downloader.py +++ b/tests/unit/asyncio/test_async_multi_range_downloader.py @@ -358,7 +358,7 @@ def test_init_raises_if_crc32c_c_extension_is_missing( ): mock_google_crc32c.implementation = "python" - with pytest.raises(exceptions.NotFound) as exc_info: + with pytest.raises(exceptions.FailedPrecondition) as exc_info: AsyncMultiRangeDownloader(mock_grpc_client, "bucket", "object") assert "The google-crc32c package is not installed with C support" in str( From 2b1d6cef5c6f0c429cf05ee65e084cd3855f3406 Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Thu, 18 Dec 2025 14:14:55 +0000 Subject: [PATCH 06/21] chore: add test cases for large objects --- tests/system/test_zonal.py | 83 +++++++++++++++++++++++++++++++++++--- 1 file changed, 78 insertions(+), 5 deletions(-) diff --git a/tests/system/test_zonal.py b/tests/system/test_zonal.py index 05bb317d7..a64c98211 100644 --- a/tests/system/test_zonal.py +++ b/tests/system/test_zonal.py @@ -3,14 +3,18 @@ import os import uuid from io import BytesIO +import random # python additional imports +import google_crc32c + import pytest # current library imports from google.cloud.storage._experimental.asyncio.async_grpc_client import AsyncGrpcClient from google.cloud.storage._experimental.asyncio.async_appendable_object_writer import ( AsyncAppendableObjectWriter, + _MAX_BUFFER_SIZE_BYTES, ) from google.cloud.storage._experimental.asyncio.async_multi_range_downloader import ( AsyncMultiRangeDownloader, @@ -28,6 +32,11 @@ _BYTES_TO_UPLOAD = b"dummy_bytes_to_write_read_and_delete_appendable_object" +def _get_equal_dist(a: int, b: int) -> tuple[int, int]: + step = (b - a) // 3 + return a + step, a + 2 * step + + async def write_one_appendable_object( bucket_name: str, object_name: str, @@ -59,11 +68,21 @@ def appendable_object(storage_client, blobs_to_delete): @pytest.mark.asyncio +@pytest.mark.parametrize( + "object_size", + [ + 256, # less than _chunk size + 10 * 1024 * 1024, # less than _MAX_BUFFER_SIZE_BYTES + 20 * 1024 * 1024, # greater than _MAX_BUFFER_SIZE + ], +) @pytest.mark.parametrize( "attempt_direct_path", [True, False], ) -async def test_basic_wrd(storage_client, blobs_to_delete, attempt_direct_path): +async def test_basic_wrd( + storage_client, blobs_to_delete, attempt_direct_path, object_size +): object_name = f"test_basic_wrd-{str(uuid.uuid4())}" # Client instantiation; it cannot be part of fixture because. @@ -74,13 +93,16 @@ async def test_basic_wrd(storage_client, blobs_to_delete, attempt_direct_path): # 2. we can keep the same event loop for entire module but that may # create issues if tests are run in parallel and one test hogs the event # loop slowing down other tests. + object_data = os.urandom(object_size) + object_checksum = google_crc32c.value(object_data) grpc_client = AsyncGrpcClient(attempt_direct_path=attempt_direct_path).grpc_client writer = AsyncAppendableObjectWriter(grpc_client, _ZONAL_BUCKET, object_name) await writer.open() - await writer.append(_BYTES_TO_UPLOAD) + await writer.append(object_data) object_metadata = await writer.close(finalize_on_close=True) - assert object_metadata.size == len(_BYTES_TO_UPLOAD) + assert object_metadata.size == object_size + assert int(object_metadata.checksums.crc32c) == object_checksum mrd = AsyncMultiRangeDownloader(grpc_client, _ZONAL_BUCKET, object_name) buffer = BytesIO() @@ -88,8 +110,59 @@ async def test_basic_wrd(storage_client, blobs_to_delete, attempt_direct_path): # (0, 0) means read the whole object await mrd.download_ranges([(0, 0, buffer)]) await mrd.close() - assert buffer.getvalue() == _BYTES_TO_UPLOAD - assert mrd.persisted_size == len(_BYTES_TO_UPLOAD) + assert buffer.getvalue() == object_data + assert mrd.persisted_size == object_size + + # Clean up; use json client (i.e. `storage_client` fixture) to delete. + blobs_to_delete.append(storage_client.bucket(_ZONAL_BUCKET).blob(object_name)) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "object_size", + [ + 20 * 1024 * 1024, # greater than _MAX_BUFFER_SIZE + ], +) +@pytest.mark.parametrize( + "attempt_direct_path", + [True], +) +async def test_basic_wrd_in_slices( + storage_client, blobs_to_delete, attempt_direct_path, object_size +): + object_name = f"test_basic_wrd-{str(uuid.uuid4())}" + + # Client instantiation; it cannot be part of fixture because. + # grpc_client's event loop and event loop of coroutine running it + # (i.e. this test) must be same. + # Note: + # 1. @pytest.mark.asyncio ensures new event loop for each test. + # 2. we can keep the same event loop for entire module but that may + # create issues if tests are run in parallel and one test hogs the event + # loop slowing down other tests. + object_data = os.urandom(object_size) + object_checksum = google_crc32c.value(object_data) + grpc_client = AsyncGrpcClient(attempt_direct_path=attempt_direct_path).grpc_client + + writer = AsyncAppendableObjectWriter(grpc_client, _ZONAL_BUCKET, object_name) + await writer.open() + mark1, mark2 = _get_equal_dist(0, object_size) + await writer.append(object_data[0:mark1]) + await writer.append(object_data[mark1:mark2]) + await writer.append(object_data[mark2:]) + object_metadata = await writer.close(finalize_on_close=True) + assert object_metadata.size == object_size + assert int(object_metadata.checksums.crc32c) == object_checksum + + mrd = AsyncMultiRangeDownloader(grpc_client, _ZONAL_BUCKET, object_name) + buffer = BytesIO() + await mrd.open() + # (0, 0) means read the whole object + await mrd.download_ranges([(0, 0, buffer)]) + await mrd.close() + assert buffer.getvalue() == object_data + assert mrd.persisted_size == object_size # Clean up; use json client (i.e. `storage_client` fixture) to delete. blobs_to_delete.append(storage_client.bucket(_ZONAL_BUCKET).blob(object_name)) From cf9597c580bc7295ff354285f6252edfd485a980 Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Thu, 18 Dec 2025 15:13:34 +0000 Subject: [PATCH 07/21] feat: provide flush size to be configurable --- .../asyncio/async_appendable_object_writer.py | 26 +++++++-- tests/system/test_zonal.py | 55 ++++++++++++++++++- .../test_async_appendable_object_writer.py | 35 ++++++++++-- 3 files changed, 106 insertions(+), 10 deletions(-) diff --git a/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py b/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py index d9095003e..071bb58f0 100644 --- a/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py +++ b/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py @@ -38,7 +38,7 @@ _MAX_CHUNK_SIZE_BYTES = 2 * 1024 * 1024 # 2 MiB -_MAX_BUFFER_SIZE_BYTES = 16 * 1024 * 1024 # 16 MiB +_DEFAULT_FLUSH_INTERVAL_BYTES = 16 * 1024 * 1024 # 16 MiB class AsyncAppendableObjectWriter: @@ -51,6 +51,7 @@ def __init__( object_name: str, generation=None, write_handle=None, + writer_options: Optional[dict] = None, ): """ Class for appending data to a GCS Appendable Object. @@ -127,6 +128,21 @@ def __init__( # Please note: `offset` and `persisted_size` are same when the stream is # opened. self.persisted_size: Optional[int] = None + if writer_options is None: + writer_options = {} + self.flush_interval = writer_options.get( + "FLUSH_INTERVAL_BYTES", _DEFAULT_FLUSH_INTERVAL_BYTES + ) + # TODO: add test case for this. + if self.flush_interval < _MAX_CHUNK_SIZE_BYTES: + raise exceptions.OutOfRange( + f"flush_interval must be >= {_MAX_CHUNK_SIZE_BYTES} , but provided {self.flush_interval}" + ) + if not self.flush_interval % _MAX_CHUNK_SIZE_BYTES == 0: + raise exceptions.OutOfRange( + f"flush interval - {self.flush_interval} should be multiple of {_MAX_CHUNK_SIZE_BYTES}" + ) + self.bytes_appended_since_last_flush = 0 async def state_lookup(self) -> int: """Returns the persisted_size @@ -195,7 +211,7 @@ async def append(self, data: bytes) -> None: self.offset = self.persisted_size start_idx = 0 - bytes_to_flush = 0 + # bytes_to_flush = 0 while start_idx < total_bytes: end_idx = min(start_idx + _MAX_CHUNK_SIZE_BYTES, total_bytes) data_chunk = data[start_idx:end_idx] @@ -210,10 +226,10 @@ async def append(self, data: bytes) -> None: ) chunk_size = end_idx - start_idx self.offset += chunk_size - bytes_to_flush += chunk_size - if bytes_to_flush >= _MAX_BUFFER_SIZE_BYTES: + self.bytes_appended_since_last_flush += chunk_size + if self.bytes_appended_since_last_flush >= self.flush_interval: await self.simple_flush() - bytes_to_flush = 0 + self.bytes_appended_since_last_flush = 0 start_idx = end_idx async def simple_flush(self) -> None: diff --git a/tests/system/test_zonal.py b/tests/system/test_zonal.py index a64c98211..8b95b6612 100644 --- a/tests/system/test_zonal.py +++ b/tests/system/test_zonal.py @@ -14,7 +14,7 @@ from google.cloud.storage._experimental.asyncio.async_grpc_client import AsyncGrpcClient from google.cloud.storage._experimental.asyncio.async_appendable_object_writer import ( AsyncAppendableObjectWriter, - _MAX_BUFFER_SIZE_BYTES, + _DEFAULT_FLUSH_INTERVAL_BYTES, ) from google.cloud.storage._experimental.asyncio.async_multi_range_downloader import ( AsyncMultiRangeDownloader, @@ -168,6 +168,59 @@ async def test_basic_wrd_in_slices( blobs_to_delete.append(storage_client.bucket(_ZONAL_BUCKET).blob(object_name)) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "flush_interval", + [2 * 1024 * 1024, 4 * 1024 * 1024, 8 * 1024 * 1024, _DEFAULT_FLUSH_INTERVAL_BYTES], +) +async def test_wrd_with_non_default_flush_interval( + storage_client, + blobs_to_delete, + flush_interval, +): + object_name = f"test_basic_wrd-{str(uuid.uuid4())}" + object_size = 9 * 1024 * 1024 + + # Client instantiation; it cannot be part of fixture because. + # grpc_client's event loop and event loop of coroutine running it + # (i.e. this test) must be same. + # Note: + # 1. @pytest.mark.asyncio ensures new event loop for each test. + # 2. we can keep the same event loop for entire module but that may + # create issues if tests are run in parallel and one test hogs the event + # loop slowing down other tests. + object_data = os.urandom(object_size) + object_checksum = google_crc32c.value(object_data) + grpc_client = AsyncGrpcClient().grpc_client + + writer = AsyncAppendableObjectWriter( + grpc_client, + _ZONAL_BUCKET, + object_name, + writer_options={"FLUSH_INTERVAL_BYTES": flush_interval}, + ) + await writer.open() + mark1, mark2 = _get_equal_dist(0, object_size) + await writer.append(object_data[0:mark1]) + await writer.append(object_data[mark1:mark2]) + await writer.append(object_data[mark2:]) + object_metadata = await writer.close(finalize_on_close=True) + assert object_metadata.size == object_size + assert int(object_metadata.checksums.crc32c) == object_checksum + + mrd = AsyncMultiRangeDownloader(grpc_client, _ZONAL_BUCKET, object_name) + buffer = BytesIO() + await mrd.open() + # (0, 0) means read the whole object + await mrd.download_ranges([(0, 0, buffer)]) + await mrd.close() + assert buffer.getvalue() == object_data + assert mrd.persisted_size == object_size + + # Clean up; use json client (i.e. `storage_client` fixture) to delete. + blobs_to_delete.append(storage_client.bucket(_ZONAL_BUCKET).blob(object_name)) + + @pytest.mark.asyncio async def test_read_unfinalized_appendable_object(storage_client, blobs_to_delete): object_name = f"read_unfinalized_appendable_object-{str(uuid.uuid4())[:4]}" diff --git a/tests/unit/asyncio/test_async_appendable_object_writer.py b/tests/unit/asyncio/test_async_appendable_object_writer.py index c2d125011..d9feb1885 100644 --- a/tests/unit/asyncio/test_async_appendable_object_writer.py +++ b/tests/unit/asyncio/test_async_appendable_object_writer.py @@ -27,6 +27,7 @@ GENERATION = 123 WRITE_HANDLE = b"test-write-handle" PERSISTED_SIZE = 456 +_ONE_MIB = 1024 * 1024 @pytest.fixture @@ -50,6 +51,7 @@ def test_init(mock_write_object_stream, mock_client): assert not writer._is_stream_open assert writer.offset is None assert writer.persisted_size is None + assert writer.bytes_appended_since_last_flush == 0 mock_write_object_stream.assert_called_once_with( client=mock_client, @@ -76,6 +78,7 @@ def test_init_with_optional_args(mock_write_object_stream, mock_client): assert writer.generation == GENERATION assert writer.write_handle == WRITE_HANDLE + assert writer.bytes_appended_since_last_flush == 0 mock_write_object_stream.assert_called_once_with( client=mock_client, @@ -86,6 +89,30 @@ def test_init_with_optional_args(mock_write_object_stream, mock_client): ) +@mock.patch( + "google.cloud.storage._experimental.asyncio.async_appendable_object_writer._AsyncWriteObjectStream" +) +def test_init_with_writer_options(mock_write_object_stream, mock_client): + """Test the constructor with optional arguments.""" + writer = AsyncAppendableObjectWriter( + mock_client, + BUCKET, + OBJECT, + writer_options={"FLUSH_INTERVAL_BYTES": 8 * _ONE_MIB}, + ) + + assert writer.flush_interval == 8 * _ONE_MIB + assert writer.bytes_appended_since_last_flush == 0 + + mock_write_object_stream.assert_called_once_with( + client=mock_client, + bucket_name=BUCKET, + object_name=OBJECT, + generation_number=None, + write_handle=None, + ) + + @mock.patch("google.cloud.storage._experimental.asyncio._utils.google_crc32c") @mock.patch( "google.cloud.storage._experimental.asyncio.async_grpc_client.AsyncGrpcClient.grpc_client" @@ -470,7 +497,7 @@ async def test_append_flushes_when_buffer_is_full( ): """Test that append flushes the stream when the buffer size is reached.""" from google.cloud.storage._experimental.asyncio.async_appendable_object_writer import ( - _MAX_BUFFER_SIZE_BYTES, + _DEFAULT_FLUSH_INTERVAL_BYTES, ) writer = AsyncAppendableObjectWriter(mock_client, BUCKET, OBJECT) @@ -480,7 +507,7 @@ async def test_append_flushes_when_buffer_is_full( mock_stream.send = mock.AsyncMock() writer.simple_flush = mock.AsyncMock() - data = b"a" * _MAX_BUFFER_SIZE_BYTES + data = b"a" * _DEFAULT_FLUSH_INTERVAL_BYTES await writer.append(data) writer.simple_flush.assert_awaited_once() @@ -493,7 +520,7 @@ async def test_append_flushes_when_buffer_is_full( async def test_append_handles_large_data(mock_write_object_stream, mock_client): """Test that append handles data larger than the buffer size.""" from google.cloud.storage._experimental.asyncio.async_appendable_object_writer import ( - _MAX_BUFFER_SIZE_BYTES, + _DEFAULT_FLUSH_INTERVAL_BYTES, ) writer = AsyncAppendableObjectWriter(mock_client, BUCKET, OBJECT) @@ -503,7 +530,7 @@ async def test_append_handles_large_data(mock_write_object_stream, mock_client): mock_stream.send = mock.AsyncMock() writer.simple_flush = mock.AsyncMock() - data = b"a" * (_MAX_BUFFER_SIZE_BYTES * 2 + 1) + data = b"a" * (_DEFAULT_FLUSH_INTERVAL_BYTES * 2 + 1) await writer.append(data) assert writer.simple_flush.await_count == 2 From b285a400f7e6d1cc851900b949d5b556e862ab7d Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Thu, 18 Dec 2025 16:08:10 +0000 Subject: [PATCH 08/21] remove unused imports --- .../_experimental/asyncio/async_appendable_object_writer.py | 1 - .../_experimental/asyncio/async_multi_range_downloader.py | 1 - tests/system/test_zonal.py | 1 - 3 files changed, 3 deletions(-) diff --git a/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py b/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py index 071bb58f0..4eb9381dc 100644 --- a/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py +++ b/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py @@ -21,7 +21,6 @@ if you want to use these Rapid Storage APIs. """ -import time from typing import Optional, Union from google.api_core import exceptions diff --git a/google/cloud/storage/_experimental/asyncio/async_multi_range_downloader.py b/google/cloud/storage/_experimental/asyncio/async_multi_range_downloader.py index 320b09fee..1beedd097 100644 --- a/google/cloud/storage/_experimental/asyncio/async_multi_range_downloader.py +++ b/google/cloud/storage/_experimental/asyncio/async_multi_range_downloader.py @@ -16,7 +16,6 @@ import asyncio from typing import List, Optional, Tuple -from google.api_core import exceptions from google_crc32c import Checksum from ._utils import raise_if_no_fast_crc32c diff --git a/tests/system/test_zonal.py b/tests/system/test_zonal.py index 8b95b6612..c0d53a001 100644 --- a/tests/system/test_zonal.py +++ b/tests/system/test_zonal.py @@ -3,7 +3,6 @@ import os import uuid from io import BytesIO -import random # python additional imports import google_crc32c From df42160abaeaab43d3bcf3a7728ca8dee13c15db Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Thu, 18 Dec 2025 16:21:57 +0000 Subject: [PATCH 09/21] add unit tests and idomatic --- .../asyncio/async_appendable_object_writer.py | 5 ++- .../test_async_appendable_object_writer.py | 33 +++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py b/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py index 4eb9381dc..4b05d1d43 100644 --- a/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py +++ b/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py @@ -137,9 +137,9 @@ def __init__( raise exceptions.OutOfRange( f"flush_interval must be >= {_MAX_CHUNK_SIZE_BYTES} , but provided {self.flush_interval}" ) - if not self.flush_interval % _MAX_CHUNK_SIZE_BYTES == 0: + if self.flush_interval % _MAX_CHUNK_SIZE_BYTES != 0: raise exceptions.OutOfRange( - f"flush interval - {self.flush_interval} should be multiple of {_MAX_CHUNK_SIZE_BYTES}" + f"flush_interval must be a multiple of {_MAX_CHUNK_SIZE_BYTES}, but provided {self.flush_interval}" ) self.bytes_appended_since_last_flush = 0 @@ -210,7 +210,6 @@ async def append(self, data: bytes) -> None: self.offset = self.persisted_size start_idx = 0 - # bytes_to_flush = 0 while start_idx < total_bytes: end_idx = min(start_idx + _MAX_CHUNK_SIZE_BYTES, total_bytes) data_chunk = data[start_idx:end_idx] diff --git a/tests/unit/asyncio/test_async_appendable_object_writer.py b/tests/unit/asyncio/test_async_appendable_object_writer.py index d9feb1885..cf5a336a8 100644 --- a/tests/unit/asyncio/test_async_appendable_object_writer.py +++ b/tests/unit/asyncio/test_async_appendable_object_writer.py @@ -19,6 +19,9 @@ from google.cloud.storage._experimental.asyncio.async_appendable_object_writer import ( AsyncAppendableObjectWriter, ) +from google.cloud.storage._experimental.asyncio.async_appendable_object_writer import ( + _MAX_CHUNK_SIZE_BYTES, +) from google.cloud import _storage_v2 @@ -113,6 +116,36 @@ def test_init_with_writer_options(mock_write_object_stream, mock_client): ) +@mock.patch( + "google.cloud.storage._experimental.asyncio.async_appendable_object_writer._AsyncWriteObjectStream" +) +def test_init_with_flush_interval_less_than_chunk_size_raises_error(mock_client): + """Test that an OutOfRange error is raised if flush_interval is less than the chunk size.""" + + with pytest.raises(exceptions.OutOfRange): + AsyncAppendableObjectWriter( + mock_client, + BUCKET, + OBJECT, + writer_options={"FLUSH_INTERVAL_BYTES": _MAX_CHUNK_SIZE_BYTES - 1}, + ) + + +@mock.patch( + "google.cloud.storage._experimental.asyncio.async_appendable_object_writer._AsyncWriteObjectStream" +) +def test_init_with_flush_interval_not_multiple_of_chunk_size_raises_error(mock_client): + """Test that an OutOfRange error is raised if flush_interval is not a multiple of the chunk size.""" + + with pytest.raises(exceptions.OutOfRange): + AsyncAppendableObjectWriter( + mock_client, + BUCKET, + OBJECT, + writer_options={"FLUSH_INTERVAL_BYTES": _MAX_CHUNK_SIZE_BYTES + 1}, + ) + + @mock.patch("google.cloud.storage._experimental.asyncio._utils.google_crc32c") @mock.patch( "google.cloud.storage._experimental.asyncio.async_grpc_client.AsyncGrpcClient.grpc_client" From 2dc9f6135a272383162f9286a6d43533df87c82d Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Thu, 18 Dec 2025 17:32:42 +0000 Subject: [PATCH 10/21] feat: implement "append_from_file" --- .../asyncio/async_appendable_object_writer.py | 16 +++++++-- .../test_async_appendable_object_writer.py | 34 +++++++++++++++---- 2 files changed, 41 insertions(+), 9 deletions(-) diff --git a/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py b/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py index 4b05d1d43..0bb9d8969 100644 --- a/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py +++ b/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py @@ -339,6 +339,16 @@ async def append_from_stream(self, stream_obj): """ raise NotImplementedError("append_from_stream is not implemented yet.") - async def append_from_file(self, file_path: str): - """Create a file object from `file_path` and call append_from_stream(file_obj)""" - raise NotImplementedError("append_from_file is not implemented yet.") + async def append_from_file( + self, file_obj: BufferedReader, block_size: int = _DEFAULT_FLUSH_INTERVAL_BYTES + ): + """ + Appends data to an Appendable Object using file_handle which is opened + for reading in binary mode. + + :type file_obj: file + :param file_obj: A file handle opened in binary mode for reading. + + """ + while block := file_obj.read(block_size): + await self.append(block) diff --git a/tests/unit/asyncio/test_async_appendable_object_writer.py b/tests/unit/asyncio/test_async_appendable_object_writer.py index cf5a336a8..cd38ccce0 100644 --- a/tests/unit/asyncio/test_async_appendable_object_writer.py +++ b/tests/unit/asyncio/test_async_appendable_object_writer.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from io import BytesIO import pytest from unittest import mock @@ -21,6 +22,7 @@ ) from google.cloud.storage._experimental.asyncio.async_appendable_object_writer import ( _MAX_CHUNK_SIZE_BYTES, + _DEFAULT_FLUSH_INTERVAL_BYTES, ) from google.cloud import _storage_v2 @@ -285,9 +287,6 @@ async def test_unimplemented_methods_raise_error(mock_client): with pytest.raises(NotImplementedError): await writer.append_from_stream(mock.Mock()) - with pytest.raises(NotImplementedError): - await writer.append_from_file("file.txt") - @pytest.mark.asyncio @mock.patch( @@ -529,9 +528,6 @@ async def test_append_flushes_when_buffer_is_full( mock_write_object_stream, mock_client ): """Test that append flushes the stream when the buffer size is reached.""" - from google.cloud.storage._experimental.asyncio.async_appendable_object_writer import ( - _DEFAULT_FLUSH_INTERVAL_BYTES, - ) writer = AsyncAppendableObjectWriter(mock_client, BUCKET, OBJECT) writer._is_stream_open = True @@ -595,3 +591,29 @@ async def test_append_data_two_times(mock_write_object_stream, mock_client): total_data_length = len(data1) + len(data2) assert writer.offset == total_data_length assert writer.simple_flush.await_count == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "file_size, block_size", + [ + (10, 4 * 1024), + (20 * 1024 * 1024, _DEFAULT_FLUSH_INTERVAL_BYTES), + (16 * 1024 * 1024, _DEFAULT_FLUSH_INTERVAL_BYTES), + ], +) +async def test_append_from_file(file_size, block_size, mock_client): + # arrange + fp = BytesIO(b"a" * file_size) + writer = AsyncAppendableObjectWriter(mock_client, BUCKET, OBJECT) + writer._is_stream_open = True + writer.append = mock.AsyncMock() + + # act + await writer.append_from_file(fp, block_size=block_size) + + # assert + if file_size % block_size == 0: + writer.append.await_count == file_size // block_size + else: + writer.append.await_count == file_size // block_size + 1 From 262480d2d8191bc1654583b7ffdd3b0c8849b6ee Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Thu, 18 Dec 2025 17:38:52 +0000 Subject: [PATCH 11/21] add imports --- .../_experimental/asyncio/async_appendable_object_writer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py b/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py index 0bb9d8969..4524108a1 100644 --- a/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py +++ b/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py @@ -21,6 +21,7 @@ if you want to use these Rapid Storage APIs. """ +from io import BytesIO, BufferedReader from typing import Optional, Union from google.api_core import exceptions From def00f36dc832d4ffb4136e3b4473ad9c7bd5585 Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Fri, 19 Dec 2025 11:28:49 +0000 Subject: [PATCH 12/21] address nit comments --- tests/unit/asyncio/test_async_appendable_object_writer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/asyncio/test_async_appendable_object_writer.py b/tests/unit/asyncio/test_async_appendable_object_writer.py index cf5a336a8..8101ab512 100644 --- a/tests/unit/asyncio/test_async_appendable_object_writer.py +++ b/tests/unit/asyncio/test_async_appendable_object_writer.py @@ -30,7 +30,7 @@ GENERATION = 123 WRITE_HANDLE = b"test-write-handle" PERSISTED_SIZE = 456 -_ONE_MIB = 1024 * 1024 +EIGHT_MIB = 8 * 1024 * 1024 @pytest.fixture @@ -101,10 +101,10 @@ def test_init_with_writer_options(mock_write_object_stream, mock_client): mock_client, BUCKET, OBJECT, - writer_options={"FLUSH_INTERVAL_BYTES": 8 * _ONE_MIB}, + writer_options={"FLUSH_INTERVAL_BYTES": EIGHT_MIB}, ) - assert writer.flush_interval == 8 * _ONE_MIB + assert writer.flush_interval == EIGHT_MIB assert writer.bytes_appended_since_last_flush == 0 mock_write_object_stream.assert_called_once_with( From 2fbe83d74c7d8f3af3d859303e1fd7557399db16 Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Fri, 19 Dec 2025 11:34:13 +0000 Subject: [PATCH 13/21] add assert statements --- .../asyncio/async_appendable_object_writer.py | 2 +- .../asyncio/test_async_appendable_object_writer.py | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py b/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py index 4524108a1..86399375a 100644 --- a/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py +++ b/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py @@ -21,7 +21,7 @@ if you want to use these Rapid Storage APIs. """ -from io import BytesIO, BufferedReader +from io import BufferedReader from typing import Optional, Union from google.api_core import exceptions diff --git a/tests/unit/asyncio/test_async_appendable_object_writer.py b/tests/unit/asyncio/test_async_appendable_object_writer.py index cd38ccce0..978f82584 100644 --- a/tests/unit/asyncio/test_async_appendable_object_writer.py +++ b/tests/unit/asyncio/test_async_appendable_object_writer.py @@ -598,6 +598,7 @@ async def test_append_data_two_times(mock_write_object_stream, mock_client): "file_size, block_size", [ (10, 4 * 1024), + (0, _DEFAULT_FLUSH_INTERVAL_BYTES), (20 * 1024 * 1024, _DEFAULT_FLUSH_INTERVAL_BYTES), (16 * 1024 * 1024, _DEFAULT_FLUSH_INTERVAL_BYTES), ], @@ -613,7 +614,9 @@ async def test_append_from_file(file_size, block_size, mock_client): await writer.append_from_file(fp, block_size=block_size) # assert - if file_size % block_size == 0: - writer.append.await_count == file_size // block_size - else: - writer.append.await_count == file_size // block_size + 1 + exepected_calls = ( + file_size // block_size + if file_size % block_size == 0 + else file_size // block_size + 1 + ) + assert writer.append.await_count == exepected_calls From 5bc78db1ea2030853958badcf5a55e0f58d2684d Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Fri, 19 Dec 2025 11:46:25 +0000 Subject: [PATCH 14/21] remove unused imports --- .../_experimental/asyncio/async_appendable_object_writer.py | 1 - tests/system/test_zonal.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py b/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py index d7eb3faa0..0fa0e467e 100644 --- a/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py +++ b/google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py @@ -21,7 +21,6 @@ if you want to use these Rapid Storage APIs. """ -import time from typing import Optional, Union from google_crc32c import Checksum diff --git a/tests/system/test_zonal.py b/tests/system/test_zonal.py index a64c98211..d662a757e 100644 --- a/tests/system/test_zonal.py +++ b/tests/system/test_zonal.py @@ -3,7 +3,6 @@ import os import uuid from io import BytesIO -import random # python additional imports import google_crc32c @@ -14,7 +13,6 @@ from google.cloud.storage._experimental.asyncio.async_grpc_client import AsyncGrpcClient from google.cloud.storage._experimental.asyncio.async_appendable_object_writer import ( AsyncAppendableObjectWriter, - _MAX_BUFFER_SIZE_BYTES, ) from google.cloud.storage._experimental.asyncio.async_multi_range_downloader import ( AsyncMultiRangeDownloader, From 2f1c0dcf8a0c5be98b8691220a16d4f2817e4ca7 Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Mon, 22 Dec 2025 05:20:23 +0000 Subject: [PATCH 15/21] increase the open file descriptors limit --- cloudbuild/run_zonal_tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudbuild/run_zonal_tests.sh b/cloudbuild/run_zonal_tests.sh index ef94e629b..a1b6594a8 100644 --- a/cloudbuild/run_zonal_tests.sh +++ b/cloudbuild/run_zonal_tests.sh @@ -23,4 +23,5 @@ echo '--- Setting up environment variables on VM ---' export ZONAL_BUCKET=${_ZONAL_BUCKET} export RUN_ZONAL_SYSTEM_TESTS=True echo '--- Running Zonal tests on VM ---' +ulimit -n 1048576 pytest -vv -s --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' tests/system/test_zonal.py From 3de9d7aa0c6ad6870ef1dc7c07c8c8ff310bc872 Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Mon, 22 Dec 2025 05:39:59 +0000 Subject: [PATCH 16/21] apply limit on cloudbuild.yaml --- cloudbuild/run_zonal_tests.sh | 1 - cloudbuild/zb-system-tests-cloudbuild.yaml | 7 +++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/cloudbuild/run_zonal_tests.sh b/cloudbuild/run_zonal_tests.sh index a1b6594a8..ef94e629b 100644 --- a/cloudbuild/run_zonal_tests.sh +++ b/cloudbuild/run_zonal_tests.sh @@ -23,5 +23,4 @@ echo '--- Setting up environment variables on VM ---' export ZONAL_BUCKET=${_ZONAL_BUCKET} export RUN_ZONAL_SYSTEM_TESTS=True echo '--- Running Zonal tests on VM ---' -ulimit -n 1048576 pytest -vv -s --log-format='%(asctime)s %(levelname)s %(message)s' --log-date-format='%H:%M:%S' tests/system/test_zonal.py diff --git a/cloudbuild/zb-system-tests-cloudbuild.yaml b/cloudbuild/zb-system-tests-cloudbuild.yaml index 383c4fa96..455854dbc 100644 --- a/cloudbuild/zb-system-tests-cloudbuild.yaml +++ b/cloudbuild/zb-system-tests-cloudbuild.yaml @@ -52,6 +52,13 @@ steps: args: - "-c" - | + ulimit -Sn 4096 + # Double-check the limit was applied + CURRENT_LIMIT=$(ulimit -n) + if [ "$CURRENT_LIMIT" -lt 4096 ]; then + echo "ERROR: Failed to set ulimit to 4096. Current limit is $CURRENT_LIMIT " + exit 1 + fi set -e # Wait for the VM to be fully initialized and SSH to be ready. for i in {1..10}; do From bff780b29522a3053719182c3acdf7b7bf8a0647 Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Mon, 22 Dec 2025 06:00:40 +0000 Subject: [PATCH 17/21] use _ in front of variable name for cloud build --- cloudbuild/zb-system-tests-cloudbuild.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cloudbuild/zb-system-tests-cloudbuild.yaml b/cloudbuild/zb-system-tests-cloudbuild.yaml index 455854dbc..6f2dbb51d 100644 --- a/cloudbuild/zb-system-tests-cloudbuild.yaml +++ b/cloudbuild/zb-system-tests-cloudbuild.yaml @@ -52,11 +52,11 @@ steps: args: - "-c" - | - ulimit -Sn 4096 + ulimit -n 4096 # Double-check the limit was applied - CURRENT_LIMIT=$(ulimit -n) - if [ "$CURRENT_LIMIT" -lt 4096 ]; then - echo "ERROR: Failed to set ulimit to 4096. Current limit is $CURRENT_LIMIT " + _CURRENT_LIMIT=$(ulimit -n) + if [ "$_CURRENT_LIMIT" -lt 4096 ]; then + echo "ERROR: Failed to set ulimit to 4096. Current limit is $_CURRENT_LIMIT " exit 1 fi set -e From af94e3a55eb26119efc8788a829f0e9c3edf88b9 Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Mon, 22 Dec 2025 06:10:22 +0000 Subject: [PATCH 18/21] print current ulimit --- cloudbuild/zb-system-tests-cloudbuild.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudbuild/zb-system-tests-cloudbuild.yaml b/cloudbuild/zb-system-tests-cloudbuild.yaml index 6f2dbb51d..b7bb2e56a 100644 --- a/cloudbuild/zb-system-tests-cloudbuild.yaml +++ b/cloudbuild/zb-system-tests-cloudbuild.yaml @@ -59,6 +59,7 @@ steps: echo "ERROR: Failed to set ulimit to 4096. Current limit is $_CURRENT_LIMIT " exit 1 fi + echo "current ulimit: $_CURRENT_LIMIT" set -e # Wait for the VM to be fully initialized and SSH to be ready. for i in {1..10}; do From cbfb53ee0e2ec6a9f590d500f4562487d1b7fba6 Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Mon, 22 Dec 2025 06:22:10 +0000 Subject: [PATCH 19/21] increase ulimit to 1048576 --- cloudbuild/zb-system-tests-cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudbuild/zb-system-tests-cloudbuild.yaml b/cloudbuild/zb-system-tests-cloudbuild.yaml index b7bb2e56a..b416d37a1 100644 --- a/cloudbuild/zb-system-tests-cloudbuild.yaml +++ b/cloudbuild/zb-system-tests-cloudbuild.yaml @@ -52,7 +52,7 @@ steps: args: - "-c" - | - ulimit -n 4096 + ulimit -n 1048576 # Double-check the limit was applied _CURRENT_LIMIT=$(ulimit -n) if [ "$_CURRENT_LIMIT" -lt 4096 ]; then From e4aef2bd00ed8f940099e56675758ebca956bcb4 Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Mon, 22 Dec 2025 06:31:05 +0000 Subject: [PATCH 20/21] use $$ instead of $ for $CURRENT_PATH --- cloudbuild/zb-system-tests-cloudbuild.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cloudbuild/zb-system-tests-cloudbuild.yaml b/cloudbuild/zb-system-tests-cloudbuild.yaml index b416d37a1..394fde22d 100644 --- a/cloudbuild/zb-system-tests-cloudbuild.yaml +++ b/cloudbuild/zb-system-tests-cloudbuild.yaml @@ -55,11 +55,11 @@ steps: ulimit -n 1048576 # Double-check the limit was applied _CURRENT_LIMIT=$(ulimit -n) - if [ "$_CURRENT_LIMIT" -lt 4096 ]; then - echo "ERROR: Failed to set ulimit to 4096. Current limit is $_CURRENT_LIMIT " + if [ "$$_CURRENT_LIMIT" -lt 1048576 ]; then + echo "ERROR: Failed to set ulimit to 1048576. Current limit is $$_CURRENT_LIMIT " exit 1 fi - echo "current ulimit: $_CURRENT_LIMIT" + echo "current ulimit: $$_CURRENT_LIMIT" set -e # Wait for the VM to be fully initialized and SSH to be ready. for i in {1..10}; do From bab767722bae65ab5824579c7fa7c167eebc73b0 Mon Sep 17 00:00:00 2001 From: Chandra Sirimala Date: Mon, 22 Dec 2025 12:18:32 +0000 Subject: [PATCH 21/21] reset .yaml file to main --- cloudbuild/zb-system-tests-cloudbuild.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/cloudbuild/zb-system-tests-cloudbuild.yaml b/cloudbuild/zb-system-tests-cloudbuild.yaml index 394fde22d..383c4fa96 100644 --- a/cloudbuild/zb-system-tests-cloudbuild.yaml +++ b/cloudbuild/zb-system-tests-cloudbuild.yaml @@ -52,14 +52,6 @@ steps: args: - "-c" - | - ulimit -n 1048576 - # Double-check the limit was applied - _CURRENT_LIMIT=$(ulimit -n) - if [ "$$_CURRENT_LIMIT" -lt 1048576 ]; then - echo "ERROR: Failed to set ulimit to 1048576. Current limit is $$_CURRENT_LIMIT " - exit 1 - fi - echo "current ulimit: $$_CURRENT_LIMIT" set -e # Wait for the VM to be fully initialized and SSH to be ready. for i in {1..10}; do