Skip to content

Commit e2a595a

Browse files
committed
feat: compute chunk wise checksum for bidi_writes
1 parent 0e2961b commit e2a595a

File tree

1 file changed

+13
-1
lines changed

1 file changed

+13
-1
lines changed

google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222
2323
"""
2424
from typing import Optional, Union
25+
from google.api_core import exceptions
26+
from google_crc32c import Checksum, implementation as crc32c_impl
2527
from google.cloud import _storage_v2
2628
from google.cloud.storage._experimental.asyncio.async_grpc_client import (
2729
AsyncGrpcClient,
@@ -100,6 +102,14 @@ def __init__(
100102
:param write_handle: (Optional) An existing handle for writing the object.
101103
If provided, opening the bidi-gRPC connection will be faster.
102104
"""
105+
# Verify that the fast, C-accelerated version of crc32c is available.
106+
# If not, raise an error to prevent silent performance degradation.
107+
if crc32c_impl != "c":
108+
raise exceptions.NotFound(
109+
"The google-crc32c package is not installed with C support. "
110+
"Bidi reads require the C extension for data integrity checks."
111+
"For more information, see https://github.com/googleapis/python-crc32c."
112+
)
103113
self.client = client
104114
self.bucket_name = bucket_name
105115
self.object_name = object_name
@@ -191,11 +201,13 @@ async def append(self, data: bytes) -> None:
191201
bytes_to_flush = 0
192202
while start_idx < total_bytes:
193203
end_idx = min(start_idx + _MAX_CHUNK_SIZE_BYTES, total_bytes)
204+
data_chunk = data[start_idx:end_idx]
194205
await self.write_obj_stream.send(
195206
_storage_v2.BidiWriteObjectRequest(
196207
write_offset=self.offset,
197208
checksummed_data=_storage_v2.ChecksummedData(
198-
content=data[start_idx:end_idx]
209+
content=data_chunk,
210+
crc32c=int.from_bytes(Checksum(data_chunk).digest(), "big"),
199211
),
200212
)
201213
)

0 commit comments

Comments
 (0)