Skip to content

Commit 4bbbc44

Browse files
committed
Handle another case of incomplete reads from S3-type storage (OVH S3)
* Ensure that any botocore ResponseStreamingErrors are handled and retried in a similar way to botocore IncompleteReadErrors. * This type of error appears to be frequently raised when using OVH S3 storage, resulting in a failure to download the file.
1 parent 9995cbc commit 4bbbc44

File tree

2 files changed

+10
-2
lines changed

2 files changed

+10
-2
lines changed

rohmu/object_storage/s3.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -443,7 +443,11 @@ def _read_object_to_fileobj(
443443
read_amount = min(read_amount, READ_BLOCK_SIZE)
444444
try:
445445
data = streaming_body.read(amt=read_amount)
446-
except (botocore.exceptions.IncompleteReadError, botocore.exceptions.ReadTimeoutError) as ex:
446+
except (
447+
botocore.exceptions.IncompleteReadError,
448+
botocore.exceptions.ReadTimeoutError,
449+
botocore.exceptions.ResponseStreamingError,
450+
) as ex:
447451
raise MaybeRecoverableError("botocore.exceptions.IncompleteReadError", position=data_read) from ex
448452

449453
fileobj.write(data)

test/object_storage/test_s3.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -245,6 +245,10 @@ def test_get_contents_to_fileobj_raises_error_on_invalid_byte_range(infra: S3Inf
245245
[
246246
botocore.exceptions.IncompleteReadError(actual_bytes=80, expected_bytes=200),
247247
botocore.exceptions.ReadTimeoutError(endpoint_url="https://example.org"),
248+
botocore.exceptions.ResponseStreamingError(
249+
endpoint_url="https://example.org",
250+
error="An error occurred while reading from response stream: ...",
251+
),
248252
],
249253
ids=type,
250254
)
@@ -424,7 +428,7 @@ class TransferMinMultipartChunkSizeTestData:
424428
expected_chunk_size=mb_to_bytes(9),
425429
),
426430
TransferMinMultipartChunkSizeTestData(
427-
description="Same as above, but min_multipart_chunk_size is set to 50 MB, " "we expect only 2 chunks of 50 MB each",
431+
description="Same as above, but min_multipart_chunk_size is set to 50 MB, we expect only 2 chunks of 50 MB each",
428432
size=mb_to_bytes(100),
429433
segment_size=mb_to_bytes(9),
430434
min_multipart_chunk_size=mb_to_bytes(50),

0 commit comments

Comments
 (0)