Merge pull request #216 from Aiven-Open/bridgetb/handle-botocore-error

Handle another case of incomplete reads from S3-type storage (OVH S3)

Author: edward-evans-aiven

pyproject.toml

@@ -87,6 +87,8 @@ docs  = [
 [tool.ruff]
 target-version = "py38"
 line-length = 125
+extend-exclude = ["rohmu/version.py"]
+
 
 [tool.ruff.lint]
 extend-select = [

rohmu/object_storage/s3.py

@@ -443,7 +443,11 @@ def _read_object_to_fileobj(
             read_amount = min(read_amount, READ_BLOCK_SIZE)
             try:
                 data = streaming_body.read(amt=read_amount)
-            except (botocore.exceptions.IncompleteReadError, botocore.exceptions.ReadTimeoutError) as ex:
+            except (
+                botocore.exceptions.IncompleteReadError,
+                botocore.exceptions.ReadTimeoutError,
+                botocore.exceptions.ResponseStreamingError,
+            ) as ex:
                 raise MaybeRecoverableError("botocore.exceptions.IncompleteReadError", position=data_read) from ex
 
             fileobj.write(data)

test/object_storage/test_s3.py

@@ -245,6 +245,10 @@ def test_get_contents_to_fileobj_raises_error_on_invalid_byte_range(infra: S3Inf
     [
         botocore.exceptions.IncompleteReadError(actual_bytes=80, expected_bytes=200),
         botocore.exceptions.ReadTimeoutError(endpoint_url="https://example.org"),
+        botocore.exceptions.ResponseStreamingError(
+            endpoint_url="https://example.org",
+            error="An error occurred while reading from response stream: ...",
+        ),
     ],
     ids=type,
 )
@@ -424,7 +428,7 @@ class TransferMinMultipartChunkSizeTestData:
         expected_chunk_size=mb_to_bytes(9),
     ),
     TransferMinMultipartChunkSizeTestData(
-        description="Same as above, but min_multipart_chunk_size is set to 50 MB, " "we expect only 2 chunks of 50 MB each",
+        description="Same as above, but min_multipart_chunk_size is set to 50 MB, we expect only 2 chunks of 50 MB each",
         size=mb_to_bytes(100),
         segment_size=mb_to_bytes(9),
         min_multipart_chunk_size=mb_to_bytes(50),