refactor

Author: Andrei Neagu

packages/aws-library/src/aws_library/s3/_client.py

@@ -19,7 +19,7 @@
 from pydantic import AnyUrl, ByteSize, TypeAdapter
 from servicelib.bytes_iters import DEFAULT_READ_CHUNK_SIZE, BytesStreamer
 from servicelib.logging_utils import log_catch, log_context
-from servicelib.s3_utils import FileLikeBytesIterReader
+from servicelib.s3_utils import FileLikeReader
 from servicelib.utils import limited_gather
 from settings_library.s3 import S3Settings
 from types_aiobotocore_s3 import S3Client
@@ -515,14 +515,14 @@ async def _() -> BytesIter:
         return BytesStreamer(data_size, _)
 
     @s3_exception_handler(_logger)
-    async def upload_object_from_bytes_iter(  # TODO: this needs to be based on file interface -> use protocol to expose read
+    async def upload_object_from_file_like(
         self,
         bucket_name: S3BucketName,
         object_key: S3ObjectKey,
-        bytes_iter: BytesIter,
+        file_like_reader: FileLikeReader,
     ) -> None:
         """streams write an object in S3 from an AsyncIterable[bytes]"""
-        await self._client.upload_fileobj(FileLikeBytesIterReader(bytes_iter), bucket_name, object_key)  # type: ignore[arg-type]
+        await self._client.upload_fileobj(file_like_reader, bucket_name, object_key)  # type: ignore[arg-type]
 
     @staticmethod
     def is_multipart(file_size: ByteSize) -> bool:

packages/aws-library/tests/test_s3_client.py

@@ -64,6 +64,7 @@
 from servicelib.bytes_iters._models import DataSize
 from servicelib.file_utils import remove_directory
 from servicelib.progress_bar import ProgressBarData
+from servicelib.s3_utils import FileLikeBytesIterReader
 from servicelib.utils import limited_as_completed, limited_gather
 from settings_library.s3 import S3Settings
 from types_aiobotocore_s3 import S3Client
@@ -1417,7 +1418,7 @@ async def test_read_object_file_stream(
     await assert_same_file_content(with_uploaded_file_on_s3.local_path, tmp_file_name)
 
 
-async def test_upload_object_from_file_stream(
+async def test_upload_object_from_file_like(
     mocked_s3_server_envs: EnvVarsDict,
     with_uploaded_file_on_s3: UploadedFile,
     simcore_s3_api: SimcoreS3API,
@@ -1429,10 +1430,14 @@ async def test_upload_object_from_file_stream(
     )
     assert isinstance(bytes_streamer.data_size, DataSize)
 
-    await simcore_s3_api.upload_object_from_bytes_iter(
-        with_s3_bucket, object_key, bytes_streamer.with_progress_bytes_iter(AsyncMock())
+    await simcore_s3_api.upload_object_from_file_like(
+        with_s3_bucket,
+        object_key,
+        FileLikeBytesIterReader(bytes_streamer.with_progress_bytes_iter(AsyncMock())),
     )
 
+    # TODO: also add a test uploading form a file instead of S3 since it's possible, open it with aiofiles
+
     await simcore_s3_api.delete_object(bucket=with_s3_bucket, object_key=object_key)
 
 
@@ -1589,13 +1594,15 @@ async def test_workflow_compress_s3_objects_and_local_files_in_a_single_archive_
         progress_report_cb=mocked_progress_bar_cb,
         description="root_bar",
     ) as progress_bar:
-        await simcore_s3_api.upload_object_from_bytes_iter(
+        await simcore_s3_api.upload_object_from_file_like(
             with_s3_bucket,
             archive_s3_object_key,
-            get_zip_bytes_iter(
-                archive_entries,
-                progress_bar=progress_bar,
-                chunk_size=MULTIPART_COPY_THRESHOLD,
+            FileLikeBytesIterReader(
+                get_zip_bytes_iter(
+                    archive_entries,
+                    progress_bar=progress_bar,
+                    chunk_size=MULTIPART_COPY_THRESHOLD,
+                )
             ),
         )
 

packages/service-library/src/servicelib/s3_utils.py

@@ -1,7 +1,16 @@
+from typing import Protocol
+
 from models_library.bytes_iters import BytesIter
 
 
-class FileLikeBytesIterReader:
+class FileLikeReader(Protocol):
+    """minimal interface for upload from file objects to S3"""
+
+    async def read(self, size: int) -> bytes:
+        ...
+
+
+class FileLikeBytesIterReader(FileLikeReader):
     def __init__(self, bytes_iter: BytesIter):
         self._bytes_iter = bytes_iter
         self._buffer = bytearray()