refactor

Author: Andrei Neagu

packages/service-library/src/servicelib/zip_stream/_constants.py

@@ -1,5 +1,5 @@
 from typing import Final
 
-_KB: Final[int] = 1024
+from pydantic import ByteSize, TypeAdapter
 
-DEFAULT_CHUNK_SIZE: Final[int] = 64 * _KB
+DEFAULT_CHUNK_SIZE: Final[int] = TypeAdapter(ByteSize).validate_python("1MiB")

packages/service-library/src/servicelib/zip_stream/_zipper.py

@@ -27,11 +27,19 @@ async def _iter_files(
 
 
 async def get_zip_archive_stream(
-    archive_files: ArchiveEntries, *, progress_bar: ProgressBarData | None = None
+    archive_files: ArchiveEntries,
+    *,
+    progress_bar: ProgressBarData | None = None,
+    chunk_size: int = DEFAULT_CHUNK_SIZE
 ) -> FileStream:
+    # NOTE: this is CPU bound task, even though the loop is not blocked,
+    # the CPU is still used for compressing the content
     if progress_bar is None:
         progress_bar = ProgressBarData(num_steps=1, description="stream archiver")
+
+    # NOTE: do not disable compression or the streams will be
+    # loaded fully in memory before yielding their content
     async for chunk in async_stream_zip(
-        _iter_files(archive_files, progress_bar), chunk_size=DEFAULT_CHUNK_SIZE
+        _iter_files(archive_files, progress_bar), chunk_size=chunk_size
     ):
         yield chunk

packages/service-library/tests/test_zip_stream.py

@@ -10,8 +10,8 @@
 from faker import Faker
 from pytest_mock import MockerFixture
 from pytest_simcore.helpers.comparing import (
-    assert_same_folder_contents,
-    get_files_in_folder,
+    assert_same_contents,
+    get_files_info_from_path,
     get_relative_to,
 )
 from servicelib.archiving_utils import unarchive_dir
@@ -121,7 +121,7 @@ async def test_get_zip_archive_stream(
     await unarchive_dir(local_archive_path, local_unpacked_archive)
 
     # 3. compare files in directories (same paths & sizes)
-    await assert_same_folder_contents(
-        get_files_in_folder(local_files_dir),
-        get_files_in_folder(local_unpacked_archive),
+    await assert_same_contents(
+        get_files_info_from_path(local_files_dir),
+        get_files_info_from_path(local_unpacked_archive),
     )