fixed zip, using zip_64 by default

Author: Andrei Neagu

packages/aws-library/src/aws_library/s3/_constants.py

@@ -9,6 +9,7 @@
 MULTIPART_COPY_THRESHOLD: Final[ByteSize] = TypeAdapter(ByteSize).validate_python(
     "100MiB"
 )
+STREAM_REDER_CHUNK: Final[ByteSize] = TypeAdapter(ByteSize).validate_python("10MiB")
 
 PRESIGNED_LINK_MAX_SIZE: Final[ByteSize] = TypeAdapter(ByteSize).validate_python("5GiB")
 S3_MAX_FILE_SIZE: Final[ByteSize] = TypeAdapter(ByteSize).validate_python("5TiB")

packages/aws-library/tests/test_s3_client.py

@@ -27,8 +27,8 @@
 from aiohttp import ClientSession
 from aws_library.s3._client import _AWS_MAX_ITEMS_PER_PAGE, S3ObjectKey, SimcoreS3API
 from aws_library.s3._constants import (
-    MULTIPART_COPY_THRESHOLD,
     MULTIPART_UPLOADS_MIN_TOTAL_SIZE,
+    STREAM_REDER_CHUNK,
 )
 from aws_library.s3._errors import (
     S3BucketInvalidError,
@@ -1902,7 +1902,7 @@ async def test_workflow_compress_s3_objects_and_local_files_in_a_single_archive_
                 get_zip_bytes_iter(
                     archive_entries,
                     progress_bar=progress_bar,
-                    chunk_size=MULTIPART_COPY_THRESHOLD,
+                    chunk_size=STREAM_REDER_CHUNK,
                 )
             ),
         )

packages/service-library/src/servicelib/bytes_iters/_stream_zip.py

@@ -1,14 +1,17 @@
+import logging
 from collections.abc import AsyncIterable
 from datetime import UTC, datetime
 from stat import S_IFREG
 from typing import TypeAlias
 
 from models_library.bytes_iters import BytesIter, DataSize
-from stream_zip import ZIP_32, AsyncMemberFile, async_stream_zip
+from stream_zip import ZIP_64, AsyncMemberFile, async_stream_zip
 
 from ..progress_bar import ProgressBarData
 from ._models import BytesStreamer
 
+_logger = logging.getLogger(__name__)
+
 FileNameInArchive: TypeAlias = str
 ArchiveFileEntry: TypeAlias = tuple[FileNameInArchive, BytesStreamer]
 ArchiveEntries: TypeAlias = list[ArchiveFileEntry]
@@ -22,7 +25,7 @@ async def _member_files_iter(
             file_name,
             datetime.now(UTC),
             S_IFREG | 0o600,
-            ZIP_32,
+            ZIP_64,
             byte_streamer.with_progress_bytes_iter(progress_bar=progress_bar),
         )
 
@@ -42,6 +45,7 @@ async def get_zip_bytes_iter(
         sum(bytes_streamer.data_size for _, bytes_streamer in archive_entries)
     )
     description = f"files: count={len(archive_entries)}, size={total_stream_lenth.human_readable()}"
+    _logger.info("Archiving %s", description)
 
     async with progress_bar.sub_progress(
         steps=total_stream_lenth, description=description, progress_unit="Byte"

services/storage/src/simcore_service_storage/utils/simcore_s3_dsm_utils.py

@@ -4,7 +4,7 @@
 
 import orjson
 from aws_library.s3 import S3MetaData, SimcoreS3API
-from aws_library.s3._constants import MULTIPART_COPY_THRESHOLD
+from aws_library.s3._constants import STREAM_REDER_CHUNK
 from models_library.api_schemas_storage.storage_schemas import S3BucketName
 from models_library.projects import ProjectID
 from models_library.projects_nodes_io import (
@@ -173,7 +173,7 @@ async def create_and_upload_export(
                 get_zip_bytes_iter(
                     archive_entries,
                     progress_bar=progress_bar,
-                    chunk_size=MULTIPART_COPY_THRESHOLD,
+                    chunk_size=STREAM_REDER_CHUNK,
                 )
             ),
         )