🎨 Maintenance: change default chunk size to compute checksums to 8MiB instead of 5KB (#7753)

Author: sanderegg

packages/service-library/src/servicelib/file_utils.py

@@ -1,10 +1,11 @@
 import asyncio
 import hashlib
 import shutil
+from collections.abc import Iterator
 from contextlib import contextmanager
 from logging import Logger
 from pathlib import Path
-from typing import Final, Iterator, Protocol
+from typing import Final, Protocol
 
 # https://docs.python.org/3/library/shutil.html#shutil.rmtree
 # https://docs.python.org/3/library/os.html#os.remove
@@ -13,11 +14,13 @@
 from pydantic import ByteSize, TypeAdapter
 
 CHUNK_4KB: Final[ByteSize] = TypeAdapter(ByteSize).validate_python("4kb")  # 4K blocks
+CHUNK_8MB: Final[ByteSize] = TypeAdapter(ByteSize).validate_python(
+    "8MiB"
+)  # 8mIB blocks
 
 
 class AsyncStream(Protocol):
-    async def read(self, size: int = -1) -> bytes:
-        ...
+    async def read(self, size: int = -1) -> bytes: ...
 
 
 _shutil_rmtree = sync_to_async(shutil.rmtree)
@@ -45,7 +48,7 @@ async def remove_directory(
 
 
 async def create_sha256_checksum(
-    async_stream: AsyncStream, *, chunk_size: ByteSize = CHUNK_4KB
+    async_stream: AsyncStream, *, chunk_size: ByteSize = CHUNK_8MB
 ) -> str:
     """
     Usage:

packages/simcore-sdk/src/simcore_sdk/node_ports_common/filemanager.py

@@ -291,7 +291,7 @@ async def _generate_checksum(
         return checksum
     if isinstance(path_to_upload, Path):
         async with aiofiles.open(path_to_upload, mode="rb") as f:
-            checksum = SHA256Str(await create_sha256_checksum(f))
+            checksum = await create_sha256_checksum(f)
     elif isinstance(path_to_upload, UploadableFileObject):
         checksum = path_to_upload.sha256_checksum
     return checksum

packages/simcore-sdk/tests/integration/conftest.py

@@ -337,7 +337,7 @@ def _assign_config(
 @pytest.fixture
 async def r_clone_settings_factory(
     minio_s3_settings: S3Settings, storage_service: URL
-) -> Awaitable[RCloneSettings]:
+) -> Callable[[], Awaitable[RCloneSettings]]:
     async def _factory() -> RCloneSettings:
         settings = RCloneSettings(
             R_CLONE_S3=minio_s3_settings, R_CLONE_PROVIDER=S3Provider.MINIO
@@ -347,35 +347,35 @@ async def _factory() -> RCloneSettings:
 
         return settings
 
-    return _factory()
+    return _factory
 
 
 @pytest.fixture
 async def aws_s3_cli_settings_factory(
     minio_s3_settings: S3Settings, storage_service: URL
-) -> Awaitable[AwsS3CliSettings]:
+) -> Callable[[], Awaitable[AwsS3CliSettings]]:
     async def _factory() -> AwsS3CliSettings:
         settings = AwsS3CliSettings(AWS_S3_CLI_S3=minio_s3_settings)
         if not await is_aws_s3_cli_available(settings):
             pytest.skip("aws cli not installed")
 
         return settings
 
-    return _factory()
+    return _factory
 
 
 @pytest.fixture
 async def r_clone_settings(
-    r_clone_settings_factory: Awaitable[RCloneSettings],
+    r_clone_settings_factory: Callable[[], Awaitable[RCloneSettings]],
 ) -> RCloneSettings:
-    return await r_clone_settings_factory
+    return await r_clone_settings_factory()
 
 
 @pytest.fixture
 async def aws_s3_cli_settings(
-    aws_s3_cli_settings_factory: Awaitable[AwsS3CliSettings],
+    aws_s3_cli_settings_factory: Callable[[], Awaitable[AwsS3CliSettings]],
 ) -> AwsS3CliSettings:
-    return await aws_s3_cli_settings_factory
+    return await aws_s3_cli_settings_factory()
 
 
 @pytest.fixture

packages/simcore-sdk/tests/integration/test_node_ports_common_filemanager.py

@@ -56,15 +56,17 @@ class _SyncSettings(BaseModel):
         "Both RClone and AwsS3Cli disabled",
     ],
 )
-def optional_sync_settings(
-    r_clone_settings: RCloneSettings,
-    aws_s3_cli_settings: AwsS3CliSettings,
+async def optional_sync_settings(
+    r_clone_settings_factory: Callable[[], Awaitable[RCloneSettings]],
+    aws_s3_cli_settings_factory: Callable[[], Awaitable[AwsS3CliSettings]],
     request: pytest.FixtureRequest,
 ) -> _SyncSettings:
     _rclone_enabled, _aws_s3_cli_enabled = request.param
 
-    _r_clone_settings = r_clone_settings if _rclone_enabled else None
-    _aws_s3_cli_settings = aws_s3_cli_settings if _aws_s3_cli_enabled else None
+    _r_clone_settings = await r_clone_settings_factory() if _rclone_enabled else None
+    _aws_s3_cli_settings = (
+        await aws_s3_cli_settings_factory() if _aws_s3_cli_enabled else None
+    )
 
     return _SyncSettings(
         r_clone_settings=_r_clone_settings, aws_s3_cli_settings=_aws_s3_cli_settings

packages/simcore-sdk/tests/integration/test_node_ports_v2_nodeports2.py

@@ -21,7 +21,6 @@
 import pytest
 import sqlalchemy as sa
 from faker import Faker
-from models_library.projects import ProjectIDStr
 from models_library.projects_nodes_io import (
     BaseFileLink,
     DownloadLink,
@@ -156,10 +155,10 @@ def config_value_symlink_path(symlink_path: Path) -> dict[str, Any]:
 
 @pytest.fixture(params=[True, False])
 async def option_r_clone_settings(
-    request, r_clone_settings_factory: Awaitable[RCloneSettings]
+    request, r_clone_settings_factory: Callable[[], Awaitable[RCloneSettings]]
 ) -> RCloneSettings | None:
     if request.param:
-        return await r_clone_settings_factory
+        return await r_clone_settings_factory()
     return None
 
 
@@ -174,7 +173,7 @@ async def test_default_configuration(
     await check_config_valid(
         await node_ports_v2.ports(
             user_id=user_id,
-            project_id=ProjectIDStr(project_id),
+            project_id=project_id,
             node_uuid=node_uuid,
             r_clone_settings=option_r_clone_settings,
         ),
@@ -192,7 +191,7 @@ async def test_invalid_ports(
     config_dict, _, _ = create_special_configuration()
     PORTS = await node_ports_v2.ports(
         user_id=user_id,
-        project_id=ProjectIDStr(project_id),
+        project_id=project_id,
         node_uuid=node_uuid,
         r_clone_settings=option_r_clone_settings,
     )
@@ -238,7 +237,7 @@ async def test_port_value_accessors(
 
     PORTS = await node_ports_v2.ports(
         user_id=user_id,
-        project_id=ProjectIDStr(project_id),
+        project_id=project_id,
         node_uuid=node_uuid,
         r_clone_settings=option_r_clone_settings,
     )
@@ -298,7 +297,7 @@ async def test_port_file_accessors(
 
     PORTS = await node_ports_v2.ports(
         user_id=user_id,
-        project_id=ProjectIDStr(project_id),
+        project_id=project_id,
         node_uuid=node_uuid,
         r_clone_settings=option_r_clone_settings,
     )
@@ -375,7 +374,7 @@ async def test_adding_new_ports(
     config_dict, project_id, node_uuid = create_special_configuration()
     PORTS = await node_ports_v2.ports(
         user_id=user_id,
-        project_id=ProjectIDStr(project_id),
+        project_id=project_id,
         node_uuid=node_uuid,
         r_clone_settings=option_r_clone_settings,
     )
@@ -429,7 +428,7 @@ async def test_removing_ports(
     )  # pylint: disable=W0612
     PORTS = await node_ports_v2.ports(
         user_id=user_id,
-        project_id=ProjectIDStr(project_id),
+        project_id=project_id,
         node_uuid=node_uuid,
         r_clone_settings=option_r_clone_settings,
     )
@@ -489,7 +488,7 @@ async def test_get_value_from_previous_node(
 
     PORTS = await node_ports_v2.ports(
         user_id=user_id,
-        project_id=ProjectIDStr(project_id),
+        project_id=project_id,
         node_uuid=node_uuid,
         r_clone_settings=option_r_clone_settings,
     )
@@ -541,7 +540,7 @@ async def test_get_file_from_previous_node(
     )
     PORTS = await node_ports_v2.ports(
         user_id=user_id,
-        project_id=ProjectIDStr(project_id),
+        project_id=project_id,
         node_uuid=node_uuid,
         r_clone_settings=option_r_clone_settings,
     )
@@ -598,7 +597,7 @@ async def test_get_file_from_previous_node_with_mapping_of_same_key_name(
     )
     PORTS = await node_ports_v2.ports(
         user_id=user_id,
-        project_id=ProjectIDStr(project_id),
+        project_id=project_id,
         node_uuid=node_uuid,
         r_clone_settings=option_r_clone_settings,
     )
@@ -659,7 +658,7 @@ async def test_file_mapping(
     )
     PORTS = await node_ports_v2.ports(
         user_id=user_id,
-        project_id=ProjectIDStr(project_id),
+        project_id=project_id,
         node_uuid=node_uuid,
         r_clone_settings=option_r_clone_settings,
     )
@@ -752,7 +751,7 @@ async def test_regression_concurrent_port_update_fails(
 
     PORTS = await node_ports_v2.ports(
         user_id=user_id,
-        project_id=ProjectIDStr(project_id),
+        project_id=project_id,
         node_uuid=node_uuid,
         r_clone_settings=option_r_clone_settings,
     )
@@ -841,7 +840,7 @@ async def test_batch_update_inputs_outputs(
 
     PORTS = await node_ports_v2.ports(
         user_id=user_id,
-        project_id=ProjectIDStr(project_id),
+        project_id=project_id,
         node_uuid=node_uuid,
         r_clone_settings=option_r_clone_settings,
     )