refactoring

Author: sanderegg

services/storage/tests/conftest.py

@@ -63,11 +63,11 @@
 from tenacity.retry import retry_if_exception_type
 from tenacity.stop import stop_after_delay
 from tenacity.wait import wait_fixed
+from tests.fixtures.data_models import FileIDDict
+from tests.helpers.utils_file_meta_data import assert_file_meta_data_in_db
 from types_aiobotocore_s3 import S3Client
 from yarl import URL
 
-from tests.helpers.utils_file_meta_data import assert_file_meta_data_in_db
-
 pytest_plugins = [
     "pytest_simcore.aws_s3_service",
     "pytest_simcore.aws_server",
@@ -290,9 +290,9 @@ async def _link_creator(
             location_id=f"{location_id}",
             file_id=file_id,
         ).with_query(**query_kwargs, user_id=user_id)
-        assert "file_size" in url.query, (
-            "V2 call to upload file must contain file_size field!"
-        )
+        assert (
+            "file_size" in url.query
+        ), "V2 call to upload file must contain file_size field!"
         response = await client.put(f"{url}")
         received_file_upload, error = assert_status(
             response, status.HTTP_200_OK, FileUploadSchema
@@ -641,14 +641,11 @@ async def with_random_project_with_files(
         Awaitable[
             tuple[
                 ProjectAtDB,
-                dict[NodeID, dict[SimcoreS3FileID, dict[str, Path | str]]],
+                dict[NodeID, dict[SimcoreS3FileID, FileIDDict]],
             ]
         ],
     ],
-) -> tuple[
-    ProjectAtDB,
-    dict[NodeID, dict[SimcoreS3FileID, dict[str, Path | str]]],
-]:
+) -> tuple[ProjectAtDB, dict[NodeID, dict[SimcoreS3FileID, FileIDDict]],]:
     return await random_project_with_files(
         file_sizes=(
             TypeAdapter(ByteSize).validate_python("1Mib"),

services/storage/tests/fixtures/data_models.py

@@ -7,7 +7,7 @@
 from contextlib import asynccontextmanager
 from pathlib import Path
 from random import choice, randint
-from typing import Any, cast
+from typing import Any, TypedDict, cast
 
 import pytest
 import sqlalchemy as sa
@@ -259,6 +259,11 @@ async def _creator(
     return _creator
 
 
+class FileIDDict(TypedDict):
+    path: Path
+    sha256_checksum: SHA256Str
+
+
 async def _upload_file_and_update_project(
     project_id: ProjectID,
     node_id: NodeID,
@@ -267,7 +272,7 @@ async def _upload_file_and_update_project(
     file_id: StorageFileID | None,
     file_sizes: tuple[ByteSize, ...],
     file_checksums: tuple[SHA256Str, ...],
-    node_to_files_mapping: dict[NodeID, dict[SimcoreS3FileID, dict[str, Path | str]]],
+    node_to_files_mapping: dict[NodeID, dict[SimcoreS3FileID, FileIDDict]],
     upload_file: Callable[..., Awaitable[tuple[Path, SimcoreS3FileID]]],
     create_simcore_file_id: Callable[
         [ProjectID, NodeID, str, Path | None], SimcoreS3FileID
@@ -304,9 +309,7 @@ async def random_project_with_files(
     faker: Faker,
 ) -> Callable[
     [int, tuple[ByteSize, ...], tuple[SHA256Str, ...]],
-    Awaitable[
-        tuple[ProjectAtDB, dict[NodeID, dict[SimcoreS3FileID, dict[str, Path | str]]]]
-    ],
+    Awaitable[tuple[ProjectAtDB, dict[NodeID, dict[SimcoreS3FileID, FileIDDict]]]],
 ]:
     async def _creator(
         num_nodes: int = 12,
@@ -326,12 +329,10 @@ async def _creator(
                 "488f3b57932803bbf644593bd46d95599b1d4da1d63bc020d7ebe6f1c255f7f3"
             ),
         ),
-    ) -> tuple[ProjectAtDB, dict[NodeID, dict[SimcoreS3FileID, dict[str, Path | str]]]]:
+    ) -> tuple[ProjectAtDB, dict[NodeID, dict[SimcoreS3FileID, FileIDDict]]]:
         assert len(file_sizes) == len(file_checksums)
         project = await create_project(name="random-project")
-        node_to_files_mapping: dict[
-            NodeID, dict[SimcoreS3FileID, dict[str, Path | str]]
-        ] = {}
+        node_to_files_mapping: dict[NodeID, dict[SimcoreS3FileID, FileIDDict]] = {}
         upload_tasks: deque[Awaitable] = deque()
         for _node_index in range(num_nodes):
             # Create a node with outputs (files and others)

services/storage/tests/unit/test_handlers_files.py

@@ -61,11 +61,11 @@
 from tenacity.retry import retry_if_exception_type
 from tenacity.stop import stop_after_delay
 from tenacity.wait import wait_fixed
+from tests.fixtures.data_models import FileIDDict
+from tests.helpers.utils_file_meta_data import assert_file_meta_data_in_db
 from types_aiobotocore_s3 import S3Client
 from yarl import URL
 
-from tests.helpers.utils_file_meta_data import assert_file_meta_data_in_db
-
 pytest_simcore_core_services_selection = ["postgres"]
 pytest_simcore_ops_services_selection = ["adminer"]
 
@@ -91,14 +91,14 @@ async def assert_multipart_uploads_in_progress(
         tuple[UploadID, S3ObjectKey]
     ] = await storage_s3_client.list_ongoing_multipart_uploads(bucket=storage_s3_bucket)
     if expected_upload_ids is None:
-        assert not list_uploads, (
-            f"expected NO multipart uploads in progress, got {list_uploads}"
-        )
+        assert (
+            not list_uploads
+        ), f"expected NO multipart uploads in progress, got {list_uploads}"
     else:
         for upload_id, _ in list_uploads:
-            assert upload_id in expected_upload_ids, (
-                f"{upload_id=} is in progress but was not expected!"
-            )
+            assert (
+                upload_id in expected_upload_ids
+            ), f"{upload_id=} is in progress but was not expected!"
 
 
 @dataclass
@@ -209,9 +209,9 @@ async def _link_creator(file_id: SimcoreS3FileID, **query_kwargs) -> PresignedLi
             location_id=f"{location_id}",
             file_id=file_id,
         ).with_query(**query_kwargs, user_id=user_id)
-        assert "file_size" not in url.query, (
-            "v1 call to upload_file MUST NOT contain file_size field, this is reserved for v2 call"
-        )
+        assert (
+            "file_size" not in url.query
+        ), "v1 call to upload_file MUST NOT contain file_size field, this is reserved for v2 call"
         response = await client.put(f"{url}")
         received_file_upload_link, error = assert_status(
             response, status.HTTP_200_OK, PresignedLink
@@ -390,9 +390,9 @@ async def test_create_upload_file_presigned_with_file_size_returns_multipart_lin
         file_size=f"{test_param.file_size}",
     )
     # number of links
-    assert len(received_file_upload.urls) == test_param.expected_num_links, (
-        f"{len(received_file_upload.urls)} vs {test_param.expected_num_links=}"
-    )
+    assert (
+        len(received_file_upload.urls) == test_param.expected_num_links
+    ), f"{len(received_file_upload.urls)} vs {test_param.expected_num_links=}"
     # all links are unique
     assert len(set(received_file_upload.urls)) == len(received_file_upload.urls)
     assert received_file_upload.chunk_size == test_param.expected_chunk_size
@@ -547,9 +547,9 @@ async def test_upload_same_file_uuid_aborts_previous_upload(
         expected_sha256_checksum=None,
     )
     if expect_upload_id and link_type == LinkType.PRESIGNED:
-        assert upload_id != new_upload_id, (
-            "There shall be a new upload id after a new call to create_upload_file"
-        )
+        assert (
+            upload_id != new_upload_id
+        ), "There shall be a new upload id after a new call to create_upload_file"
     elif expect_upload_id and link_type == LinkType.S3:
         assert upload_id == new_upload_id
         assert upload_id == S3_UNDEFINED_OR_EXTERNAL_MULTIPART_ID
@@ -1483,7 +1483,7 @@ async def test_listing_with_project_id_filter(
         Awaitable[
             tuple[
                 ProjectAtDB,
-                dict[NodeID, dict[SimcoreS3FileID, dict[str, Path | str]]],
+                dict[NodeID, dict[SimcoreS3FileID, FileIDDict]],
             ]
         ],
     ],

services/storage/tests/unit/test_handlers_paths.py

@@ -19,6 +19,7 @@
 from pytest_simcore.helpers.fastapi import url_from_operation_id
 from pytest_simcore.helpers.httpx_assert_checks import assert_status
 from servicelib.aiohttp import status
+from tests.fixtures.data_models import FileIDDict
 
 pytest_simcore_core_services_selection = ["postgres"]
 pytest_simcore_ops_services_selection = ["adminer"]
@@ -55,7 +56,7 @@ async def test_list_paths_root_folder(
     user_id: UserID,
     with_random_project_with_files: tuple[
         ProjectAtDB,
-        dict[NodeID, dict[SimcoreS3FileID, dict[str, Path | str]]],
+        dict[NodeID, dict[SimcoreS3FileID, FileIDDict]],
     ],
 ):
     project_in_db, list_of_files = with_random_project_with_files

services/storage/tests/unit/test_handlers_simcore_s3.py

@@ -37,6 +37,7 @@
 from simcore_service_storage.models import SearchFilesQueryParams
 from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager
 from sqlalchemy.ext.asyncio import AsyncEngine
+from tests.fixtures.data_models import FileIDDict
 from tests.helpers.utils_file_meta_data import assert_file_meta_data_in_db
 from tests.helpers.utils_project import clone_project_data
 from yarl import URL
@@ -213,7 +214,7 @@ async def test_copy_folders_from_valid_project_with_one_large_file(
         Awaitable[
             tuple[
                 ProjectAtDB,
-                dict[NodeID, dict[SimcoreS3FileID, dict[str, Path | str]]],
+                dict[NodeID, dict[SimcoreS3FileID, FileIDDict]],
             ]
         ],
     ],
@@ -283,7 +284,7 @@ async def test_copy_folders_from_valid_project(
         Awaitable[
             tuple[
                 ProjectAtDB,
-                dict[NodeID, dict[SimcoreS3FileID, dict[str, Path | SHA256Str]]],
+                dict[NodeID, dict[SimcoreS3FileID, FileIDDict]],
             ]
         ],
     ],