fix: fixtures still using workbench

Author: giancarloromeo

packages/pytest-simcore/src/pytest_simcore/simcore_storage_data_models.py

@@ -215,10 +215,10 @@ async def _() -> None:
 @pytest.fixture
 async def create_project_node(
     user_id: UserID, sqlalchemy_async_engine: AsyncEngine, faker: Faker
-) -> Callable[..., Awaitable[NodeID]]:
+) -> Callable[..., Awaitable[tuple[NodeID, dict[str, Any]]]]:
     async def _creator(
         project_id: ProjectID, node_id: NodeID | None = None, **kwargs
-    ) -> NodeID:
+    ) -> tuple[NodeID, dict[str, Any]]:
         async with sqlalchemy_async_engine.begin() as conn:
             new_node_id = node_id or NodeID(faker.uuid4())
             node_values = {
@@ -227,13 +227,16 @@ async def _creator(
                 "label": "pytest_fake_node",
             }
             node_values.update(**kwargs)
-            await conn.execute(
-                projects_nodes.insert().values(
+            result = await conn.execute(
+                projects_nodes.insert()
+                .values(
                     node_id=f"{new_node_id}",
                     project_uuid=f"{project_id}",
                     **node_values,
                 )
+                .returning(sa.literal_column("*"))
             )
-        return new_node_id
+            row = result.one()
+            return new_node_id, row._asdict()
 
     return _creator

services/storage/tests/conftest.py

@@ -259,9 +259,13 @@ async def client(
 
 @pytest.fixture
 async def node_id(
-    project_id: ProjectID, create_project_node: Callable[[ProjectID], Awaitable[NodeID]]
+    project_id: ProjectID,
+    create_project_node: Callable[
+        [ProjectID], Awaitable[tuple[NodeID, dict[str, Any]]]
+    ],
 ) -> NodeID:
-    return await create_project_node(project_id)
+    node_id, _ = await create_project_node(project_id)
+    return node_id
 
 
 @pytest.fixture
@@ -784,7 +788,7 @@ async def _upload_folder_task(
 async def random_project_with_files(
     sqlalchemy_async_engine: AsyncEngine,
     create_project: Callable[..., Awaitable[dict[str, Any]]],
-    create_project_node: Callable[..., Awaitable[NodeID]],
+    create_project_node: Callable[..., Awaitable[tuple[NodeID, dict[str, Any]]]],
     create_simcore_file_id: Callable[
         [ProjectID, NodeID, str, Path | None], SimcoreS3FileID
     ],
@@ -798,17 +802,28 @@ async def random_project_with_files(
     upload_file: Callable[..., Awaitable[tuple[Path, SimcoreS3FileID]]],
 ) -> Callable[
     [ProjectWithFilesParams],
-    Awaitable[tuple[dict[str, Any], dict[NodeID, dict[SimcoreS3FileID, FileIDDict]]]],
+    Awaitable[
+        tuple[
+            dict[str, Any],
+            dict[NodeID, dict[str, Any]],
+            dict[NodeID, dict[SimcoreS3FileID, FileIDDict]],
+        ]
+    ],
 ]:
     async def _creator(
         project_params: ProjectWithFilesParams,
-    ) -> tuple[dict[str, Any], dict[NodeID, dict[SimcoreS3FileID, FileIDDict]]]:
+    ) -> tuple[
+        dict[str, Any],
+        dict[NodeID, dict[str, Any]],
+        dict[NodeID, dict[SimcoreS3FileID, FileIDDict]],
+    ]:
         assert len(project_params.allowed_file_sizes) == len(
             project_params.allowed_file_checksums
         )
         project = await create_project(name="random-project")
         node_to_files_mapping: dict[NodeID, dict[SimcoreS3FileID, FileIDDict]] = {}
         upload_tasks = []
+        nodes: dict[NodeID, dict[str, Any]] = {}
         for _ in range(project_params.num_nodes):
             # Create a node with outputs (files and others)
             project_id = ProjectID(project["uuid"])
@@ -818,7 +833,7 @@ async def _creator(
             output3_file_id = create_simcore_file_id(
                 project_id, node_id, output3_file_name, Path("outputs/output_3")
             )
-            created_node_id = await create_project_node(
+            created_node_id, created_node = await create_project_node(
                 ProjectID(project["uuid"]),
                 node_id,
                 outputs={
@@ -828,6 +843,7 @@ async def _creator(
                 },
             )
             assert created_node_id == node_id
+            nodes[created_node_id] = created_node
 
             upload_tasks.append(
                 _upload_one_file_task(
@@ -878,7 +894,7 @@ async def _creator(
                 node_to_files_mapping[node_id][file_id] = file_dict
 
         project = await get_updated_project(sqlalchemy_async_engine, project["uuid"])
-        return project, node_to_files_mapping
+        return project, nodes, node_to_files_mapping
 
     return _creator
 
@@ -933,7 +949,7 @@ async def output_file(
     yield file
 
     async with sqlalchemy_async_engine.begin() as conn:
-        result = await conn.execute(
+        await conn.execute(
             file_meta_data.delete().where(file_meta_data.c.file_id == row.file_id)
         )
 

services/storage/tests/unit/test_handlers_paths.py

@@ -157,16 +157,16 @@ async def test_list_paths_pagination(
     user_id: UserID,
     with_random_project_with_files: tuple[
         dict[str, Any],
+        dict[NodeID, dict[str, Any]],
         dict[NodeID, dict[SimcoreS3FileID, FileIDDict]],
     ],
 ):
-    project, list_of_files = with_random_project_with_files
-    num_nodes = len(list(project["workbench"]))
+    project, nodes, list_of_files = with_random_project_with_files
 
     # ls the nodes (DB-based)
     file_filter = Path(project["uuid"])
     expected_paths = sorted(
-        ((file_filter / node_key, False) for node_key in project["workbench"]),
+        ((file_filter / f"{node_id}", False) for node_id in nodes),
         key=lambda x: x[0],
     )
     await _assert_list_paths(
@@ -176,12 +176,12 @@ async def test_list_paths_pagination(
         user_id,
         file_filter=file_filter,
         expected_paths=expected_paths,
-        limit=int(num_nodes / 2 + 0.5),
+        limit=int(len(nodes) / 2 + 0.5),
     )
 
     # ls in the workspace (S3-based)
     # ls in the workspace
-    selected_node_id = NodeID(random.choice(list(project["workbench"])))  # noqa: S311
+    selected_node_id = random.choice(list(nodes.keys()))  # noqa: S311
     selected_node_s3_keys = [
         Path(s3_object_id) for s3_object_id in list_of_files[selected_node_id]
     ]
@@ -240,11 +240,12 @@ async def test_list_paths_pagination_large_page(
     user_id: UserID,
     with_random_project_with_files: tuple[
         dict[str, Any],
+        dict[NodeID, dict[str, Any]],
         dict[NodeID, dict[SimcoreS3FileID, FileIDDict]],
     ],
 ):
-    project, list_of_files = with_random_project_with_files
-    selected_node_id = NodeID(random.choice(list(project["workbench"])))  # noqa: S311
+    project, nodes, list_of_files = with_random_project_with_files
+    selected_node_id = random.choice(list(nodes.keys()))  # noqa: S311
     selected_node_s3_keys = [
         Path(s3_object_id) for s3_object_id in list_of_files[selected_node_id]
     ]
@@ -292,7 +293,11 @@ async def test_list_paths(
     random_project_with_files: Callable[
         [ProjectWithFilesParams],
         Awaitable[
-            tuple[dict[str, Any], dict[NodeID, dict[SimcoreS3FileID, FileIDDict]]]
+            tuple[
+                dict[str, Any],
+                dict[NodeID, dict[str, Any]],
+                dict[NodeID, dict[SimcoreS3FileID, FileIDDict]],
+            ]
         ],
     ],
     project_params: ProjectWithFilesParams,
@@ -305,7 +310,10 @@ async def test_list_paths(
 
     # ls root returns our projects
     expected_paths = sorted(
-        ((Path(f"{prj_db['uuid']}"), False) for prj_db, _ in project_to_files_mapping),
+        (
+            (Path(f"{prj_db['uuid']}"), False)
+            for prj_db, _, _ in project_to_files_mapping
+        ),
         key=lambda x: x[0],
     )
     await _assert_list_paths(
@@ -318,9 +326,9 @@ async def test_list_paths(
     )
 
     # ls with only some part of the path should return only the projects that match
-    selected_project, selected_project_files = random.choice(  # noqa: S311
+    selected_project, selected_nodes, selected_project_files = random.choice(
         project_to_files_mapping
-    )
+    )  # noqa: S311
     partial_file_filter = Path(
         selected_project["uuid"][: len(selected_project["uuid"]) // 2]
     )
@@ -340,7 +348,7 @@ async def test_list_paths(
     # now we ls inside one of the projects returns the nodes
     file_filter = Path(selected_project["uuid"])
     expected_paths = sorted(
-        ((file_filter / node_key, False) for node_key in selected_project["workbench"]),
+        ((file_filter / f"{node_id}", False) for node_id in selected_nodes),
         key=lambda x: x[0],
     )
     await _assert_list_paths(
@@ -353,9 +361,7 @@ async def test_list_paths(
     )
 
     # now we ls in one of the nodes
-    selected_node_id = NodeID(
-        random.choice(list(selected_project["workbench"]))  # noqa: S311
-    )
+    selected_node_id = random.choice(list(selected_nodes))  # noqa: S311
     selected_node_s3_keys = [
         Path(s3_object_id) for s3_object_id in selected_project_files[selected_node_id]
     ]
@@ -623,14 +629,15 @@ async def test_path_compute_size(
     user_id: UserID,
     with_random_project_with_files: tuple[
         dict[str, Any],
+        dict[NodeID, dict[str, Any]],
         dict[NodeID, dict[SimcoreS3FileID, FileIDDict]],
     ],
     project_params: ProjectWithFilesParams,
 ):
     assert (
         len(project_params.allowed_file_sizes) == 1
     ), "test preconditions are not filled! allowed file sizes should have only 1 option for this test"
-    project, list_of_files = with_random_project_with_files
+    project, nodes, list_of_files = with_random_project_with_files
 
     total_num_files = sum(
         len(files_in_node) for files_in_node in list_of_files.values()
@@ -649,7 +656,7 @@ async def test_path_compute_size(
     )
 
     # get size of one of the nodes
-    selected_node_id = NodeID(random.choice(list(project["workbench"])))  # noqa: S311
+    selected_node_id = random.choice(list(nodes.keys()))  # noqa: S311
     path = Path(project["uuid"]) / f"{selected_node_id}"
     selected_node_s3_keys = [
         Path(s3_object_id) for s3_object_id in list_of_files[selected_node_id]