fix: webserver

Author: giancarloromeo

packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py

@@ -4,6 +4,8 @@
 
 import json
 import uuid as uuidlib
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
 from pathlib import Path
 from typing import Any
 
@@ -73,7 +75,9 @@ async def create_project(
 
     db: ProjectDBAPI = app[APP_PROJECT_DBAPI]
 
-    new_project = await db.insert_project(
+    workbench: dict[str, Any] = project_data.pop("workbench", {})
+
+    project_created = await db.insert_project(
         project_data,
         user_id,
         product_name=product_name,
@@ -86,11 +90,9 @@ async def create_project(
                 required_resources=ServiceResourcesDictHelpers.model_config[
                     "json_schema_extra"
                 ]["examples"][0],
-                key=node_info.get("key"),
-                version=node_info.get("version"),
-                label=node_info.get("label"),
+                **node_info,
             )
-            for node_id, node_info in project_data.get("workbench", {}).items()
+            for node_id, node_info in workbench.items()
         },
     )
 
@@ -103,7 +105,7 @@ async def create_project(
         for group_id, permissions in _access_rights.items():
             await update_or_insert_project_group(
                 app,
-                project_id=new_project["uuid"],
+                project_id=project_created["uuid"],
                 group_id=int(group_id),
                 read=permissions["read"],
                 write=permissions["write"],
@@ -112,20 +114,22 @@ async def create_project(
 
     try:
         uuidlib.UUID(str(project_data["uuid"]))
-        assert new_project["uuid"] == project_data["uuid"]
+        assert project_created["uuid"] == project_data["uuid"]
     except (ValueError, AssertionError):
         # in that case the uuid gets replaced
-        assert new_project["uuid"] != project_data["uuid"]
-        project_data["uuid"] = new_project["uuid"]
+        assert project_created["uuid"] != project_data["uuid"]
+        project_data["uuid"] = project_created["uuid"]
 
     for key in DB_EXCLUSIVE_COLUMNS:
         project_data.pop(key, None)
 
-    new_project: ProjectDict = remap_keys(
-        new_project,
+    project_created: ProjectDict = remap_keys(
+        project_created,
         rename={"trashed": "trashedAt"},
     )
-    return new_project
+
+    project_created["workbench"] = workbench  # NOTE: restore workbench
+    return project_created
 
 
 async def delete_all_projects(app: web.Application):
@@ -137,50 +141,35 @@ async def delete_all_projects(app: web.Application):
         await conn.execute(query)
 
 
-class NewProject:
-    def __init__(
-        self,
-        params_override: dict | None = None,
-        app: web.Application | None = None,
-        *,
-        user_id: int,
-        product_name: str,
-        tests_data_dir: Path,
-        force_uuid: bool = False,
-        as_template: bool = False,
-    ):
-        assert app  # nosec
-
-        self.params_override = params_override
-        self.user_id = user_id
-        self.product_name = product_name
-        self.app = app
-        self.prj = {}
-        self.force_uuid = force_uuid
-        self.tests_data_dir = tests_data_dir
-        self.as_template = as_template
-
-        assert tests_data_dir.exists()
-        assert tests_data_dir.is_dir()
-
-    async def __aenter__(self) -> ProjectDict:
-        assert self.app  # nosec
-
-        self.prj = await create_project(
-            self.app,
-            self.params_override,
-            self.user_id,
-            product_name=self.product_name,
-            force_uuid=self.force_uuid,
-            default_project_json=self.tests_data_dir / "fake-project.json",
-            as_template=self.as_template,
-        )
-
-        return self.prj
+@asynccontextmanager
+async def new_project(
+    params_override: dict | None = None,
+    app: web.Application | None = None,
+    *,
+    user_id: int,
+    product_name: str,
+    tests_data_dir: Path,
+    force_uuid: bool = False,
+    as_template: bool = False,
+) -> AsyncIterator[ProjectDict]:
+    assert app  # nosec
+    assert tests_data_dir.exists()
+    assert tests_data_dir.is_dir()
+
+    project = await create_project(
+        app,
+        params_override,
+        user_id,
+        product_name=product_name,
+        force_uuid=force_uuid,
+        default_project_json=tests_data_dir / "fake-project.json",
+        as_template=as_template,
+    )
 
-    async def __aexit__(self, *args):
-        assert self.app  # nosec
-        await delete_all_projects(self.app)
+    try:
+        yield project
+    finally:
+        await delete_all_projects(app)
 
 
 async def assert_get_same_project(

services/web/server/src/simcore_service_webserver/director_v2/_computations_service.py

@@ -1,3 +1,4 @@
+from collections.abc import Awaitable
 from decimal import Decimal
 
 from aiohttp import web
@@ -14,6 +15,8 @@
 )
 from models_library.products import ProductName
 from models_library.projects import ProjectID
+from models_library.projects_nodes import Node
+from models_library.projects_nodes_io import NodeID
 from models_library.rest_ordering import OrderBy
 from models_library.services_types import ServiceRunID
 from models_library.users import UserID
@@ -29,6 +32,7 @@
     CreditTransactionNotFoundError,
 )
 from servicelib.utils import limited_gather
+from simcore_service_webserver.projects._projects_nodes_repository import get_by_project
 
 from ..products.products_service import is_product_billable
 from ..projects.api import (
@@ -248,15 +252,23 @@ async def list_computations_latest_iteration_tasks(
     unique_project_uuids = {task.project_uuid for task in _tasks_get.items}
     # Fetch projects metadata concurrently
     # NOTE: MD: can be improved with a single batch call
-    project_dicts = await limited_gather(
+
+    async def _wrap_with_id(
+        project_id: ProjectID, coro: Awaitable[list[tuple[NodeID, Node]]]
+    ) -> tuple[ProjectID, dict[NodeID, Node]]:
+        nodes = await coro
+        return project_id, dict(nodes)
+
+    results = await limited_gather(
         *[
-            get_project_dict_legacy(app, project_uuid=project_uuid)
+            _wrap_with_id(project_uuid, get_by_project(app, project_id=project_uuid))
             for project_uuid in unique_project_uuids
         ],
         limit=20,
     )
+
     # Build a dict: project_uuid -> workbench
-    project_uuid_to_workbench = {prj["uuid"]: prj["workbench"] for prj in project_dicts}
+    project_uuid_to_workbench: dict[ProjectID, dict[NodeID, Node]] = dict(results)
 
     _service_run_ids = [item.service_run_id for item in _tasks_get.items]
     _is_product_billable = await is_product_billable(app, product_name=product_name)
@@ -286,9 +298,8 @@ async def list_computations_latest_iteration_tasks(
             started_at=item.started_at,
             ended_at=item.ended_at,
             log_download_link=item.log_download_link,
-            node_name=project_uuid_to_workbench[f"{item.project_uuid}"][
-                f"{item.node_id}"
-            ].get("label", ""),
+            node_name=project_uuid_to_workbench[item.project_uuid][item.node_id].label
+            or "",
             osparc_credits=credits_or_none,
         )
         for item, credits_or_none in zip(

services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py

@@ -1,25 +1,30 @@
 import logging
 
 import sqlalchemy as sa
-
 from aiohttp import web
 from models_library.projects import ProjectID
 from models_library.projects_nodes import Node, PartialNode
 from models_library.projects_nodes_io import NodeID
-from simcore_postgres_database.utils_repos import transaction_context
+from simcore_postgres_database.utils_repos import (
+    pass_or_acquire_connection,
+    transaction_context,
+)
 from simcore_postgres_database.webserver_models import projects_nodes
 from sqlalchemy.ext.asyncio import AsyncConnection
 
-from .exceptions import NodeNotFoundError
 from ..db.plugin import get_asyncpg_engine
+from .exceptions import NodeNotFoundError
 
 _logger = logging.getLogger(__name__)
 
 
 _SELECTION_PROJECTS_NODES_DB_ARGS = [
+    projects_nodes.c.node_id,
     projects_nodes.c.key,
     projects_nodes.c.version,
     projects_nodes.c.label,
+    projects_nodes.c.created,
+    projects_nodes.c.modified,
     projects_nodes.c.progress,
     projects_nodes.c.thumbnail,
     projects_nodes.c.input_access,
@@ -43,27 +48,56 @@ async def get(
     project_id: ProjectID,
     node_id: NodeID,
 ) -> Node:
-    async with transaction_context(get_asyncpg_engine(app), connection) as conn:
-        get_stmt = sa.select(
-            *_SELECTION_PROJECTS_NODES_DB_ARGS
-        ).where(
+    async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn:
+        query = sa.select(*_SELECTION_PROJECTS_NODES_DB_ARGS).where(
             (projects_nodes.c.project_uuid == f"{project_id}")
             & (projects_nodes.c.node_id == f"{node_id}")
         )
 
-        result = await conn.stream(get_stmt)
+        result = await conn.stream(query)
         assert result  # nosec
 
         row = await result.first()
         if row is None:
             raise NodeNotFoundError(
-                project_uuid=f"{project_id}",
-                node_uuid=f"{node_id}"
+                project_uuid=f"{project_id}", node_uuid=f"{node_id}"
             )
         assert row  # nosec
         return Node.model_validate(row, from_attributes=True)
 
 
+async def get_by_project(
+    app: web.Application,
+    connection: AsyncConnection | None = None,
+    *,
+    project_id: ProjectID,
+) -> list[tuple[NodeID, Node]]:
+    async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn:
+        query = sa.select(*_SELECTION_PROJECTS_NODES_DB_ARGS).where(
+            projects_nodes.c.project_uuid == f"{project_id}"
+        )
+
+        result = await conn.stream(query)
+        assert result  # nosec
+
+        from simcore_postgres_database.utils_projects_nodes import ProjectNode
+
+        rows = await result.all()
+        return [
+            (
+                NodeID(row.node_id),
+                Node.model_validate(
+                    ProjectNode.model_validate(row, from_attributes=True).model_dump(
+                        exclude_none=True,
+                        exclude_unset=True,
+                        exclude={"node_id", "created", "modified"},
+                    )
+                ),
+            )
+            for row in rows
+        ]
+
+
 async def update(
     app: web.Application,
     connection: AsyncConnection | None = None,

services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py

@@ -234,41 +234,10 @@ def _reraise_if_not_unique_uuid_error(err: UniqueViolation):
                         )
                         selected_values["tags"] = project_tag_ids
 
-                        # NOTE: this will at some point completely replace workbench in the DB
-                        if selected_values["workbench"]:
-                            project_nodes_repo = ProjectNodesRepo(
-                                project_uuid=project_uuid
+                        if project_nodes:
+                            await ProjectNodesRepo(project_uuid=project_uuid).add(
+                                conn, nodes=list(project_nodes.values())
                             )
-                            if project_nodes is None:
-                                project_nodes = {
-                                    NodeID(node_id): ProjectNodeCreate(
-                                        node_id=NodeID(node_id),
-                                        required_resources={},
-                                        key=node_info.get("key"),
-                                        version=node_info.get("version"),
-                                        label=node_info.get("label"),
-                                    )
-                                    for node_id, node_info in selected_values[
-                                        "workbench"
-                                    ].items()
-                                }
-
-                            nodes = [
-                                project_nodes.get(
-                                    NodeID(node_id),
-                                    ProjectNodeCreate(
-                                        node_id=NodeID(node_id),
-                                        required_resources={},
-                                        key=node_info.get("key"),
-                                        version=node_info.get("version"),
-                                        label=node_info.get("label"),
-                                    ),
-                                )
-                                for node_id, node_info in selected_values[
-                                    "workbench"
-                                ].items()
-                            ]
-                            await project_nodes_repo.add(conn, nodes=nodes)
         return selected_values
 
     async def insert_project(
@@ -333,7 +302,6 @@ async def insert_project(
         # ensure we have the minimal amount of data here
         # All non-default in projects table
         insert_values.setdefault("name", "New Study")
-        insert_values.setdefault("workbench", {})
         insert_values.setdefault("workspace_id", None)
 
         # must be valid uuid
@@ -738,7 +706,6 @@ async def get_project_db(self, project_uuid: ProjectID) -> ProjectDBGet:
             result = await conn.execute(
                 sa.select(
                     *PROJECT_DB_COLS,
-                    projects.c.workbench,
                 ).where(projects.c.uuid == f"{project_uuid}")
             )
             row = await result.fetchone()

services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py

@@ -224,7 +224,6 @@ async def _get_project(
         query = (
             sa.select(
                 *PROJECT_DB_COLS,
-                projects.c.workbench,
                 users.c.primary_gid.label("trashed_by_primary_gid"),
                 access_rights_subquery.c.access_rights,
             )

services/web/server/tests/integration/02/conftest.py

@@ -9,7 +9,7 @@
 
 import pytest
 from models_library.projects import ProjectID
-from pytest_simcore.helpers.webserver_projects import NewProject
+from pytest_simcore.helpers.webserver_projects import new_project
 
 
 @pytest.fixture(scope="session")
@@ -45,7 +45,7 @@ async def user_project(
     fake_project["prjOwner"] = logged_user["name"]
     fake_project["uuid"] = f"{project_id}"
 
-    async with NewProject(
+    async with new_project(
         fake_project,
         client.app,
         user_id=logged_user["id"],