Merge branch 'is5646/use-project-nodes-table-instead-of-workbench' of github.com:giancarloromeo/osparc-simcore into pr/giancarloromeo/8141

Author: pcrespov

packages/postgres-database/src/simcore_postgres_database/utils_projects.py

@@ -7,7 +7,7 @@
 from sqlalchemy.ext.asyncio import AsyncConnection
 
 from .models.projects import projects
-from .utils_repos import transaction_context
+from .utils_repos import pass_or_acquire_connection, transaction_context
 
 
 class DBBaseProjectError(OsparcErrorMixin, Exception):
@@ -22,6 +22,23 @@ class ProjectsRepo:
     def __init__(self, engine):
         self.engine = engine
 
+    async def exists(
+        self,
+        project_uuid: uuid.UUID,
+        *,
+        connection: AsyncConnection | None = None,
+    ) -> bool:
+        async with pass_or_acquire_connection(self.engine, connection) as conn:
+            return (
+                await conn.scalar(
+                    sa.select(1)
+                    .select_from(projects)
+                    .where(projects.c.uuid == project_uuid)
+                    .limit(1)
+                )
+                is not None
+            )
+
     async def get_project_last_change_date(
         self,
         project_uuid: uuid.UUID,

packages/pytest-simcore/src/pytest_simcore/simcore_storage_data_models.py

@@ -14,6 +14,7 @@
 from models_library.users import UserID
 from pydantic import TypeAdapter
 from simcore_postgres_database.models.project_to_groups import project_to_groups
+from simcore_postgres_database.models.projects_nodes import projects_nodes
 from simcore_postgres_database.storage_models import projects, users
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine
@@ -219,26 +220,19 @@ async def _creator(
         project_id: ProjectID, node_id: NodeID | None = None, **kwargs
     ) -> NodeID:
         async with sqlalchemy_async_engine.begin() as conn:
-            result = await conn.execute(
-                sa.select(projects.c.workbench).where(
-                    projects.c.uuid == f"{project_id}"
-                )
-            )
-            row = result.fetchone()
-            assert row
-            project_workbench: dict[str, Any] = row.workbench
-            new_node_id = node_id or NodeID(f"{faker.uuid4()}")
-            node_data = {
+            new_node_id = node_id or NodeID(faker.uuid4())
+            node_values = {
                 "key": "simcore/services/frontend/file-picker",
                 "version": "1.0.0",
                 "label": "pytest_fake_node",
             }
-            node_data.update(**kwargs)
-            project_workbench.update({f"{new_node_id}": node_data})
+            node_values.update(**kwargs)
             await conn.execute(
-                projects.update()
-                .where(projects.c.uuid == f"{project_id}")
-                .values(workbench=project_workbench)
+                projects_nodes.insert().values(
+                    node_id=f"{new_node_id}",
+                    project_uuid=f"{project_id}",
+                    **node_values,
+                )
             )
         return new_node_id
 

services/storage/src/simcore_service_storage/modules/db/projects.py

@@ -5,6 +5,7 @@
 from models_library.projects import ProjectAtDB, ProjectID, ProjectIDStr
 from models_library.projects_nodes_io import NodeIDStr
 from pydantic import ValidationError
+from simcore_postgres_database.models.projects_nodes import projects_nodes
 from simcore_postgres_database.storage_models import projects
 from simcore_postgres_database.utils_repos import pass_or_acquire_connection
 from sqlalchemy.ext.asyncio import AsyncConnection
@@ -32,38 +33,34 @@ async def list_valid_projects_in(
                 with suppress(ValidationError):
                     yield ProjectAtDB.model_validate(row)
 
-    async def project_exists(
-        self,
-        *,
-        connection: AsyncConnection | None = None,
-        project_uuid: ProjectID,
-    ) -> bool:
-        async with pass_or_acquire_connection(self.db_engine, connection) as conn:
-            return bool(
-                await conn.scalar(
-                    sa.select(sa.func.count())
-                    .select_from(projects)
-                    .where(projects.c.uuid == f"{project_uuid}")
-                )
-                == 1
-            )
-
     async def get_project_id_and_node_id_to_names_map(
         self,
         *,
         connection: AsyncConnection | None = None,
         project_uuids: list[ProjectID],
     ) -> dict[ProjectID, dict[ProjectIDStr | NodeIDStr, str]]:
-        mapping = {}
+        names_map = {}
         async with pass_or_acquire_connection(self.db_engine, connection) as conn:
             async for row in await conn.stream(
-                sa.select(projects.c.uuid, projects.c.name, projects.c.workbench).where(
-                    projects.c.uuid.in_(f"{pid}" for pid in project_uuids)
+                sa.select(projects.c.uuid, projects.c.name).where(
+                    projects.c.uuid.in_(
+                        [f"{project_uuid}" for project_uuid in project_uuids]
+                    )
+                )
+            ):
+                names_map[ProjectID(row.uuid)] = {f"{row.uuid}": row.name}
+
+            async for row in await conn.stream(
+                sa.select(
+                    projects_nodes.c.node_id,
+                    projects_nodes.c.project_uuid,
+                    projects_nodes.c.label,
+                ).where(
+                    projects_nodes.c.project_uuid.in_(
+                        [f"{project_uuid}" for project_uuid in project_uuids]
+                    )
                 )
             ):
-                mapping[ProjectID(f"{row.uuid}")] = {f"{row.uuid}": row.name} | {
-                    f"{node_id}": node["label"]
-                    for node_id, node in row.workbench.items()
-                }
+                names_map[ProjectID(row.project_uuid)] |= {f"{row.node_id}": row.label}
 
-        return mapping
+        return names_map

services/storage/src/simcore_service_storage/simcore_s3_dsm.py

@@ -41,6 +41,7 @@
 from servicelib.logging_utils import log_context
 from servicelib.progress_bar import ProgressBarData
 from servicelib.utils import ensure_ends_with, limited_gather
+from simcore_postgres_database.utils_projects import ProjectsRepo
 from simcore_postgres_database.utils_repos import transaction_context
 from sqlalchemy.ext.asyncio import AsyncEngine
 
@@ -792,9 +793,9 @@ async def deep_copy_project_simcore_s3(
             task_progress.description = "Checking study access rights..."
 
             for prj_uuid in [src_project_uuid, dst_project_uuid]:
-                if not await ProjectRepository.instance(
-                    get_db_engine(self.app)
-                ).project_exists(project_uuid=prj_uuid):
+                if not await ProjectsRepo(get_db_engine(self.app)).exists(
+                    project_uuid=prj_uuid
+                ):
                     raise ProjectNotFoundError(project_id=prj_uuid)
             source_access_rights = await AccessLayerRepository.instance(
                 get_db_engine(self.app)