fix: improve listing

Author: giancarloromeo

packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py

@@ -116,7 +116,7 @@ def model_dump_as_node(self) -> dict[str, Any]:
         )
 
 
-def create_workbench_subquery() -> Subquery:
+def create_workbench_subquery(project_id: str) -> Subquery:
     return (
         sa.select(
             projects_nodes.c.project_uuid,
@@ -163,6 +163,7 @@ def create_workbench_subquery() -> Subquery:
                 projects, projects_nodes.c.project_uuid == projects.c.uuid
             )
         )
+        .where(projects.c.uuid == project_id)
         .group_by(projects_nodes.c.project_uuid)
         .subquery()
     )

services/director-v2/src/simcore_service_director_v2/modules/db/repositories/projects.py

@@ -18,7 +18,7 @@
 
 class ProjectsRepository(BaseRepository):
     async def get_project(self, project_id: ProjectID) -> ProjectAtDB:
-        workbench_subquery = create_workbench_subquery()
+        workbench_subquery = create_workbench_subquery(f"{project_id}")
 
         async with self.db_engine.connect() as conn:
             query = (

services/web/server/src/simcore_service_webserver/projects/_jobs_repository.py

@@ -4,14 +4,12 @@
 from models_library.products import ProductName
 from models_library.projects import ProjectID
 from models_library.users import UserID
-from pydantic import TypeAdapter
 from simcore_postgres_database.models.groups import user_to_groups
 from simcore_postgres_database.models.project_to_groups import project_to_groups
 from simcore_postgres_database.models.projects import projects
 from simcore_postgres_database.models.projects_metadata import projects_metadata
 from simcore_postgres_database.models.projects_to_jobs import projects_to_jobs
 from simcore_postgres_database.models.projects_to_products import projects_to_products
-from simcore_postgres_database.utils_projects_nodes import create_workbench_subquery
 from simcore_postgres_database.utils_repos import (
     get_columns_from_db_model,
     pass_or_acquire_connection,
@@ -20,6 +18,8 @@
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.ext.asyncio import AsyncConnection
 
+from simcore_service_webserver.projects._projects_repository_legacy_utils import get_project_workbench
+
 from ..db.base_repository import BaseRepository
 from .models import ProjectDBGet, ProjectJobDBGet
 
@@ -170,25 +170,16 @@ async def list_projects_marked_as_jobs(
         # Step 5: Query to get the total count
         total_query = sa.select(sa.func.count()).select_from(base_query)
 
-        # Step 6: Create subquery to aggregate project nodes into workbench structure
-        workbench_subquery = create_workbench_subquery()
-
-        # Step 7: Query to get the paginated list with full selection
+        # Step 6: Query to get the paginated list with full selection
         list_query = (
             sa.select(
                 *_PROJECT_DB_COLS,
-                sa.func.coalesce(
-                    workbench_subquery.c.workbench, sa.text("'{}'::json")
-                ).label("workbench"),
                 base_query.c.job_parent_resource_name,
             )
             .select_from(
                 base_query.join(
                     projects,
                     projects.c.uuid == base_query.c.project_uuid,
-                ).outerjoin(
-                    workbench_subquery,
-                    projects.c.uuid == workbench_subquery.c.project_uuid,
                 )
             )
             .order_by(
@@ -204,9 +195,9 @@ async def list_projects_marked_as_jobs(
             total_count = await conn.scalar(total_query)
             assert isinstance(total_count, int)  # nosec
 
-            result = await conn.execute(list_query)
-            projects_list = TypeAdapter(list[ProjectJobDBGet]).validate_python(
-                result.fetchall()
-            )
+            projects_list = []
+            async for project_row in await conn.stream(list_query):
+                workbench = await get_project_workbench(conn, project_row.uuid)
+                projects_list.append(ProjectJobDBGet.model_validate({**project_row, "workbench": workbench}))
 
             return total_count, projects_list

services/web/server/src/simcore_service_webserver/projects/_projects_repository.py

@@ -143,7 +143,7 @@ async def get_project_with_workbench(
     project_uuid: ProjectID,
 ) -> ProjectWithWorkbenchDBGet:
     async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn:
-        workbench_subquery = create_workbench_subquery()
+        workbench_subquery = create_workbench_subquery(f"{project_uuid}")
         query = (
             sql.select(
                 *PROJECT_DB_COLS,

services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py

@@ -60,7 +60,6 @@
     ProjectNode,
     ProjectNodeCreate,
     ProjectNodesRepo,
-    create_workbench_subquery,
 )
 from simcore_postgres_database.webserver_models import (
     ProjectType,
@@ -96,6 +95,7 @@
     convert_to_schema_names,
     create_project_access_rights,
     patch_workbench,
+    get_project_workbench
 )
 from ._socketio_service import notify_project_document_updated
 from .exceptions import (
@@ -395,14 +395,9 @@ def _create_private_workspace_query(
                 .group_by(project_to_groups.c.project_uuid)
             ).subquery("my_access_rights_subquery")
 
-            workbench_subquery = create_workbench_subquery()
-
             private_workspace_query = (
                 sa.select(
                     *PROJECT_DB_COLS,
-                    sa.func.coalesce(
-                        workbench_subquery.c.workbench, sa.text("'{}'::json")
-                    ).label("workbench"),
                     projects_to_products.c.product_name,
                     projects_to_folders.c.folder_id,
                 )
@@ -417,10 +412,6 @@ def _create_private_workspace_query(
                         ),
                         isouter=True,
                     )
-                    .outerjoin(
-                        workbench_subquery,
-                        workbench_subquery.c.project_uuid == projects.c.uuid,
-                    )
                 )
                 .where(
                     (projects.c.workspace_id.is_(None))  # <-- Private workspace
@@ -462,14 +453,9 @@ def _create_shared_workspace_query(
                 .group_by(workspaces_access_rights.c.workspace_id)
             ).subquery("my_workspace_access_rights_subquery")
 
-            workbench_subquery = create_workbench_subquery()
-
             shared_workspace_query = (
                 sa.select(
                     *PROJECT_DB_COLS,
-                    sa.func.coalesce(
-                        workbench_subquery.c.workbench, sa.text("'{}'::json")
-                    ).label("workbench"),
                     projects_to_products.c.product_name,
                     projects_to_folders.c.folder_id,
                 )
@@ -488,10 +474,6 @@ def _create_shared_workspace_query(
                         ),
                         isouter=True,
                     )
-                    .outerjoin(
-                        workbench_subquery,
-                        workbench_subquery.c.project_uuid == projects.c.uuid,
-                    )
                 )
                 .where(projects_to_products.c.product_name == product_name)
             )
@@ -698,7 +680,7 @@ async def list_projects_dicts(  # pylint: disable=too-many-arguments,too-many-st
                 # with the frontend. The frontend would need to check and adapt how it handles default values in
                 # Workbench nodes, which are currently not returned if not set in the DB.
                 prj_dict = dict(row.items()) | {
-                    "workbench": await self._get_workbench(conn, row.uuid),
+                    "workbench": await get_project_workbench(conn, row.uuid),
                 }
                 ProjectListAtDB.model_validate(prj_dict)
                 prjs_output.append(prj_dict)

services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py

@@ -191,23 +191,6 @@ async def _upsert_tags_in_project(
                 .on_conflict_do_nothing()
             )
 
-    @staticmethod
-    async def _get_workbench(
-        connection: SAConnection,
-        project_uuid: str,
-    ) -> dict[str, Any]:
-        project_nodes_repo = ProjectNodesRepo(project_uuid=ProjectID(project_uuid))
-        exclude_fields = {"node_id", "required_resources", "created", "modified"}
-        workbench: dict[str, Any] = {}
-
-        project_nodes = await project_nodes_repo.list(connection)  # type: ignore
-        for project_node in project_nodes:
-            node_data = project_node.model_dump(
-                exclude=exclude_fields, exclude_none=True, exclude_unset=True
-            )
-            workbench[f"{project_node.node_id}"] = node_data
-        return workbench
-
     async def _get_project(
         self,
         connection: SAConnection,
@@ -238,11 +221,11 @@ async def _get_project(
                     ),
                 ).label("access_rights"),
             )
-            .where(project_to_groups.c.project_uuid == f"{project_uuid}")
+            .where(project_to_groups.c.project_uuid == project_uuid)
             .group_by(project_to_groups.c.project_uuid)
         ).subquery("access_rights_subquery")
 
-        workbench_subquery = create_workbench_subquery()
+        workbench_subquery = create_workbench_subquery(project_uuid)
 
         query = (
             sa.select(
@@ -393,3 +376,20 @@ def patch_workbench(
             # patch
             current_node_data.update(new_node_data)
     return (patched_project, changed_entries)
+
+
+async def get_project_workbench(
+    connection: SAConnection,
+    project_uuid: str,
+) -> dict[str, Any]:
+    project_nodes_repo = ProjectNodesRepo(project_uuid=ProjectID(project_uuid))
+    exclude_fields = {"node_id", "required_resources", "created", "modified"}
+    workbench: dict[str, Any] = {}
+
+    project_nodes = await project_nodes_repo.list(connection)  # type: ignore
+    for project_node in project_nodes:
+        node_data = project_node.model_dump(
+            exclude=exclude_fields, exclude_none=True, exclude_unset=True
+        )
+        workbench[f"{project_node.node_id}"] = node_data
+    return workbench