♻️ Maintenance/remove legacy db listing code 🚨🚨

services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py

@@ -21,13 +21,14 @@
     ProjectTemplateType as ProjectTemplateTypeDB,
 )
 from simcore_postgres_database.webserver_models import ProjectType as ProjectTypeDB
+from simcore_service_webserver.users.api import get_user_email_legacy
 
 from ..folders import _folders_repository
 from ..workspaces.api import check_user_workspace_access
 from . import _projects_service
 from ._access_rights_repository import batch_get_project_access_rights
 from ._projects_repository import batch_get_trashed_by_primary_gid
-from ._projects_repository_legacy import ProjectDBAPI
+from ._projects_repository_legacy import ProjectDBAPI, convert_to_schema_names
 from .models import ProjectDict, ProjectTypeAPI
 
 
@@ -53,7 +54,6 @@ async def _aggregate_data_to_projects_from_other_sources(
     app: web.Application,
     *,
     db_projects: list[ProjectDict],
-    db_project_types: list[ProjectTypeDB],
     user_id: UserID,
 ) -> list[ProjectDict]:
     """
@@ -79,10 +79,10 @@ async def _aggregate_data_to_projects_from_other_sources(
         _projects_service.add_project_states_for_user(
             user_id=user_id,
             project=prj,
-            is_template=prj_type == ProjectTypeDB.TEMPLATE,
+            is_template=prj["type"] == ProjectTypeDB.TEMPLATE,
             app=app,
         )
-        for prj, prj_type in zip(db_projects, db_project_types, strict=False)
+        for prj in db_projects
     ]
 
     updated_projects: list[ProjectDict] = await _paralell_update(
@@ -95,6 +95,24 @@ async def _aggregate_data_to_projects_from_other_sources(
     return updated_projects
 
 
+async def _convert_db_projects_to_api_projects(
+    app: web.Application,
+    db,
+    db_projects: list,
+) -> list[dict]:
+    """
+    Converts db schema projects to API schema (legacy postprocessing).
+    """
+    api_projects: list[dict] = []
+    for db_prj in db_projects:
+        db_prj_dict = db_prj.model_dump()
+        db_prj_dict.pop("product_name", None)
+        db_prj_dict["tags"] = await db.get_tags_by_project(project_id=f"{db_prj.id}")
+        user_email = await get_user_email_legacy(app, db_prj.prj_owner)
+        api_projects.append(convert_to_schema_names(db_prj_dict, user_email))
+    return api_projects
+
+
 async def list_projects(  # pylint: disable=too-many-arguments
     app: web.Application,
     user_id: UserID,
@@ -140,7 +158,7 @@ async def list_projects(  # pylint: disable=too-many-arguments
             workspace_id=workspace_id,
         )
 
-    db_projects, db_project_types, total_number_projects = await db.list_projects_dicts(
+    db_projects, total_number_projects = await db.list_projects_dicts(
         product_name=product_name,
         user_id=user_id,
         workspace_query=(
@@ -172,11 +190,13 @@ async def list_projects(  # pylint: disable=too-many-arguments
         order_by=order_by,
     )
 
-    projects = await _aggregate_data_to_projects_from_other_sources(
-        app, db_projects=db_projects, db_project_types=db_project_types, user_id=user_id
+    api_projects = await _convert_db_projects_to_api_projects(app, db, db_projects)
+
+    final_projects = await _aggregate_data_to_projects_from_other_sources(
+        app, db_projects=api_projects, user_id=user_id
     )
 
-    return projects, total_number_projects
+    return final_projects, total_number_projects
 
 
 async def list_projects_full_depth(
@@ -196,7 +216,7 @@ async def list_projects_full_depth(
 ) -> tuple[list[ProjectDict], int]:
     db = ProjectDBAPI.get_from_app_context(app)
 
-    db_projects, db_project_types, total_number_projects = await db.list_projects_dicts(
+    db_projects, total_number_projects = await db.list_projects_dicts(
         product_name=product_name,
         user_id=user_id,
         workspace_query=WorkspaceQuery(workspace_scope=WorkspaceScope.ALL),
@@ -210,8 +230,10 @@ async def list_projects_full_depth(
         order_by=order_by,
     )
 
-    projects = await _aggregate_data_to_projects_from_other_sources(
-        app, db_projects=db_projects, db_project_types=db_project_types, user_id=user_id
+    api_projects = await _convert_db_projects_to_api_projects(app, db, db_projects)
+
+    final_projects = await _aggregate_data_to_projects_from_other_sources(
+        app, db_projects=api_projects, user_id=user_id
     )
 
-    return projects, total_number_projects
+    return final_projects, total_number_projects

services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py

@@ -19,7 +19,7 @@
 from models_library.folders import FolderQuery, FolderScope
 from models_library.groups import GroupID
 from models_library.products import ProductName
-from models_library.projects import ProjectID, ProjectIDStr
+from models_library.projects import ProjectAtDB, ProjectID, ProjectIDStr
 from models_library.projects_comments import CommentID, ProjectsCommentsDB
 from models_library.projects_nodes import Node
 from models_library.projects_nodes_io import NodeID, NodeIDStr
@@ -584,7 +584,7 @@ async def list_projects_dicts(  # pylint: disable=too-many-arguments,too-many-st
         limit: int | None = None,
         # order
         order_by: OrderBy = DEFAULT_ORDER_BY,
-    ) -> tuple[list[ProjectDict], list[ProjectType], int]:
+    ) -> tuple[list[ProjectAtDB], int]:
         async with self.engine.acquire() as conn:
             user_groups_proxy: list[RowProxy] = await self._list_user_groups(
                 conn, user_id
@@ -667,14 +667,15 @@ async def list_projects_dicts(  # pylint: disable=too-many-arguments,too-many-st
                     projects.c.id,
                 )
 
-            prjs, prj_types = await self._execute_without_permission_check(
-                conn,
-                select_projects_query=combined_query.offset(offset).limit(limit),
-            )
+            prjs_at_db = [
+                ProjectAtDB.model_validate(row)
+                async for row in conn.execute(
+                    combined_query.offset(offset).limit(limit)
+                )
+            ]
 
             return (
-                prjs,
-                prj_types,
+                prjs_at_db,
                 cast(int, total_count),
             )
 
@@ -1249,6 +1250,13 @@ async def remove_tag(
                     project["tags"].remove(tag_id)
                 return convert_to_schema_names(project, user_email)
 
+    async def get_tags_by_project(self, project_id: str) -> list:
+        async with self.engine.acquire() as conn:
+            query = sa.select(projects_tags.c.tag_id).where(
+                projects_tags.c.project_id == project_id
+            )
+            return [row.tag_id async for row in conn.execute(query)]
+
     #
     # Project Comments
     #

services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py

@@ -1,4 +1,3 @@
-import asyncio
 import logging
 from collections.abc import Mapping
 from copy import deepcopy
@@ -9,23 +8,21 @@
 import sqlalchemy as sa
 from aiopg.sa.connection import SAConnection
 from aiopg.sa.result import RowProxy
-from models_library.projects import ProjectAtDB, ProjectID, ProjectTemplateType
+from models_library.projects import ProjectID, ProjectTemplateType
 from models_library.projects_nodes import Node
 from models_library.projects_nodes_io import NodeIDStr
 from models_library.utils.change_case import camel_to_snake, snake_to_camel
 from pydantic import ValidationError
 from simcore_postgres_database.models.project_to_groups import project_to_groups
 from simcore_postgres_database.webserver_models import ProjectType, projects
 from sqlalchemy.dialects.postgresql import insert as pg_insert
-from sqlalchemy.sql.selectable import CompoundSelect, Select
 
 from ..db.models import GroupType, groups, projects_tags, user_to_groups, users
 from ..users.exceptions import UserNotFoundError
 from ..utils import format_datetime
 from ._projects_repository import PROJECT_DB_COLS
 from .exceptions import (
     NodeNotFoundError,
-    ProjectInvalidRightsError,
     ProjectInvalidUsageError,
     ProjectNotFoundError,
 )
@@ -184,50 +181,6 @@ async def _upsert_tags_in_project(
                 .on_conflict_do_nothing()
             )
 
-    async def _execute_without_permission_check(
-        self,
-        conn: SAConnection,
-        *,
-        select_projects_query: Select | CompoundSelect,
-    ) -> tuple[list[dict[str, Any]], list[ProjectType]]:
-        api_projects: list[dict] = []  # API model-compatible projects
-        db_projects: list[dict] = []  # DB model-compatible projects
-        project_types: list[ProjectType] = []
-        async for row in conn.execute(select_projects_query):
-            assert isinstance(row, RowProxy)  # nosec
-            try:
-                await asyncio.get_event_loop().run_in_executor(
-                    None, ProjectAtDB.model_validate, row
-                )
-
-            except ProjectInvalidRightsError:
-                continue
-
-            except ValidationError as exc:
-                logger.warning(
-                    "project  %s  failed validation, please check. error: %s",
-                    f"{row.id=}",
-                    exc,
-                )
-                continue
-
-            prj: dict[str, Any] = dict(row.items())
-            prj.pop("product_name", None)
-
-            db_projects.append(prj)
-
-        # NOTE: DO NOT nest _get_tags_by_project in async loop above !!!
-        # FIXME: temporary avoids inner async loops issue https://github.com/aio-libs/aiopg/issues/535
-        for db_prj in db_projects:
-            db_prj["tags"] = await self._get_tags_by_project(
-                conn, project_id=db_prj["id"]
-            )
-            user_email = await self._get_user_email(conn, db_prj["prj_owner"])
-            api_projects.append(convert_to_schema_names(db_prj, user_email))
-            project_types.append(db_prj["type"])
-
-        return (api_projects, project_types)
-
     async def _get_project(
         self,
         connection: SAConnection,

services/web/server/src/simcore_service_webserver/users/_users_repository.py

@@ -199,6 +199,18 @@ async def get_user_id_from_pgid(app: web.Application, *, primary_gid: int) -> Us
         return user_id
 
 
+async def get_user_email_legacy(engine: AsyncEngine, *, user_id: UserID | None) -> str:
+    if not user_id:
+        return "[email protected]"
+    async with pass_or_acquire_connection(engine=engine) as conn:
+        email: str | None = await conn.scalar(
+            sa.select(
+                users.c.email,
+            ).where(users.c.id == user_id)
+        )
+        return email or "Unknown"
+
+
 async def get_user_fullname(app: web.Application, *, user_id: UserID) -> FullNameDict:
     """
     :raises UserNotFoundError:

services/web/server/src/simcore_service_webserver/users/_users_service.py

@@ -124,6 +124,15 @@ async def get_user(app: web.Application, user_id: UserID) -> dict[str, Any]:
     )
 
 
+async def get_user_email_legacy(app: web.Application, user_id: UserID | None) -> str:
+    """
+    :raises UserNotFoundError: if missing but NOT if marked for deletion!
+    """
+    return await _users_repository.get_user_email_legacy(
+        engine=get_asyncpg_engine(app), user_id=user_id
+    )
+
+
 async def get_user_primary_group_id(app: web.Application, user_id: UserID) -> GroupID:
     return await _users_repository.get_user_primary_group_id(
         engine=get_asyncpg_engine(app), user_id=user_id

services/web/server/src/simcore_service_webserver/users/api.py

@@ -7,6 +7,7 @@
     get_user,
     get_user_credentials,
     get_user_display_and_id_names,
+    get_user_email_legacy,
     get_user_fullname,
     get_user_id_from_gid,
     get_user_invoice_address,
@@ -28,6 +29,7 @@
     "get_user",
     "get_user_credentials",
     "get_user_display_and_id_names",
+    "get_user_email_legacy",
     "get_user_fullname",
     "get_user_id_from_gid",
     "get_user_invoice_address",