diff --git a/packages/models-library/src/models_library/projects.py b/packages/models-library/src/models_library/projects.py index 0c4dd0884b9..2092d1fce93 100644 --- a/packages/models-library/src/models_library/projects.py +++ b/packages/models-library/src/models_library/projects.py @@ -8,9 +8,6 @@ from uuid import UUID from common_library.basic_types import DEFAULT_FACTORY -from models_library.basic_types import ConstrainedStr -from models_library.folders import FolderID -from models_library.workspaces import WorkspaceID from pydantic import ( BaseModel, ConfigDict, @@ -21,8 +18,11 @@ ) from .basic_regex import DATE_RE, UUID_RE_BASE +from .basic_types import ConstrainedStr from .emails import LowerCaseEmailStr +from .folders import FolderID from .groups import GroupID +from .products import ProductName from .projects_access import AccessRights, GroupIDStr from .projects_nodes import Node from .projects_nodes_io import NodeIDStr @@ -33,6 +33,7 @@ none_to_empty_str_pre_validator, ) from .utils.enums import StrAutoEnum +from .workspaces import WorkspaceID ProjectID: TypeAlias = UUID CommitID: TypeAlias = int @@ -147,6 +148,25 @@ def _convert_sql_alchemy_enum(cls, v): ) +class ProjectListAtDB(BaseProjectModel): + id: int + type: ProjectType + template_type: ProjectTemplateType | None + prj_owner: int | None + ui: dict[str, Any] | None + classifiers: list[ClassifierID] | None + dev: dict[str, Any] | None + quality: dict[str, Any] + published: bool | None + hidden: bool + workspace_id: WorkspaceID | None + trashed: datetime | None + trashed_by: UserID | None + trashed_explicitly: bool + product_name: ProductName + folder_id: FolderID | None + + class Project(BaseProjectModel): # NOTE: This is the pydantic pendant of project-v0.0.1.json used in the API of the webserver/webclient # NOT for usage with DB!! diff --git a/packages/models-library/src/models_library/projects_nodes.py b/packages/models-library/src/models_library/projects_nodes.py index 66683369f35..9792ba33ec5 100644 --- a/packages/models-library/src/models_library/projects_nodes.py +++ b/packages/models-library/src/models_library/projects_nodes.py @@ -168,7 +168,7 @@ class Node(BaseModel): ), ] = None - thumbnail: Annotated[ + thumbnail: Annotated[ # <-- (DEPRECATED) Can be removed str | HttpUrl | None, Field( description="url of the latest screenshot of the node", @@ -232,10 +232,10 @@ class Node(BaseModel): ] = DEFAULT_FACTORY output_node: Annotated[bool | None, Field(deprecated=True, alias="outputNode")] = ( - None + None # <-- (DEPRECATED) Can be removed ) - output_nodes: Annotated[ + output_nodes: Annotated[ # <-- (DEPRECATED) Can be removed list[NodeID] | None, Field( description="Used in group-nodes. Node IDs of those connected to the output", @@ -243,7 +243,7 @@ class Node(BaseModel): ), ] = None - parent: Annotated[ + parent: Annotated[ # <-- (DEPRECATED) Can be removed NodeID | None, Field( description="Parent's (group-nodes') node ID s. Used to group", diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py index b1775f88b65..a4a533b542c 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py @@ -10,7 +10,7 @@ from aiohttp import web from models_library.folders import FolderID, FolderQuery, FolderScope -from models_library.projects import ProjectID, ProjectTemplateType +from models_library.projects import ProjectTemplateType from models_library.rest_ordering import OrderBy from models_library.users import UserID from models_library.workspaces import WorkspaceID, WorkspaceQuery, WorkspaceScope @@ -23,11 +23,13 @@ from simcore_postgres_database.webserver_models import ProjectType as ProjectTypeDB from ..folders import _folders_repository +from ..users.api import get_user_email_legacy from ..workspaces.api import check_user_workspace_access from . import _projects_service from ._access_rights_repository import batch_get_project_access_rights from ._projects_repository import batch_get_trashed_by_primary_gid from ._projects_repository_legacy import ProjectDBAPI +from ._projects_repository_legacy_utils import convert_to_schema_names from .models import ProjectDict, ProjectTypeAPI @@ -53,7 +55,6 @@ async def _aggregate_data_to_projects_from_other_sources( app: web.Application, *, db_projects: list[ProjectDict], - db_project_types: list[ProjectTypeDB], user_id: UserID, ) -> list[ProjectDict]: """ @@ -61,7 +62,7 @@ async def _aggregate_data_to_projects_from_other_sources( """ # updating `project.trashed_by_primary_gid` trashed_by_primary_gid_values = await batch_get_trashed_by_primary_gid( - app, projects_uuids=[ProjectID(p["uuid"]) for p in db_projects] + app, projects_uuids=[p["uuid"] for p in db_projects] ) _batch_update("trashed_by_primary_gid", trashed_by_primary_gid_values, db_projects) @@ -70,7 +71,7 @@ async def _aggregate_data_to_projects_from_other_sources( project_to_access_rights = await batch_get_project_access_rights( app=app, projects_uuids_with_workspace_id=[ - (ProjectID(p["uuid"]), p["workspaceId"]) for p in db_projects + (p["uuid"], p["workspaceId"]) for p in db_projects ], ) @@ -79,10 +80,10 @@ async def _aggregate_data_to_projects_from_other_sources( _projects_service.add_project_states_for_user( user_id=user_id, project=prj, - is_template=prj_type == ProjectTypeDB.TEMPLATE, + is_template=prj["type"] == ProjectTypeDB.TEMPLATE.value, app=app, ) - for prj, prj_type in zip(db_projects, db_project_types, strict=False) + for prj in db_projects ] updated_projects: list[ProjectDict] = await _paralell_update( @@ -90,11 +91,29 @@ async def _aggregate_data_to_projects_from_other_sources( ) for project in updated_projects: - project["accessRights"] = project_to_access_rights[project["uuid"]] + project["accessRights"] = project_to_access_rights[f"{project['uuid']}"] return updated_projects +async def _legacy_convert_db_projects_to_api_projects( + app: web.Application, + db, + db_projects: list[dict[str, Any]], +) -> list[dict]: + """ + Converts db schema projects to API schema (legacy postprocessing). + """ + api_projects: list[dict] = [] + for db_prj in db_projects: + db_prj_dict = db_prj + db_prj_dict.pop("product_name", None) + db_prj_dict["tags"] = await db.get_tags_by_project(project_id=f"{db_prj['id']}") + user_email = await get_user_email_legacy(app, db_prj["prj_owner"]) + api_projects.append(convert_to_schema_names(db_prj_dict, user_email)) + return api_projects + + async def list_projects( # pylint: disable=too-many-arguments app: web.Application, user_id: UserID, @@ -140,7 +159,7 @@ async def list_projects( # pylint: disable=too-many-arguments workspace_id=workspace_id, ) - db_projects, db_project_types, total_number_projects = await db.list_projects_dicts( + db_projects, total_number_projects = await db.list_projects_dicts( product_name=product_name, user_id=user_id, workspace_query=( @@ -172,11 +191,15 @@ async def list_projects( # pylint: disable=too-many-arguments order_by=order_by, ) - projects = await _aggregate_data_to_projects_from_other_sources( - app, db_projects=db_projects, db_project_types=db_project_types, user_id=user_id + api_projects = await _legacy_convert_db_projects_to_api_projects( + app, db, db_projects ) - return projects, total_number_projects + final_projects = await _aggregate_data_to_projects_from_other_sources( + app, db_projects=api_projects, user_id=user_id + ) + + return final_projects, total_number_projects async def list_projects_full_depth( @@ -196,7 +219,7 @@ async def list_projects_full_depth( ) -> tuple[list[ProjectDict], int]: db = ProjectDBAPI.get_from_app_context(app) - db_projects, db_project_types, total_number_projects = await db.list_projects_dicts( + db_projects, total_number_projects = await db.list_projects_dicts( product_name=product_name, user_id=user_id, workspace_query=WorkspaceQuery(workspace_scope=WorkspaceScope.ALL), @@ -210,8 +233,12 @@ async def list_projects_full_depth( order_by=order_by, ) - projects = await _aggregate_data_to_projects_from_other_sources( - app, db_projects=db_projects, db_project_types=db_project_types, user_id=user_id + api_projects = await _legacy_convert_db_projects_to_api_projects( + app, db, db_projects + ) + + final_projects = await _aggregate_data_to_projects_from_other_sources( + app, db_projects=api_projects, user_id=user_id ) - return projects, total_number_projects + return final_projects, total_number_projects diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py index ee92f5d79e9..f9a2340db76 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py @@ -19,7 +19,11 @@ from models_library.folders import FolderQuery, FolderScope from models_library.groups import GroupID from models_library.products import ProductName -from models_library.projects import ProjectID, ProjectIDStr +from models_library.projects import ( + ProjectID, + ProjectIDStr, + ProjectListAtDB, +) from models_library.projects_comments import CommentID, ProjectsCommentsDB from models_library.projects_nodes import Node from models_library.projects_nodes_io import NodeID, NodeIDStr @@ -584,7 +588,7 @@ async def list_projects_dicts( # pylint: disable=too-many-arguments,too-many-st limit: int | None = None, # order order_by: OrderBy = DEFAULT_ORDER_BY, - ) -> tuple[list[ProjectDict], list[ProjectType], int]: + ) -> tuple[list[dict[str, Any]], int]: async with self.engine.acquire() as conn: user_groups_proxy: list[RowProxy] = await self._list_user_groups( conn, user_id @@ -667,14 +671,18 @@ async def list_projects_dicts( # pylint: disable=too-many-arguments,too-many-st projects.c.id, ) - prjs, prj_types = await self._execute_without_permission_check( - conn, - select_projects_query=combined_query.offset(offset).limit(limit), - ) + prjs_output = [] + async for row in conn.execute(combined_query.offset(offset).limit(limit)): + # NOTE: Historically, projects were returned as a dictionary. I have created a model that + # validates the DB row, but this model includes some default values inside the Workbench Node model. + # Therefore, if we use this model, it will return those default values, which is not backward-compatible + # with the frontend. The frontend would need to check and adapt how it handles default values in + # Workbench nodes, which are currently not returned if not set in the DB. + ProjectListAtDB.model_validate(row) + prjs_output.append(dict(row.items())) return ( - prjs, - prj_types, + prjs_output, cast(int, total_count), ) @@ -1249,6 +1257,13 @@ async def remove_tag( project["tags"].remove(tag_id) return convert_to_schema_names(project, user_email) + async def get_tags_by_project(self, project_id: str) -> list[int]: + async with self.engine.acquire() as conn: + query = sa.select(projects_tags.c.tag_id).where( + projects_tags.c.project_id == project_id + ) + return [row.tag_id async for row in conn.execute(query)] + # # Project Comments # diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py index 0ff06f4ec0e..5b482452625 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py @@ -1,4 +1,3 @@ -import asyncio import logging from collections.abc import Mapping from copy import deepcopy @@ -9,15 +8,20 @@ import sqlalchemy as sa from aiopg.sa.connection import SAConnection from aiopg.sa.result import RowProxy -from models_library.projects import ProjectAtDB, ProjectID, ProjectTemplateType +from models_library.projects import ProjectID, ProjectType from models_library.projects_nodes import Node from models_library.projects_nodes_io import NodeIDStr from models_library.utils.change_case import camel_to_snake, snake_to_camel from pydantic import ValidationError from simcore_postgres_database.models.project_to_groups import project_to_groups -from simcore_postgres_database.webserver_models import ProjectType, projects +from simcore_postgres_database.webserver_models import ( + ProjectTemplateType as ProjectTemplateTypeDB, +) +from simcore_postgres_database.webserver_models import ProjectType as ProjectTypeDB +from simcore_postgres_database.webserver_models import ( + projects, +) from sqlalchemy.dialects.postgresql import insert as pg_insert -from sqlalchemy.sql.selectable import CompoundSelect, Select from ..db.models import GroupType, groups, projects_tags, user_to_groups, users from ..users.exceptions import UserNotFoundError @@ -25,7 +29,6 @@ from ._projects_repository import PROJECT_DB_COLS from .exceptions import ( NodeNotFoundError, - ProjectInvalidRightsError, ProjectInvalidUsageError, ProjectNotFoundError, ) @@ -91,9 +94,9 @@ def convert_to_schema_names( if col_name == "prj_owner": # this entry has to be converted to the owner e-mail address converted_value = user_email - if col_name == "type" and isinstance(col_value, ProjectType): + if col_name == "type" and isinstance(col_value, ProjectTypeDB): converted_value = col_value.value - if col_name == "template_type" and isinstance(col_value, ProjectTemplateType): + if col_name == "template_type" and isinstance(col_value, ProjectTemplateTypeDB): converted_value = col_value.value if col_name in SCHEMA_NON_NULL_KEYS and col_value is None: @@ -184,50 +187,6 @@ async def _upsert_tags_in_project( .on_conflict_do_nothing() ) - async def _execute_without_permission_check( - self, - conn: SAConnection, - *, - select_projects_query: Select | CompoundSelect, - ) -> tuple[list[dict[str, Any]], list[ProjectType]]: - api_projects: list[dict] = [] # API model-compatible projects - db_projects: list[dict] = [] # DB model-compatible projects - project_types: list[ProjectType] = [] - async for row in conn.execute(select_projects_query): - assert isinstance(row, RowProxy) # nosec - try: - await asyncio.get_event_loop().run_in_executor( - None, ProjectAtDB.model_validate, row - ) - - except ProjectInvalidRightsError: - continue - - except ValidationError as exc: - logger.warning( - "project %s failed validation, please check. error: %s", - f"{row.id=}", - exc, - ) - continue - - prj: dict[str, Any] = dict(row.items()) - prj.pop("product_name", None) - - db_projects.append(prj) - - # NOTE: DO NOT nest _get_tags_by_project in async loop above !!! - # FIXME: temporary avoids inner async loops issue https://github.com/aio-libs/aiopg/issues/535 - for db_prj in db_projects: - db_prj["tags"] = await self._get_tags_by_project( - conn, project_id=db_prj["id"] - ) - user_email = await self._get_user_email(conn, db_prj["prj_owner"]) - api_projects.append(convert_to_schema_names(db_prj, user_email)) - project_types.append(db_prj["type"]) - - return (api_projects, project_types) - async def _get_project( self, connection: SAConnection, diff --git a/services/web/server/src/simcore_service_webserver/users/_users_repository.py b/services/web/server/src/simcore_service_webserver/users/_users_repository.py index 41c358cb520..f49c961b440 100644 --- a/services/web/server/src/simcore_service_webserver/users/_users_repository.py +++ b/services/web/server/src/simcore_service_webserver/users/_users_repository.py @@ -199,6 +199,18 @@ async def get_user_id_from_pgid(app: web.Application, *, primary_gid: int) -> Us return user_id +async def get_user_email_legacy(engine: AsyncEngine, *, user_id: UserID | None) -> str: + if not user_id: + return "not_a_user@unknown.com" + async with pass_or_acquire_connection(engine=engine) as conn: + email: str | None = await conn.scalar( + sa.select( + users.c.email, + ).where(users.c.id == user_id) + ) + return email or "Unknown" + + async def get_user_fullname(app: web.Application, *, user_id: UserID) -> FullNameDict: """ :raises UserNotFoundError: diff --git a/services/web/server/src/simcore_service_webserver/users/_users_service.py b/services/web/server/src/simcore_service_webserver/users/_users_service.py index 0889763c2e7..b7a6668244c 100644 --- a/services/web/server/src/simcore_service_webserver/users/_users_service.py +++ b/services/web/server/src/simcore_service_webserver/users/_users_service.py @@ -124,6 +124,15 @@ async def get_user(app: web.Application, user_id: UserID) -> dict[str, Any]: ) +async def get_user_email_legacy(app: web.Application, user_id: UserID | None) -> str: + """ + :raises UserNotFoundError: if missing but NOT if marked for deletion! + """ + return await _users_repository.get_user_email_legacy( + engine=get_asyncpg_engine(app), user_id=user_id + ) + + async def get_user_primary_group_id(app: web.Application, user_id: UserID) -> GroupID: return await _users_repository.get_user_primary_group_id( engine=get_asyncpg_engine(app), user_id=user_id diff --git a/services/web/server/src/simcore_service_webserver/users/api.py b/services/web/server/src/simcore_service_webserver/users/api.py index 238cd68b20e..f19108debe5 100644 --- a/services/web/server/src/simcore_service_webserver/users/api.py +++ b/services/web/server/src/simcore_service_webserver/users/api.py @@ -7,6 +7,7 @@ get_user, get_user_credentials, get_user_display_and_id_names, + get_user_email_legacy, get_user_fullname, get_user_id_from_gid, get_user_invoice_address, @@ -28,6 +29,7 @@ "get_user", "get_user_credentials", "get_user_display_and_id_names", + "get_user_email_legacy", "get_user_fullname", "get_user_id_from_gid", "get_user_invoice_address",