♻️ Refactor webserver projects: drops request dependency on service layer functions (#7228)

Author: pcrespov

.vscode/settings.template.json

@@ -59,9 +59,7 @@
   },
   "hadolint.hadolintPath": "${workspaceFolder}/scripts/hadolint.bash",
   "hadolint.cliOptions": [],
-  "ruff.lint.args": [
-    "--config=${workspaceFolder}/.ruff.toml"
-  ],
+  "ruff.configuration": "${workspaceFolder}/.ruff.toml",
   "ruff.path": [
     "${workspaceFolder}/.venv/bin/ruff"
   ],

services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py

@@ -42,7 +42,7 @@
 from . import _folders_db as project_to_folders_db
 from . import projects_service
 from ._metadata_api import set_project_ancestors
-from ._permalink_api import update_or_pop_permalink_in_project
+from ._permalink_service import update_or_pop_permalink_in_project
 from .db import ProjectDBAPI
 from .exceptions import (
     ParentNodeNotFoundError,

services/web/server/src/simcore_service_webserver/projects/_crud_api_delete.py

@@ -78,7 +78,7 @@ async def delete_project(
     app: web.Application,
     project_uuid: ProjectID,
     user_id: UserID,
-    simcore_user_agent,
+    simcore_user_agent: str,
     remove_project_dynamic_services: RemoveProjectServicesCallable,
 ) -> None:
     """Stops dynamic services, deletes data and finally deletes project

services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py

@@ -5,6 +5,8 @@
 
 """
 
+from typing import Any, Coroutine
+
 from aiohttp import web
 from models_library.folders import FolderID, FolderQuery, FolderScope
 from models_library.projects import ProjectID
@@ -23,51 +25,65 @@
 from ..folders import _folders_repository as _folders_repository
 from ..workspaces._workspaces_service import check_user_workspace_access
 from . import projects_service
-from ._permalink_api import update_or_pop_permalink_in_project
 from .db import ProjectDBAPI
 from .models import ProjectDict, ProjectTypeAPI
 
 
-async def _update_project_dict(
-    request: web.Request,
-    *,
-    user_id: UserID,
-    project: ProjectDict,
-    is_template: bool,
-) -> ProjectDict:
-    # state
-    await projects_service.add_project_states_for_user(
-        user_id=user_id,
-        project=project,
-        is_template=is_template,
-        app=request.app,
-    )
+def _batch_update(
+    key: str,
+    value_per_object: list[Any],
+    objects: list[dict[str, Any]],
+) -> list[dict[str, Any]]:
+    for obj, value in zip(objects, value_per_object, strict=True):
+        obj[key] = value
+    return objects
 
-    # permalink
-    await update_or_pop_permalink_in_project(request, project)
 
-    return project
+async def _paralell_update(*update_per_object: Coroutine) -> list[Any]:
+    return await logged_gather(
+        *update_per_object,
+        reraise=True,
+        max_concurrency=100,
+    )
 
 
-async def _batch_update_list_of_project_dict(
-    app: web.Application, list_of_project_dict: list[ProjectDict]
+async def _aggregate_data_to_projects_from_other_sources(
+    app: web.Application,
+    *,
+    db_projects: list[ProjectDict],
+    db_project_types: list[ProjectTypeDB],
+    user_id: UserID,
 ) -> list[ProjectDict]:
-
-    # updating `trashed_by_primary_gid`
+    """
+    Aggregates data to each project from other sources, first as a batch-update and then as a parallel-update.
+    """
+    # updating `project.trashed_by_primary_gid`
     trashed_by_primary_gid_values = await batch_get_trashed_by_primary_gid(
-        app, projects_uuids=[ProjectID(p["uuid"]) for p in list_of_project_dict]
+        app, projects_uuids=[ProjectID(p["uuid"]) for p in db_projects]
     )
 
-    for project_dict, value in zip(
-        list_of_project_dict, trashed_by_primary_gid_values, strict=True
-    ):
-        project_dict.update(trashed_by_primary_gid=value)
+    _batch_update("trashed_by_primary_gid", trashed_by_primary_gid_values, db_projects)
+
+    # udpating `project.state`
+    update_state_per_project = [
+        projects_service.add_project_states_for_user(
+            user_id=user_id,
+            project=prj,
+            is_template=prj_type == ProjectTypeDB.TEMPLATE,
+            app=app,
+        )
+        for prj, prj_type in zip(db_projects, db_project_types, strict=False)
+    ]
+
+    updated_projects: list[ProjectDict] = await _paralell_update(
+        *update_state_per_project,
+    )
 
-    return list_of_project_dict
+    return updated_projects
 
 
 async def list_projects(  # pylint: disable=too-many-arguments
-    request: web.Request,
+    app: web.Application,
     user_id: UserID,
     product_name: str,
     *,
@@ -87,7 +103,6 @@ async def list_projects(  # pylint: disable=too-many-arguments
     # ordering
     order_by: OrderBy,
 ) -> tuple[list[ProjectDict], int]:
-    app = request.app
     db = ProjectDBAPI.get_from_app_context(app)
 
     user_available_services: list[dict] = await get_services_for_user_in_product(
@@ -145,27 +160,15 @@ async def list_projects(  # pylint: disable=too-many-arguments
         order_by=order_by,
     )
 
-    db_projects = await _batch_update_list_of_project_dict(app, db_projects)
-
-    projects: list[ProjectDict] = await logged_gather(
-        *(
-            _update_project_dict(
-                request,
-                user_id=user_id,
-                project=prj,
-                is_template=prj_type == ProjectTypeDB.TEMPLATE,
-            )
-            for prj, prj_type in zip(db_projects, db_project_types, strict=False)
-        ),
-        reraise=True,
-        max_concurrency=100,
+    projects = await _aggregate_data_to_projects_from_other_sources(
+        app, db_projects=db_projects, db_project_types=db_project_types, user_id=user_id
     )
 
     return projects, total_number_projects
 
 
 async def list_projects_full_depth(
-    request,
+    app: web.Application,
     *,
     user_id: UserID,
     product_name: str,
@@ -180,10 +183,10 @@ async def list_projects_full_depth(
     search_by_multi_columns: str | None,
     search_by_project_name: str | None,
 ) -> tuple[list[ProjectDict], int]:
-    db = ProjectDBAPI.get_from_app_context(request.app)
+    db = ProjectDBAPI.get_from_app_context(app)
 
     user_available_services: list[dict] = await get_services_for_user_in_product(
-        request.app, user_id, product_name, only_key_versions=True
+        app, user_id, product_name, only_key_versions=True
     )
 
     db_projects, db_project_types, total_number_projects = await db.list_projects_dicts(
@@ -202,30 +205,8 @@ async def list_projects_full_depth(
         order_by=order_by,
     )
 
-    db_projects = await _batch_update_list_of_project_dict(request.app, db_projects)
-
-    projects: list[ProjectDict] = await logged_gather(
-        *(
-            _update_project_dict(
-                request,
-                user_id=user_id,
-                project=prj,
-                is_template=prj_type == ProjectTypeDB.TEMPLATE,
-            )
-            for prj, prj_type in zip(db_projects, db_project_types, strict=False)
-        ),
-        reraise=True,
-        max_concurrency=100,
+    projects = await _aggregate_data_to_projects_from_other_sources(
+        app, db_projects=db_projects, db_project_types=db_project_types, user_id=user_id
     )
 
     return projects, total_number_projects
-
-
-async def get_project(
-    request: web.Request,
-    user_id: UserID,
-    product_name: str,
-    project_uuid: ProjectID,
-    project_type: ProjectTypeAPI,
-):
-    raise NotImplementedError

services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py

@@ -14,14 +14,11 @@
     ProjectCopyOverride,
     ProjectCreateNew,
     ProjectGet,
-    ProjectListItem,
     ProjectPatch,
 )
 from models_library.generics import Envelope
 from models_library.projects_state import ProjectLocked
 from models_library.rest_ordering import OrderBy
-from models_library.rest_pagination import Page
-from models_library.rest_pagination_utils import paginate_data
 from models_library.utils.fastapi_encoders import jsonable_encoder
 from servicelib.aiohttp import status
 from servicelib.aiohttp.long_running_tasks.server import start_long_running_task
@@ -36,9 +33,7 @@
     UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE,
     X_SIMCORE_USER_AGENT,
 )
-from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON
 from servicelib.redis import get_project_locked_state
-from servicelib.rest_constants import RESPONSE_MODEL_POLICY
 from simcore_service_webserver.projects.models import ProjectDict
 
 from .._meta import API_VTAG as VTAG
@@ -51,7 +46,7 @@
 from ..security.decorators import permission_required
 from ..users.api import get_user_fullname
 from ..workspaces.errors import WorkspaceAccessForbiddenError, WorkspaceNotFoundError
-from . import _crud_api_create, _crud_api_read, projects_service
+from . import _crud_api_create, _crud_api_read, _crud_handlers_utils, projects_service
 from ._common.models import ProjectPathParams, RequestContext
 from ._crud_handlers_models import (
     ProjectActiveQueryParams,
@@ -61,7 +56,7 @@
     ProjectsListQueryParams,
     ProjectsSearchQueryParams,
 )
-from ._permalink_api import update_or_pop_permalink_in_project
+from ._permalink_service import update_or_pop_permalink_in_project
 from .exceptions import (
     ProjectDeleteError,
     ProjectInvalidRightsError,
@@ -167,27 +162,6 @@ async def create_project(request: web.Request):
     )
 
 
-def _create_page_response(projects, request_url, total, limit, offset) -> web.Response:
-    page = Page[ProjectListItem].model_validate(
-        paginate_data(
-            chunk=[
-                ProjectListItem.from_domain_model(prj).model_dump(
-                    by_alias=True, exclude_unset=True
-                )
-                for prj in projects
-            ],
-            request_url=request_url,
-            total=total,
-            limit=limit,
-            offset=offset,
-        )
-    )
-    return web.Response(
-        text=page.model_dump_json(**RESPONSE_MODEL_POLICY),
-        content_type=MIMETYPE_APPLICATION_JSON,
-    )
-
-
 @routes.get(f"/{VTAG}/projects", name="list_projects")
 @login_required
 @permission_required("project.read")
@@ -213,7 +187,7 @@ async def list_projects(request: web.Request):
     assert query_params.filters  # nosec
 
     projects, total_number_of_projects = await _crud_api_read.list_projects(
-        request,
+        request.app,
         user_id=req_ctx.user_id,
         product_name=req_ctx.product_name,
         project_type=query_params.project_type,
@@ -228,7 +202,11 @@ async def list_projects(request: web.Request):
         order_by=OrderBy.model_construct(**query_params.order_by.model_dump()),
     )
 
-    return _create_page_response(
+    projects = await _crud_handlers_utils.aggregate_data_to_projects_from_request(
+        request, projects
+    )
+
+    return _crud_handlers_utils.create_page_response(
         projects=projects,
         request_url=request.url,
         total=total_number_of_projects,
@@ -252,7 +230,7 @@ async def list_projects_full_search(request: web.Request):
     tag_ids_list = query_params.tag_ids_list()
 
     projects, total_number_of_projects = await _crud_api_read.list_projects_full_depth(
-        request,
+        request.app,
         user_id=req_ctx.user_id,
         product_name=req_ctx.product_name,
         trashed=query_params.filters.trashed,
@@ -264,7 +242,11 @@ async def list_projects_full_search(request: web.Request):
         order_by=OrderBy.model_construct(**query_params.order_by.model_dump()),
     )
 
-    return _create_page_response(
+    projects = await _crud_handlers_utils.aggregate_data_to_projects_from_request(
+        request, projects
+    )
+
+    return _crud_handlers_utils.create_page_response(
         projects=projects,
         request_url=request.url,
         total=total_number_of_projects,
@@ -486,9 +468,9 @@ async def delete_project(request: web.Request):
 
         await projects_service.submit_delete_project_task(
             request.app,
-            path_params.project_id,
-            req_ctx.user_id,
-            request.headers.get(
+            project_uuid=path_params.project_id,
+            user_id=req_ctx.user_id,
+            simcore_user_agent=request.headers.get(
                 X_SIMCORE_USER_AGENT, UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE
             ),
         )

services/web/server/src/simcore_service_webserver/projects/_crud_handlers_utils.py

@@ -0,0 +1,48 @@
+from aiohttp import web
+from models_library.api_schemas_webserver.projects import ProjectListItem
+from models_library.rest_pagination import Page
+from models_library.rest_pagination_utils import paginate_data
+from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON
+from servicelib.rest_constants import RESPONSE_MODEL_POLICY
+
+from . import _permalink_service
+from ._crud_api_read import _paralell_update
+from .models import ProjectDict
+
+
+async def aggregate_data_to_projects_from_request(
+    request: web.Request,
+    projects: list[ProjectDict],
+) -> list[ProjectDict]:
+
+    update_permalink_per_project = [
+        # permalink
+        _permalink_service.aggregate_permalink_in_project(request, project=prj)
+        for prj in projects
+    ]
+
+    updated_projects: list[ProjectDict] = await _paralell_update(
+        *update_permalink_per_project,
+    )
+    return updated_projects
+
+
+def create_page_response(projects, request_url, total, limit, offset) -> web.Response:
+    page = Page[ProjectListItem].model_validate(
+        paginate_data(
+            chunk=[
+                ProjectListItem.from_domain_model(prj).model_dump(
+                    by_alias=True, exclude_unset=True
+                )
+                for prj in projects
+            ],
+            request_url=request_url,
+            total=total,
+            limit=limit,
+            offset=offset,
+        )
+    )
+    return web.Response(
+        text=page.model_dump_json(**RESPONSE_MODEL_POLICY),
+        content_type=MIMETYPE_APPLICATION_JSON,
+    )