🐛 Mitigates "Projects not being listed on osparc-staging.io" (#7836)

Author: pcrespov

services/web/server/src/simcore_service_webserver/catalog/_catalog_rest_client_service.py

@@ -2,7 +2,7 @@
 
 import logging
 import urllib.parse
-from collections.abc import Callable, Iterator
+from collections.abc import Iterator
 from contextlib import contextmanager
 from typing import Any, Final
 
@@ -30,20 +30,11 @@
 from yarl import URL
 
 from .._meta import api_version_prefix
+from ._models import ServiceKeyVersionDict
 from .settings import CatalogSettings, get_plugin_settings
 
 _logger = logging.getLogger(__name__)
 
-# Cache settings
-_SECOND = 1  # in seconds
-_MINUTE = 60 * _SECOND
-_CACHE_TTL: Final = 1 * _MINUTE
-
-
-def _create_service_cache_key(_f: Callable[..., Any], *_args, **kw):
-    assert len(_args) == 1, f"Expected only app, got {_args}"  # nosec
-    return f"get_service_{kw['user_id']}_{kw['service_key']}_{kw['service_version']}_{kw['product_name']}"
-
 
 @contextmanager
 def _handle_client_exceptions(app: web.Application) -> Iterator[ClientSession]:
@@ -96,10 +87,27 @@ def to_backend_service(rel_url: URL, origin: URL, version_prefix: str) -> URL:
     return origin.with_path(new_path).with_query(rel_url.query)
 
 
+# Cache settings for services rest API
+_SECOND = 1  # in seconds
+_MINUTE = 60 * _SECOND
+_CACHE_TTL: Final = 1 * _MINUTE
+
+
+@cached(
+    ttl=_CACHE_TTL,
+    key_builder=lambda _f, *_args, **kw: f"get_services_for_user_in_product_{kw['user_id']}_{kw['product_name']}",
+    cache=Cache.MEMORY,
+)
 async def get_services_for_user_in_product(
-    app: web.Application, user_id: UserID, product_name: str, *, only_key_versions: bool
-) -> list[dict]:
+    app: web.Application, *, user_id: UserID, product_name: str
+) -> list[ServiceKeyVersionDict]:
+    """
+    DEPRECATED: see instead RPC interface.
+    SEE https://github.com/ITISFoundation/osparc-simcore/issues/7838
+    """
     settings: CatalogSettings = get_plugin_settings(app)
+    only_key_versions = True
+
     url = (URL(settings.api_base_url) / "services").with_query(
         {"user_id": user_id, "details": f"{not only_key_versions}"}
     )
@@ -115,13 +123,18 @@ async def get_services_for_user_in_product(
                     user_id,
                 )
                 return []
-            body: list[dict] = await response.json()
-            return body
+            services: list[dict] = await response.json()
+
+            # This reduces the size cached in the memory
+            return [
+                ServiceKeyVersionDict(key=service["key"], version=service["version"])
+                for service in services
+            ]
 
 
 @cached(
     ttl=_CACHE_TTL,
-    key_builder=_create_service_cache_key,
+    key_builder=lambda _f, *_args, **kw: f"get_service_{kw['user_id']}_{kw['service_key']}_{kw['service_version']}_{kw['product_name']}",
     cache=Cache.MEMORY,
     # SEE https://github.com/ITISFoundation/osparc-simcore/pull/7802
 )
@@ -133,6 +146,10 @@ async def get_service(
     service_version: ServiceVersion,
     product_name: ProductName,
 ) -> dict[str, Any]:
+    """
+    DEPRECATED: see instead RPC interface.
+    SEE https://github.com/ITISFoundation/osparc-simcore/issues/7838
+    """
     settings: CatalogSettings = get_plugin_settings(app)
     url = URL(
         f"{settings.api_base_url}/services/{urllib.parse.quote_plus(service_key)}/{service_version}",
@@ -144,8 +161,8 @@ async def get_service(
             url, headers={X_PRODUCT_NAME_HEADER: product_name}
         ) as response:
             response.raise_for_status()
-            body: dict[str, Any] = await response.json()
-            return body
+            service: dict[str, Any] = await response.json()
+            return service
 
 
 async def get_service_resources(

services/web/server/src/simcore_service_webserver/catalog/_controller_rest.py

@@ -37,7 +37,6 @@
 from ..utils_aiohttp import envelope_json_response
 from . import _catalog_rest_client_service, _service
 from ._controller_rest_exceptions import (
-    DefaultPricingUnitForServiceNotFoundError,
     handle_plugin_requests_exceptions,
 )
 from ._controller_rest_schemas import (
@@ -50,6 +49,7 @@
     ServiceTagPathParams,
     ToServiceInputsQueryParams,
 )
+from .errors import DefaultPricingUnitForServiceNotFoundError
 
 _logger = logging.getLogger(__name__)
 

services/web/server/src/simcore_service_webserver/catalog/_controller_rest_exceptions.py

@@ -105,19 +105,12 @@ async def _handler_catalog_client_errors(
 }
 
 
-_exceptions_handlers_map: ExceptionHandlersMap = {
+catalog_exceptions_handlers_map: ExceptionHandlersMap = {
     CatalogResponseError: _handler_catalog_client_errors,
     CatalogConnectionError: _handler_catalog_client_errors,
 }
-_exceptions_handlers_map.update(to_exceptions_handlers_map(_TO_HTTP_ERROR_MAP))
+catalog_exceptions_handlers_map.update(to_exceptions_handlers_map(_TO_HTTP_ERROR_MAP))
 
 handle_plugin_requests_exceptions = exception_handling_decorator(
-    _exceptions_handlers_map
-)
-
-
-__all__: tuple[str, ...] = (
-    "CatalogForbiddenError",
-    "CatalogItemNotFoundError",
-    "DefaultPricingUnitForServiceNotFoundError",
+    catalog_exceptions_handlers_map
 )

services/web/server/src/simcore_service_webserver/catalog/_models.py

@@ -1 +1,6 @@
-# NOTE: missing. @bisgaard-itis will follow up here
+from typing import TypedDict
+
+
+class ServiceKeyVersionDict(TypedDict):
+    key: str
+    version: str

services/web/server/src/simcore_service_webserver/catalog/catalog_service.py

@@ -6,6 +6,7 @@
     is_catalog_service_responsive,
     to_backend_service,
 )
+from ._models import ServiceKeyVersionDict
 from ._service import batch_get_my_services
 
 __all__: tuple[str, ...] = (
@@ -16,5 +17,6 @@
     "get_services_for_user_in_product",
     "is_catalog_service_responsive",
     "to_backend_service",
+    "ServiceKeyVersionDict",
 )
 # nopycln: file

services/web/server/src/simcore_service_webserver/projects/_controller/_rest_exceptions.py

@@ -8,7 +8,9 @@
     CatalogItemNotFoundError,
     CatalogNotAvailableError,
 )
+from simcore_service_webserver.exception_handling._base import ExceptionHandlersMap
 
+from ...catalog._controller_rest_exceptions import catalog_exceptions_handlers_map
 from ...conversations.errors import (
     ConversationErrorNotFoundError,
     ConversationMessageErrorNotFoundError,
@@ -239,6 +241,9 @@ def _assert_duplicate():
 }
 
 
-handle_plugin_requests_exceptions = exception_handling_decorator(
-    to_exceptions_handlers_map(_TO_HTTP_ERROR_MAP)
-)
+_handlers: ExceptionHandlersMap = {
+    **catalog_exceptions_handlers_map,
+    **to_exceptions_handlers_map(_TO_HTTP_ERROR_MAP),
+}
+
+handle_plugin_requests_exceptions = exception_handling_decorator(_handlers)

services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py

@@ -39,7 +39,7 @@
 from .. import _crud_api_create, _crud_api_read, _projects_service
 from .._permalink_service import update_or_pop_permalink_in_project
 from ..models import ProjectDict
-from ..utils import get_project_unavailable_services, project_uses_available_services
+from ..utils import are_project_services_available, get_project_unavailable_services
 from . import _rest_utils
 from ._rest_exceptions import handle_plugin_requests_exceptions
 from ._rest_schemas import (
@@ -55,12 +55,6 @@
     ProjectsSearchQueryParams,
 )
 
-# When the user requests a project with a repo, the working copy might differ from
-# the repo project. A middleware in the meta module (if active) will resolve
-# the working copy and redirect to the appropriate project entrypoint. Nonetheless, the
-# response needs to refer to the uuid of the request and this is passed through this request key
-RQ_REQUESTED_REPO_PROJECT_UUID_KEY = f"{__name__}.RQT_REQUESTED_REPO_PROJECT_UUID_KEY"
-
 _logger = logging.getLogger(__name__)
 
 
@@ -277,10 +271,8 @@ async def get_project(request: web.Request):
     req_ctx = RequestContext.model_validate(request)
     path_params = parse_request_path_parameters_as(ProjectPathParams, request)
 
-    user_available_services: list[dict] = (
-        await catalog_service.get_services_for_user_in_product(
-            request.app, req_ctx.user_id, req_ctx.product_name, only_key_versions=True
-        )
+    user_available_services = await catalog_service.get_services_for_user_in_product(
+        request.app, user_id=req_ctx.user_id, product_name=req_ctx.product_name
     )
 
     project = await _projects_service.get_project_for_user(
@@ -290,7 +282,8 @@ async def get_project(request: web.Request):
         include_state=True,
         include_trashed_by_primary_gid=True,
     )
-    if not await project_uses_available_services(project, user_available_services):
+
+    if not are_project_services_available(project, user_available_services):
         unavilable_services = get_project_unavailable_services(
             project, user_available_services
         )
@@ -305,9 +298,6 @@ async def get_project(request: web.Request):
             )
         )
 
-    if new_uuid := request.get(RQ_REQUESTED_REPO_PROJECT_UUID_KEY):
-        project["uuid"] = new_uuid
-
     # Adds permalink
     await update_or_pop_permalink_in_project(request, project)
 

services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py

@@ -120,10 +120,8 @@ async def list_projects(  # pylint: disable=too-many-arguments
 ) -> tuple[list[ProjectDict], int]:
     db = ProjectDBAPI.get_from_app_context(app)
 
-    user_available_services: list[dict] = (
-        await catalog_service.get_services_for_user_in_product(
-            app, user_id, product_name, only_key_versions=True
-        )
+    user_available_services = await catalog_service.get_services_for_user_in_product(
+        app, user_id=user_id, product_name=product_name
     )
 
     workspace_is_private = True
@@ -204,10 +202,8 @@ async def list_projects_full_depth(
 ) -> tuple[list[ProjectDict], int]:
     db = ProjectDBAPI.get_from_app_context(app)
 
-    user_available_services: list[dict] = (
-        await catalog_service.get_services_for_user_in_product(
-            app, user_id, product_name, only_key_versions=True
-        )
+    user_available_services = await catalog_service.get_services_for_user_in_product(
+        app, user_id=user_id, product_name=product_name
     )
 
     db_projects, db_project_types, total_number_projects = await db.list_projects_dicts(

services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy_utils.py

@@ -32,7 +32,7 @@
     ProjectNotFoundError,
 )
 from .models import ProjectDict
-from .utils import find_changed_node_keys, project_uses_available_services
+from .utils import are_project_services_available, find_changed_node_keys
 
 logger = logging.getLogger(__name__)
 
@@ -222,7 +222,7 @@ async def _execute_without_permission_check(
                 filter_by_services is not None
                 # This checks only old projects that are not in the projects_to_products table.
                 and row[projects_to_products.c.product_name] is None
-                and not await project_uses_available_services(prj, filter_by_services)
+                and not are_project_services_available(prj, filter_by_services)
             ):
                 logger.warning(
                     "Project %s will not be listed for user %s since it has no access rights"

services/web/server/src/simcore_service_webserver/projects/utils.py

@@ -189,20 +189,21 @@ def is_graph_equal(
     return True
 
 
-async def project_uses_available_services(
-    project: dict[str, Any], available_services: list[dict[str, Any]]
+def are_project_services_available(
+    project: dict[str, Any], available_services: list[dict[str, str]]
 ) -> bool:
     if not project["workbench"]:
         # empty project
         return True
-    # get project services
-    needed_services: set[tuple[str, str]] = {
-        (s["key"], s["version"]) for _, s in project["workbench"].items()
+
+    # list services in project
+    needed_services = {
+        (srv["key"], srv["version"]) for _, srv in project["workbench"].items()
     }
 
-    # get available services
-    available_services_set: set[tuple[str, str]] = {
-        (s["key"], s["version"]) for s in available_services
+    # list available services
+    available_services_set = {
+        (srv["key"], srv["version"]) for srv in available_services
     }
 
     return needed_services.issubset(available_services_set)