🐛 Fixes webserver-api error on get_project_services when project services are missing in the catalog by allowing partial failure (ITISFoundation#8486)

Author: pcrespov

packages/models-library/src/models_library/api_schemas_catalog/services.py

@@ -6,6 +6,7 @@
 from pydantic import ConfigDict, Field, HttpUrl, NonNegativeInt
 from pydantic.config import JsonDict
 
+from ..batch_operations import BatchGetEnvelope
 from ..boot_options import BootOptions
 from ..emails import LowerCaseEmailStr
 from ..groups import GroupID
@@ -420,6 +421,46 @@ class MyServiceGet(CatalogOutputSchema):
     my_access_rights: ServiceGroupAccessRightsV2
 
 
+class MyServicesRpcBatchGet(
+    CatalogOutputSchema,
+    BatchGetEnvelope[MyServiceGet, tuple[ServiceKey, ServiceVersion]],
+):
+    """Result for batch get user services operations"""
+
+    @staticmethod
+    def _update_json_schema_extra(schema: JsonDict) -> None:
+        missing: Any = [("simcore/services/comp/itis/sleeper", "100.2.3")]
+        schema.update(
+            {
+                "examples": [
+                    {
+                        "found_items": [
+                            {
+                                "key": missing[0][0],
+                                "release": {
+                                    "version": "2.2.1",
+                                    "version_display": "Winter Release",
+                                    "released": "2026-07-21T15:00:00",
+                                },
+                                "owner": 42,
+                                "my_access_rights": {
+                                    "execute": True,
+                                    "write": False,
+                                },
+                            }
+                        ],
+                        "missing_identifiers": missing,
+                    }
+                ]
+            }
+        )
+
+    model_config = ConfigDict(
+        extra="forbid",
+        json_schema_extra=_update_json_schema_extra,
+    )
+
+
 class ServiceListFilters(Filters):
     service_type: Annotated[
         ServiceType | None,

packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py

@@ -12,6 +12,7 @@
 from ..projects_nodes import InputID, InputsDict, PartialNode
 from ..projects_nodes_io import NodeID
 from ..services import ServiceKey, ServicePortKey, ServiceVersion
+from ..services_base import ServiceKeyVersion
 from ..services_enums import ServiceState
 from ..services_history import ServiceRelease
 from ..services_resources import ServiceResourcesDict
@@ -225,3 +226,9 @@ class NodeServiceGet(OutputSchema):
 class ProjectNodeServicesGet(OutputSchema):
     project_uuid: ProjectID
     services: list[NodeServiceGet]
+    missing: Annotated[
+        list[ServiceKeyVersion] | None,
+        Field(
+            description="List of services defined in the project but that were not found in the catalog"
+        ),
+    ] = None

packages/models-library/src/models_library/batch_operations.py

@@ -0,0 +1,62 @@
+from typing import Annotated, Generic, TypeVar
+
+from common_library.basic_types import DEFAULT_FACTORY
+from pydantic import BaseModel, BeforeValidator, Field, TypeAdapter
+
+ResourceT = TypeVar("ResourceT")
+IdentifierT = TypeVar("IdentifierT")
+SchemaT = TypeVar("SchemaT")
+
+
+def _deduplicate_preserving_order(identifiers: list[IdentifierT]) -> list[IdentifierT]:
+    """Remove duplicates while preserving order of first occurrence."""
+    return list(dict.fromkeys(identifiers))
+
+
+def create_batch_ids_validator(identifier_type: type[IdentifierT]) -> TypeAdapter:
+    """Create a TypeAdapter for validating batch identifiers.
+
+    This validator ensures:
+    - At least one identifier is provided (empty list is invalid for batch operations)
+    - Duplicates are removed while preserving order
+
+    Args:
+        identifier_type: The type of identifiers in the batch
+
+    Returns:
+        TypeAdapter configured for the specific identifier type
+    """
+    return TypeAdapter(
+        Annotated[
+            list[identifier_type],  # type: ignore[valid-type]
+            BeforeValidator(_deduplicate_preserving_order),
+            Field(
+                min_length=1,
+                description="List of identifiers to batch process. Empty list is not allowed for batch operations.",
+            ),
+        ]
+    )
+
+
+class BatchGetEnvelope(BaseModel, Generic[ResourceT, IdentifierT]):
+    """Generic envelope model for batch-get operations that can contain partial results.
+
+    This model represents the result of a batch operation where some items might be found
+    and others might be missing. It enforces that at least one item must be found,
+    as an empty batch operation is considered a client error.
+    """
+
+    found_items: Annotated[
+        list[ResourceT],
+        Field(
+            min_length=1,
+            description="List of successfully retrieved items. Must contain at least one item.",
+        ),
+    ]
+    missing_identifiers: Annotated[
+        list[IdentifierT],
+        Field(
+            default_factory=list,
+            description="List of identifiers for items that were not found",
+        ),
+    ] = DEFAULT_FACTORY

packages/models-library/tests/test_batch_operations.py

@@ -0,0 +1,108 @@
+import pytest
+from faker import Faker
+from models_library.api_schemas_webserver._base import (
+    OutputSchema as WebServerOutputSchema,
+)
+from models_library.api_schemas_webserver.projects import (
+    ProjectGet,
+)
+from models_library.batch_operations import BatchGetEnvelope, create_batch_ids_validator
+from models_library.generics import Envelope
+from models_library.projects import ProjectID
+from pydantic import TypeAdapter, ValidationError
+
+
+@pytest.mark.parametrize(
+    "identifier_type,input_ids,expected_output,should_raise",
+    [
+        # Valid cases - successful validation
+        pytest.param(
+            str, ["a", "b", "c"], ["a", "b", "c"], False, id="str_valid_no_duplicates"
+        ),
+        pytest.param(int, [1, 2, 3], [1, 2, 3], False, id="int_valid_no_duplicates"),
+        pytest.param(
+            tuple,
+            [("a", 1), ("b", 2)],
+            [("a", 1), ("b", 2)],
+            False,
+            id="tuple_valid_no_duplicates",
+        ),
+        # Deduplication cases - preserving order
+        pytest.param(
+            str, ["a", "b", "a", "c"], ["a", "b", "c"], False, id="str_with_duplicates"
+        ),
+        pytest.param(int, [1, 2, 1, 3, 2], [1, 2, 3], False, id="int_with_duplicates"),
+        pytest.param(
+            tuple,
+            [("a", 1), ("b", 2), ("a", 1)],
+            [("a", 1), ("b", 2)],
+            False,
+            id="tuple_with_duplicates",
+        ),
+        # Single item cases
+        pytest.param(str, ["single"], ["single"], False, id="str_single_item"),
+        pytest.param(int, [42], [42], False, id="int_single_item"),
+        # Edge case - all duplicates resolve to single item
+        pytest.param(
+            str, ["same", "same", "same"], ["same"], False, id="str_all_duplicates"
+        ),
+        # Error cases - empty list should raise ValidationError
+        pytest.param(str, [], None, True, id="str_empty_list_error"),
+        pytest.param(int, [], None, True, id="int_empty_list_error"),
+        pytest.param(tuple, [], None, True, id="tuple_empty_list_error"),
+    ],
+)
+def test_create_batch_ids_validator(
+    identifier_type, input_ids, expected_output, should_raise
+):
+    validator = create_batch_ids_validator(identifier_type)
+
+    if should_raise:
+        with pytest.raises(ValidationError) as exc_info:
+            validator.validate_python(input_ids)
+
+        # Verify the error is about minimum length
+        assert "at least 1" in str(exc_info.value).lower()
+    else:
+        result = validator.validate_python(input_ids)
+        assert result == expected_output
+        assert len(result) >= 1  # Ensure minimum length constraint
+        # Verify order preservation by checking first occurrence positions
+        if len(set(input_ids)) != len(input_ids):  # Had duplicates
+            original_first_positions = {
+                item: input_ids.index(item) for item in set(input_ids)
+            }
+            # Items should appear in the same relative order as their first occurrence
+            sorted_by_original = sorted(
+                result, key=lambda x: original_first_positions[x]
+            )
+            assert result == sorted_by_original
+
+
+def test_composing_schemas_for_batch_operations(faker: Faker):
+
+    # inner schema model
+    class WebServerProjectBatchGetSchema(
+        WebServerOutputSchema, BatchGetEnvelope[ProjectGet, ProjectID]
+    ): ...
+
+    some_projects = ProjectGet.model_json_schema()["examples"]
+
+    # response model
+    response_model = Envelope[WebServerProjectBatchGetSchema].model_validate(
+        {
+            # NOTE: how camelcase (from WebServerOutputSchema.model_config) applies here
+            "data": {
+                "foundItems": some_projects,
+                "missingIdentifiers": [ProjectID(faker.uuid4())],
+            }
+        }
+    )
+
+    assert response_model.data is not None
+
+    assert response_model.data.found_items == TypeAdapter(
+        list[ProjectGet]
+    ).validate_python(some_projects)
+
+    assert len(response_model.data.missing_identifiers) == 1

packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/errors.py

@@ -1,21 +1,29 @@
 from common_library.errors_classes import OsparcErrorMixin
 
 
-class CatalogApiBaseError(OsparcErrorMixin, Exception):
+class CatalogRpcError(OsparcErrorMixin, Exception):
     pass
 
 
-class CatalogInconsistentError(CatalogApiBaseError):
+class CatalogInconsistentRpcError(CatalogRpcError):
     msg_template = "Catalog is inconsistent: The following  services are in the database but missing in the registry manifest {missing_services}"
 
 
-class CatalogItemNotFoundError(CatalogApiBaseError):
+class CatalogItemNotFoundRpcError(CatalogRpcError):
     msg_template = "{name} was not found"
 
 
-class CatalogForbiddenError(CatalogApiBaseError):
+class CatalogBatchNotFoundRpcError(CatalogRpcError):
+    msg_template = "{name} were not found"
+
+
+class CatalogForbiddenRpcError(CatalogRpcError):
     msg_template = "Insufficient access rights for {name}"
 
 
-class CatalogNotAvailableError(CatalogApiBaseError):
-    msg_template = "Catalog service failed unexpectedly"
+class CatalogNotAvailableRpcError(CatalogRpcError):
+    msg_template = "Catalog service is currently not available"
+
+
+class CatalogBadRequestRpcError(CatalogRpcError):
+    msg_template = "Bad request on {name}: {reason}"

packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/services.py

@@ -14,7 +14,7 @@
 from models_library.api_schemas_catalog import CATALOG_RPC_NAMESPACE
 from models_library.api_schemas_catalog.services import (
     LatestServiceGet,
-    MyServiceGet,
+    MyServicesRpcBatchGet,
     PageRpcLatestServiceGet,
     PageRpcServiceRelease,
     PageRpcServiceSummary,
@@ -177,7 +177,7 @@ async def batch_get_my_services(
             ServiceVersion,
         ]
     ],
-) -> list[MyServiceGet]:
+) -> MyServicesRpcBatchGet:
     """
     Raises:
         ValidationError: on invalid arguments
@@ -191,8 +191,7 @@ async def batch_get_my_services(
         ids=ids,
         timeout_s=40 * RPC_REQUEST_DEFAULT_TIMEOUT_S,
     )
-    assert TypeAdapter(list[MyServiceGet]).validate_python(result) is not None  # nosec
-    return cast(list[MyServiceGet], result)
+    return TypeAdapter(MyServicesRpcBatchGet).validate_python(result)
 
 
 @validate_call(config={"arbitrary_types_allowed": True})

services/api-server/src/simcore_service_api_server/services_rpc/catalog.py

@@ -23,8 +23,8 @@
 from servicelib.rabbitmq import RabbitMQRPCClient
 from servicelib.rabbitmq.rpc_interfaces.catalog import services as catalog_rpc
 from servicelib.rabbitmq.rpc_interfaces.catalog.errors import (
-    CatalogForbiddenError,
-    CatalogItemNotFoundError,
+    CatalogForbiddenRpcError,
+    CatalogItemNotFoundRpcError,
 )
 
 from ..exceptions.backend_errors import (
@@ -134,8 +134,8 @@ async def list_all_services_summaries(
 
     @_exception_mapper(
         rpc_exception_map={
-            CatalogItemNotFoundError: ProgramOrSolverOrStudyNotFoundError,
-            CatalogForbiddenError: ServiceForbiddenAccessError,
+            CatalogItemNotFoundRpcError: ProgramOrSolverOrStudyNotFoundError,
+            CatalogForbiddenRpcError: ServiceForbiddenAccessError,
             ValidationError: InvalidInputError,
         }
     )
@@ -156,8 +156,8 @@ async def get(
 
     @_exception_mapper(
         rpc_exception_map={
-            CatalogItemNotFoundError: ProgramOrSolverOrStudyNotFoundError,
-            CatalogForbiddenError: ServiceForbiddenAccessError,
+            CatalogItemNotFoundRpcError: ProgramOrSolverOrStudyNotFoundError,
+            CatalogForbiddenRpcError: ServiceForbiddenAccessError,
             ValidationError: InvalidInputError,
         }
     )

services/catalog/src/simcore_service_catalog/api/rest/_services.py

@@ -154,7 +154,7 @@ async def cached_registry_services() -> dict[str, Any]:
         services_owner_emails,
     ) = await asyncio.gather(
         cached_registry_services(),
-        services_repo.batch_get_services_access_rights(
+        services_repo.batch_get_services_access_rights_or_none(
             key_versions=services_in_db,
             product_name=x_simcore_products_name,
         ),
@@ -163,6 +163,8 @@ async def cached_registry_services() -> dict[str, Any]:
         ),
     )
 
+    services_access_rights = services_access_rights or {}
+
     # NOTE: for the details of the services:
     # 1. we get all the services from the director-v0 (TODO: move the registry to the catalog)
     # 2. we filter the services using the visible ones from the db
@@ -176,7 +178,7 @@ async def cached_registry_services() -> dict[str, Any]:
                 _compose_service_details,
                 s,
                 services_in_db[s["key"], s["version"]],
-                services_access_rights[s["key"], s["version"]],
+                services_access_rights.get((s["key"], s["version"])) or [],
                 services_owner_emails.get(
                     services_in_db[s["key"], s["version"]].owner or 0
                 ),