diff --git a/packages/models-library/src/models_library/api_schemas_catalog/services.py b/packages/models-library/src/models_library/api_schemas_catalog/services.py index f94d6be84efb..a6a41ec49326 100644 --- a/packages/models-library/src/models_library/api_schemas_catalog/services.py +++ b/packages/models-library/src/models_library/api_schemas_catalog/services.py @@ -6,6 +6,7 @@ from pydantic import ConfigDict, Field, HttpUrl, NonNegativeInt from pydantic.config import JsonDict +from ..batch_operations import BatchGetEnvelope from ..boot_options import BootOptions from ..emails import LowerCaseEmailStr from ..groups import GroupID @@ -420,6 +421,46 @@ class MyServiceGet(CatalogOutputSchema): my_access_rights: ServiceGroupAccessRightsV2 +class MyServicesRpcBatchGet( + CatalogOutputSchema, + BatchGetEnvelope[MyServiceGet, tuple[ServiceKey, ServiceVersion]], +): + """Result for batch get user services operations""" + + @staticmethod + def _update_json_schema_extra(schema: JsonDict) -> None: + missing: Any = [("simcore/services/comp/itis/sleeper", "100.2.3")] + schema.update( + { + "examples": [ + { + "found_items": [ + { + "key": missing[0][0], + "release": { + "version": "2.2.1", + "version_display": "Winter Release", + "released": "2026-07-21T15:00:00", + }, + "owner": 42, + "my_access_rights": { + "execute": True, + "write": False, + }, + } + ], + "missing_identifiers": missing, + } + ] + } + ) + + model_config = ConfigDict( + extra="forbid", + json_schema_extra=_update_json_schema_extra, + ) + + class ServiceListFilters(Filters): service_type: Annotated[ ServiceType | None, diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py b/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py index c180bec8c506..837712a42f65 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py @@ -12,6 +12,7 @@ from ..projects_nodes import InputID, InputsDict, PartialNode from ..projects_nodes_io import NodeID from ..services import ServiceKey, ServicePortKey, ServiceVersion +from ..services_base import ServiceKeyVersion from ..services_enums import ServiceState from ..services_history import ServiceRelease from ..services_resources import ServiceResourcesDict @@ -225,3 +226,9 @@ class NodeServiceGet(OutputSchema): class ProjectNodeServicesGet(OutputSchema): project_uuid: ProjectID services: list[NodeServiceGet] + missing: Annotated[ + list[ServiceKeyVersion] | None, + Field( + description="List of services defined in the project but that were not found in the catalog" + ), + ] = None diff --git a/packages/models-library/src/models_library/batch_operations.py b/packages/models-library/src/models_library/batch_operations.py new file mode 100644 index 000000000000..9951ee614760 --- /dev/null +++ b/packages/models-library/src/models_library/batch_operations.py @@ -0,0 +1,62 @@ +from typing import Annotated, Generic, TypeVar + +from common_library.basic_types import DEFAULT_FACTORY +from pydantic import BaseModel, BeforeValidator, Field, TypeAdapter + +ResourceT = TypeVar("ResourceT") +IdentifierT = TypeVar("IdentifierT") +SchemaT = TypeVar("SchemaT") + + +def _deduplicate_preserving_order(identifiers: list[IdentifierT]) -> list[IdentifierT]: + """Remove duplicates while preserving order of first occurrence.""" + return list(dict.fromkeys(identifiers)) + + +def create_batch_ids_validator(identifier_type: type[IdentifierT]) -> TypeAdapter: + """Create a TypeAdapter for validating batch identifiers. + + This validator ensures: + - At least one identifier is provided (empty list is invalid for batch operations) + - Duplicates are removed while preserving order + + Args: + identifier_type: The type of identifiers in the batch + + Returns: + TypeAdapter configured for the specific identifier type + """ + return TypeAdapter( + Annotated[ + list[identifier_type], # type: ignore[valid-type] + BeforeValidator(_deduplicate_preserving_order), + Field( + min_length=1, + description="List of identifiers to batch process. Empty list is not allowed for batch operations.", + ), + ] + ) + + +class BatchGetEnvelope(BaseModel, Generic[ResourceT, IdentifierT]): + """Generic envelope model for batch-get operations that can contain partial results. + + This model represents the result of a batch operation where some items might be found + and others might be missing. It enforces that at least one item must be found, + as an empty batch operation is considered a client error. + """ + + found_items: Annotated[ + list[ResourceT], + Field( + min_length=1, + description="List of successfully retrieved items. Must contain at least one item.", + ), + ] + missing_identifiers: Annotated[ + list[IdentifierT], + Field( + default_factory=list, + description="List of identifiers for items that were not found", + ), + ] = DEFAULT_FACTORY diff --git a/packages/models-library/tests/test_batch_operations.py b/packages/models-library/tests/test_batch_operations.py new file mode 100644 index 000000000000..0343a9e3b3c2 --- /dev/null +++ b/packages/models-library/tests/test_batch_operations.py @@ -0,0 +1,108 @@ +import pytest +from faker import Faker +from models_library.api_schemas_webserver._base import ( + OutputSchema as WebServerOutputSchema, +) +from models_library.api_schemas_webserver.projects import ( + ProjectGet, +) +from models_library.batch_operations import BatchGetEnvelope, create_batch_ids_validator +from models_library.generics import Envelope +from models_library.projects import ProjectID +from pydantic import TypeAdapter, ValidationError + + +@pytest.mark.parametrize( + "identifier_type,input_ids,expected_output,should_raise", + [ + # Valid cases - successful validation + pytest.param( + str, ["a", "b", "c"], ["a", "b", "c"], False, id="str_valid_no_duplicates" + ), + pytest.param(int, [1, 2, 3], [1, 2, 3], False, id="int_valid_no_duplicates"), + pytest.param( + tuple, + [("a", 1), ("b", 2)], + [("a", 1), ("b", 2)], + False, + id="tuple_valid_no_duplicates", + ), + # Deduplication cases - preserving order + pytest.param( + str, ["a", "b", "a", "c"], ["a", "b", "c"], False, id="str_with_duplicates" + ), + pytest.param(int, [1, 2, 1, 3, 2], [1, 2, 3], False, id="int_with_duplicates"), + pytest.param( + tuple, + [("a", 1), ("b", 2), ("a", 1)], + [("a", 1), ("b", 2)], + False, + id="tuple_with_duplicates", + ), + # Single item cases + pytest.param(str, ["single"], ["single"], False, id="str_single_item"), + pytest.param(int, [42], [42], False, id="int_single_item"), + # Edge case - all duplicates resolve to single item + pytest.param( + str, ["same", "same", "same"], ["same"], False, id="str_all_duplicates" + ), + # Error cases - empty list should raise ValidationError + pytest.param(str, [], None, True, id="str_empty_list_error"), + pytest.param(int, [], None, True, id="int_empty_list_error"), + pytest.param(tuple, [], None, True, id="tuple_empty_list_error"), + ], +) +def test_create_batch_ids_validator( + identifier_type, input_ids, expected_output, should_raise +): + validator = create_batch_ids_validator(identifier_type) + + if should_raise: + with pytest.raises(ValidationError) as exc_info: + validator.validate_python(input_ids) + + # Verify the error is about minimum length + assert "at least 1" in str(exc_info.value).lower() + else: + result = validator.validate_python(input_ids) + assert result == expected_output + assert len(result) >= 1 # Ensure minimum length constraint + # Verify order preservation by checking first occurrence positions + if len(set(input_ids)) != len(input_ids): # Had duplicates + original_first_positions = { + item: input_ids.index(item) for item in set(input_ids) + } + # Items should appear in the same relative order as their first occurrence + sorted_by_original = sorted( + result, key=lambda x: original_first_positions[x] + ) + assert result == sorted_by_original + + +def test_composing_schemas_for_batch_operations(faker: Faker): + + # inner schema model + class WebServerProjectBatchGetSchema( + WebServerOutputSchema, BatchGetEnvelope[ProjectGet, ProjectID] + ): ... + + some_projects = ProjectGet.model_json_schema()["examples"] + + # response model + response_model = Envelope[WebServerProjectBatchGetSchema].model_validate( + { + # NOTE: how camelcase (from WebServerOutputSchema.model_config) applies here + "data": { + "foundItems": some_projects, + "missingIdentifiers": [ProjectID(faker.uuid4())], + } + } + ) + + assert response_model.data is not None + + assert response_model.data.found_items == TypeAdapter( + list[ProjectGet] + ).validate_python(some_projects) + + assert len(response_model.data.missing_identifiers) == 1 diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/errors.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/errors.py index 906bc641665c..4ffdf53d021a 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/errors.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/errors.py @@ -1,21 +1,29 @@ from common_library.errors_classes import OsparcErrorMixin -class CatalogApiBaseError(OsparcErrorMixin, Exception): +class CatalogRpcError(OsparcErrorMixin, Exception): pass -class CatalogInconsistentError(CatalogApiBaseError): +class CatalogInconsistentRpcError(CatalogRpcError): msg_template = "Catalog is inconsistent: The following services are in the database but missing in the registry manifest {missing_services}" -class CatalogItemNotFoundError(CatalogApiBaseError): +class CatalogItemNotFoundRpcError(CatalogRpcError): msg_template = "{name} was not found" -class CatalogForbiddenError(CatalogApiBaseError): +class CatalogBatchNotFoundRpcError(CatalogRpcError): + msg_template = "{name} were not found" + + +class CatalogForbiddenRpcError(CatalogRpcError): msg_template = "Insufficient access rights for {name}" -class CatalogNotAvailableError(CatalogApiBaseError): - msg_template = "Catalog service failed unexpectedly" +class CatalogNotAvailableRpcError(CatalogRpcError): + msg_template = "Catalog service is currently not available" + + +class CatalogBadRequestRpcError(CatalogRpcError): + msg_template = "Bad request on {name}: {reason}" diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/services.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/services.py index 7ac19275fb67..adf4c4e5a2b7 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/services.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/services.py @@ -14,7 +14,7 @@ from models_library.api_schemas_catalog import CATALOG_RPC_NAMESPACE from models_library.api_schemas_catalog.services import ( LatestServiceGet, - MyServiceGet, + MyServicesRpcBatchGet, PageRpcLatestServiceGet, PageRpcServiceRelease, PageRpcServiceSummary, @@ -177,7 +177,7 @@ async def batch_get_my_services( ServiceVersion, ] ], -) -> list[MyServiceGet]: +) -> MyServicesRpcBatchGet: """ Raises: ValidationError: on invalid arguments @@ -191,8 +191,7 @@ async def batch_get_my_services( ids=ids, timeout_s=40 * RPC_REQUEST_DEFAULT_TIMEOUT_S, ) - assert TypeAdapter(list[MyServiceGet]).validate_python(result) is not None # nosec - return cast(list[MyServiceGet], result) + return TypeAdapter(MyServicesRpcBatchGet).validate_python(result) @validate_call(config={"arbitrary_types_allowed": True}) diff --git a/services/api-server/src/simcore_service_api_server/services_rpc/catalog.py b/services/api-server/src/simcore_service_api_server/services_rpc/catalog.py index f8dbe035e6bc..b47ceec17688 100644 --- a/services/api-server/src/simcore_service_api_server/services_rpc/catalog.py +++ b/services/api-server/src/simcore_service_api_server/services_rpc/catalog.py @@ -23,8 +23,8 @@ from servicelib.rabbitmq import RabbitMQRPCClient from servicelib.rabbitmq.rpc_interfaces.catalog import services as catalog_rpc from servicelib.rabbitmq.rpc_interfaces.catalog.errors import ( - CatalogForbiddenError, - CatalogItemNotFoundError, + CatalogForbiddenRpcError, + CatalogItemNotFoundRpcError, ) from ..exceptions.backend_errors import ( @@ -134,8 +134,8 @@ async def list_all_services_summaries( @_exception_mapper( rpc_exception_map={ - CatalogItemNotFoundError: ProgramOrSolverOrStudyNotFoundError, - CatalogForbiddenError: ServiceForbiddenAccessError, + CatalogItemNotFoundRpcError: ProgramOrSolverOrStudyNotFoundError, + CatalogForbiddenRpcError: ServiceForbiddenAccessError, ValidationError: InvalidInputError, } ) @@ -156,8 +156,8 @@ async def get( @_exception_mapper( rpc_exception_map={ - CatalogItemNotFoundError: ProgramOrSolverOrStudyNotFoundError, - CatalogForbiddenError: ServiceForbiddenAccessError, + CatalogItemNotFoundRpcError: ProgramOrSolverOrStudyNotFoundError, + CatalogForbiddenRpcError: ServiceForbiddenAccessError, ValidationError: InvalidInputError, } ) diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services.py b/services/catalog/src/simcore_service_catalog/api/rest/_services.py index 55108b8c2c68..6ee01f2cad91 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services.py @@ -154,7 +154,7 @@ async def cached_registry_services() -> dict[str, Any]: services_owner_emails, ) = await asyncio.gather( cached_registry_services(), - services_repo.batch_get_services_access_rights( + services_repo.batch_get_services_access_rights_or_none( key_versions=services_in_db, product_name=x_simcore_products_name, ), @@ -163,6 +163,8 @@ async def cached_registry_services() -> dict[str, Any]: ), ) + services_access_rights = services_access_rights or {} + # NOTE: for the details of the services: # 1. we get all the services from the director-v0 (TODO: move the registry to the catalog) # 2. we filter the services using the visible ones from the db @@ -176,7 +178,7 @@ async def cached_registry_services() -> dict[str, Any]: _compose_service_details, s, services_in_db[s["key"], s["version"]], - services_access_rights[s["key"], s["version"]], + services_access_rights.get((s["key"], s["version"])) or [], services_owner_emails.get( services_in_db[s["key"], s["version"]].owner or 0 ), diff --git a/services/catalog/src/simcore_service_catalog/api/rpc/_services.py b/services/catalog/src/simcore_service_catalog/api/rpc/_services.py index 7b5173a5383d..7470b486a93f 100644 --- a/services/catalog/src/simcore_service_catalog/api/rpc/_services.py +++ b/services/catalog/src/simcore_service_catalog/api/rpc/_services.py @@ -4,7 +4,7 @@ from fastapi import FastAPI from models_library.api_schemas_catalog.services import ( - MyServiceGet, + MyServicesRpcBatchGet, PageRpcLatestServiceGet, PageRpcServiceRelease, PageRpcServiceSummary, @@ -22,10 +22,12 @@ from pyinstrument import Profiler from servicelib.rabbitmq import RPCRouter from servicelib.rabbitmq.rpc_interfaces.catalog.errors import ( - CatalogForbiddenError, - CatalogItemNotFoundError, + CatalogBatchNotFoundRpcError, + CatalogForbiddenRpcError, + CatalogItemNotFoundRpcError, ) +from ...errors import BatchNotFoundError from ...models.services_db import ServiceDBFilters from ...repository.groups import GroupsRepository from ...repository.services import ServicesRepository @@ -58,7 +60,12 @@ async def _wrapper(app: FastAPI, **kwargs): return _wrapper -@router.expose(reraise_if_error_type=(CatalogForbiddenError, ValidationError)) +@router.expose( + reraise_if_error_type=( + CatalogForbiddenRpcError, + ValidationError, + ) +) @_profile_rpc_call @validate_call(config={"arbitrary_types_allowed": True}) async def list_services_paginated( @@ -100,8 +107,8 @@ async def list_services_paginated( @router.expose( reraise_if_error_type=( - CatalogItemNotFoundError, - CatalogForbiddenError, + CatalogItemNotFoundRpcError, + CatalogForbiddenRpcError, ValidationError, ) ) @@ -134,8 +141,8 @@ async def get_service( @router.expose( reraise_if_error_type=( - CatalogItemNotFoundError, - CatalogForbiddenError, + CatalogItemNotFoundRpcError, + CatalogForbiddenRpcError, ValidationError, ) ) @@ -171,8 +178,8 @@ async def update_service( @router.expose( reraise_if_error_type=( - CatalogItemNotFoundError, - CatalogForbiddenError, + CatalogItemNotFoundRpcError, + CatalogForbiddenRpcError, ValidationError, ) ) @@ -198,7 +205,13 @@ async def check_for_service( ) -@router.expose(reraise_if_error_type=(CatalogForbiddenError, ValidationError)) +@router.expose( + reraise_if_error_type=( + CatalogForbiddenRpcError, + CatalogBatchNotFoundRpcError, + ValidationError, + ) +) @validate_call(config={"arbitrary_types_allowed": True}) async def batch_get_my_services( app: FastAPI, @@ -211,20 +224,32 @@ async def batch_get_my_services( ServiceVersion, ] ], -) -> list[MyServiceGet]: +) -> MyServicesRpcBatchGet: assert app.state.engine # nosec - services_batch = await catalog_services.batch_get_user_services( - repo=ServicesRepository(app.state.engine), - groups_repo=GroupsRepository(app.state.engine), - product_name=product_name, - user_id=user_id, - ids=ids, - ) + try: - assert [(sv.key, sv.release.version) for sv in services_batch] == ids # nosec + batch_got = await catalog_services.batch_get_user_services( + repo=ServicesRepository(app.state.engine), + groups_repo=GroupsRepository(app.state.engine), + product_name=product_name, + user_id=user_id, + ids=ids, + ) + + except BatchNotFoundError as e: + ctx = e.error_context() + ctx["name"] = f"{ctx.get('missing_services',[])}" + raise CatalogBatchNotFoundRpcError(**ctx) from e - return services_batch + assert [ + (sv.key, sv.release.version) for sv in batch_got.found_items + ] == ids # nosec + + return MyServicesRpcBatchGet( + found_items=batch_got.found_items, + missing_identifiers=batch_got.missing_identifiers, + ) @router.expose(reraise_if_error_type=(ValidationError,)) @@ -270,8 +295,8 @@ async def list_my_service_history_latest_first( @router.expose( reraise_if_error_type=( - CatalogItemNotFoundError, - CatalogForbiddenError, + CatalogItemNotFoundRpcError, + CatalogForbiddenRpcError, ValidationError, ) ) @@ -306,7 +331,7 @@ async def get_service_ports( ] -@router.expose(reraise_if_error_type=(CatalogForbiddenError, ValidationError)) +@router.expose(reraise_if_error_type=(CatalogForbiddenRpcError, ValidationError)) @_profile_rpc_call @validate_call(config={"arbitrary_types_allowed": True}) async def list_all_services_summaries_paginated( diff --git a/services/catalog/src/simcore_service_catalog/errors.py b/services/catalog/src/simcore_service_catalog/errors.py index 7e33eb08d0d7..805262a97a84 100644 --- a/services/catalog/src/simcore_service_catalog/errors.py +++ b/services/catalog/src/simcore_service_catalog/errors.py @@ -1,10 +1,10 @@ from common_library.errors_classes import OsparcErrorMixin -class CatalogBaseError(OsparcErrorMixin, Exception): ... +class BaseCatalogError(OsparcErrorMixin, Exception): ... -class RepositoryError(CatalogBaseError): +class RepositoryError(BaseCatalogError): msg_template = "Unexpected error in {repo_cls}" @@ -12,7 +12,7 @@ class UninitializedGroupError(RepositoryError): msg_tempalte = "{group} groups was never initialized" -class BaseDirectorError(CatalogBaseError): ... +class BaseDirectorError(BaseCatalogError): ... class DirectorUnresponsiveError(BaseDirectorError): @@ -20,3 +20,7 @@ class DirectorUnresponsiveError(BaseDirectorError): class DirectorStatusError(BaseDirectorError): ... + + +class BatchNotFoundError(BaseCatalogError): + msg_template = "None of the batch services were found in the catalog. Missing: {missing_services}" diff --git a/services/catalog/src/simcore_service_catalog/models/catalog_services.py b/services/catalog/src/simcore_service_catalog/models/catalog_services.py new file mode 100644 index 000000000000..7b0766b34ec2 --- /dev/null +++ b/services/catalog/src/simcore_service_catalog/models/catalog_services.py @@ -0,0 +1,7 @@ +from typing import TypeAlias + +from models_library.api_schemas_catalog.services import MyServicesRpcBatchGet + +# NOTE: for now schema and domain are identical. If they differ in the future +# this indirection will allow us to transform between the two +BatchGetUserServicesResult: TypeAlias = MyServicesRpcBatchGet diff --git a/services/catalog/src/simcore_service_catalog/repository/services.py b/services/catalog/src/simcore_service_catalog/repository/services.py index 12e0d88a0fe6..e7809977ba1f 100644 --- a/services/catalog/src/simcore_service_catalog/repository/services.py +++ b/services/catalog/src/simcore_service_catalog/repository/services.py @@ -657,12 +657,14 @@ async def get_service_access_rights( async for row in await conn.stream(query) ] - async def batch_get_services_access_rights( + async def batch_get_services_access_rights_or_none( self, key_versions: Iterable[tuple[str, str]], product_name: str | None = None, - ) -> dict[tuple[str, str], list[ServiceAccessRightsDB]]: - """Batch version of get_service_access_rights""" + ) -> dict[tuple[str, str], list[ServiceAccessRightsDB]] | None: + """ + Returns only found. If None found, then None + """ service_to_access_rights = defaultdict(list) query = ( sa.select(services_access_rights) @@ -681,7 +683,7 @@ async def batch_get_services_access_rights( service_to_access_rights[(row.key, row.version)].append( ServiceAccessRightsDB.model_validate(row) ) - return service_to_access_rights + return service_to_access_rights or None async def upsert_service_access_rights( self, new_access_rights: list[ServiceAccessRightsDB] diff --git a/services/catalog/src/simcore_service_catalog/service/catalog_services.py b/services/catalog/src/simcore_service_catalog/service/catalog_services.py index f97b6da792d6..22daf22cc20b 100644 --- a/services/catalog/src/simcore_service_catalog/service/catalog_services.py +++ b/services/catalog/src/simcore_service_catalog/service/catalog_services.py @@ -14,6 +14,7 @@ ) from models_library.api_schemas_directorv2.services import ServiceExtras from models_library.basic_types import VersionStr +from models_library.batch_operations import create_batch_ids_validator from models_library.groups import GroupID from models_library.products import ProductName from models_library.rest_pagination import PageLimitInt, PageOffsetInt, PageTotalCount @@ -24,12 +25,14 @@ from models_library.users import UserID from pydantic import HttpUrl from servicelib.rabbitmq.rpc_interfaces.catalog.errors import ( - CatalogForbiddenError, - CatalogInconsistentError, - CatalogItemNotFoundError, + CatalogForbiddenRpcError, + CatalogInconsistentRpcError, + CatalogItemNotFoundRpcError, ) from ..clients.director import DirectorClient +from ..errors import BatchNotFoundError +from ..models.catalog_services import BatchGetUserServicesResult from ..models.services_db import ( ServiceAccessRightsDB, ServiceDBFilters, @@ -171,11 +174,11 @@ async def _get_services_with_access_rights( return {} # Inject access-rights - access_rights = await repo.batch_get_services_access_rights( + access_rights = await repo.batch_get_services_access_rights_or_none( ((sc.key, sc.version) for sc in services), product_name=product_name ) if not access_rights: - raise CatalogForbiddenError( + raise CatalogForbiddenRpcError( name="any service", user_id=user_id, product_name=product_name, @@ -238,7 +241,7 @@ async def _get_services_manifests( _logger.warning( **create_troubleshooting_log_kwargs( msg, - error=CatalogInconsistentError( + error=CatalogInconsistentRpcError( missing_services=missing_services, user_id=user_id, product_name=product_name, @@ -411,7 +414,7 @@ async def get_catalog_service( ) if not service: # no service found provided `access_rights` - raise CatalogForbiddenError( + raise CatalogForbiddenRpcError( name=f"{service_key}:{service_version}", service_key=service_key, service_version=service_version, @@ -446,7 +449,7 @@ async def update_catalog_service( update: ServiceUpdateV2, ) -> ServiceGetV2: if is_function_service(service_key): - raise CatalogForbiddenError( + raise CatalogForbiddenRpcError( name=f"function service {service_key}:{service_version}", service_key=service_key, service_version=service_version, @@ -547,7 +550,7 @@ async def check_catalog_service_permissions( product_name=product_name, ) if not access_rights: - raise CatalogItemNotFoundError( + raise CatalogItemNotFoundRpcError( name=f"{service_key}:{service_version}", service_key=service_key, service_version=service_version, @@ -572,7 +575,7 @@ async def check_catalog_service_permissions( ) if not has_permission: - raise CatalogForbiddenError( + raise CatalogForbiddenRpcError( name=f"{service_key}:{service_version}", service_key=service_key, service_version=service_version, @@ -583,92 +586,171 @@ async def check_catalog_service_permissions( return access_rights +_BatchIdsValidator = create_batch_ids_validator(tuple[ServiceKey, ServiceVersion]) + + +def _evaluate_user_access_rights( + access_rights: list, user_group_ids: set[GroupID] +) -> ServiceGroupAccessRightsV2: + """Evaluate user's access rights based on their group memberships.""" + my_access_rights = ServiceGroupAccessRightsV2(execute=False, write=False) + for ar in access_rights: + if ar.gid in user_group_ids: + my_access_rights.execute |= ar.execute_access + my_access_rights.write |= ar.write_access + return my_access_rights + + +def _find_service_owner(service_db, access_rights: list) -> GroupID | None: + """Find service owner from database or access rights.""" + owner: GroupID | None = service_db.owner + if not owner: + # NOTE can be more than one. Just get first. + with suppress(StopIteration): + owner = next( + ar.gid for ar in access_rights if ar.write_access and ar.execute_access + ) + return owner + + +async def _get_service_compatibility( + repo: ServicesRepository, + product_name: ProductName, + user_id: UserID, + service_key: ServiceKey, + service_version: ServiceVersion, + my_access_rights: ServiceGroupAccessRightsV2, +) -> Compatibility | None: + """Get service compatibility if user has access rights.""" + if not (my_access_rights.execute or my_access_rights.write): + return None + + history = await repo.get_service_history( + product_name=product_name, + user_id=user_id, + key=service_key, + ) + compatibility_map = await evaluate_service_compatibility_map( + repo, + product_name=product_name, + user_id=user_id, + service_release_history=history, + ) + return compatibility_map.get(service_version) + + +async def _process_single_service( + repo: ServicesRepository, + product_name: ProductName, + user_id: UserID, + service_key: ServiceKey, + service_version: ServiceVersion, + services_access_rights: dict, + user_group_ids: set[GroupID], +) -> MyServiceGet | None: + """Process a single service and return MyServiceGet or None if missing.""" + # Check access rights + access_rights = services_access_rights.get((service_key, service_version), []) + if not access_rights: + return None + + # Evaluate user's access rights + my_access_rights = _evaluate_user_access_rights(access_rights, user_group_ids) + + # Get service metadata + service_db = await repo.get_service( + product_name=product_name, + key=service_key, + version=service_version, + ) + if not service_db: + return None + + # Find service owner + owner = _find_service_owner(service_db, access_rights) + + # Evaluate compatibility + compatibility = await _get_service_compatibility( + repo, product_name, user_id, service_key, service_version, my_access_rights + ) + + return MyServiceGet( + key=service_db.key, + release=ServiceRelease( + version=service_db.version, + version_display=service_db.version_display, + released=service_db.created, + retired=service_db.deprecated, + compatibility=compatibility, + ), + owner=owner, + my_access_rights=my_access_rights, + ) + + async def batch_get_user_services( repo: ServicesRepository, groups_repo: GroupsRepository, *, product_name: ProductName, user_id: UserID, - ids: list[ - tuple[ - ServiceKey, - ServiceVersion, - ] - ], -) -> list[MyServiceGet]: - services_access_rights = await repo.batch_get_services_access_rights( - key_versions=ids, product_name=product_name - ) + ids: list[tuple[ServiceKey, ServiceVersion]], +) -> BatchGetUserServicesResult: + """Batch get user services. - user_groups = await groups_repo.list_user_groups(user_id=user_id) - my_group_ids = {g.gid for g in user_groups} + - Allows partial success, i.e. some services might be found while others not. + - Silently deduplicates ids while preserving order. + + Raises: + CatalogItemNotFoundError: When no services are found at all + """ + unique_service_identifiers = _BatchIdsValidator.validate_python(ids) - my_services = [] - for service_key, service_version in ids: - # Evaluate user's access-rights to this service key:version - access_rights = services_access_rights.get((service_key, service_version), []) - my_access_rights = ServiceGroupAccessRightsV2(execute=False, write=False) - for ar in access_rights: - if ar.gid in my_group_ids: - my_access_rights.execute |= ar.execute_access - my_access_rights.write |= ar.write_access - - # Get service metadata - service_db = await repo.get_service( + services_access_rights = await repo.batch_get_services_access_rights_or_none( + key_versions=unique_service_identifiers, product_name=product_name + ) + if not services_access_rights: + raise BatchNotFoundError( + missing_services=unique_service_identifiers, + user_id=user_id, product_name=product_name, - key=service_key, - version=service_version, ) - assert service_db # nosec - - # Find service owner (if defined!) - owner: GroupID | None = service_db.owner - if not owner: - # NOTE can be more than one. Just get first. - with suppress(StopIteration): - owner = next( - ar.gid - for ar in access_rights - if ar.write_access and ar.execute_access - ) - # Evaluate `compatibility` - compatibility: Compatibility | None = None - if my_access_rights.execute or my_access_rights.write: - history = await repo.get_service_history( - # NOTE: that the service history might be different for each user - # since access rights are defined on a version basis (i.e. one use can have access to v1 but ot to v2) - product_name=product_name, - user_id=user_id, - key=service_key, - ) - assert history # nosec + user_groups = await groups_repo.list_user_groups(user_id=user_id) + my_group_ids = {g.gid for g in user_groups} - compatibility_map = await evaluate_service_compatibility_map( - repo, - product_name=product_name, - user_id=user_id, - service_release_history=history, - ) + found = [] + missing = [] + + for service_key, service_version in unique_service_identifiers: + # NOTE: parallel? + service_result = await _process_single_service( + repo, + product_name, + user_id, + service_key, + service_version, + services_access_rights, + my_group_ids, + ) - compatibility = compatibility_map.get(service_db.version) - - my_services.append( - MyServiceGet( - key=service_db.key, - release=ServiceRelease( - version=service_db.version, - version_display=service_db.version_display, - released=service_db.created, - retired=service_db.deprecated, - compatibility=compatibility, - ), - owner=owner, - my_access_rights=my_access_rights, - ) + if service_result: + found.append(service_result) + else: + missing.append((service_key, service_version)) + + # Check for complete failure scenarios and raise appropriate exceptions + if not found: + # None of the services found + assert len(unique_service_identifiers) == len(missing) # nosec + raise BatchNotFoundError( + missing_services=missing, + user_id=user_id, + product_name=product_name, ) - return my_services + # Success or partial success - return the result model + return BatchGetUserServicesResult(found_items=found, missing_identifiers=missing) async def list_user_service_release_history( diff --git a/services/catalog/tests/unit/with_dbs/test_api_rpc.py b/services/catalog/tests/unit/with_dbs/test_api_rpc.py index 4cc4c452fdb1..2384bdd01d89 100644 --- a/services/catalog/tests/unit/with_dbs/test_api_rpc.py +++ b/services/catalog/tests/unit/with_dbs/test_api_rpc.py @@ -34,8 +34,8 @@ from servicelib.rabbitmq import RabbitMQRPCClient from servicelib.rabbitmq.rpc_interfaces.catalog import services as catalog_rpc from servicelib.rabbitmq.rpc_interfaces.catalog.errors import ( - CatalogForbiddenError, - CatalogItemNotFoundError, + CatalogForbiddenRpcError, + CatalogItemNotFoundRpcError, ) pytest_simcore_core_services_selection = [ @@ -371,7 +371,7 @@ async def test_rpc_get_service_not_found_error( user_id: UserID, ): - with pytest.raises(CatalogItemNotFoundError, match="unknown"): + with pytest.raises(CatalogItemNotFoundRpcError, match="unknown"): await catalog_rpc.get_service( rpc_client, product_name=product_name, @@ -408,7 +408,7 @@ async def test_rpc_check_for_service( product_name: ProductName, user_id: UserID, ): - with pytest.raises(CatalogItemNotFoundError, match="unknown"): + with pytest.raises(CatalogItemNotFoundRpcError, match="unknown"): await catalog_rpc.check_for_service( rpc_client, product_name=product_name, @@ -450,7 +450,7 @@ async def test_rpc_get_service_access_rights( assert other_user["primary_gid"] not in service.access_rights # other_user does not have EXECUTE access ----------------- - with pytest.raises(CatalogForbiddenError, match=service_key): + with pytest.raises(CatalogForbiddenRpcError, match=service_key): await catalog_rpc.get_service( rpc_client, product_name=product_name, @@ -460,7 +460,7 @@ async def test_rpc_get_service_access_rights( ) # other_user does not have WRITE access - with pytest.raises(CatalogForbiddenError, match=service_key): + with pytest.raises(CatalogForbiddenRpcError, match=service_key): await catalog_rpc.update_service( rpc_client, product_name=product_name, @@ -501,7 +501,7 @@ async def test_rpc_get_service_access_rights( service_version=service_version, ) - with pytest.raises(CatalogForbiddenError, match=service_key): + with pytest.raises(CatalogForbiddenRpcError, match=service_key): await catalog_rpc.update_service( rpc_client, product_name=product_name, @@ -608,13 +608,16 @@ async def test_rpc_batch_get_my_services( (other_service_key, other_service_version), ] - my_services = await catalog_rpc.batch_get_my_services( + batch_got = await catalog_rpc.batch_get_my_services( rpc_client, product_name=product_name, user_id=user_id, ids=ids, ) + my_services = batch_got.found_items + assert batch_got.missing_identifiers == [] + assert len(my_services) == 2 # Check access rights to all of them @@ -757,7 +760,7 @@ async def test_rpc_get_service_ports_not_found( non_existent_key = "simcore/services/comp/non-existent-service" # Test service not found scenario - with pytest.raises(CatalogItemNotFoundError, match="non-existent-service"): + with pytest.raises(CatalogItemNotFoundRpcError, match="non-existent-service"): await catalog_rpc.get_service_ports( rpc_client, product_name=product_name, @@ -808,7 +811,7 @@ async def test_rpc_get_service_ports_permission_denied( await services_db_tables_injector([fake_restricted_service]) # Attempt to access without permission - with pytest.raises(CatalogForbiddenError): + with pytest.raises(CatalogForbiddenRpcError): await catalog_rpc.get_service_ports( rpc_client, product_name=product_name, diff --git a/services/catalog/tests/unit/with_dbs/test_service_catalog_services.py b/services/catalog/tests/unit/with_dbs/test_service_catalog_services.py index a7f106483a10..78f16ed8febb 100644 --- a/services/catalog/tests/unit/with_dbs/test_service_catalog_services.py +++ b/services/catalog/tests/unit/with_dbs/test_service_catalog_services.py @@ -13,11 +13,12 @@ from models_library.api_schemas_catalog.services import MyServiceGet, ServiceSummary from models_library.products import ProductName from models_library.users import UserID -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError from pytest_simcore.helpers.catalog_services import CreateFakeServiceDataCallable from respx.router import MockRouter from simcore_service_catalog.api._dependencies.director import get_director_client from simcore_service_catalog.clients.director import DirectorClient +from simcore_service_catalog.errors import BatchNotFoundError from simcore_service_catalog.repository.groups import GroupsRepository from simcore_service_catalog.repository.services import ServicesRepository from simcore_service_catalog.service import catalog_services, manifest @@ -125,7 +126,7 @@ async def test_list_latest_catalog_services( assert not mocked_director_rest_api["get_service"].called - total_count, page_items = await catalog_services.list_latest_catalog_services( + total_count, page_found_items = await catalog_services.list_latest_catalog_services( services_repo, director_client, product_name=target_product, @@ -135,12 +136,12 @@ async def test_list_latest_catalog_services( ) assert total_count == num_services - assert page_items - assert len(page_items) <= limit + assert page_found_items + assert len(page_found_items) <= limit assert mocked_director_rest_api["get_service"].called assert mocked_director_rest_api["get_service"].call_count == limit - for item in page_items: + for item in page_found_items: assert item.access_rights assert item.owner is not None @@ -219,7 +220,7 @@ async def test_batch_get_my_services( # Inject fake services into the database await services_db_tables_injector([fake_service_1, fake_service_2, fake_service_3]) - # UNDER TEST ------------------------------- + # ACT ------------------------------- # Batch get services e.g. services in a project services_ids = [ @@ -227,7 +228,7 @@ async def test_batch_get_my_services( (other_service_key, other_service_version), ] - my_services = await catalog_services.batch_get_user_services( + result = await catalog_services.batch_get_user_services( services_repo, groups_repo, product_name=target_product, @@ -235,7 +236,12 @@ async def test_batch_get_my_services( ids=services_ids, ) - # CHECKS ------------------------------- + my_services = result.found_items + + # ASSERT ------------------------------- + + assert result.missing_identifiers == [] + assert len(my_services) == 2 # assert returned order and length as ids assert services_ids == [(sc.key, sc.release.version) for sc in my_services] @@ -275,6 +281,183 @@ async def test_batch_get_my_services( ) +async def test_batch_get_my_services_partial_success( + background_task_lifespan_disabled: None, + rabbitmq_and_rpc_setup_disabled: None, + mocked_director_rest_api: MockRouter, + target_product: ProductName, + services_repo: ServicesRepository, + groups_repo: GroupsRepository, + user_id: UserID, + user: dict[str, Any], + create_fake_service_data: CreateFakeServiceDataCallable, + services_db_tables_injector: Callable, +): + """Test batch get with some services found and some missing.""" + + # Create only one service in the database + service_key = "simcore/services/comp/existing-service" + service_version = "1.0.0" + + fake_service = create_fake_service_data( + service_key, + service_version, + team_access=None, + everyone_access=None, + product=target_product, + ) + + # Inject only this service into the database + await services_db_tables_injector([fake_service]) + + # Request both existing and non-existing services + services_ids = [ + (service_key, service_version), # exists + ("simcore/services/comp/missing-service", "2.0.0"), # does not exist + ("simcore/services/comp/another-missing", "3.0.0"), # does not exist + ] + + # ACT + result = await catalog_services.batch_get_user_services( + services_repo, + groups_repo, + product_name=target_product, + user_id=user_id, + ids=services_ids, + ) + + # ASSERT + assert len(result.found_items) == 1 # Only one service found + assert len(result.missing_identifiers) == 2 # Two services missing + + # Check the found service + found_service = result.found_items[0] + assert found_service.key == service_key + assert found_service.release.version == service_version + assert found_service.owner == user["primary_gid"] + assert found_service.my_access_rights.execute is True + assert found_service.my_access_rights.write is True + + # Check missing services + expected_missing = [ + ("simcore/services/comp/missing-service", "2.0.0"), + ("simcore/services/comp/another-missing", "3.0.0"), + ] + assert result.missing_identifiers == expected_missing + + +async def test_batch_get_my_services_none_found_raises_error( + background_task_lifespan_disabled: None, + rabbitmq_and_rpc_setup_disabled: None, + mocked_director_rest_api: MockRouter, + target_product: ProductName, + services_repo: ServicesRepository, + groups_repo: GroupsRepository, + user_id: UserID, +): + """Test batch get with no services found raises CatalogServiceNotFoundError.""" + + # Request non-existing services only (no services in database) + services_ids = [ + ("simcore/services/comp/missing-service-1", "1.0.0"), + ("simcore/services/comp/missing-service-2", "2.0.0"), + ] + + # ACT & ASSERT + with pytest.raises(BatchNotFoundError) as exc_info: + await catalog_services.batch_get_user_services( + services_repo, + groups_repo, + product_name=target_product, + user_id=user_id, + ids=services_ids, + ) + + # Verify the exception contains the missing services information + assert exc_info.value.missing_services == services_ids + assert exc_info.value.user_id == user_id + assert exc_info.value.product_name == target_product + + +async def test_batch_get_my_services_empty_ids_raises_validation_error( + target_product: ProductName, + services_repo: ServicesRepository, + groups_repo: GroupsRepository, + user_id: UserID, +): + """Test batch get with empty ids list raises ValidationError.""" + + # ACT & ASSERT + with pytest.raises(ValidationError) as exc_info: + await catalog_services.batch_get_user_services( + services_repo, + groups_repo, + product_name=target_product, + user_id=user_id, + ids=[], # Empty list should raise ValidationError due to Field(min_length=1) + ) + + # Verify it's a validation error related to the min_length constraint + assert "at least 1 item" in str(exc_info.value) or "min_length" in str( + exc_info.value + ) + + +async def test_batch_get_my_services_deduplication( + background_task_lifespan_disabled: None, + rabbitmq_and_rpc_setup_disabled: None, + mocked_director_rest_api: MockRouter, + target_product: ProductName, + services_repo: ServicesRepository, + groups_repo: GroupsRepository, + user_id: UserID, + user: dict[str, Any], + create_fake_service_data: CreateFakeServiceDataCallable, + services_db_tables_injector: Callable, +): + """Test that duplicate service identifiers are silently deduplicated while preserving order.""" + + # Create a service in the database + service_key = "simcore/services/comp/test-service" + service_version = "1.0.0" + + fake_service = create_fake_service_data( + service_key, + service_version, + team_access=None, + everyone_access=None, + product=target_product, + ) + + await services_db_tables_injector([fake_service]) + + # Request the same service multiple times with duplicates + services_ids = [ + (service_key, service_version), # first occurrence + (service_key, service_version), # duplicate + (service_key, service_version), # another duplicate + ] + + # ACT + result = await catalog_services.batch_get_user_services( + services_repo, + groups_repo, + product_name=target_product, + user_id=user_id, + ids=services_ids, + ) + + # ASSERT + assert ( + len(result.found_items) == 1 + ) # Only one service should be returned despite duplicates + assert len(result.missing_identifiers) == 0 + + found_service = result.found_items[0] + assert found_service.key == service_key + assert found_service.release.version == service_version + + async def test_list_all_vs_latest_services( background_sync_task_mocked: None, rabbitmq_and_rpc_setup_disabled: None, @@ -294,7 +477,7 @@ async def test_list_all_vs_latest_services( offset = 0 # Get latest services first - latest_total_count, latest_items = ( + latest_total_count, latest_found_items = ( await catalog_services.list_latest_catalog_services( services_repo, director_client, @@ -306,13 +489,15 @@ async def test_list_all_vs_latest_services( ) # Get all services as summaries - all_total_count, all_items = await catalog_services.list_all_service_summaries( - services_repo, - director_client, - product_name=target_product, - user_id=user_id, - limit=limit, - offset=offset, + all_total_count, all_found_items = ( + await catalog_services.list_all_service_summaries( + services_repo, + director_client, + product_name=target_product, + user_id=user_id, + limit=limit, + offset=offset, + ) ) # Verify counts @@ -321,24 +506,24 @@ async def test_list_all_vs_latest_services( assert latest_total_count == num_services assert all_total_count == num_services * num_versions_per_service - # Verify we got the expected number of items - assert len(latest_items) == num_services - assert len(all_items) == num_services * num_versions_per_service + # Verify we got the expected number of found_items + assert len(latest_found_items) == num_services + assert len(all_found_items) == num_services * num_versions_per_service - # Collect all service keys from latest items - latest_keys = {item.key for item in latest_items} + # Collect all service keys from latest found_items + latest_keys = {item.key for item in latest_found_items} - # Verify all returned items have the expected structure - for item in all_items: + # Verify all returned found_items have the expected structure + for item in all_found_items: # Each summary should have the basic fields assert item.key in latest_keys assert item.name assert item.description is not None assert isinstance(item, ServiceSummary) - # Group all items by key + # Group all found_items by key key_to_all_versions = {} - for item in all_items: + for item in all_found_items: if item.key not in key_to_all_versions: key_to_all_versions[item.key] = [] key_to_all_versions[item.key].append(item) @@ -347,7 +532,7 @@ async def test_list_all_vs_latest_services( for key, versions in key_to_all_versions.items(): assert len(versions) == num_versions_per_service - # Find this service in latest_items - latest_item = next(item for item in latest_items if item.key == key) + # Find this service in latest_found_items + latest_item = next(item for item in latest_found_items if item.key == key) # Verify there's a summary item with the same version as the latest assert any(item.version == latest_item.version for item in versions) diff --git a/services/web/server/VERSION b/services/web/server/VERSION index 870aec4a7caa..b53d377d9099 100644 --- a/services/web/server/VERSION +++ b/services/web/server/VERSION @@ -1 +1 @@ -0.79.0 +0.80.0 diff --git a/services/web/server/setup.cfg b/services/web/server/setup.cfg index 9c4a56f0e785..c80a97c9ad26 100644 --- a/services/web/server/setup.cfg +++ b/services/web/server/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.79.0 +current_version = 0.80.0 commit = True message = services/webserver api version: {current_version} → {new_version} tag = False diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 07364b9219ad..6460d11a9211 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -2,7 +2,7 @@ openapi: 3.1.0 info: title: simcore-service-webserver description: Main service with an interface (http-API & websockets) to the web front-end - version: 0.79.0 + version: 0.80.0 servers: - url: '' description: webserver @@ -16236,6 +16236,15 @@ components: $ref: '#/components/schemas/NodeServiceGet' type: array title: Services + missing: + anyOf: + - items: + $ref: '#/components/schemas/ServiceKeyVersion' + type: array + - type: 'null' + title: Missing + description: List of services defined in the project but that were not found + in the catalog type: object required: - projectUuid diff --git a/services/web/server/src/simcore_service_webserver/catalog/_controller_rest_exceptions.py b/services/web/server/src/simcore_service_webserver/catalog/_controller_rest_exceptions.py index b3728e979658..d14bca213e3b 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/_controller_rest_exceptions.py +++ b/services/web/server/src/simcore_service_webserver/catalog/_controller_rest_exceptions.py @@ -10,8 +10,8 @@ from servicelib.aiohttp import status from servicelib.rabbitmq._errors import RemoteMethodNotRegisteredError from servicelib.rabbitmq.rpc_interfaces.catalog.errors import ( - CatalogForbiddenError, - CatalogItemNotFoundError, + CatalogForbiddenRpcError, + CatalogItemNotFoundRpcError, ) from ..exception_handling import ( @@ -32,8 +32,8 @@ ) # mypy: disable-error-code=truthy-function -assert CatalogForbiddenError # nosec -assert CatalogItemNotFoundError # nosec +assert CatalogForbiddenRpcError # nosec +assert CatalogItemNotFoundRpcError # nosec _logger = logging.getLogger(__name__) @@ -90,14 +90,14 @@ async def _handler_catalog_client_errors( _version=2, ), ), - CatalogForbiddenError: HttpErrorInfo( + CatalogForbiddenRpcError: HttpErrorInfo( status.HTTP_403_FORBIDDEN, user_message( "Access denied: You don't have permission to view this catalog item.", _version=2, ), ), - CatalogItemNotFoundError: HttpErrorInfo( + CatalogItemNotFoundRpcError: HttpErrorInfo( status.HTTP_404_NOT_FOUND, user_message( "This catalog item does not exist or has been removed.", _version=2 diff --git a/services/web/server/src/simcore_service_webserver/catalog/_models.py b/services/web/server/src/simcore_service_webserver/catalog/_models.py index 18dd24dad6f8..15c83a3e0269 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/_models.py +++ b/services/web/server/src/simcore_service_webserver/catalog/_models.py @@ -1,6 +1,11 @@ -from typing import TypedDict +from typing import TypeAlias, TypedDict + +from models_library.api_schemas_catalog.services import MyServicesRpcBatchGet class ServiceKeyVersionDict(TypedDict): key: str version: str + + +MyServicesBatchGetResult: TypeAlias = MyServicesRpcBatchGet diff --git a/services/web/server/src/simcore_service_webserver/catalog/_service.py b/services/web/server/src/simcore_service_webserver/catalog/_service.py index 128aad83cb3f..d8121bd75946 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/_service.py +++ b/services/web/server/src/simcore_service_webserver/catalog/_service.py @@ -3,7 +3,7 @@ from typing import Any, cast from aiohttp import web -from models_library.api_schemas_catalog.services import MyServiceGet, ServiceUpdateV2 +from models_library.api_schemas_catalog.services import ServiceUpdateV2 from models_library.api_schemas_webserver.catalog import ( ServiceInputGet, ServiceInputKey, @@ -27,7 +27,9 @@ from pint import UnitRegistry from servicelib.rabbitmq._errors import RPCServerError from servicelib.rabbitmq.rpc_interfaces.catalog import services as catalog_rpc -from servicelib.rabbitmq.rpc_interfaces.catalog.errors import CatalogNotAvailableError +from servicelib.rabbitmq.rpc_interfaces.catalog.errors import ( + CatalogNotAvailableRpcError, +) from servicelib.rest_constants import RESPONSE_MODEL_POLICY from ..rabbitmq import get_rabbitmq_rpc_client @@ -37,6 +39,7 @@ ServiceInputGetFactory, ServiceOutputGetFactory, ) +from ._models import MyServicesBatchGetResult from ._units_service import can_connect, replace_service_input_outputs _logger = logging.getLogger(__name__) @@ -97,7 +100,7 @@ async def batch_get_my_services( user_id: UserID, product_name: ProductName, services_ids: list[tuple[ServiceKey, ServiceVersion]], -) -> list[MyServiceGet]: +) -> MyServicesBatchGetResult: try: return await catalog_rpc.batch_get_my_services( @@ -108,7 +111,7 @@ async def batch_get_my_services( ) except RPCServerError as err: - raise CatalogNotAvailableError( + raise CatalogNotAvailableRpcError( user_id=user_id, product_name=product_name, services_ids=services_ids, diff --git a/services/web/server/src/simcore_service_webserver/catalog/catalog_service.py b/services/web/server/src/simcore_service_webserver/catalog/catalog_service.py index 3bce6a913869..b9dcdc1ce4f3 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/catalog_service.py +++ b/services/web/server/src/simcore_service_webserver/catalog/catalog_service.py @@ -10,6 +10,7 @@ from ._service import batch_get_my_services __all__: tuple[str, ...] = ( + "ServiceKeyVersionDict", "batch_get_my_services", "get_service", "get_service_access_rights", @@ -17,6 +18,5 @@ "get_services_for_user_in_product", "is_catalog_service_responsive", "to_backend_service", - "ServiceKeyVersionDict", ) # nopycln: file diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/_rest_exceptions.py b/services/web/server/src/simcore_service_webserver/projects/_controller/_rest_exceptions.py index 60f24a3c6f69..23218e3a1b9d 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/_rest_exceptions.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/_rest_exceptions.py @@ -5,9 +5,10 @@ from common_library.user_messages import user_message from servicelib.aiohttp import status from servicelib.rabbitmq.rpc_interfaces.catalog.errors import ( - CatalogForbiddenError, - CatalogItemNotFoundError, - CatalogNotAvailableError, + CatalogBatchNotFoundRpcError, + CatalogForbiddenRpcError, + CatalogItemNotFoundRpcError, + CatalogNotAvailableRpcError, ) from ...catalog._controller_rest_exceptions import catalog_exceptions_handlers_map @@ -247,7 +248,7 @@ _OTHER_ERRORS: ExceptionToHttpErrorMap = { - CatalogNotAvailableError: HttpErrorInfo( + CatalogNotAvailableRpcError: HttpErrorInfo( status.HTTP_503_SERVICE_UNAVAILABLE, user_message("The catalog service is currently unavailable.", _version=1), ), @@ -257,17 +258,21 @@ "The clusters-keeper service is currently unavailable.", _version=1 ), ), - CatalogForbiddenError: HttpErrorInfo( + CatalogForbiddenRpcError: HttpErrorInfo( status.HTTP_403_FORBIDDEN, user_message( "Access denied: You do not have sufficient permissions for {name}.", _version=1, ), ), - CatalogItemNotFoundError: HttpErrorInfo( + CatalogItemNotFoundRpcError: HttpErrorInfo( status.HTTP_404_NOT_FOUND, user_message("The requested item '{name}' was not found.", _version=1), ), + CatalogBatchNotFoundRpcError: HttpErrorInfo( + status.HTTP_404_NOT_FOUND, + user_message("None of these items '{name}' could be found.", _version=1), + ), } diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/nodes_rest.py b/services/web/server/src/simcore_service_webserver/projects/_controller/nodes_rest.py index 925ca516f2c0..0468583d1938 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/nodes_rest.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/nodes_rest.py @@ -9,7 +9,6 @@ from models_library.api_schemas_catalog.service_access_rights import ( ServiceAccessRightsGet, ) -from models_library.api_schemas_catalog.services import MyServiceGet from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceGet from models_library.api_schemas_dynamic_scheduler.dynamic_services import ( DynamicServiceStop, @@ -547,7 +546,7 @@ async def get_project_services(request: web.Request) -> web.Response: ) ) - services: list[MyServiceGet] = await catalog_service.batch_get_my_services( + batch_got = await catalog_service.batch_get_my_services( request.app, product_name=req_ctx.product_name, user_id=req_ctx.user_id, @@ -559,8 +558,16 @@ async def get_project_services(request: web.Request) -> web.Response: project_uuid=path_params.project_id, services=[ NodeServiceGet.model_validate(sv, from_attributes=True) - for sv in services + for sv in batch_got.found_items ], + missing=( + [ + ServiceKeyVersion(key=k, version=v) + for k, v in batch_got.missing_identifiers + ] + if batch_got.missing_identifiers + else None + ), ) ) diff --git a/services/web/server/src/simcore_service_webserver/socketio/messages.py b/services/web/server/src/simcore_service_webserver/socketio/messages.py index 5d6a973ea304..b3fa6496323b 100644 --- a/services/web/server/src/simcore_service_webserver/socketio/messages.py +++ b/services/web/server/src/simcore_service_webserver/socketio/messages.py @@ -7,7 +7,7 @@ from aiohttp.web import Application from models_library.api_schemas_webserver.socketio import SocketIORoomStr -from models_library.groups import GroupID +from models_library.groups import StandardGroupID from models_library.projects import ProjectID from models_library.socketio import SocketMessageDict from models_library.users import UserID @@ -82,7 +82,7 @@ async def send_message_to_user( async def send_message_to_standard_group( app: Application, - group_id: GroupID, + group_id: StandardGroupID, message: SocketMessageDict, ) -> None: """ diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__patch.py b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__patch.py index 59958de8acf2..a93480d7e607 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__patch.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__patch.py @@ -18,8 +18,8 @@ from pytest_simcore.helpers.webserver_users import UserInfoDict from servicelib.aiohttp import status from servicelib.rabbitmq.rpc_interfaces.catalog.errors import ( - CatalogForbiddenError, - CatalogItemNotFoundError, + CatalogForbiddenRpcError, + CatalogItemNotFoundRpcError, ) from simcore_service_webserver._meta import api_version_prefix from simcore_service_webserver.db.models import UserRole @@ -345,14 +345,14 @@ async def test_patch_project_node_service_key_with_error( with mocker.patch( "simcore_service_webserver.projects._projects_service.catalog_rpc.check_for_service", - side_effect=CatalogForbiddenError(name="test"), + side_effect=CatalogForbiddenRpcError(name="test"), ): resp = await client.patch(f"{url}", json=_patch_version) assert resp.status == status.HTTP_403_FORBIDDEN with mocker.patch( "simcore_service_webserver.projects._projects_service.catalog_rpc.check_for_service", - side_effect=CatalogItemNotFoundError(name="test"), + side_effect=CatalogItemNotFoundRpcError(name="test"), ): resp = await client.patch(f"{url}", json=_patch_version) assert resp.status == status.HTTP_404_NOT_FOUND diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__services_access.py b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__services_access.py index af61022e28b2..3cb8bb46fb7f 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__services_access.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__services_access.py @@ -13,7 +13,10 @@ from models_library.api_schemas_catalog.service_access_rights import ( ServiceAccessRightsGet, ) -from models_library.api_schemas_catalog.services import MyServiceGet +from models_library.api_schemas_catalog.services import ( + MyServiceGet, + MyServicesRpcBatchGet, +) from models_library.services_history import ServiceRelease from pytest_mock import MockerFixture from pytest_simcore.helpers.assert_checks import assert_status @@ -481,21 +484,23 @@ async def test_get_project_services( mocker.patch( "simcore_service_webserver.catalog._service.catalog_rpc.batch_get_my_services", spec=True, - return_value=[ - MyServiceGet( - key=service_key, - release=ServiceRelease( - version=service_version, - version_display=f"v{service_version}", - released="2023-01-01T00:00:00Z", - retired=None, - compatibility=None, - ), - owner=logged_user["primary_gid"], - my_access_rights={"execute": True, "write": False}, - ) - for service_key, service_version in fake_services_in_project - ], + return_value=MyServicesRpcBatchGet( + found_items=[ + MyServiceGet( + key=service_key, + release=ServiceRelease( + version=service_version, + version_display=f"v{service_version}", + released="2023-01-01T00:00:00Z", + retired=None, + compatibility=None, + ), + owner=logged_user["primary_gid"], + my_access_rights={"execute": True, "write": False}, + ) + for service_key, service_version in fake_services_in_project + ] + ), ) assert client.app @@ -550,6 +555,7 @@ async def test_get_project_services( }, }, ], + "missing": None, }