diff --git a/packages/models-library/src/models_library/api_schemas_catalog/services.py b/packages/models-library/src/models_library/api_schemas_catalog/services.py index 565cd814a805..5b66f46351f8 100644 --- a/packages/models-library/src/models_library/api_schemas_catalog/services.py +++ b/packages/models-library/src/models_library/api_schemas_catalog/services.py @@ -172,23 +172,57 @@ def _update_json_schema_extra(schema: JsonDict) -> None: ) -class _BaseServiceGetV2(CatalogOutputSchema): - # Model used in catalog's rpc and rest interfaces +class ServiceSummary(CatalogOutputSchema): key: ServiceKey version: ServiceVersion - name: str - thumbnail: HttpUrl | None = None - icon: HttpUrl | None = None description: str - description_ui: bool = False - version_display: str | None = None + contact: LowerCaseEmailStr | None + + @staticmethod + def _update_json_schema_extra(schema: JsonDict) -> None: + schema.update( + { + "examples": [ + { + "key": _EXAMPLE_SLEEPER["key"], + "version": _EXAMPLE_SLEEPER["version"], + "name": _EXAMPLE_SLEEPER["name"], + "description": _EXAMPLE_SLEEPER["description"], + "version_display": _EXAMPLE_SLEEPER["version_display"], + "contact": _EXAMPLE_SLEEPER["contact"], + }, + { + "key": _EXAMPLE_FILEPICKER["key"], + "version": _EXAMPLE_FILEPICKER["version"], + "name": _EXAMPLE_FILEPICKER["name"], + "description": _EXAMPLE_FILEPICKER["description"], + "version_display": None, + "contact": _EXAMPLE_FILEPICKER["contact"], + }, + ] + } + ) + + model_config = ConfigDict( + extra="ignore", + populate_by_name=True, + alias_generator=snake_to_camel, + json_schema_extra=_update_json_schema_extra, + ) + + +class _BaseServiceGetV2(ServiceSummary): service_type: Annotated[ServiceType, Field(alias="type")] - contact: LowerCaseEmailStr | None + thumbnail: HttpUrl | None = None + icon: HttpUrl | None = None + + description_ui: bool = False + authors: Annotated[list[Author], Field(min_length=1)] owner: Annotated[ LowerCaseEmailStr | None, @@ -217,6 +251,7 @@ class _BaseServiceGetV2(CatalogOutputSchema): extra="forbid", populate_by_name=True, alias_generator=snake_to_camel, + json_schema_extra={"example": _EXAMPLE_SLEEPER}, ) @@ -249,7 +284,6 @@ def _update_json_schema_extra(schema: JsonDict) -> None: class ServiceGetV2(_BaseServiceGetV2): - # Model used in catalog's rpc and rest interfaces history: Annotated[ list[ServiceRelease], Field( @@ -338,6 +372,9 @@ def _update_json_schema_extra(schema: JsonDict) -> None: ServiceRelease ] +# Create PageRpc types +PageRpcServiceSummary = PageRpc[ServiceSummary] + ServiceResourcesGet: TypeAlias = ServiceResourcesDict diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/catalog_rpc_server.py b/packages/pytest-simcore/src/pytest_simcore/helpers/catalog_rpc_server.py index 0dc9145c2442..0e06f7d261ed 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/catalog_rpc_server.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/catalog_rpc_server.py @@ -13,6 +13,7 @@ LatestServiceGet, ServiceGetV2, ServiceListFilters, + ServiceSummary, ServiceUpdateV2, ) from models_library.api_schemas_catalog.services_ports import ServicePortGet @@ -200,6 +201,66 @@ async def get_service_ports( ServicePortGet.model_json_schema()["examples"], ) + @validate_call(config={"arbitrary_types_allowed": True}) + async def list_all_services_summaries_paginated( + self, + rpc_client: RabbitMQRPCClient | MockType, + *, + product_name: ProductName, + user_id: UserID, + limit: PageLimitInt, + offset: NonNegativeInt, + filters: ServiceListFilters | None = None, + ): + assert rpc_client + assert product_name + assert user_id + + service_summaries = TypeAdapter(list[ServiceSummary]).validate_python( + ServiceSummary.model_json_schema()["examples"], + ) + if filters: + filtered_summaries = [] + for summary in service_summaries: + # Match service type if specified + if ( + filters.service_type + and { + ServiceType.COMPUTATIONAL: "/comp/", + ServiceType.DYNAMIC: "/dynamic/", + }[filters.service_type] + not in summary.key + ): + continue + + # Match service key pattern if specified + if filters.service_key_pattern and not fnmatch.fnmatch( + summary.key, filters.service_key_pattern + ): + continue + + # Match version display pattern if specified + if filters.version_display_pattern and ( + summary.version_display is None + or not fnmatch.fnmatch( + summary.version_display, filters.version_display_pattern + ) + ): + continue + + filtered_summaries.append(summary) + + service_summaries = filtered_summaries + + total_count = len(service_summaries) + + return PageRpc[ServiceSummary].create( + service_summaries[offset : offset + limit], + total=total_count, + limit=limit, + offset=offset, + ) + @dataclass class ZeroListingCatalogRpcSideEffects: @@ -216,3 +277,11 @@ async def list_my_service_history_latest_first(self, *args, **kwargs): limit=10, offset=0, ) + + async def list_all_services_summaries_paginated(self, *args, **kwargs): + return PageRpc[ServiceSummary].create( + [], + total=0, + limit=10, + offset=0, + ) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/services.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/services.py index 58a8f289ebfb..7ac19275fb67 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/services.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/services.py @@ -17,9 +17,11 @@ MyServiceGet, PageRpcLatestServiceGet, PageRpcServiceRelease, + PageRpcServiceSummary, ServiceGetV2, ServiceListFilters, ServiceRelease, + ServiceSummary, ServiceUpdateV2, ) from models_library.api_schemas_catalog.services_ports import ServicePortGet @@ -256,3 +258,51 @@ async def get_service_ports( TypeAdapter(list[ServicePortGet]).validate_python(result) is not None ) # nosec return cast(list[ServicePortGet], result) + + +@validate_call(config={"arbitrary_types_allowed": True}) +async def list_all_services_summaries_paginated( # pylint: disable=too-many-arguments + rpc_client: RabbitMQRPCClient, + *, + product_name: ProductName, + user_id: UserID, + limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, + offset: PageOffsetInt = 0, + filters: ServiceListFilters | None = None, +) -> PageRpcServiceSummary: + """Lists all services with pagination, including all versions of each service. + + Returns a lightweight summary view of services for better performance. + + Args: + rpc_client: RPC client instance + product_name: Product name + user_id: User ID + limit: Maximum number of items to return + offset: Number of items to skip + filters: Optional filters to apply + + Returns: + Paginated list of all services as summaries + + Raises: + ValidationError: on invalid arguments + CatalogForbiddenError: no access-rights to list services + """ + result = await rpc_client.request( + CATALOG_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python( + list_all_services_summaries_paginated.__name__ + ), + product_name=product_name, + user_id=user_id, + limit=limit, + offset=offset, + filters=filters, + timeout_s=40 * RPC_REQUEST_DEFAULT_TIMEOUT_S, + ) + + assert ( + TypeAdapter(PageRpc[ServiceSummary]).validate_python(result) is not None + ) # nosec + return cast(PageRpc[ServiceSummary], result) diff --git a/services/api-server/docs/api-server.drawio.svg b/services/api-server/docs/api-server.drawio.svg index 3f6efeb32725..98f7dcfdfc6e 100644 --- a/services/api-server/docs/api-server.drawio.svg +++ b/services/api-server/docs/api-server.drawio.svg @@ -1,6 +1,6 @@ - + - + @@ -285,28 +285,26 @@
- postgres -
- asyncpg + sa[asyngpg]
- postgres... + sa[asyngpg] - + -
+
CONTROLLER @@ -314,20 +312,20 @@
- + CONTROLLER - + -
+
SERVICE @@ -335,20 +333,20 @@
- + SERVICE - + -
+
REPOSITORY @@ -356,20 +354,20 @@
- + REPOSITORY - + -
+
CLIENTS @@ -377,20 +375,20 @@
- + CLIENTS - + -
+
rest @@ -398,20 +396,24 @@
- + rest - + + + + + -
+
rpc @@ -419,12 +421,83 @@
- + rpc + + + + + + + + + + + +
+
+
+ services +
+
+
+ + + services + + + + + + + + + + + + + + + +
+
+
+ catalog_srv +
+
+
+ + + catalog_srv + + + + + + + + + + + +
+
+
+ sa[asyncpg] +
+
+
+ + + sa[asyncpg] + + + + @@ -565,15 +638,13 @@
- postgres -
- asyncpg + sa[asyncg]
- postgres... + sa[asyncg] @@ -796,8 +867,8 @@ - - + + @@ -884,7 +955,7 @@ - + @@ -908,6 +979,9 @@ + + + diff --git a/services/api-server/src/simcore_service_api_server/_service_solvers.py b/services/api-server/src/simcore_service_api_server/_service_solvers.py index 73b9b71633cc..28c767cbeea8 100644 --- a/services/api-server/src/simcore_service_api_server/_service_solvers.py +++ b/services/api-server/src/simcore_service_api_server/_service_solvers.py @@ -125,15 +125,15 @@ async def solver_release_history( self, *, solver_key: SolverKeyId, - offset: NonNegativeInt, - limit: PositiveInt, + pagination_offset: NonNegativeInt, + pagination_limit: PositiveInt, ) -> tuple[list[Solver], PageMetaInfoLimitOffset]: releases, page_meta = ( await self.catalog_service.list_release_history_latest_first( filter_by_service_key=solver_key, - pagination_offset=offset, - pagination_limit=limit, + pagination_offset=pagination_offset, + pagination_limit=pagination_limit, ) ) @@ -153,21 +153,62 @@ async def solver_release_history( for service in releases ], page_meta + async def list_all_solvers( + self, + *, + pagination_offset: NonNegativeInt, + pagination_limit: PositiveInt, + filter_by_solver_key_pattern: str | None = None, + filter_by_version_display_pattern: str | None = None, + ) -> tuple[list[Solver], PageMetaInfoLimitOffset]: + """Lists all solvers with pagination and filtering, including all versions. + + Unlike `latest_solvers` which only shows the latest version of each solver, + this method returns all versions of solvers that match the filters. + + Args: + pagination_offset: Pagination offset + pagination_limit: Pagination limit + filter_by_solver_key_pattern: Optional pattern to filter solvers by key e.g. "simcore/service/my_solver*" + filter_by_version_display_pattern: Optional pattern to filter by version display e.g. "1.0.*-beta" + + Returns: + A tuple with the list of filtered solvers and pagination metadata + """ + filters = ServiceListFilters(service_type=ServiceType.COMPUTATIONAL) + + # Add key_pattern filter for solver ID if provided + if filter_by_solver_key_pattern: + filters.service_key_pattern = filter_by_solver_key_pattern + + # Add version_display_pattern filter if provided + if filter_by_version_display_pattern: + filters.version_display_pattern = filter_by_version_display_pattern + + services, page_meta = await self.catalog_service.list_all_services_summaries( + pagination_offset=pagination_offset, + pagination_limit=pagination_limit, + filters=filters, + ) + + solvers = [Solver.create_from_service(service) for service in services] + return solvers, page_meta + async def latest_solvers( self, *, pagination_offset: NonNegativeInt, pagination_limit: PositiveInt, - filter_by_solver_id: str | None = None, - filter_by_version_display: str | None = None, + filter_by_solver_key_pattern: str | None = None, + filter_by_version_display_pattern: str | None = None, ) -> tuple[list[Solver], PageMetaInfoLimitOffset]: """Lists the latest solvers with pagination and filtering. Args: - offset: Pagination offset - limit: Pagination limit - solver_id_pattern: Optional pattern to filter solvers by ID - version_display_pattern: Optional pattern to filter by version display + pagination_offset: Pagination offset + pagination_limit: Pagination limit + filter_by_solver_key_pattern: Optional pattern to filter solvers by key e.g. "simcore/service/my_solver*" + filter_by_version_display_pattern: Optional pattern to filter by version display e.g. "1.0.*-beta" Returns: A tuple with the list of filtered solvers and pagination metadata @@ -175,12 +216,12 @@ async def latest_solvers( filters = ServiceListFilters(service_type=ServiceType.COMPUTATIONAL) # Add key_pattern filter for solver ID if provided - if filter_by_solver_id: - filters.service_key_pattern = filter_by_solver_id + if filter_by_solver_key_pattern: + filters.service_key_pattern = filter_by_solver_key_pattern # Add version_display_pattern filter if provided - if filter_by_version_display: - filters.version_display_pattern = filter_by_version_display + if filter_by_version_display_pattern: + filters.version_display_pattern = filter_by_version_display_pattern services, page_meta = await self.catalog_service.list_latest_releases( pagination_offset=pagination_offset, diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers.py index a6092743bf53..3f4942499d24 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers.py @@ -45,7 +45,9 @@ **DEFAULT_BACKEND_SERVICE_STATUS_CODES, } -router = APIRouter() +router = APIRouter( + # /v0/solvers/ +) @router.get( @@ -82,24 +84,24 @@ async def list_solvers( "/page", response_model=Page[Solver], description=create_route_description( - base="Lists the latest version of all available solvers (paginated)", + base="Lists all available solvers (paginated)", changelog=[ FMSG_CHANGELOG_NEW_IN_VERSION.format("0.9-rc1"), ], ), include_in_schema=False, # TO BE RELEASED in 0.9 ) -async def list_solvers_paginated( +async def list_all_solvers_paginated( page_params: Annotated[PaginationParams, Depends()], solver_service: Annotated[SolverService, Depends(get_solver_service)], filters: Annotated[SolversListFilters, Depends(get_solvers_filters)], url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], ): - solvers, page_meta = await solver_service.latest_solvers( + solvers, page_meta = await solver_service.list_all_solvers( pagination_offset=page_params.offset, pagination_limit=page_params.limit, - filter_by_solver_id=filters.solver_id, - filter_by_version_display=filters.version_display, + filter_by_solver_key_pattern=filters.solver_id, + filter_by_version_display_pattern=filters.version_display, ) for solver in solvers: @@ -145,8 +147,8 @@ async def list_solvers_releases( for page_params in iter_pagination_params(limit=DEFAULT_PAGINATION_LIMIT): solvers, page_meta = await solver_service.solver_release_history( solver_key=solver.id, - offset=page_params.offset, - limit=page_params.limit, + pagination_offset=page_params.offset, + pagination_limit=page_params.limit, ) page_params.total_number_of_items = page_meta.total all_solvers.extend(solvers) @@ -216,8 +218,8 @@ async def list_solver_releases( for page_params in iter_pagination_params(limit=DEFAULT_PAGINATION_LIMIT): solvers, page_meta = await solver_service.solver_release_history( solver_key=solver_key, - offset=page_params.offset, - limit=page_params.limit, + pagination_offset=page_params.offset, + pagination_limit=page_params.limit, ) page_params.total_number_of_items = page_meta.total all_releases.extend(solvers) @@ -249,8 +251,8 @@ async def list_solver_releases_paginated( ): solvers, page_meta = await solver_service.solver_release_history( solver_key=solver_key, - offset=page_params.offset, - limit=page_params.limit, + pagination_offset=page_params.offset, + pagination_limit=page_params.limit, ) for solver in solvers: diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/solvers.py b/services/api-server/src/simcore_service_api_server/models/schemas/solvers.py index 19228766f943..63a383c4f2b1 100644 --- a/services/api-server/src/simcore_service_api_server/models/schemas/solvers.py +++ b/services/api-server/src/simcore_service_api_server/models/schemas/solvers.py @@ -1,6 +1,10 @@ from typing import Annotated, Any, Literal, Self, TypeAlias -from models_library.api_schemas_catalog.services import LatestServiceGet, ServiceGetV2 +from models_library.api_schemas_catalog.services import ( + LatestServiceGet, + ServiceGetV2, + ServiceSummary, +) from models_library.basic_regex import PUBLIC_VARIABLE_NAME_RE from models_library.emails import LowerCaseEmailStr from models_library.services import ServiceMetaDataPublished @@ -72,14 +76,23 @@ def create_from_image(cls, image_meta: ServiceMetaDataPublished) -> Self: ) @classmethod - def create_from_service(cls, service: ServiceGetV2 | LatestServiceGet) -> Self: + def create_from_service( + cls, service: ServiceGetV2 | LatestServiceGet | ServiceSummary + ) -> Self: + # Common fields in all service types + maintainer = "" + if hasattr(service, "contact") and service.contact: + maintainer = service.contact + return cls( id=service.key, version=service.version, title=service.name, description=service.description, - maintainer=service.contact or "UNDEFINED", - version_display=service.version_display, + maintainer=maintainer or "UNDEFINED", + version_display=( + service.version_display if hasattr(service, "version_display") else None + ), url=None, ) diff --git a/services/api-server/src/simcore_service_api_server/services_rpc/catalog.py b/services/api-server/src/simcore_service_api_server/services_rpc/catalog.py index a428ccb33bae..32a4b9ce17e0 100644 --- a/services/api-server/src/simcore_service_api_server/services_rpc/catalog.py +++ b/services/api-server/src/simcore_service_api_server/services_rpc/catalog.py @@ -5,6 +5,7 @@ LatestServiceGet, ServiceGetV2, ServiceListFilters, + ServiceSummary, ) from models_library.api_schemas_catalog.services_ports import ServicePortGet from models_library.products import ProductName @@ -89,6 +90,41 @@ async def list_release_history_latest_first( ) return page.data, meta + async def list_all_services_summaries( + self, + *, + pagination_offset: PageOffsetInt = 0, + pagination_limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, + filters: ServiceListFilters | None = None, + ) -> tuple[list[ServiceSummary], PageMetaInfoLimitOffset]: + """Lists all services with pagination, including all versions of each service. + + Returns a lightweight summary view of services for better performance. + + Args: + pagination_offset: Number of items to skip + pagination_limit: Maximum number of items to return + filters: Optional filters to apply + + Returns: + Tuple containing list of service summaries and pagination metadata + """ + page = await catalog_rpc.list_all_services_summaries_paginated( + self._rpc_client, + product_name=self.product_name, + user_id=self.user_id, + offset=pagination_offset, + limit=pagination_limit, + filters=filters, + ) + meta = PageMetaInfoLimitOffset( + limit=page.meta.limit, + offset=page.meta.offset, + total=page.meta.total, + count=page.meta.count, + ) + return page.data, meta + @_exception_mapper( rpc_exception_map={ CatalogItemNotFoundError: ProgramOrSolverOrStudyNotFoundError, diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers.py index b6be360f26b4..83fa6a0ce523 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers.py @@ -6,12 +6,12 @@ import httpx +from fastapi import status from pydantic import TypeAdapter from pytest_mock import MockType from simcore_service_api_server._meta import API_VTAG from simcore_service_api_server.models.pagination import OnePage from simcore_service_api_server.models.schemas.solvers import Solver, SolverPort -from starlette import status async def test_list_all_solvers( diff --git a/services/api-server/tests/unit/conftest.py b/services/api-server/tests/unit/conftest.py index 7c3936d9bdb5..174bf1bd6013 100644 --- a/services/api-server/tests/unit/conftest.py +++ b/services/api-server/tests/unit/conftest.py @@ -509,38 +509,27 @@ def mocked_catalog_rpc_api( services as catalog_rpc, # keep import here ) - return { - "list_services_paginated": mocker.patch.object( - catalog_rpc, - "list_services_paginated", - autospec=True, - side_effect=catalog_rpc_side_effects.list_services_paginated, - ), - "get_service": mocker.patch.object( - catalog_rpc, - "get_service", - autospec=True, - side_effect=catalog_rpc_side_effects.get_service, - ), - "update_service": mocker.patch.object( - catalog_rpc, - "update_service", - autospec=True, - side_effect=catalog_rpc_side_effects.update_service, - ), - "list_my_service_history_latest_first": mocker.patch.object( - catalog_rpc, - "list_my_service_history_latest_first", - autospec=True, - side_effect=catalog_rpc_side_effects.list_my_service_history_latest_first, - ), - "get_service_ports": mocker.patch.object( - catalog_rpc, - "get_service_ports", - autospec=True, - side_effect=catalog_rpc_side_effects.get_service_ports, - ), - } + mocks = {} + + # Get all callable methods from the side effects class that are not built-ins + side_effect_methods = [ + method_name + for method_name in dir(catalog_rpc_side_effects) + if not method_name.startswith("_") + and callable(getattr(catalog_rpc_side_effects, method_name)) + ] + + # Create mocks for each method in catalog_rpc that has a corresponding side effect + for method_name in side_effect_methods: + if hasattr(catalog_rpc, method_name): + mocks[method_name] = mocker.patch.object( + catalog_rpc, + method_name, + autospec=True, + side_effect=getattr(catalog_rpc_side_effects, method_name), + ) + + return mocks # diff --git a/services/catalog/src/simcore_service_catalog/api/rpc/_services.py b/services/catalog/src/simcore_service_catalog/api/rpc/_services.py index 356ae89f74ce..73d5bc3e562f 100644 --- a/services/catalog/src/simcore_service_catalog/api/rpc/_services.py +++ b/services/catalog/src/simcore_service_catalog/api/rpc/_services.py @@ -7,6 +7,7 @@ MyServiceGet, PageRpcLatestServiceGet, PageRpcServiceRelease, + PageRpcServiceSummary, ServiceGetV2, ServiceListFilters, ServiceUpdateV2, @@ -252,8 +253,8 @@ async def list_my_service_history_latest_first( product_name=product_name, user_id=user_id, service_key=service_key, - limit=limit, - offset=offset, + pagination_limit=limit, + pagination_offset=offset, filters=TypeAdapter(ServiceDBFilters | None).validate_python( filters, from_attributes=True ), @@ -310,3 +311,59 @@ async def get_service_ports( ) for port in service_ports ] + + +@router.expose(reraise_if_error_type=(CatalogForbiddenError, ValidationError)) +@_profile_rpc_call +@validate_call(config={"arbitrary_types_allowed": True}) +async def list_all_services_summaries_paginated( + app: FastAPI, + *, + product_name: ProductName, + user_id: UserID, + limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, + offset: PageOffsetInt = 0, + filters: ServiceListFilters | None = None, +) -> PageRpcServiceSummary: + """Lists all services with pagination, including all versions of each service. + + Returns a lightweight summary view of services for better performance compared to + full service details. This is useful for listings where complete details aren't needed. + + Args: + app: FastAPI application + product_name: Product name + user_id: User ID + limit: Maximum number of items to return + offset: Number of items to skip + filters: Optional filters to apply + + Returns: + Paginated list of all services as summaries + """ + assert app.state.engine # nosec + + total_count, items = await catalog_services.list_all_service_summaries( + repo=ServicesRepository(app.state.engine), + director_api=get_director_client(app), + product_name=product_name, + user_id=user_id, + limit=limit, + offset=offset, + filters=TypeAdapter(ServiceDBFilters | None).validate_python( + filters, from_attributes=True + ), + ) + + assert len(items) <= total_count # nosec + assert len(items) <= limit if limit is not None else True # nosec + + return cast( + PageRpcServiceSummary, + PageRpcServiceSummary.create( + items, + total=total_count, + limit=limit, + offset=offset, + ), + ) diff --git a/services/catalog/src/simcore_service_catalog/repository/_services_sql.py b/services/catalog/src/simcore_service_catalog/repository/_services_sql.py index 716a1276f927..ca05641a5923 100644 --- a/services/catalog/src/simcore_service_catalog/repository/_services_sql.py +++ b/services/catalog/src/simcore_service_catalog/repository/_services_sql.py @@ -436,3 +436,79 @@ def get_service_history_stmt( .select_from(history_subquery) .group_by(history_subquery.c.key) ) + + +def all_services_total_count_stmt( + *, + product_name: ProductName, + user_id: UserID, + access_rights: AccessRightsClauses, + filters: ServiceDBFilters | None = None, +) -> sa.sql.Select: + """Statement to count all services""" + stmt = ( + sa.select(sa.func.count()) + .select_from( + services_meta_data.join( + services_access_rights, + (services_meta_data.c.key == services_access_rights.c.key) + & (services_meta_data.c.version == services_access_rights.c.version), + ).join( + user_to_groups, + (user_to_groups.c.gid == services_access_rights.c.gid), + ) + ) + .where( + (services_access_rights.c.product_name == product_name) + & (user_to_groups.c.uid == user_id) + & access_rights + ) + ) + + if filters: + stmt = apply_services_filters(stmt, filters) + + return stmt + + +def list_all_services_stmt( + *, + product_name: ProductName, + user_id: UserID, + access_rights: AccessRightsClauses, + limit: int | None = None, + offset: int | None = None, + filters: ServiceDBFilters | None = None, +) -> sa.sql.Select: + """Statement to list all services with pagination""" + stmt = ( + sa.select(*SERVICES_META_DATA_COLS) + .select_from( + services_meta_data.join( + services_access_rights, + (services_meta_data.c.key == services_access_rights.c.key) + & (services_meta_data.c.version == services_access_rights.c.version), + ).join( + user_to_groups, + (user_to_groups.c.gid == services_access_rights.c.gid), + ) + ) + .where( + (services_access_rights.c.product_name == product_name) + & (user_to_groups.c.uid == user_id) + & access_rights + ) + .order_by( + services_meta_data.c.key, sa.desc(by_version(services_meta_data.c.version)) + ) + ) + + if filters: + stmt = apply_services_filters(stmt, filters) + + if offset is not None: + stmt = stmt.offset(offset) + if limit is not None: + stmt = stmt.limit(limit) + + return stmt diff --git a/services/catalog/src/simcore_service_catalog/repository/services.py b/services/catalog/src/simcore_service_catalog/repository/services.py index 3d24521df4d6..12e0d88a0fe6 100644 --- a/services/catalog/src/simcore_service_catalog/repository/services.py +++ b/services/catalog/src/simcore_service_catalog/repository/services.py @@ -376,6 +376,83 @@ async def get_service_with_history( ) return None + async def list_all_services( + self, + *, + # access-rights + product_name: ProductName, + user_id: UserID, + # list args: pagination + pagination_limit: int | None = None, + pagination_offset: int | None = None, + filters: ServiceDBFilters | None = None, + ) -> tuple[PositiveInt, list[ServiceMetaDataDBGet]]: + # Create base query that's common to both count and content queries + base_query = ( + sa.select(services_meta_data.c.key, services_meta_data.c.version) + .select_from( + services_meta_data.join( + services_access_rights, + (services_meta_data.c.key == services_access_rights.c.key) + & ( + services_meta_data.c.version == services_access_rights.c.version + ), + ).join( + user_to_groups, + (user_to_groups.c.gid == services_access_rights.c.gid), + ) + ) + .where( + (services_access_rights.c.product_name == product_name) + & (user_to_groups.c.uid == user_id) + & AccessRightsClauses.can_read + ) + .distinct() + ) + + if filters: + base_query = _services_sql.apply_services_filters(base_query, filters) + + # Subquery for efficient counting and further joins + subquery = base_query.subquery() + + # Count query - only counts distinct key/version pairs + stmt_total = sa.select(sa.func.count()).select_from(subquery) + + # Content query - gets all details with pagination + stmt_page = ( + sa.select(*SERVICES_META_DATA_COLS) + .select_from( + subquery.join( + services_meta_data, + (subquery.c.key == services_meta_data.c.key) + & (subquery.c.version == services_meta_data.c.version), + ) + ) + .order_by( + services_meta_data.c.key, + sa.desc(_services_sql.by_version(services_meta_data.c.version)), + ) + ) + + # Apply pagination to content query + if pagination_offset is not None: + stmt_page = stmt_page.offset(pagination_offset) + if pagination_limit is not None: + stmt_page = stmt_page.limit(pagination_limit) + + # Execute both queries + async with self.db_engine.connect() as conn: + result = await conn.execute(stmt_total) + total_count = result.scalar() or 0 + + items_page = [ + ServiceMetaDataDBGet.model_validate(row) + async for row in await conn.stream(stmt_page) + ] + + return (total_count, items_page) + async def list_latest_services( self, *, @@ -383,8 +460,8 @@ async def list_latest_services( product_name: ProductName, user_id: UserID, # list args: pagination - limit: int | None = None, - offset: int | None = None, + pagination_limit: int | None = None, + pagination_offset: int | None = None, filters: ServiceDBFilters | None = None, ) -> tuple[PositiveInt, list[ServiceWithHistoryDBGet]]: @@ -399,8 +476,8 @@ async def list_latest_services( product_name=product_name, user_id=user_id, access_rights=AccessRightsClauses.can_read, - limit=limit, - offset=offset, + limit=pagination_limit, + offset=pagination_offset, filters=filters, ) @@ -475,8 +552,8 @@ async def get_service_history_page( # get args key: ServiceKey, # list args: pagination - limit: int | None = None, - offset: int | None = None, + pagination_limit: int | None = None, + pagination_offset: int | None = None, filters: ServiceDBFilters | None = None, ) -> tuple[PositiveInt, list[ReleaseDBGet]]: @@ -540,8 +617,8 @@ async def get_service_history_page( ) ) .order_by(sql.desc(_services_sql.by_version(services_meta_data.c.version))) - .offset(offset) - .limit(limit) + .offset(pagination_offset) + .limit(pagination_limit) ) async with pass_or_acquire_connection(self.db_engine) as conn: diff --git a/services/catalog/src/simcore_service_catalog/service/catalog_services.py b/services/catalog/src/simcore_service_catalog/service/catalog_services.py index 579396faae55..d1377bb4db6c 100644 --- a/services/catalog/src/simcore_service_catalog/service/catalog_services.py +++ b/services/catalog/src/simcore_service_catalog/service/catalog_services.py @@ -2,25 +2,26 @@ import logging from contextlib import suppress -from typing import Literal +from typing import Literal, TypeVar from models_library.api_schemas_catalog.services import ( LatestServiceGet, MyServiceGet, ServiceGetV2, + ServiceSummary, ServiceUpdateV2, ) from models_library.api_schemas_directorv2.services import ServiceExtras from models_library.basic_types import VersionStr from models_library.groups import GroupID from models_library.products import ProductName -from models_library.rest_pagination import PageLimitInt, PageTotalCount +from models_library.rest_pagination import PageLimitInt, PageOffsetInt, PageTotalCount from models_library.services_access import ServiceGroupAccessRightsV2 from models_library.services_history import Compatibility, ServiceRelease from models_library.services_metadata_published import ServiceMetaDataPublished from models_library.services_types import ServiceKey, ServiceVersion from models_library.users import UserID -from pydantic import HttpUrl, NonNegativeInt +from pydantic import HttpUrl from servicelib.logging_errors import ( create_troubleshotting_log_kwargs, ) @@ -34,6 +35,7 @@ from ..models.services_db import ( ServiceAccessRightsDB, ServiceDBFilters, + ServiceMetaDataDBGet, ServiceMetaDataDBPatch, ServiceWithHistoryDBGet, ) @@ -46,9 +48,12 @@ _logger = logging.getLogger(__name__) +# Type variable for service models that can be returned from list functions +T = TypeVar("T", ServiceGetV2, LatestServiceGet) + def _aggregate( - service_db: ServiceWithHistoryDBGet, + service_db: ServiceWithHistoryDBGet | ServiceMetaDataDBGet, access_rights_db: list[ServiceAccessRightsDB], service_manifest: ServiceMetaDataPublished, ) -> dict: @@ -64,7 +69,8 @@ def _aggregate( "service_type": service_manifest.service_type, "contact": service_manifest.contact, "authors": service_manifest.authors, - "owner": (service_db.owner_email if service_db.owner_email else None), + # Check if owner attribute is available in the service_db object + "owner": getattr(service_db, "owner_email", None), "inputs": service_manifest.inputs or {}, "outputs": service_manifest.outputs or {}, "boot_options": service_manifest.boot_options, @@ -82,6 +88,21 @@ def _aggregate( } +def _aggregate_summary( + service_db: ServiceWithHistoryDBGet | ServiceMetaDataDBGet, + service_manifest: ServiceMetaDataPublished, +) -> dict: + """Creates a minimal dictionary with only the fields needed for ServiceSummary""" + return { + "key": service_db.key, + "version": service_db.version, + "name": service_db.name, + "description": service_db.description, + "version_display": service_db.version_display, + "contact": service_manifest.contact, + } + + def _to_latest_get_schema( service_db: ServiceWithHistoryDBGet, access_rights_db: list[ServiceAccessRightsDB], @@ -129,40 +150,71 @@ def _to_get_schema( ) -async def list_latest_catalog_services( +async def _get_services_with_access_rights( repo: ServicesRepository, + services: list[ServiceWithHistoryDBGet] | list[ServiceMetaDataDBGet], + product_name: ProductName, + user_id: UserID, +) -> dict[tuple[str, str], list[ServiceAccessRightsDB]]: + """Common function to get access rights for a list of services. + + Args: + repo: Repository for services + services: List of services to get access rights for + product_name: Product name + user_id: User ID + + Returns: + Dictionary mapping (key, version) to list of access rights + + Raises: + CatalogForbiddenError: If no access rights are found for any service + """ + if not services: + return {} + + # Inject access-rights + access_rights = await repo.batch_get_services_access_rights( + ((sc.key, sc.version) for sc in services), product_name=product_name + ) + if not access_rights: + raise CatalogForbiddenError( + name="any service", + user_id=user_id, + product_name=product_name, + ) + + return access_rights + + +async def _get_services_manifests( + services: list[ServiceWithHistoryDBGet] | list[ServiceMetaDataDBGet], + access_rights: dict[tuple[str, str], list[ServiceAccessRightsDB]], director_api: DirectorClient, - *, product_name: ProductName, user_id: UserID, + filters: ServiceDBFilters | None, limit: PageLimitInt | None, - offset: NonNegativeInt = 0, - filters: ServiceDBFilters | None = None, -) -> tuple[PageTotalCount, list[LatestServiceGet]]: + offset: PageOffsetInt | None, +) -> dict[tuple[str, str], ServiceMetaDataPublished]: + """Common function to get service manifests from director. - # defines the order - total_count, services = await repo.list_latest_services( - product_name=product_name, - user_id=user_id, - limit=limit, - offset=offset, - filters=filters, - ) + Args: + services: List of services to get manifests for + access_rights: Dictionary mapping (key, version) to list of access rights + director_api: Director API client + product_name: Product name + user_id: User ID + filters: Filters that were applied + limit: Pagination limit that was applied + offset: Pagination offset that was applied - if services: - # injects access-rights - access_rights: dict[tuple[str, str], list[ServiceAccessRightsDB]] = ( - await repo.batch_get_services_access_rights( - ((sc.key, sc.version) for sc in services), product_name=product_name - ) - ) - if not access_rights: - raise CatalogForbiddenError( - name="any service", - user_id=user_id, - product_name=product_name, - ) + Returns: + Dictionary mapping (key, version) to manifest + Raises: + CatalogInconsistentError: Logs warning if some services are missing from manifest + """ # Get manifest of those with access rights got = await manifest.get_batch_services( [ @@ -203,6 +255,123 @@ async def list_latest_catalog_services( # NOTE: tests should fail if this happens but it is not a critical error so it is ignored in production assert len(missing_services) == 0, msg # nosec + return service_manifest + + +async def list_all_service_summaries( + repo: ServicesRepository, + director_api: DirectorClient, + *, + product_name: ProductName, + user_id: UserID, + limit: PageLimitInt | None, + offset: PageOffsetInt = 0, + filters: ServiceDBFilters | None = None, +) -> tuple[PageTotalCount, list[ServiceSummary]]: + """Lists all catalog services with minimal information. + + This is different from list_latest_catalog_services which only returns the latest version of each service + and includes complete service information. + + Args: + repo: Repository for services + director_api: Director API client + product_name: Product name + user_id: User ID + limit: Pagination limit + offset: Pagination offset + filters: Filters to apply + + Returns: + Tuple of total count and list of service summaries + """ + # Get all services with pagination + total_count, services = await repo.list_all_services( + product_name=product_name, + user_id=user_id, + pagination_limit=limit, + pagination_offset=offset, + filters=filters, + ) + + # Get access rights and manifests + access_rights = await _get_services_with_access_rights( + repo, services, product_name, user_id + ) + service_manifest = await _get_services_manifests( + services, + access_rights, + director_api, + product_name, + user_id, + filters, + limit, + offset, + ) + + # Create service summaries + items = [] + for sc in services: + sm = service_manifest.get((sc.key, sc.version)) + if access_rights.get((sc.key, sc.version)) and sm: + # Create a minimal ServiceSummary + service_data = _aggregate_summary( + service_db=sc, + service_manifest=sm, + ) + items.append(ServiceSummary.model_validate(service_data)) + + return total_count, items + + +async def list_latest_catalog_services( + repo: ServicesRepository, + director_api: DirectorClient, + *, + product_name: ProductName, + user_id: UserID, + limit: PageLimitInt | None, + offset: PageOffsetInt = 0, + filters: ServiceDBFilters | None = None, +) -> tuple[PageTotalCount, list[LatestServiceGet]]: + """Lists latest versions of catalog services. + + Args: + repo: Repository for services + director_api: Director API client + product_name: Product name + user_id: UserID + limit: Pagination limit + offset: Pagination offset + filters: Filters to apply + + Returns: + Tuple of total count and list of latest services + """ + # defines the order + total_count, services = await repo.list_latest_services( + product_name=product_name, + user_id=user_id, + pagination_limit=limit, + pagination_offset=offset, + filters=filters, + ) + + # Get access rights and manifests using shared functions + access_rights = await _get_services_with_access_rights( + repo, services, product_name, user_id + ) + service_manifest = await _get_services_manifests( + services, + access_rights, + director_api, + product_name, + user_id, + filters, + limit, + offset, + ) + # Aggregate the services manifest and access-rights items = [ _to_latest_get_schema( @@ -519,11 +688,11 @@ async def list_user_service_release_history( # target service service_key: ServiceKey, # pagination - limit: PageLimitInt | None = None, - offset: NonNegativeInt | None = None, + pagination_limit: PageLimitInt | None = None, + pagination_offset: PageOffsetInt | None = None, # filters filters: ServiceDBFilters | None = None, - # options + # result options include_compatibility: bool = False, ) -> tuple[PageTotalCount, list[ServiceRelease]]: @@ -533,8 +702,8 @@ async def list_user_service_release_history( product_name=product_name, user_id=user_id, key=service_key, - limit=limit, - offset=offset, + pagination_limit=pagination_limit, + pagination_offset=pagination_offset, filters=filters, ) diff --git a/services/catalog/tests/unit/with_dbs/test_api_rpc.py b/services/catalog/tests/unit/with_dbs/test_api_rpc.py index 1cb346f3cd7c..5b43b728261b 100644 --- a/services/catalog/tests/unit/with_dbs/test_api_rpc.py +++ b/services/catalog/tests/unit/with_dbs/test_api_rpc.py @@ -16,7 +16,10 @@ ServiceUpdateV2, ) from models_library.products import ProductName -from models_library.rest_pagination import MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE +from models_library.rest_pagination import ( + DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, + MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE, +) from models_library.services_enums import ServiceType from models_library.services_history import ServiceRelease from models_library.services_types import ServiceKey, ServiceVersion @@ -143,7 +146,8 @@ async def test_rpc_list_services_paginated_with_filters( user_id=user_id, filters={"service_type": "computational"}, ) - assert page.meta.total == page.meta.count + # Fixed: Count might be capped by page limit + assert page.meta.count <= page.meta.total assert page.meta.total > 0 page = await catalog_rpc.list_services_paginated( @@ -821,7 +825,7 @@ async def test_rpc_get_service_ports_validation_error( user_id: UserID, app: FastAPI, ): - """Tests validation error handling for invalid service key format""" + """Tests validation error handling for list_all_services_summaries_paginated.""" assert app # Test with invalid service key format @@ -833,3 +837,207 @@ async def test_rpc_get_service_ports_validation_error( service_key="invalid-service-key-format", service_version="1.0.0", ) + + +async def test_rpc_list_all_services_summaries_paginated_with_no_services_returns_empty_page( + background_sync_task_mocked: None, + mocked_director_rest_api: MockRouter, + rpc_client: RabbitMQRPCClient, + user_id: UserID, + app: FastAPI, +): + """Tests that requesting summaries for non-existing services returns an empty page.""" + assert app + + page = await catalog_rpc.list_all_services_summaries_paginated( + rpc_client, product_name="not_existing_returns_no_services", user_id=user_id + ) + assert page.data == [] + assert page.links.next is None + assert page.links.prev is None + assert page.meta.count == 0 + assert page.meta.total == 0 + + +async def test_rpc_list_all_services_summaries_paginated_with_filters( + background_sync_task_mocked: None, + mocked_director_rest_api: MockRouter, + rpc_client: RabbitMQRPCClient, + product_name: ProductName, + user_id: UserID, + app: FastAPI, +): + """Tests that service summaries can be filtered by service type.""" + assert app + + # Get all computational services introduced by the background_sync_task_mocked + page = await catalog_rpc.list_all_services_summaries_paginated( + rpc_client, + product_name=product_name, + user_id=user_id, + filters={"service_type": "computational"}, + ) + # Fixed: Count might be capped by page limit + assert page.meta.count <= page.meta.total + assert page.meta.total > 0 + + # All items should be service summaries with the expected minimal fields + for item in page.data: + assert "key" in item.model_dump() + assert "name" in item.model_dump() + assert "version" in item.model_dump() + assert "description" in item.model_dump() + + # Filter for a service type that doesn't exist + page = await catalog_rpc.list_all_services_summaries_paginated( + rpc_client, + product_name=product_name, + user_id=user_id, + filters=ServiceListFilters(service_type=ServiceType.DYNAMIC), + ) + assert page.meta.total == 0 + + +async def test_rpc_list_all_services_summaries_paginated_with_pagination( + background_sync_task_mocked: None, + mocked_director_rest_api: MockRouter, + rpc_client: RabbitMQRPCClient, + product_name: ProductName, + user_id: UserID, + app: FastAPI, + num_services: int, + num_versions_per_service: int, +): + """Tests pagination of service summaries.""" + assert app + + total_services = num_services * num_versions_per_service + + # Get first page with default page size + first_page = await catalog_rpc.list_all_services_summaries_paginated( + rpc_client, + product_name=product_name, + user_id=user_id, + ) + + # Verify total count is correct + assert first_page.meta.total == total_services + + # Maximum items per page is constrained by DEFAULT_NUMBER_OF_ITEMS_PER_PAGE + assert len(first_page.data) <= DEFAULT_NUMBER_OF_ITEMS_PER_PAGE + + # Test with small page size + page_size = 5 + first_small_page = await catalog_rpc.list_all_services_summaries_paginated( + rpc_client, + product_name=product_name, + user_id=user_id, + limit=page_size, + offset=0, + ) + assert len(first_small_page.data) == page_size + assert first_small_page.meta.total == total_services + assert first_small_page.links.next is not None + assert first_small_page.links.prev is None + + # Get next page and verify different content + next_page = await catalog_rpc.list_all_services_summaries_paginated( + rpc_client, + product_name=product_name, + user_id=user_id, + limit=page_size, + offset=page_size, + ) + assert len(next_page.data) == page_size + assert next_page.meta.total == first_small_page.meta.total + + # Check that first and second page contain different items + first_page_keys = {(item.key, item.version) for item in first_small_page.data} + next_page_keys = {(item.key, item.version) for item in next_page.data} + assert not first_page_keys.intersection(next_page_keys) + + +async def test_rpc_compare_latest_vs_all_services_summaries( + background_sync_task_mocked: None, + mocked_director_rest_api: MockRouter, + rpc_client: RabbitMQRPCClient, + product_name: ProductName, + user_id: UserID, + app: FastAPI, + num_services: int, + num_versions_per_service: int, +): + """Compares results of list_services_paginated vs list_all_services_summaries_paginated.""" + assert app + + total_expected_services = num_services * num_versions_per_service + + # Get all latest services (should fit in one page) + latest_page = await catalog_rpc.list_services_paginated( + rpc_client, + product_name=product_name, + user_id=user_id, + ) + assert latest_page.meta.total == num_services + + # For all services (all versions), we might need multiple requests + # First page to get metadata + first_all_page = await catalog_rpc.list_all_services_summaries_paginated( + rpc_client, + product_name=product_name, + user_id=user_id, + ) + assert first_all_page.meta.total == total_expected_services + + # Collect all items across multiple pages if needed + all_items = list(first_all_page.data) + offset = len(all_items) + + # Continue fetching pages until we have all items + while offset < total_expected_services: + next_page = await catalog_rpc.list_all_services_summaries_paginated( + rpc_client, + product_name=product_name, + user_id=user_id, + offset=offset, + ) + all_items.extend(next_page.data) + offset += len(next_page.data) + if not next_page.links.next: + break + + # Verify we got all items + assert len(all_items) == total_expected_services + + # Collect unique keys from both responses + latest_keys = {item.key for item in latest_page.data} + all_keys = {item.key for item in all_items} + + # All service keys in latest should be in all services + assert latest_keys.issubset(all_keys) + + # For each key in latest, there should be exactly num_versions_per_service entries in all + for key in latest_keys: + versions_in_all = [item.version for item in all_items if item.key == key] + assert len(versions_in_all) == num_versions_per_service + + # Get the latest version from latest_page + latest_version = next( + item.version for item in latest_page.data if item.key == key + ) + + # Verify this version exists in versions_in_all + assert latest_version in versions_in_all + + # Verify all items are ServiceSummary objects with just the essential fields + for item in all_items: + item_dict = item.model_dump() + assert "key" in item_dict + assert "version" in item_dict + assert "name" in item_dict + assert "description" in item_dict + assert "thumbnail" not in item_dict + assert "service_type" not in item_dict + assert "inputs" not in item_dict + assert "outputs" not in item_dict + assert "access_rights" not in item_dict diff --git a/services/catalog/tests/unit/with_dbs/test_repositories.py b/services/catalog/tests/unit/with_dbs/test_repositories.py index 9f6f54d4a02f..ec8fca128256 100644 --- a/services/catalog/tests/unit/with_dbs/test_repositories.py +++ b/services/catalog/tests/unit/with_dbs/test_repositories.py @@ -390,7 +390,7 @@ async def test_list_latest_services_with_pagination( assert service.version == expected_latest_version _, services_items = await services_repo.list_latest_services( - product_name=target_product, user_id=user_id, limit=2 + product_name=target_product, user_id=user_id, pagination_limit=2 ) assert len(services_items) == 2 @@ -711,8 +711,8 @@ async def test_get_service_history_page( product_name=target_product, user_id=user_id, key=service_key, - limit=limit, - offset=offset, + pagination_limit=limit, + pagination_offset=offset, ) assert total_count == num_versions assert len(paginated_history) == limit @@ -900,3 +900,263 @@ async def test_list_services_from_published_templates_with_invalid_service( "service {'key': 'simcore/services/dynamic/invalid-service', 'version': 'invalid'} could not be validated" in caplog.text ) + + +async def test_compare_list_all_and_latest_services( + target_product: ProductName, + create_fake_service_data: CreateFakeServiceDataCallable, + services_db_tables_injector: Callable, + services_repo: ServicesRepository, + user_id: UserID, +): + # Setup: Create multiple versions of the same service and a few distinct services + service_data: list[tuple] = [] + + # Service 1 with multiple versions + service_key_1 = "simcore/services/dynamic/multi-version" + service_versions_1 = ["1.0.0", "1.1.0", "2.0.0"] + service_data.extend( + [ + create_fake_service_data( + service_key_1, + version_, + team_access=None, + everyone_access=None, + product=target_product, + ) + for version_ in service_versions_1 + ] + ) + + # Service 2 with single version + service_key_2 = "simcore/services/dynamic/single-version" + service_data.append( + create_fake_service_data( + service_key_2, + "1.0.0", + team_access=None, + everyone_access=None, + product=target_product, + ) + ) + + # Service 3 with computational type + service_key_3 = "simcore/services/comp/computational-service" + service_versions_3 = ["0.5.0", "1.0.0"] + service_data.extend( + [ + create_fake_service_data( + service_key_3, + version_, + team_access=None, + everyone_access=None, + product=target_product, + ) + for version_ in service_versions_3 + ] + ) + + await services_db_tables_injector(service_data) + + # Test 1: Compare all services vs latest without filters + total_all, all_services = await services_repo.list_all_services( + product_name=target_product, user_id=user_id + ) + total_latest, latest_services = await services_repo.list_latest_services( + product_name=target_product, user_id=user_id + ) + + # Verify counts + # All services should be 6 (3 versions of service 1, 1 of service 2, 2 of service 3) + assert total_all == 6 + # Latest services should be 3 (one latest for each distinct service key) + assert total_latest == 3 + + # Verify latest services are contained in all services + latest_key_versions = {(s.key, s.version) for s in latest_services} + all_key_versions = {(s.key, s.version) for s in all_services} + assert latest_key_versions.issubset(all_key_versions) + + # Verify latest versions are correct + latest_versions_by_key = {s.key: s.version for s in latest_services} + assert latest_versions_by_key[service_key_1] == "2.0.0" + assert latest_versions_by_key[service_key_2] == "1.0.0" + assert latest_versions_by_key[service_key_3] == "1.0.0" + + # Test 2: Using service_type filter to get only dynamic services + filters = ServiceDBFilters(service_type=ServiceType.DYNAMIC) + + total_all_filtered, all_services_filtered = await services_repo.list_all_services( + product_name=target_product, user_id=user_id, filters=filters + ) + total_latest_filtered, latest_services_filtered = ( + await services_repo.list_latest_services( + product_name=target_product, user_id=user_id, filters=filters + ) + ) + + # Verify counts with filter + assert total_all_filtered == 4 # 3 versions of service 1, 1 of service 2 + assert total_latest_filtered == 2 # 1 latest each for service 1 and 2 + + # Verify service types are correct after filtering + assert all( + s.key.startswith(DYNAMIC_SERVICE_KEY_PREFIX) for s in all_services_filtered + ) + assert all( + s.key.startswith(DYNAMIC_SERVICE_KEY_PREFIX) for s in latest_services_filtered + ) + + # Verify latest versions are correct + latest_versions_by_key = {s.key: s.version for s in latest_services_filtered} + assert latest_versions_by_key[service_key_1] == "2.0.0" + assert latest_versions_by_key[service_key_2] == "1.0.0" + assert service_key_3 not in latest_versions_by_key # Filtered out + + # Test 3: Using service_key_pattern to find specific service + filters = ServiceDBFilters(service_key_pattern="*/multi-*") + + total_all_filtered, all_services_filtered = await services_repo.list_all_services( + product_name=target_product, user_id=user_id, filters=filters + ) + total_latest_filtered, latest_services_filtered = ( + await services_repo.list_latest_services( + product_name=target_product, user_id=user_id, filters=filters + ) + ) + + # Verify counts with key pattern filter + assert total_all_filtered == 3 # All 3 versions of service 1 + assert total_latest_filtered == 1 # Only latest version of service 1 + + # Verify service key pattern is matched + assert all(s.key == service_key_1 for s in all_services_filtered) + assert all(s.key == service_key_1 for s in latest_services_filtered) + + # Test 4: Pagination + # Get first page (limit=2) + total_all_page1, all_services_page1 = await services_repo.list_all_services( + product_name=target_product, + user_id=user_id, + pagination_limit=2, + pagination_offset=0, + ) + + # Get second page (limit=2, offset=2) + total_all_page2, all_services_page2 = await services_repo.list_all_services( + product_name=target_product, + user_id=user_id, + pagination_limit=2, + pagination_offset=2, + ) + + # Verify pagination + assert total_all_page1 == 6 # Total count should still be total + assert total_all_page2 == 6 + assert len(all_services_page1) == 2 # But only 2 items on first page + assert len(all_services_page2) == 2 # And 2 items on second page + + # Ensure pages have different items + page1_key_versions = {(s.key, s.version) for s in all_services_page1} + page2_key_versions = {(s.key, s.version) for s in all_services_page2} + assert not page1_key_versions.intersection(page2_key_versions) + + +async def test_list_all_services_empty_database( + target_product: ProductName, + services_repo: ServicesRepository, + user_id: UserID, +): + """Test list_all_services and list_latest_services with an empty database.""" + # Test with empty database + total_all, all_services = await services_repo.list_all_services( + product_name=target_product, user_id=user_id + ) + total_latest, latest_services = await services_repo.list_latest_services( + product_name=target_product, user_id=user_id + ) + + assert total_all == 0 + assert len(all_services) == 0 + assert total_latest == 0 + assert len(latest_services) == 0 + + +async def test_list_all_services_deprecated_versions( + target_product: ProductName, + create_fake_service_data: CreateFakeServiceDataCallable, + services_db_tables_injector: Callable, + services_repo: ServicesRepository, + user_id: UserID, +): + """Test that list_all_services includes deprecated versions while list_latest_services ignores them.""" + from datetime import datetime, timedelta + + # Create a service with regular and deprecated versions + service_key = "simcore/services/dynamic/with-deprecated" + service_data = [] + + # Add regular version + service_data.append( + create_fake_service_data( + service_key, + "1.0.0", + team_access=None, + everyone_access=None, + product=target_product, + ) + ) + + # Add deprecated version (with higher version number) + deprecated_service = create_fake_service_data( + service_key, + "2.0.0", + team_access=None, + everyone_access=None, + product=target_product, + ) + # Set deprecated timestamp to yesterday + deprecated_service[0]["deprecated"] = datetime.now() - timedelta(days=1) + service_data.append(deprecated_service) + + # Add newer non-deprecated version + service_data.append( + create_fake_service_data( + service_key, + "3.0.0", + team_access=None, + everyone_access=None, + product=target_product, + ) + ) + + await services_db_tables_injector(service_data) + + # Get all services - should include both deprecated and non-deprecated + total_all, all_services = await services_repo.list_all_services( + product_name=target_product, user_id=user_id + ) + + # Get latest services - should only show latest non-deprecated + total_latest, latest_services = await services_repo.list_latest_services( + product_name=target_product, user_id=user_id + ) + + # Verify counts + assert total_all == 3 # All 3 versions + + # Verify latest is the newest non-deprecated version + assert len(latest_services) == 1 + assert latest_services[0].key == service_key + assert latest_services[0].version == "3.0.0" + + # Get versions from all services + versions = [s.version for s in all_services if s.key == service_key] + assert sorted(versions) == ["1.0.0", "2.0.0", "3.0.0"] + + # Verify the deprecated status is correctly set + for service in all_services: + if service.key == service_key and service.version == "2.0.0": + assert service.deprecated is not None + else: + assert service.deprecated is None diff --git a/services/catalog/tests/unit/with_dbs/test_service_catalog_services.py b/services/catalog/tests/unit/with_dbs/test_service_catalog_services.py index 039b5f41ccdd..5faaf6b384d4 100644 --- a/services/catalog/tests/unit/with_dbs/test_service_catalog_services.py +++ b/services/catalog/tests/unit/with_dbs/test_service_catalog_services.py @@ -10,7 +10,7 @@ import pytest from fastapi import FastAPI -from models_library.api_schemas_catalog.services import MyServiceGet +from models_library.api_schemas_catalog.services import MyServiceGet, ServiceSummary from models_library.products import ProductName from models_library.users import UserID from pydantic import TypeAdapter @@ -108,7 +108,7 @@ async def director_client(app: FastAPI) -> DirectorClient: return director_api -async def test_list_services_paginated( +async def test_list_latest_catalog_services( background_sync_task_mocked: None, rabbitmq_and_rpc_setup_disabled: None, mocked_director_rest_api: MockRouter, @@ -273,3 +273,81 @@ async def test_batch_get_my_services( }, ] ) + + +async def test_list_all_vs_latest_services( + background_sync_task_mocked: None, + rabbitmq_and_rpc_setup_disabled: None, + mocked_director_rest_api: MockRouter, + target_product: ProductName, + services_repo: ServicesRepository, + user_id: UserID, + director_client: DirectorClient, + num_services: int, + num_versions_per_service: int, +): + """Test that list_all_catalog_services returns all services as summaries while + list_latest_catalog_services returns only the latest version of each service with full details. + """ + # No pagination to get all services + limit = None + offset = 0 + + # Get latest services first + latest_total_count, latest_items = ( + await catalog_services.list_latest_catalog_services( + services_repo, + director_client, + product_name=target_product, + user_id=user_id, + limit=limit, + offset=offset, + ) + ) + + # Get all services as summaries + all_total_count, all_items = await catalog_services.list_all_service_summaries( + services_repo, + director_client, + product_name=target_product, + user_id=user_id, + limit=limit, + offset=offset, + ) + + # Verify counts + # - latest_total_count should equal num_services since we only get the latest version of each service + # - all_total_count should equal num_services * num_versions_per_service since we get all versions + assert latest_total_count == num_services + assert all_total_count == num_services * num_versions_per_service + + # Verify we got the expected number of items + assert len(latest_items) == num_services + assert len(all_items) == num_services * num_versions_per_service + + # Collect all service keys from latest items + latest_keys = {item.key for item in latest_items} + + # Verify all returned items have the expected structure + for item in all_items: + # Each summary should have the basic fields + assert item.key in latest_keys + assert item.name + assert item.description is not None + assert isinstance(item, ServiceSummary) + + # Group all items by key + key_to_all_versions = {} + for item in all_items: + if item.key not in key_to_all_versions: + key_to_all_versions[item.key] = [] + key_to_all_versions[item.key].append(item) + + # For each service key, verify we have the expected number of versions + for key, versions in key_to_all_versions.items(): + assert len(versions) == num_versions_per_service + + # Find this service in latest_items + latest_item = next(item for item in latest_items if item.key == key) + # Verify there's a summary item with the same version as the latest + assert any(item.version == latest_item.version for item in versions) diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index e60f7169086a..29e6a9c423d8 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -8616,36 +8616,36 @@ components: name: type: string title: Name - thumbnail: - anyOf: - - type: string - - type: 'null' - title: Thumbnail - icon: - anyOf: - - type: string - - type: 'null' - title: Icon description: type: string title: Description - descriptionUi: - type: boolean - title: Descriptionui - default: false versionDisplay: anyOf: - type: string - type: 'null' title: Versiondisplay - type: - $ref: '#/components/schemas/ServiceType' contact: anyOf: - type: string format: email - type: 'null' title: Contact + type: + $ref: '#/components/schemas/ServiceType' + thumbnail: + anyOf: + - type: string + - type: 'null' + title: Thumbnail + icon: + anyOf: + - type: string + - type: 'null' + title: Icon + descriptionUi: + type: boolean + title: Descriptionui + default: false authors: items: $ref: '#/components/schemas/Author' @@ -8702,8 +8702,8 @@ components: - version - name - description - - type - contact + - type - authors - owner - inputs @@ -8835,36 +8835,36 @@ components: name: type: string title: Name - thumbnail: - anyOf: - - type: string - - type: 'null' - title: Thumbnail - icon: - anyOf: - - type: string - - type: 'null' - title: Icon description: type: string title: Description - descriptionUi: - type: boolean - title: Descriptionui - default: false versionDisplay: anyOf: - type: string - type: 'null' title: Versiondisplay - type: - $ref: '#/components/schemas/ServiceType' contact: anyOf: - type: string format: email - type: 'null' title: Contact + type: + $ref: '#/components/schemas/ServiceType' + thumbnail: + anyOf: + - type: string + - type: 'null' + title: Thumbnail + icon: + anyOf: + - type: string + - type: 'null' + title: Icon + descriptionUi: + type: boolean + title: Descriptionui + default: false authors: items: $ref: '#/components/schemas/Author' @@ -8928,8 +8928,8 @@ components: - version - name - description - - type - contact + - type - authors - owner - inputs