diff --git a/packages/models-library/src/models_library/api_schemas_catalog/services.py b/packages/models-library/src/models_library/api_schemas_catalog/services.py index a80490fad36a..363b74c84768 100644 --- a/packages/models-library/src/models_library/api_schemas_catalog/services.py +++ b/packages/models-library/src/models_library/api_schemas_catalog/services.py @@ -323,11 +323,20 @@ def _update_json_schema_extra(schema: JsonDict) -> None: ) -PageRpcServicesGetV2: TypeAlias = PageRpc[ +PageRpcLatestServiceGet: TypeAlias = PageRpc[ # WARNING: keep this definition in models_library and not in the RPC interface + # otherwise the metaclass PageRpc[*] will create *different* classes in server/client side + # and will fail to serialize/deserialize these parameters when transmitted/received LatestServiceGet ] +PageRpcServiceRelease: TypeAlias = PageRpc[ + # WARNING: keep this definition in models_library and not in the RPC interface + # otherwise the metaclass PageRpc[*] will create *different* classes in server/client side + # and will fail to serialize/deserialize these parameters when transmitted/received + ServiceRelease +] + ServiceResourcesGet: TypeAlias = ServiceResourcesDict @@ -365,3 +374,6 @@ class MyServiceGet(CatalogOutputSchema): owner: GroupID | None my_access_rights: ServiceGroupAccessRightsV2 + + +__all__: tuple[str, ...] = ("ServiceRelease",) diff --git a/packages/models-library/src/models_library/rest_pagination.py b/packages/models-library/src/models_library/rest_pagination.py index 0b7e68e52227..e4a815777420 100644 --- a/packages/models-library/src/models_library/rest_pagination.py +++ b/packages/models-library/src/models_library/rest_pagination.py @@ -28,6 +28,8 @@ PageOffsetInt: TypeAlias = NonNegativeInt +PageTotalCount: TypeAlias = NonNegativeInt + DEFAULT_NUMBER_OF_ITEMS_PER_PAGE: Final[PageLimitInt] = TypeAdapter( PageLimitInt ).validate_python(20) @@ -70,7 +72,7 @@ class PageQueryParameters(RequestParameters): class PageMetaInfoLimitOffset(BaseModel): limit: PositiveInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE - total: NonNegativeInt + total: PageTotalCount offset: NonNegativeInt = 0 count: NonNegativeInt diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/catalog_rpc_server.py b/packages/pytest-simcore/src/pytest_simcore/helpers/catalog_rpc_server.py new file mode 100644 index 000000000000..aa6db574410a --- /dev/null +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/catalog_rpc_server.py @@ -0,0 +1,117 @@ +# pylint: disable=not-context-manager +# pylint: disable=protected-access +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable + + +from models_library.api_schemas_catalog.services import LatestServiceGet, ServiceGetV2 +from models_library.api_schemas_webserver.catalog import ( + CatalogServiceUpdate, +) +from models_library.products import ProductName +from models_library.rest_pagination import PageOffsetInt +from models_library.rpc_pagination import PageLimitInt, PageRpc +from models_library.services_history import ServiceRelease +from models_library.services_types import ServiceKey, ServiceVersion +from models_library.users import UserID +from pydantic import NonNegativeInt, TypeAdapter +from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient + + +class CatalogRpcSideEffects: + # pylint: disable=no-self-use + async def list_services_paginated( + self, + rpc_client: RabbitMQRPCClient, + *, + product_name: ProductName, + user_id: UserID, + limit: PageLimitInt, + offset: NonNegativeInt, + ): + assert rpc_client + assert product_name + assert user_id + + items = TypeAdapter(list[LatestServiceGet]).validate_python( + LatestServiceGet.model_json_schema()["examples"], + ) + total_count = len(items) + + return PageRpc[LatestServiceGet].create( + items[offset : offset + limit], + total=total_count, + limit=limit, + offset=offset, + ) + + async def get_service( + self, + rpc_client: RabbitMQRPCClient, + *, + product_name: ProductName, + user_id: UserID, + service_key: ServiceKey, + service_version: ServiceVersion, + ): + assert rpc_client + assert product_name + assert user_id + + got = ServiceGetV2.model_validate( + ServiceGetV2.model_json_schema()["examples"][0] + ) + got.version = service_version + got.key = service_key + + return got + + async def update_service( + self, + rpc_client: RabbitMQRPCClient, + *, + product_name: ProductName, + user_id: UserID, + service_key: ServiceKey, + service_version: ServiceVersion, + update: CatalogServiceUpdate, + ): + assert rpc_client + assert product_name + assert user_id + + got = ServiceGetV2.model_validate( + ServiceGetV2.model_json_schema()["examples"][0] + ) + got.version = service_version + got.key = service_key + return got.model_copy(update=update.model_dump(exclude_unset=True)) + + async def list_my_service_history_paginated( + self, + rpc_client: RabbitMQRPCClient, + *, + product_name: ProductName, + user_id: UserID, + service_key: ServiceKey, + offset: PageOffsetInt, + limit: PageLimitInt, + ) -> PageRpc[ServiceRelease]: + + assert rpc_client + assert product_name + assert user_id + assert service_key + + items = TypeAdapter(list[ServiceRelease]).validate_python( + ServiceRelease.model_json_schema()["examples"], + ) + total_count = len(items) + + return PageRpc[ServiceRelease].create( + items[offset : offset + limit], + total=total_count, + limit=limit, + offset=offset, + ) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/services.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/services.py index ca4f8876f597..3a5392cfdd83 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/services.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/services.py @@ -7,11 +7,15 @@ from models_library.api_schemas_catalog.services import ( LatestServiceGet, MyServiceGet, + PageRpcLatestServiceGet, + PageRpcServiceRelease, ServiceGetV2, + ServiceRelease, ServiceUpdateV2, ) from models_library.products import ProductName from models_library.rabbitmq_basic_types import RPCMethodName +from models_library.rest_pagination import PageOffsetInt from models_library.rpc_pagination import ( DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, PageLimitInt, @@ -19,7 +23,7 @@ ) from models_library.services_types import ServiceKey, ServiceVersion from models_library.users import UserID -from pydantic import NonNegativeInt, TypeAdapter, validate_call +from pydantic import TypeAdapter, validate_call from servicelib.logging_utils import log_decorator from servicelib.rabbitmq._constants import RPC_REQUEST_DEFAULT_TIMEOUT_S @@ -34,8 +38,8 @@ async def list_services_paginated( # pylint: disable=too-many-arguments product_name: ProductName, user_id: UserID, limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, - offset: NonNegativeInt = 0, -) -> PageRpc[LatestServiceGet]: + offset: PageOffsetInt = 0, +) -> PageRpcLatestServiceGet: """ Raises: ValidationError: on invalid arguments @@ -47,7 +51,7 @@ async def _call( product_name: ProductName, user_id: UserID, limit: PageLimitInt, - offset: NonNegativeInt, + offset: PageOffsetInt, ): return await rpc_client.request( CATALOG_RPC_NAMESPACE, @@ -235,3 +239,51 @@ async def _call( result = await _call(product_name=product_name, user_id=user_id, ids=ids) assert TypeAdapter(list[MyServiceGet]).validate_python(result) is not None # nosec return cast(list[MyServiceGet], result) + + +async def list_my_service_history_paginated( # pylint: disable=too-many-arguments + rpc_client: RabbitMQRPCClient, + *, + product_name: ProductName, + user_id: UserID, + service_key: ServiceKey, + limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, + offset: PageOffsetInt = 0, +) -> PageRpcServiceRelease: + """ + Raises: + ValidationError: on invalid arguments + """ + + @validate_call() + async def _call( + product_name: ProductName, + user_id: UserID, + service_key: ServiceKey, + limit: PageLimitInt, + offset: PageOffsetInt, + ): + return await rpc_client.request( + CATALOG_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python( + "list_my_service_history_paginated" + ), + product_name=product_name, + user_id=user_id, + service_key=service_key, + limit=limit, + offset=offset, + ) + + result = await _call( + product_name=product_name, + user_id=user_id, + service_key=service_key, + limit=limit, + offset=offset, + ) + + assert ( # nosec + TypeAdapter(PageRpcServiceRelease).validate_python(result) is not None + ) + return cast(PageRpc[ServiceRelease], result) diff --git a/services/api-server/src/simcore_service_api_server/services_http/catalog.py b/services/api-server/src/simcore_service_api_server/services_http/catalog.py index f20264087e10..b6607342724c 100644 --- a/services/api-server/src/simcore_service_api_server/services_http/catalog.py +++ b/services/api-server/src/simcore_service_api_server/services_http/catalog.py @@ -9,7 +9,9 @@ from fastapi import FastAPI, status from models_library.emails import LowerCaseEmailStr +from models_library.products import ProductName from models_library.services import ServiceMetaDataPublished, ServiceType +from models_library.users import UserID from pydantic import ConfigDict, TypeAdapter, ValidationError from settings_library.catalog import CatalogSettings from settings_library.tracing import TracingSettings @@ -70,9 +72,9 @@ def to_solver(self) -> Solver: _exception_mapper = partial(service_exception_mapper, service_name="Catalog") -TruncatedCatalogServiceOutAdapter: Final[ - TypeAdapter[TruncatedCatalogServiceOut] -] = TypeAdapter(TruncatedCatalogServiceOut) +TruncatedCatalogServiceOutAdapter: Final[TypeAdapter[TruncatedCatalogServiceOut]] = ( + TypeAdapter(TruncatedCatalogServiceOut) +) TruncatedCatalogServiceOutListAdapter: Final[ TypeAdapter[list[TruncatedCatalogServiceOut]] ] = TypeAdapter(list[TruncatedCatalogServiceOut]) @@ -97,8 +99,8 @@ class CatalogApi(BaseServiceClientApi): async def list_solvers( self, *, - user_id: int, - product_name: str, + user_id: UserID, + product_name: ProductName, predicate: Callable[[Solver], bool] | None = None, ) -> list[Solver]: @@ -140,7 +142,12 @@ async def list_solvers( http_status_map={status.HTTP_404_NOT_FOUND: SolverOrStudyNotFoundError} ) async def get_service( - self, *, user_id: int, name: SolverKeyId, version: VersionStr, product_name: str + self, + *, + user_id: UserID, + name: SolverKeyId, + version: VersionStr, + product_name: ProductName, ) -> Solver: assert version != LATEST_VERSION # nosec @@ -171,8 +178,13 @@ async def get_service( http_status_map={status.HTTP_404_NOT_FOUND: SolverOrStudyNotFoundError} ) async def get_service_ports( - self, *, user_id: int, name: SolverKeyId, version: VersionStr, product_name: str - ): + self, + *, + user_id: UserID, + name: SolverKeyId, + version: VersionStr, + product_name: ProductName, + ) -> list[SolverPort]: assert version != LATEST_VERSION # nosec @@ -190,7 +202,7 @@ async def get_service_ports( return TypeAdapter(list[SolverPort]).validate_python(response.json()) async def list_latest_releases( - self, *, user_id: int, product_name: str + self, *, user_id: UserID, product_name: ProductName ) -> list[Solver]: solvers: list[Solver] = await self.list_solvers( user_id=user_id, product_name=product_name @@ -205,7 +217,7 @@ async def list_latest_releases( return list(latest_releases.values()) async def list_solver_releases( - self, *, user_id: int, solver_key: SolverKeyId, product_name: str + self, *, user_id: UserID, solver_key: SolverKeyId, product_name: ProductName ) -> list[Solver]: def _this_solver(solver: Solver) -> bool: return solver.id == solver_key @@ -216,7 +228,7 @@ def _this_solver(solver: Solver) -> bool: return releases async def get_latest_release( - self, *, user_id: int, solver_key: SolverKeyId, product_name: str + self, *, user_id: UserID, solver_key: SolverKeyId, product_name: ProductName ) -> Solver: releases = await self.list_solver_releases( user_id=user_id, solver_key=solver_key, product_name=product_name diff --git a/services/api-server/src/simcore_service_api_server/services_rpc/catalog.py b/services/api-server/src/simcore_service_api_server/services_rpc/catalog.py new file mode 100644 index 000000000000..0a9fde924dee --- /dev/null +++ b/services/api-server/src/simcore_service_api_server/services_rpc/catalog.py @@ -0,0 +1,89 @@ +from dataclasses import dataclass + +from models_library.api_schemas_catalog.services import LatestServiceGet, ServiceGetV2 +from models_library.products import ProductName +from models_library.rest_pagination import ( + DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, + PageLimitInt, + PageMetaInfoLimitOffset, + PageOffsetInt, +) +from models_library.services_history import ServiceRelease +from models_library.services_types import ServiceKey, ServiceVersion +from models_library.users import UserID +from servicelib.fastapi.app_state import SingletonInAppStateMixin +from servicelib.rabbitmq import RabbitMQRPCClient +from servicelib.rabbitmq.rpc_interfaces.catalog import services as catalog_rpc + + +@dataclass +class CatalogService(SingletonInAppStateMixin): + app_state_name = "CatalogService" + _client: RabbitMQRPCClient + + async def list_latest_releases( + self, + *, + product_name: ProductName, + user_id: UserID, + offset: PageOffsetInt = 0, + limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, + ) -> tuple[list[LatestServiceGet], PageMetaInfoLimitOffset]: + + page = await catalog_rpc.list_services_paginated( + self._client, + product_name=product_name, + user_id=user_id, + offset=offset, + limit=limit, + ) + meta = PageMetaInfoLimitOffset( + limit=page.meta.limit, + offset=page.meta.offset, + total=page.meta.total, + count=page.meta.count, + ) + return page.data, meta + + async def list_release_history( + self, + *, + product_name: ProductName, + user_id: UserID, + service_key: ServiceKey, + offset: PageOffsetInt = 0, + limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, + ) -> tuple[list[ServiceRelease], PageMetaInfoLimitOffset]: + + page = await catalog_rpc.list_my_service_history_paginated( + self._client, + product_name=product_name, + user_id=user_id, + service_key=service_key, + offset=offset, + limit=limit, + ) + meta = PageMetaInfoLimitOffset( + limit=page.meta.limit, + offset=page.meta.offset, + total=page.meta.total, + count=page.meta.count, + ) + return page.data, meta + + async def get( + self, + *, + product_name: ProductName, + user_id: UserID, + service_key: ServiceKey, + service_version: ServiceVersion, + ) -> ServiceGetV2: + + return await catalog_rpc.get_service( + self._client, + product_name=product_name, + user_id=user_id, + service_key=service_key, + service_version=service_version, + ) diff --git a/services/api-server/tests/unit/test_services_catalog.py b/services/api-server/tests/unit/test_services_catalog.py new file mode 100644 index 000000000000..99c5dc58b4cd --- /dev/null +++ b/services/api-server/tests/unit/test_services_catalog.py @@ -0,0 +1,118 @@ +# pylint: disable=protected-access +# pylint: disable=redefined-outer-name +# pylint: disable=too-many-arguments +# pylint: disable=unused-argument +# pylint: disable=unused-variable + +import pytest +from fastapi import FastAPI +from models_library.api_schemas_catalog.services import LatestServiceGet, ServiceGetV2 +from models_library.products import ProductName +from models_library.services_history import ServiceRelease +from models_library.users import UserID +from pydantic import HttpUrl +from pytest_mock import MockerFixture, MockType +from pytest_simcore.helpers.catalog_rpc_server import CatalogRpcSideEffects +from simcore_service_api_server.models.schemas.solvers import Solver +from simcore_service_api_server.services_rpc.catalog import CatalogService, catalog_rpc + + +@pytest.fixture +def product_name() -> ProductName: + return "osparc" + + +@pytest.fixture +def mocked_rpc_catalog_service_api(mocker: MockerFixture) -> dict[str, MockType]: + + side_effects = CatalogRpcSideEffects() + + return { + "list_services_paginated": mocker.patch.object( + catalog_rpc, + "list_services_paginated", + autospec=True, + side_effect=side_effects.list_services_paginated, + ), + "get_service": mocker.patch.object( + catalog_rpc, + "get_service", + autospec=True, + side_effect=side_effects.get_service, + ), + "update_service": mocker.patch.object( + catalog_rpc, + "update_service", + autospec=True, + side_effect=side_effects.update_service, + ), + "list_my_service_history_paginated": mocker.patch.object( + catalog_rpc, + "list_my_service_history_paginated", + autospec=True, + side_effect=side_effects.list_my_service_history_paginated, + ), + } + + +def to_solver( + service: LatestServiceGet | ServiceGetV2, href_self: HttpUrl | None = None +) -> Solver: + # NOTE: this is an adapter around models on CatalogService interface + return Solver( + id=service.key, + version=service.version, + title=service.name, + maintainer=service.owner or service.contact or "UNKNOWN", + url=href_self, + description=service.description, + ) + + +async def test_catalog_service_read_solvers( + product_name: ProductName, + user_id: UserID, + mocker: MockerFixture, + mocked_rpc_catalog_service_api: dict[str, MockType], +): + catalog_service = CatalogService(_client=mocker.MagicMock()) + catalog_service.set_to_app_state(app=FastAPI()) + + # Step 1: List latest releases in a page + latest_releases, meta = await catalog_service.list_latest_releases( + product_name=product_name, user_id=user_id + ) + solver_releases_page = [to_solver(srv) for srv in latest_releases] + + assert solver_releases_page, "Releases page should not be empty" + assert meta.offset == 0 + + # Step 2: Select one release and list solver releases + selected_solver = solver_releases_page[0] + releases, meta = await catalog_service.list_release_history( + product_name=product_name, + user_id=user_id, + service_key=selected_solver.id, + ) + assert releases, "Solver releases should not be empty" + assert meta.offset == 0 + + # Step 3: Take the latest solver release and get solver details + oldest_release: ServiceRelease = releases[-1] + + service: ServiceGetV2 = await catalog_service.get( + product_name=product_name, + user_id=user_id, + service_key=selected_solver.id, + service_version=oldest_release.version, + ) + solver = to_solver(service) + assert solver.id == selected_solver.id + assert solver.version == oldest_release.version + + # checks calls to rpc + mocked_rpc_catalog_service_api["list_services_paginated"].assert_called_once() + mocked_rpc_catalog_service_api[ + "list_my_service_history_paginated" + ].assert_called_once() + mocked_rpc_catalog_service_api["get_service"].assert_called_once() diff --git a/services/catalog/VERSION b/services/catalog/VERSION index 8adc70fdd9d6..c18d72be3037 100644 --- a/services/catalog/VERSION +++ b/services/catalog/VERSION @@ -1 +1 @@ -0.8.0 \ No newline at end of file +0.8.1 \ No newline at end of file diff --git a/services/catalog/openapi.json b/services/catalog/openapi.json index 9fbd49878986..0173c51913d0 100644 --- a/services/catalog/openapi.json +++ b/services/catalog/openapi.json @@ -3,7 +3,7 @@ "info": { "title": "simcore-service-catalog", "description": "Manages and maintains a catalog of all published components (e.g. macro-algorithms, scripts, etc)", - "version": "0.8.0" + "version": "0.8.1" }, "paths": { "/": { @@ -473,7 +473,9 @@ "services" ], "summary": "List Services", + "description": "Use instead rpc._service.list_services_paginated -> PageRpcServicesGetV2", "operationId": "list_services_v0_services_get", + "deprecated": true, "parameters": [ { "name": "user_id", @@ -540,7 +542,9 @@ "services" ], "summary": "Get Service", + "description": "Use instead rpc._service.get_service -> ServiceGetV2", "operationId": "get_service_v0_services__service_key___service_version__get", + "deprecated": true, "parameters": [ { "name": "service_key", diff --git a/services/catalog/setup.cfg b/services/catalog/setup.cfg index 031198b2fdd1..ac92051993c2 100644 --- a/services/catalog/setup.cfg +++ b/services/catalog/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.8.0 +current_version = 0.8.1 commit = True message = services/catalog version: {current_version} → {new_version} tag = False diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services.py b/services/catalog/src/simcore_service_catalog/api/rest/_services.py index 4f77c0f8a492..c97a5545d8c5 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services.py @@ -64,21 +64,23 @@ def _build_cache_key(fct, *_, **kwargs): return f"{fct.__name__}_{kwargs['user_id']}_{kwargs['x_simcore_products_name']}_{kwargs['details']}" -# -# Routes -# - router = APIRouter() -# NOTE: this call is pretty expensive and can be called several times -# (when e2e runs or by the webserver when listing projects) therefore -# a cache is setup here -@router.get("", response_model=list[ServiceGet], **RESPONSE_MODEL_POLICY) +@router.get( + "", + response_model=list[ServiceGet], + **RESPONSE_MODEL_POLICY, + deprecated=True, + description="Use instead rpc._service.list_services_paginated -> PageRpcServicesGetV2", +) @cancel_on_disconnect @cached( ttl=LIST_SERVICES_CACHING_TTL, key_builder=_build_cache_key, + # NOTE: this call is pretty expensive and can be called several times + # (when e2e runs or by the webserver when listing projects) therefore + # a cache is setup here ) async def list_services( request: Request, # pylint:disable=unused-argument @@ -192,6 +194,8 @@ async def cached_registry_services() -> dict[str, Any]: "/{service_key:path}/{service_version}", response_model=ServiceGet, **RESPONSE_MODEL_POLICY, + deprecated=True, + description="Use instead rpc._service.get_service -> ServiceGetV2", ) async def get_service( user_id: int, @@ -226,12 +230,12 @@ async def get_service( ) if service_in_db: # we have full access, let's add the access to the output - service_access_rights: list[ - ServiceAccessRightsAtDB - ] = await services_repo.get_service_access_rights( - service_in_manifest.key, - service_in_manifest.version, - product_name=x_simcore_products_name, + service_access_rights: list[ServiceAccessRightsAtDB] = ( + await services_repo.get_service_access_rights( + service_in_manifest.key, + service_in_manifest.version, + product_name=x_simcore_products_name, + ) ) service_data["access_rights"] = { rights.gid: rights for rights in service_access_rights diff --git a/services/catalog/src/simcore_service_catalog/api/rpc/_services.py b/services/catalog/src/simcore_service_catalog/api/rpc/_services.py index 7568b18e351b..874ac9bb45fa 100644 --- a/services/catalog/src/simcore_service_catalog/api/rpc/_services.py +++ b/services/catalog/src/simcore_service_catalog/api/rpc/_services.py @@ -5,15 +5,17 @@ from fastapi import FastAPI from models_library.api_schemas_catalog.services import ( MyServiceGet, - PageRpcServicesGetV2, + PageRpcLatestServiceGet, + PageRpcServiceRelease, ServiceGetV2, ServiceUpdateV2, ) from models_library.products import ProductName +from models_library.rest_pagination import PageOffsetInt from models_library.rpc_pagination import DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, PageLimitInt from models_library.services_types import ServiceKey, ServiceVersion from models_library.users import UserID -from pydantic import NonNegativeInt, ValidationError, validate_call +from pydantic import ValidationError, validate_call from pyinstrument import Profiler from servicelib.logging_utils import log_decorator from servicelib.rabbitmq import RPCRouter @@ -62,8 +64,8 @@ async def list_services_paginated( product_name: ProductName, user_id: UserID, limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, - offset: NonNegativeInt = 0, -) -> PageRpcServicesGetV2: + offset: PageOffsetInt = 0, +) -> PageRpcLatestServiceGet: assert app.state.engine # nosec total_count, items = await services_api.list_latest_services( @@ -79,8 +81,8 @@ async def list_services_paginated( assert len(items) <= limit # nosec return cast( - PageRpcServicesGetV2, - PageRpcServicesGetV2.create( + PageRpcLatestServiceGet, + PageRpcLatestServiceGet.create( items, total=total_count, limit=limit, @@ -219,3 +221,40 @@ async def batch_get_my_services( assert [(sv.key, sv.release.version) for sv in services] == ids # nosec return services + + +@router.expose(reraise_if_error_type=(ValidationError,)) +@log_decorator(_logger, level=logging.DEBUG) +@validate_call(config={"arbitrary_types_allowed": True}) +async def list_my_service_history_paginated( + app: FastAPI, + *, + product_name: ProductName, + user_id: UserID, + service_key: ServiceKey, + limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, + offset: PageOffsetInt = 0, +) -> PageRpcServiceRelease: + assert app.state.engine # nosec + + total_count, items = await services_api.list_my_service_release_history( + repo=ServicesRepository(app.state.engine), + product_name=product_name, + user_id=user_id, + service_key=service_key, + limit=limit, + offset=offset, + ) + + assert len(items) <= total_count # nosec + assert len(items) <= limit # nosec + + return cast( + PageRpcServiceRelease, + PageRpcServiceRelease.create( + items, + total=total_count, + limit=limit, + offset=offset, + ), + ) diff --git a/services/catalog/src/simcore_service_catalog/db/repositories/_services_sql.py b/services/catalog/src/simcore_service_catalog/db/repositories/_services_sql.py index d83dffbdb220..935a38349761 100644 --- a/services/catalog/src/simcore_service_catalog/db/repositories/_services_sql.py +++ b/services/catalog/src/simcore_service_catalog/db/repositories/_services_sql.py @@ -66,7 +66,7 @@ def list_services_stmt( return stmt -def _version(column_or_value): +def by_version(column_or_value): # converts version value string to array[integer] that can be compared # i.e. '1.2.3' -> [1, 2, 3] return sa.func.string_to_array(column_or_value, ".").cast(ARRAY(INTEGER)) @@ -111,7 +111,7 @@ def _has_access_rights( ) -def total_count_stmt( +def latest_services_total_count_stmt( *, product_name: ProductName, user_id: UserID, @@ -165,7 +165,7 @@ def list_latest_services_stmt( .where(access_rights) .order_by( services_meta_data.c.key, - sa.desc(_version(services_meta_data.c.version)), # latest first + sa.desc(by_version(services_meta_data.c.version)), # latest first ) .distinct(services_meta_data.c.key) # get only first .limit(limit) @@ -313,7 +313,6 @@ def get_service_history_stmt( access_rights: sa.sql.ClauseElement, service_key: ServiceKey, ): - _sq = ( sa.select( services_meta_data.c.key, @@ -352,7 +351,7 @@ def get_service_history_stmt( history_subquery = ( sa.select(_sq) .order_by( - sa.desc(_version(_sq.c.version)), # latest version first + sa.desc(by_version(_sq.c.version)), # latest version first ) .alias("history_subquery") ) diff --git a/services/catalog/src/simcore_service_catalog/db/repositories/services.py b/services/catalog/src/simcore_service_catalog/db/repositories/services.py index 509a23d68d69..06a166659981 100644 --- a/services/catalog/src/simcore_service_catalog/db/repositories/services.py +++ b/services/catalog/src/simcore_service_catalog/db/repositories/services.py @@ -16,7 +16,10 @@ from models_library.users import UserID from psycopg2.errors import ForeignKeyViolation from pydantic import PositiveInt, TypeAdapter, ValidationError +from simcore_postgres_database.utils import as_postgres_sql_query_str +from simcore_postgres_database.utils_repos import pass_or_acquire_connection from simcore_postgres_database.utils_services import create_select_latest_services_query +from sqlalchemy import sql from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.sql import and_, or_ from sqlalchemy.sql.expression import tuple_ @@ -30,17 +33,24 @@ ServiceWithHistoryDBGet, ) from ...models.services_specifications import ServiceSpecificationsAtDB -from ..tables import services_access_rights, services_meta_data, services_specifications +from ..tables import ( + services_access_rights, + services_compatibility, + services_meta_data, + services_specifications, + user_to_groups, +) from ._base import BaseRepository from ._services_sql import ( SERVICES_META_DATA_COLS, AccessRightsClauses, + by_version, can_get_service_stmt, get_service_history_stmt, get_service_stmt, + latest_services_total_count_stmt, list_latest_services_stmt, list_services_stmt, - total_count_stmt, ) _logger = logging.getLogger(__name__) @@ -381,7 +391,7 @@ async def list_latest_services( ) -> tuple[PositiveInt, list[ServiceWithHistoryDBGet]]: # get page - stmt_total = total_count_stmt( + stmt_total = latest_services_total_count_stmt( product_name=product_name, user_id=user_id, access_rights=AccessRightsClauses.can_read, @@ -438,8 +448,10 @@ async def get_service_history( user_id: UserID, # get args key: ServiceKey, - ) -> list[ReleaseDBGet] | None: - + ) -> list[ReleaseDBGet]: + """ + DEPRECATED: use get_service_history_page instead! + """ stmt_history = get_service_history_stmt( product_name=product_name, user_id=user_id, @@ -451,10 +463,93 @@ async def get_service_history( row = result.one_or_none() return ( - TypeAdapter(list[ReleaseDBGet]).validate_python(row.history) - if row - else None + TypeAdapter(list[ReleaseDBGet]).validate_python(row.history) if row else [] + ) + + async def get_service_history_page( + self, + *, + # access-rights + product_name: ProductName, + user_id: UserID, + # get args + key: ServiceKey, + # list args: pagination + limit: int | None = None, + offset: int | None = None, + ) -> tuple[PositiveInt, list[ReleaseDBGet]]: + + base_subquery = ( + # Search on service (key, *) for (product_name, user_id w/ access) + sql.select( + services_meta_data.c.key, + services_meta_data.c.version, + ) + .select_from( + services_meta_data.join( + services_access_rights, + (services_meta_data.c.key == services_access_rights.c.key) + & ( + services_meta_data.c.version == services_access_rights.c.version + ), + ).join( + user_to_groups, + (user_to_groups.c.gid == services_access_rights.c.gid), + ) + ) + .where( + (services_meta_data.c.key == key) + & (services_access_rights.c.product_name == product_name) + & (user_to_groups.c.uid == user_id) + & AccessRightsClauses.can_read + ) + ).subquery() + + # Query to count the TOTAL number of rows + count_query = sql.select(sql.func.count()).select_from(base_subquery) + _logger.debug("count_query=\n%s", as_postgres_sql_query_str(count_query)) + + # Query to retrieve page with additional columns, ordering, offset, and limit + page_query = ( + sql.select( + services_meta_data.c.key, + services_meta_data.c.version, + services_meta_data.c.version_display, + services_meta_data.c.deprecated, + services_meta_data.c.created, + # CompatiblePolicyDict | None + services_compatibility.c.custom_policy.label("compatibility_policy"), + ) + .select_from( + # NOTE: these joins are avoided in count_query + base_subquery.join( + services_meta_data, + (base_subquery.c.key == services_meta_data.c.key) + & (base_subquery.c.version == services_meta_data.c.version), + ).outerjoin( + services_compatibility, + (services_meta_data.c.key == services_compatibility.c.key) + & ( + services_meta_data.c.version == services_compatibility.c.version + ), + ) + ) + .order_by(sql.desc(by_version(services_meta_data.c.version))) + .offset(offset) + .limit(limit) ) + _logger.debug("page_query=\n%s", as_postgres_sql_query_str(page_query)) + + async with pass_or_acquire_connection(self.db_engine) as conn: + total_count: PositiveInt = await conn.scalar(count_query) or 0 + + result = await conn.stream(page_query) + items: list[ReleaseDBGet] = [ + ReleaseDBGet.model_validate(row, from_attributes=True) + async for row in result + ] + + return total_count, items # Service Access Rights ---- diff --git a/services/catalog/src/simcore_service_catalog/services/services_api.py b/services/catalog/src/simcore_service_catalog/services/services_api.py index 843a91fc713c..43a70f45bf27 100644 --- a/services/catalog/src/simcore_service_catalog/services/services_api.py +++ b/services/catalog/src/simcore_service_catalog/services/services_api.py @@ -9,7 +9,7 @@ ) from models_library.groups import GroupID from models_library.products import ProductName -from models_library.rest_pagination import PageLimitInt +from models_library.rest_pagination import PageLimitInt, PageTotalCount from models_library.services_access import ServiceGroupAccessRightsV2 from models_library.services_history import Compatibility, ServiceRelease from models_library.services_metadata_published import ServiceMetaDataPublished @@ -121,11 +121,12 @@ def _to_get_schema( async def list_latest_services( repo: ServicesRepository, director_api: DirectorApi, + *, product_name: ProductName, user_id: UserID, limit: PageLimitInt | None, offset: NonNegativeInt = 0, -) -> tuple[NonNegativeInt, list[LatestServiceGet]]: +) -> tuple[PageTotalCount, list[LatestServiceGet]]: # defines the order total_count, services = await repo.list_latest_services( @@ -464,3 +465,48 @@ async def batch_get_my_services( ) return my_services + + +async def list_my_service_release_history( + repo: ServicesRepository, + *, + # access-rights + product_name: ProductName, + user_id: UserID, + # target service + service_key: ServiceKey, + # pagination + limit: PageLimitInt | None = None, + offset: NonNegativeInt | None = None, + # options + include_compatibility: bool = False, +) -> tuple[PageTotalCount, list[ServiceRelease]]: + + total_count, history = await repo.get_service_history_page( + # NOTE: that the service history might be different for each user + # since access-rights are defined on a k:v basis + product_name=product_name, + user_id=user_id, + key=service_key, + limit=limit, + offset=offset, + ) + + compatibility_map: dict[ServiceVersion, Compatibility | None] = {} + if include_compatibility: + msg = "This operation is heavy and for the moment is not necessary" + raise NotImplementedError(msg) + + items = [ + # domain -> domain + ServiceRelease.model_construct( + version=h.version, + version_display=h.version_display, + released=h.created, + retired=h.deprecated, + compatibility=compatibility_map.get(h.version), + ) + for h in history + ] + + return total_count, items diff --git a/services/catalog/tests/unit/test_db_repositories_services_sql.py b/services/catalog/tests/unit/test_db_repositories_services_sql.py index 09891d5fe5e1..27b8db9e770d 100644 --- a/services/catalog/tests/unit/test_db_repositories_services_sql.py +++ b/services/catalog/tests/unit/test_db_repositories_services_sql.py @@ -9,8 +9,8 @@ can_get_service_stmt, get_service_history_stmt, get_service_stmt, + latest_services_total_count_stmt, list_latest_services_stmt, - total_count_stmt, ) @@ -70,7 +70,7 @@ def _check(func_smt, **kwargs): ) _check( - total_count_stmt, + latest_services_total_count_stmt, product_name=product_name, user_id=user_id, access_rights=AccessRightsClauses.can_read, diff --git a/services/catalog/tests/unit/with_dbs/test_api_rpc.py b/services/catalog/tests/unit/with_dbs/test_api_rpc.py index 830650729baa..b0faa1734d69 100644 --- a/services/catalog/tests/unit/with_dbs/test_api_rpc.py +++ b/services/catalog/tests/unit/with_dbs/test_api_rpc.py @@ -13,8 +13,10 @@ from fastapi import FastAPI from models_library.products import ProductName from models_library.rest_pagination import MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE +from models_library.services_history import ServiceRelease from models_library.services_types import ServiceKey, ServiceVersion from models_library.users import UserID +from packaging import version from pydantic import ValidationError from pytest_simcore.helpers.faker_factories import random_icon_url from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict @@ -29,6 +31,7 @@ batch_get_my_services, check_for_service, get_service, + list_my_service_history_paginated, list_services_paginated, update_service, ) @@ -477,3 +480,76 @@ async def test_rpc_batch_get_my_services( "write": True, } assert my_services[1].owner == user["primary_gid"] + assert my_services[1].key == other_service_key + assert my_services[1].release.version == other_service_version + + +async def test_rpc_get_my_service_history( + background_sync_task_mocked: None, + mocked_director_service_api: MockRouter, + rpc_client: RabbitMQRPCClient, + product_name: ProductName, + user_id: UserID, + app: FastAPI, + create_fake_service_data: Callable, + services_db_tables_injector: Callable, +): + assert app + + service_key = "simcore/services/comp/test-service-release-history" + service_version_1 = "1.0.0" + service_version_2 = "1.1.0" + + assert version.Version(service_version_1) < version.Version(service_version_2) + + # Inject fake service releases for the target service + fake_releases = [ + create_fake_service_data( + service_key, + srv_version, + team_access=None, + everyone_access=None, + product=product_name, + ) + for srv_version in (service_version_1, service_version_2) + ] + + # Inject unrelated fake service releases + unrelated_service_key_1 = "simcore/services/comp/unrelated-service-1" + unrelated_service_key_2 = "simcore/services/comp/unrelated-service-2" + unrelated_releases = [ + *[ + create_fake_service_data( + unrelated_service_key_1, + srv_version, + team_access=None, + everyone_access=None, + product=product_name, + ) + for srv_version in (service_version_1, service_version_2) + ], + create_fake_service_data( + unrelated_service_key_2, + "2.0.0", + team_access=None, + everyone_access=None, + product=product_name, + ), + ] + + await services_db_tables_injector(fake_releases + unrelated_releases) + + # Call the RPC function + page = await list_my_service_history_paginated( + rpc_client, + product_name=product_name, + user_id=user_id, + service_key=service_key, + ) + release_history: list[ServiceRelease] = page.data + + # Validate the response + assert isinstance(release_history, list) + assert len(release_history) == 2 + assert release_history[0].version == service_version_2, "expected newest first" + assert release_history[1].version == service_version_1 diff --git a/services/catalog/tests/unit/with_dbs/test_db_repositories.py b/services/catalog/tests/unit/with_dbs/test_db_repositories.py index e8990527c047..8618a67c25e4 100644 --- a/services/catalog/tests/unit/with_dbs/test_db_repositories.py +++ b/services/catalog/tests/unit/with_dbs/test_db_repositories.py @@ -3,6 +3,7 @@ # pylint: disable=unused-variable # pylint: disable=too-many-arguments +import random from collections import Counter from collections.abc import Callable from dataclasses import dataclass, field @@ -476,3 +477,75 @@ async def test_can_get_service( key=service_key, version=service_version, ) + + +async def test_get_service_history_page( + target_product: ProductName, + create_fake_service_data: Callable, + services_db_tables_injector: Callable, + services_repo: ServicesRepository, + user_id: UserID, +): + # inject services with multiple versions + service_key = "simcore/services/dynamic/test-service" + num_versions = 10 + + release_versions = [ + f"{random.randint(0, 2)}.{random.randint(0, 9)}.{random.randint(0, 9)}" # noqa: S311 + for _ in range(num_versions) + ] + await services_db_tables_injector( + [ + create_fake_service_data( + service_key, + service_version, + team_access=None, + everyone_access=None, + product=target_product, + ) + for service_version in release_versions + ] + ) + # sorted AFTER injecting + release_versions = sorted(release_versions, key=version.Version, reverse=True) + assert version.Version(release_versions[0]) > version.Version(release_versions[-1]) + + # fetch full history using get_service_history_page + total_count, history = await services_repo.get_service_history_page( + product_name=target_product, + user_id=user_id, + key=service_key, + ) + assert total_count == num_versions + assert len(history) == num_versions + assert [release.version for release in history] == release_versions + + # fetch full history using deprecated get_service_history + deprecated_history = await services_repo.get_service_history( + product_name=target_product, + user_id=user_id, + key=service_key, + ) + assert len(deprecated_history) == len(history) + assert [release.version for release in deprecated_history] == [ + release.version for release in history + ] + + # fetch paginated history + limit = 3 + offset = 2 + total_count, paginated_history = await services_repo.get_service_history_page( + product_name=target_product, + user_id=user_id, + key=service_key, + limit=limit, + offset=offset, + ) + assert total_count == num_versions + assert len(paginated_history) == limit + assert [release.version for release in paginated_history] == release_versions[ + offset : offset + limit + ] + + # compare paginated results with the corresponding slice of the full history + assert paginated_history == history[offset : offset + limit] diff --git a/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py b/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py index e005192edaea..881282abe6fe 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py +++ b/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py @@ -4,27 +4,22 @@ import re import urllib.parse -from unittest.mock import MagicMock import pytest -from aiohttp import web from aiohttp.test_utils import TestClient from aioresponses import aioresponses as AioResponsesMock from faker import Faker -from models_library.api_schemas_catalog.services import LatestServiceGet, ServiceGetV2 +from models_library.api_schemas_catalog.services import ServiceGetV2 from models_library.api_schemas_webserver.catalog import ( CatalogServiceGet, CatalogServiceUpdate, ) -from models_library.products import ProductName from models_library.rest_pagination import Page -from models_library.rpc_pagination import PageLimitInt, PageRpc -from models_library.services_types import ServiceKey, ServiceVersion -from models_library.users import UserID from models_library.utils.fastapi_encoders import jsonable_encoder -from pydantic import NonNegativeInt, TypeAdapter -from pytest_mock import MockerFixture +from pydantic import TypeAdapter +from pytest_mock import MockerFixture, MockType from pytest_simcore.helpers.assert_checks import assert_status +from pytest_simcore.helpers.catalog_rpc_server import CatalogRpcSideEffects from pytest_simcore.helpers.faker_factories import random_icon_url from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict @@ -50,86 +45,25 @@ def app_environment( @pytest.fixture -def mocked_rpc_catalog_service_api(mocker: MockerFixture) -> dict[str, MagicMock]: - async def _list( - app: web.Application, - *, - product_name: ProductName, - user_id: UserID, - limit: PageLimitInt, - offset: NonNegativeInt, - ): - assert app - assert product_name - assert user_id - - items = TypeAdapter(list[LatestServiceGet]).validate_python( - LatestServiceGet.model_json_schema()["examples"], - ) - total_count = len(items) - - return PageRpc[LatestServiceGet].create( - items[offset : offset + limit], - total=total_count, - limit=limit, - offset=offset, - ) +def mocked_rpc_catalog_service_api(mocker: MockerFixture) -> dict[str, MockType]: - async def _get( - app: web.Application, - *, - product_name: ProductName, - user_id: UserID, - service_key: ServiceKey, - service_version: ServiceVersion, - ): - assert app - assert product_name - assert user_id - - got = ServiceGetV2.model_validate( - ServiceGetV2.model_json_schema()["examples"][0] - ) - got.version = service_version - got.key = service_key - - return got - - async def _update( - app: web.Application, - *, - product_name: ProductName, - user_id: UserID, - service_key: ServiceKey, - service_version: ServiceVersion, - update: CatalogServiceUpdate, - ): - assert app - assert product_name - assert user_id - - got = ServiceGetV2.model_validate( - ServiceGetV2.model_json_schema()["examples"][0] - ) - got.version = service_version - got.key = service_key - return got.model_copy(update=update.model_dump(exclude_unset=True)) + side_effects = CatalogRpcSideEffects() return { "list_services_paginated": mocker.patch( "simcore_service_webserver.catalog._service.catalog_rpc.list_services_paginated", autospec=True, - side_effect=_list, + side_effect=side_effects.list_services_paginated, ), "get_service": mocker.patch( "simcore_service_webserver.catalog._service.catalog_rpc.get_service", autospec=True, - side_effect=_get, + side_effect=side_effects.get_service, ), "update_service": mocker.patch( "simcore_service_webserver.catalog._service.catalog_rpc.update_service", autospec=True, - side_effect=_update, + side_effect=side_effects.update_service, ), } @@ -141,7 +75,7 @@ async def _update( async def test_list_services_latest( client: TestClient, logged_user: UserInfoDict, - mocked_rpc_catalog_service_api: dict[str, MagicMock], + mocked_rpc_catalog_service_api: dict[str, MockType], ): assert client.app assert client.app.router @@ -359,7 +293,7 @@ async def test_get_compatible_outputs_given_target_inptuts( async def test_get_and_patch_service( client: TestClient, logged_user: UserInfoDict, - mocked_rpc_catalog_service_api: dict[str, MagicMock], + mocked_rpc_catalog_service_api: dict[str, MockType], faker: Faker, ): assert client.app @@ -426,7 +360,7 @@ async def test_get_and_patch_service( async def test_tags_in_services( client: TestClient, logged_user: UserInfoDict, - mocked_rpc_catalog_service_api: dict[str, MagicMock], + mocked_rpc_catalog_service_api: dict[str, MockType], ): assert client.app assert client.app.router