diff --git a/.coveragerc b/.coveragerc index ebf1465b0fb7..c5ea6a88430d 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,6 +3,7 @@ branch = True omit = */tests/* */generated_code/* + */_original_fastapi_encoders.py parallel = True [report] diff --git a/packages/models-library/src/models_library/api_schemas_catalog/services.py b/packages/models-library/src/models_library/api_schemas_catalog/services.py index 363b74c84768..24b2a2bbb4ed 100644 --- a/packages/models-library/src/models_library/api_schemas_catalog/services.py +++ b/packages/models-library/src/models_library/api_schemas_catalog/services.py @@ -9,6 +9,7 @@ from ..boot_options import BootOptions from ..emails import LowerCaseEmailStr from ..groups import GroupID +from ..rest_filters import Filters from ..services_access import ServiceAccessRights, ServiceGroupAccessRightsV2 from ..services_authoring import Author from ..services_enums import ServiceType @@ -376,4 +377,13 @@ class MyServiceGet(CatalogOutputSchema): my_access_rights: ServiceGroupAccessRightsV2 +class ServiceListFilters(Filters): + service_type: Annotated[ + ServiceType | None, + Field( + description="Filter only services of a given type. If None, then all types are returned" + ), + ] = None + + __all__: tuple[str, ...] = ("ServiceRelease",) diff --git a/packages/models-library/src/models_library/function_services_catalog/_key_labels.py b/packages/models-library/src/models_library/function_services_catalog/_key_labels.py index 408515bacea1..599b2998d2ee 100644 --- a/packages/models-library/src/models_library/function_services_catalog/_key_labels.py +++ b/packages/models-library/src/models_library/function_services_catalog/_key_labels.py @@ -1,10 +1,11 @@ from typing import Final from ..services import ServiceKey +from ..services_constants import FRONTEND_SERVICE_KEY_PREFIX # NOTE: due to legacy reasons, the name remains with 'frontend' in it but # it now refers to a more general group: function sections that contains front-end services as well -FUNCTION_SERVICE_KEY_PREFIX: Final[str] = "simcore/services/frontend" +FUNCTION_SERVICE_KEY_PREFIX: Final[str] = FRONTEND_SERVICE_KEY_PREFIX def is_function_service(service_key: ServiceKey) -> bool: diff --git a/packages/models-library/src/models_library/rpc_filters.py b/packages/models-library/src/models_library/rpc_filters.py new file mode 100644 index 000000000000..ffc7c77f1c60 --- /dev/null +++ b/packages/models-library/src/models_library/rpc_filters.py @@ -0,0 +1,5 @@ +from .rest_filters import Filters + +__all__: tuple[str, ...] = ("Filters",) + +# nopycln:file diff --git a/packages/models-library/src/models_library/services_constants.py b/packages/models-library/src/models_library/services_constants.py index 049370611dd4..c3779791bd81 100644 --- a/packages/models-library/src/models_library/services_constants.py +++ b/packages/models-library/src/models_library/services_constants.py @@ -1,6 +1,27 @@ +from types import MappingProxyType from typing import Final -LATEST_INTEGRATION_VERSION: Final[str] = "1.0.0" +from .services_enums import ServiceType +LATEST_INTEGRATION_VERSION: Final[str] = "1.0.0" ANY_FILETYPE: Final[str] = "data:*/*" + +SERVICE_TYPE_TO_NAME_MAP = MappingProxyType( + { + ServiceType.COMPUTATIONAL: "comp", + ServiceType.DYNAMIC: "dynamic", + ServiceType.FRONTEND: "frontend", + } +) + + +def _create_key_prefix(service_type: ServiceType) -> str: + return f"simcore/services/{SERVICE_TYPE_TO_NAME_MAP[service_type]}" + + +COMPUTATIONAL_SERVICE_KEY_PREFIX: Final[str] = _create_key_prefix( + ServiceType.COMPUTATIONAL +) +DYNAMIC_SERVICE_KEY_PREFIX: Final[str] = _create_key_prefix(ServiceType.DYNAMIC) +FRONTEND_SERVICE_KEY_PREFIX: Final[str] = _create_key_prefix(ServiceType.FRONTEND) diff --git a/packages/models-library/src/models_library/services_regex.py b/packages/models-library/src/models_library/services_regex.py index c4c9e84d2e09..08154982df48 100644 --- a/packages/models-library/src/models_library/services_regex.py +++ b/packages/models-library/src/models_library/services_regex.py @@ -1,6 +1,15 @@ import re +from types import MappingProxyType from typing import Final +from .services_constants import ( + COMPUTATIONAL_SERVICE_KEY_PREFIX, + DYNAMIC_SERVICE_KEY_PREFIX, + FRONTEND_SERVICE_KEY_PREFIX, + SERVICE_TYPE_TO_NAME_MAP, +) +from .services_enums import ServiceType + PROPERTY_TYPE_RE = r"^(number|integer|boolean|string|ref_contentSchema|data:([^/\s,]+/[^/\s,]+|\[[^/\s,]+/[^/\s,]+(,[^/\s]+/[^/,\s]+)*\]))$" PROPERTY_TYPE_TO_PYTHON_TYPE_MAP = { "integer": int, @@ -11,34 +20,59 @@ FILENAME_RE = r".+" - # e.g. simcore/services/comp/opencor SERVICE_KEY_RE: Final[re.Pattern[str]] = re.compile( r"^simcore/services/" - r"(?P(comp|dynamic|frontend))/" + rf"(?P({ '|'.join(SERVICE_TYPE_TO_NAME_MAP.values()) }))/" r"(?P[a-z0-9][a-z0-9_.-]*/)*" r"(?P[a-z0-9-_]+[a-z0-9])$" ) + # e.g. simcore%2Fservices%2Fcomp%2Fopencor SERVICE_ENCODED_KEY_RE: Final[re.Pattern[str]] = re.compile( r"^simcore%2Fservices%2F" - r"(?P(comp|dynamic|frontend))%2F" + rf"(?P({'|'.join(SERVICE_TYPE_TO_NAME_MAP.values())}))%2F" r"(?P[a-z0-9][a-z0-9_.-]*%2F)*" r"(?P[a-z0-9-_]+[a-z0-9])$" ) -DYNAMIC_SERVICE_KEY_RE: Final[re.Pattern[str]] = re.compile( - r"^simcore/services/dynamic/" - r"(?P[a-z0-9][a-z0-9_.-]*/)*" - r"(?P[a-z0-9-_]+[a-z0-9])$" + +def _create_key_regex(service_type: ServiceType) -> re.Pattern[str]: + return re.compile( + rf"^simcore/services/{SERVICE_TYPE_TO_NAME_MAP[service_type]}/" + r"(?P[a-z0-9][a-z0-9_.-]*/)*" + r"(?P[a-z0-9-_]+[a-z0-9])$" + ) + + +def _create_key_format(service_type: ServiceType) -> str: + return f"simcore/services/{SERVICE_TYPE_TO_NAME_MAP[service_type]}/{{service_name}}" + + +COMPUTATIONAL_SERVICE_KEY_RE: Final[re.Pattern[str]] = _create_key_regex( + ServiceType.COMPUTATIONAL +) +COMPUTATIONAL_SERVICE_KEY_FORMAT: Final[str] = _create_key_format( + ServiceType.COMPUTATIONAL ) -DYNAMIC_SERVICE_KEY_FORMAT = "simcore/services/dynamic/{service_name}" +DYNAMIC_SERVICE_KEY_RE: Final[re.Pattern[str]] = _create_key_regex(ServiceType.DYNAMIC) +DYNAMIC_SERVICE_KEY_FORMAT: Final[str] = _create_key_format(ServiceType.DYNAMIC) -# Computational regex & format -COMPUTATIONAL_SERVICE_KEY_RE: Final[re.Pattern[str]] = re.compile( - r"^simcore/services/comp/" - r"(?P[a-z0-9][a-z0-9_.-]*/)*" - r"(?P[a-z0-9-_]+[a-z0-9])$" +FRONTEND_SERVICE_KEY_RE: Final[re.Pattern[str]] = _create_key_regex( + ServiceType.FRONTEND ) -COMPUTATIONAL_SERVICE_KEY_FORMAT: Final[str] = "simcore/services/comp/{service_name}" +FRONTEND_SERVICE_KEY_FORMAT: Final[str] = _create_key_format(ServiceType.FRONTEND) + + +SERVICE_TYPE_TO_PREFIX_MAP = MappingProxyType( + { + ServiceType.COMPUTATIONAL: COMPUTATIONAL_SERVICE_KEY_PREFIX, + ServiceType.DYNAMIC: DYNAMIC_SERVICE_KEY_PREFIX, + ServiceType.FRONTEND: FRONTEND_SERVICE_KEY_PREFIX, + } +) + +assert all( # nosec + not prefix.endswith("/") for prefix in SERVICE_TYPE_TO_PREFIX_MAP.values() +), "Service type prefixes must not end with '/'" diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/catalog_rpc_server.py b/packages/pytest-simcore/src/pytest_simcore/helpers/catalog_rpc_server.py index fe112184b0ee..02deb7f1ca0a 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/catalog_rpc_server.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/catalog_rpc_server.py @@ -5,7 +5,11 @@ # pylint: disable=unused-variable -from models_library.api_schemas_catalog.services import LatestServiceGet, ServiceGetV2 +from models_library.api_schemas_catalog.services import ( + LatestServiceGet, + ServiceGetV2, + ServiceListFilters, +) from models_library.api_schemas_webserver.catalog import ( CatalogServiceUpdate, ) @@ -34,10 +38,12 @@ async def list_services_paginated( user_id: UserID, limit: PageLimitInt, offset: NonNegativeInt, + filters: ServiceListFilters | None = None, ): assert rpc_client assert product_name assert user_id + assert filters is None, "filters not mocked yet" items = TypeAdapter(list[LatestServiceGet]).validate_python( LatestServiceGet.model_json_schema()["examples"], @@ -110,12 +116,14 @@ async def list_my_service_history_paginated( service_key: ServiceKey, offset: PageOffsetInt, limit: PageLimitInt, + filters: ServiceListFilters | None = None, ) -> PageRpc[ServiceRelease]: assert rpc_client assert product_name assert user_id assert service_key + assert filters is None, "filters not mocked yet" items = TypeAdapter(list[ServiceRelease]).validate_python( ServiceRelease.model_json_schema()["examples"], diff --git a/packages/service-integration/src/service_integration/osparc_config.py b/packages/service-integration/src/service_integration/osparc_config.py index 9a3f2e0c116a..12413dcd130d 100644 --- a/packages/service-integration/src/service_integration/osparc_config.py +++ b/packages/service-integration/src/service_integration/osparc_config.py @@ -1,4 +1,4 @@ -""" 'osparc config' is a set of stardard file forms (yaml) that the user fills to describe how his/her service works and +"""'osparc config' is a set of stardard file forms (yaml) that the user fills to describe how his/her service works and integrates with osparc. - config files are stored under '.osparc/' folder in the root repo folder (analogous to other configs like .github, .vscode, etc) @@ -26,11 +26,7 @@ RestartPolicy, ) from models_library.service_settings_nat_rule import NATRule -from models_library.services import BootOptions, ServiceMetaDataPublished, ServiceType -from models_library.services_regex import ( - COMPUTATIONAL_SERVICE_KEY_FORMAT, - DYNAMIC_SERVICE_KEY_FORMAT, -) +from models_library.services import BootOptions, ServiceMetaDataPublished from models_library.services_types import ServiceKey from models_library.utils.labels_annotations import ( OSPARC_LABEL_PREFIXES, @@ -62,12 +58,6 @@ OSPARC_CONFIG_RUNTIME_NAME: Final[str] = "runtime.yml" -SERVICE_KEY_FORMATS = { - ServiceType.COMPUTATIONAL: COMPUTATIONAL_SERVICE_KEY_FORMAT, - ServiceType.DYNAMIC: DYNAMIC_SERVICE_KEY_FORMAT, -} - - class DockerComposeOverwriteConfig(ComposeSpecification): """Content of docker-compose.overwrite.yml configuration file""" @@ -231,9 +221,9 @@ class RuntimeConfig(BaseModel): containers_allowed_outgoing_internet: set[str] | None = None - settings: Annotated[ - list[SettingsItem], Field(default_factory=list) - ] = DEFAULT_FACTORY + settings: Annotated[list[SettingsItem], Field(default_factory=list)] = ( + DEFAULT_FACTORY + ) @model_validator(mode="before") @classmethod diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/services.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/services.py index 3a5392cfdd83..f30f85a4d882 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/services.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/services.py @@ -1,7 +1,15 @@ -"""RPC client-side for the RPC server at the payments service""" +"""RPC client-side for the RPC server at the payments service + +In this interface (and all belows), the context of the caller is passed in the following arguments: +- `user_id` is intended for the caller's identifer. Do not add other user_id that is not the callers!. + - Ideally this could be injected by an authentication layer (as in the rest API) + but for now we are passing it as an argument. +- `product_name` is the name of the product at the caller's context as well + +""" import logging -from typing import Any, cast +from typing import cast from models_library.api_schemas_catalog import CATALOG_RPC_NAMESPACE from models_library.api_schemas_catalog.services import ( @@ -10,6 +18,7 @@ PageRpcLatestServiceGet, PageRpcServiceRelease, ServiceGetV2, + ServiceListFilters, ServiceRelease, ServiceUpdateV2, ) @@ -24,14 +33,15 @@ from models_library.services_types import ServiceKey, ServiceVersion from models_library.users import UserID from pydantic import TypeAdapter, validate_call -from servicelib.logging_utils import log_decorator -from servicelib.rabbitmq._constants import RPC_REQUEST_DEFAULT_TIMEOUT_S +from ....logging_utils import log_decorator from ..._client_rpc import RabbitMQRPCClient +from ..._constants import RPC_REQUEST_DEFAULT_TIMEOUT_S _logger = logging.getLogger(__name__) +@validate_call(config={"arbitrary_types_allowed": True}) async def list_services_paginated( # pylint: disable=too-many-arguments rpc_client: RabbitMQRPCClient, *, @@ -39,6 +49,7 @@ async def list_services_paginated( # pylint: disable=too-many-arguments user_id: UserID, limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, offset: PageOffsetInt = 0, + filters: ServiceListFilters | None = None, ) -> PageRpcLatestServiceGet: """ Raises: @@ -46,32 +57,24 @@ async def list_services_paginated( # pylint: disable=too-many-arguments CatalogForbiddenError: no access-rights to list services """ - @validate_call() - async def _call( - product_name: ProductName, - user_id: UserID, - limit: PageLimitInt, - offset: PageOffsetInt, - ): - return await rpc_client.request( - CATALOG_RPC_NAMESPACE, - TypeAdapter(RPCMethodName).validate_python("list_services_paginated"), - product_name=product_name, - user_id=user_id, - limit=limit, - offset=offset, - timeout_s=40 * RPC_REQUEST_DEFAULT_TIMEOUT_S, - ) - - result = await _call( - product_name=product_name, user_id=user_id, limit=limit, offset=offset + result = await rpc_client.request( + CATALOG_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("list_services_paginated"), + product_name=product_name, + user_id=user_id, + limit=limit, + offset=offset, + filters=filters, + timeout_s=40 * RPC_REQUEST_DEFAULT_TIMEOUT_S, ) + assert ( # nosec TypeAdapter(PageRpc[LatestServiceGet]).validate_python(result) is not None ) return cast(PageRpc[LatestServiceGet], result) +@validate_call(config={"arbitrary_types_allowed": True}) @log_decorator(_logger, level=logging.DEBUG) async def get_service( rpc_client: RabbitMQRPCClient, @@ -87,34 +90,20 @@ async def get_service( CatalogItemNotFoundError: service not found in catalog CatalogForbiddenError: not access rights to read this service """ - - @validate_call() - async def _call( - product_name: ProductName, - user_id: UserID, - service_key: ServiceKey, - service_version: ServiceVersion, - ) -> Any: - return await rpc_client.request( - CATALOG_RPC_NAMESPACE, - TypeAdapter(RPCMethodName).validate_python("get_service"), - product_name=product_name, - user_id=user_id, - service_key=service_key, - service_version=service_version, - timeout_s=4 * RPC_REQUEST_DEFAULT_TIMEOUT_S, - ) - - result = await _call( + result = await rpc_client.request( + CATALOG_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("get_service"), product_name=product_name, user_id=user_id, service_key=service_key, service_version=service_version, + timeout_s=4 * RPC_REQUEST_DEFAULT_TIMEOUT_S, ) assert TypeAdapter(ServiceGetV2).validate_python(result) is not None # nosec return cast(ServiceGetV2, result) +@validate_call(config={"arbitrary_types_allowed": True}) @log_decorator(_logger, level=logging.DEBUG) async def update_service( rpc_client: RabbitMQRPCClient, @@ -132,26 +121,9 @@ async def update_service( CatalogItemNotFoundError: service not found in catalog CatalogForbiddenError: not access rights to read this service """ - - @validate_call() - async def _call( - product_name: ProductName, - user_id: UserID, - service_key: ServiceKey, - service_version: ServiceVersion, - update: ServiceUpdateV2, - ): - return await rpc_client.request( - CATALOG_RPC_NAMESPACE, - TypeAdapter(RPCMethodName).validate_python("update_service"), - product_name=product_name, - user_id=user_id, - service_key=service_key, - service_version=service_version, - update=update, - ) - - result = await _call( + result = await rpc_client.request( + CATALOG_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("update_service"), product_name=product_name, user_id=user_id, service_key=service_key, @@ -162,6 +134,7 @@ async def _call( return cast(ServiceGetV2, result) +@validate_call(config={"arbitrary_types_allowed": True}) @log_decorator(_logger, level=logging.DEBUG) async def check_for_service( rpc_client: RabbitMQRPCClient, @@ -172,29 +145,15 @@ async def check_for_service( service_version: ServiceVersion, ) -> None: """ + Raises: ValidationError: on invalid arguments CatalogItemNotFoundError: service not found in catalog CatalogForbiddenError: not access rights to read this service """ - - @validate_call() - async def _call( - product_name: ProductName, - user_id: UserID, - service_key: ServiceKey, - service_version: ServiceVersion, - ): - return await rpc_client.request( - CATALOG_RPC_NAMESPACE, - TypeAdapter(RPCMethodName).validate_python("check_for_service"), - product_name=product_name, - user_id=user_id, - service_key=service_key, - service_version=service_version, - ) - - await _call( + await rpc_client.request( + CATALOG_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("check_for_service"), product_name=product_name, user_id=user_id, service_key=service_key, @@ -202,6 +161,7 @@ async def _call( ) +@validate_call(config={"arbitrary_types_allowed": True}) @log_decorator(_logger, level=logging.DEBUG) async def batch_get_my_services( rpc_client: RabbitMQRPCClient, @@ -220,27 +180,19 @@ async def batch_get_my_services( ValidationError: on invalid arguments CatalogForbiddenError: no access-rights to list services """ - - @validate_call() - async def _call( - product_name: ProductName, - user_id: UserID, - ids: list[tuple[ServiceKey, ServiceVersion]], - ): - return await rpc_client.request( - CATALOG_RPC_NAMESPACE, - TypeAdapter(RPCMethodName).validate_python("batch_get_my_services"), - product_name=product_name, - user_id=user_id, - ids=ids, - timeout_s=40 * RPC_REQUEST_DEFAULT_TIMEOUT_S, - ) - - result = await _call(product_name=product_name, user_id=user_id, ids=ids) + result = await rpc_client.request( + CATALOG_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("batch_get_my_services"), + product_name=product_name, + user_id=user_id, + ids=ids, + timeout_s=40 * RPC_REQUEST_DEFAULT_TIMEOUT_S, + ) assert TypeAdapter(list[MyServiceGet]).validate_python(result) is not None # nosec return cast(list[MyServiceGet], result) +@validate_call(config={"arbitrary_types_allowed": True}) async def list_my_service_history_paginated( # pylint: disable=too-many-arguments rpc_client: RabbitMQRPCClient, *, @@ -249,40 +201,23 @@ async def list_my_service_history_paginated( # pylint: disable=too-many-argumen service_key: ServiceKey, limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, offset: PageOffsetInt = 0, + filters: ServiceListFilters | None = None, ) -> PageRpcServiceRelease: """ + Raises: ValidationError: on invalid arguments """ - - @validate_call() - async def _call( - product_name: ProductName, - user_id: UserID, - service_key: ServiceKey, - limit: PageLimitInt, - offset: PageOffsetInt, - ): - return await rpc_client.request( - CATALOG_RPC_NAMESPACE, - TypeAdapter(RPCMethodName).validate_python( - "list_my_service_history_paginated" - ), - product_name=product_name, - user_id=user_id, - service_key=service_key, - limit=limit, - offset=offset, - ) - - result = await _call( + result = await rpc_client.request( + CATALOG_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("list_my_service_history_paginated"), product_name=product_name, user_id=user_id, service_key=service_key, limit=limit, offset=offset, + filters=filters, ) - assert ( # nosec TypeAdapter(PageRpcServiceRelease).validate_python(result) is not None ) diff --git a/services/catalog/src/simcore_service_catalog/api/rpc/_services.py b/services/catalog/src/simcore_service_catalog/api/rpc/_services.py index a662adfcfaf0..45f0d4c0a714 100644 --- a/services/catalog/src/simcore_service_catalog/api/rpc/_services.py +++ b/services/catalog/src/simcore_service_catalog/api/rpc/_services.py @@ -8,6 +8,7 @@ PageRpcLatestServiceGet, PageRpcServiceRelease, ServiceGetV2, + ServiceListFilters, ServiceUpdateV2, ) from models_library.products import ProductName @@ -23,8 +24,9 @@ CatalogForbiddenError, CatalogItemNotFoundError, ) -from simcore_service_catalog.repository.groups import GroupsRepository +from ...models.services_db import ServiceFiltersDB +from ...repository.groups import GroupsRepository from ...repository.services import ServicesRepository from ...service import catalog_services from .._dependencies.director import get_director_client @@ -55,6 +57,16 @@ async def _wrapper(app: FastAPI, **kwargs): return _wrapper +def _type_adapter_to_domain( + filters: ServiceListFilters | None, +) -> ServiceFiltersDB | None: + return ( + ServiceFiltersDB.model_validate(filters, from_attributes=True) + if filters + else None + ) + + @router.expose(reraise_if_error_type=(CatalogForbiddenError, ValidationError)) @_profile_rpc_call @validate_call(config={"arbitrary_types_allowed": True}) @@ -65,6 +77,7 @@ async def list_services_paginated( user_id: UserID, limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, offset: PageOffsetInt = 0, + filters: ServiceListFilters | None = None, ) -> PageRpcLatestServiceGet: assert app.state.engine # nosec @@ -75,6 +88,7 @@ async def list_services_paginated( user_id=user_id, limit=limit, offset=offset, + filters=_type_adapter_to_domain(filters), ) assert len(items) <= total_count # nosec @@ -234,6 +248,7 @@ async def list_my_service_history_paginated( service_key: ServiceKey, limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, offset: PageOffsetInt = 0, + filters: ServiceListFilters | None = None, ) -> PageRpcServiceRelease: assert app.state.engine # nosec @@ -244,6 +259,7 @@ async def list_my_service_history_paginated( service_key=service_key, limit=limit, offset=offset, + filters=_type_adapter_to_domain(filters), ) assert len(items) <= total_count # nosec diff --git a/services/catalog/src/simcore_service_catalog/models/services_db.py b/services/catalog/src/simcore_service_catalog/models/services_db.py index 2ad800d2b44d..899dcc86dcf0 100644 --- a/services/catalog/src/simcore_service_catalog/models/services_db.py +++ b/services/catalog/src/simcore_service_catalog/models/services_db.py @@ -5,8 +5,10 @@ from models_library.basic_types import IdInt from models_library.groups import GroupID from models_library.products import ProductName +from models_library.rest_filters import Filters from models_library.services_access import ServiceGroupAccessRights from models_library.services_base import ServiceKeyVersion +from models_library.services_enums import ServiceType from models_library.services_types import ServiceKey, ServiceVersion from models_library.utils.common_validators import empty_str_to_none_pre_validator from pydantic import ( @@ -244,3 +246,19 @@ def _update_json_schema_extra(schema: JsonDict) -> None: model_config = ConfigDict( from_attributes=True, json_schema_extra=_update_json_schema_extra ) + + +class ServiceFiltersDB(Filters): + service_type: ServiceType | None = None + + @staticmethod + def _update_json_schema_extra(schema: JsonDict) -> None: + schema.update( + { + "example": { + "by_service_type": "computational", + } + } + ) + + model_config = ConfigDict(json_schema_extra=_update_json_schema_extra) diff --git a/services/catalog/src/simcore_service_catalog/repository/_services_sql.py b/services/catalog/src/simcore_service_catalog/repository/_services_sql.py index 2495bd314723..0f4bc6c46265 100644 --- a/services/catalog/src/simcore_service_catalog/repository/_services_sql.py +++ b/services/catalog/src/simcore_service_catalog/repository/_services_sql.py @@ -2,6 +2,9 @@ import sqlalchemy as sa from models_library.products import ProductName +from models_library.services_regex import ( + SERVICE_TYPE_TO_PREFIX_MAP, +) from models_library.services_types import ServiceKey, ServiceVersion from models_library.users import UserID from simcore_postgres_database.models.groups import user_to_groups @@ -19,7 +22,7 @@ from sqlalchemy.sql.expression import func from sqlalchemy.sql.selectable import Select -from ..models.services_db import ServiceMetaDataDBGet +from ..models.services_db import ServiceFiltersDB, ServiceMetaDataDBGet SERVICES_META_DATA_COLS = get_columns_from_db_model( services_meta_data, ServiceMetaDataDBGet @@ -113,13 +116,29 @@ def _has_access_rights( ) +def _apply_services_filters( + stmt: sa.sql.Select, + filters: ServiceFiltersDB, +) -> sa.sql.Select: + if filters.service_type: + prefix = SERVICE_TYPE_TO_PREFIX_MAP.get(filters.service_type) + if prefix is None: + msg = f"Undefined service type {filters.service_type}. Please update prefix expressions" + raise ValueError(msg) + + assert not prefix.endswith("/") # nosec + return stmt.where(services_meta_data.c.key.like(f"{prefix}/%")) + return stmt + + def latest_services_total_count_stmt( *, product_name: ProductName, user_id: UserID, access_rights: sa.sql.ClauseElement, + filters: ServiceFiltersDB | None = None, ): - return ( + stmt = ( sa.select(func.count(sa.distinct(services_meta_data.c.key))) .select_from( services_meta_data.join( @@ -136,6 +155,11 @@ def latest_services_total_count_stmt( .where(access_rights) ) + if filters: + stmt = _apply_services_filters(stmt, filters) + + return stmt + def list_latest_services_stmt( *, @@ -144,10 +168,11 @@ def list_latest_services_stmt( access_rights: sa.sql.ClauseElement, limit: int | None, offset: int | None, + filters: ServiceFiltersDB | None = None, ): # get all distinct services key fitting a page # and its corresponding latest version - cte = ( + cte_stmt = ( sa.select( services_meta_data.c.key, services_meta_data.c.version.label("latest_version"), @@ -172,9 +197,13 @@ def list_latest_services_stmt( .distinct(services_meta_data.c.key) # get only first .limit(limit) .offset(offset) - .cte("cte") ) + if filters: + cte_stmt = _apply_services_filters(cte_stmt, filters) + + cte = cte_stmt.cte("cte") + # get all information of latest's services listed in CTE latest_stmt = ( sa.select( diff --git a/services/catalog/src/simcore_service_catalog/repository/services.py b/services/catalog/src/simcore_service_catalog/repository/services.py index 8c3aabda39f2..e0c0fd083430 100644 --- a/services/catalog/src/simcore_service_catalog/repository/services.py +++ b/services/catalog/src/simcore_service_catalog/repository/services.py @@ -28,7 +28,6 @@ from simcore_postgres_database.models.services_specifications import ( services_specifications, ) -from simcore_postgres_database.utils import as_postgres_sql_query_str from simcore_postgres_database.utils_repos import pass_or_acquire_connection from simcore_postgres_database.utils_services import create_select_latest_services_query from sqlalchemy import sql @@ -37,6 +36,7 @@ from ..models.services_db import ( ReleaseDBGet, ServiceAccessRightsAtDB, + ServiceFiltersDB, ServiceMetaDataDBCreate, ServiceMetaDataDBGet, ServiceMetaDataDBPatch, @@ -47,6 +47,7 @@ from ._services_sql import ( SERVICES_META_DATA_COLS, AccessRightsClauses, + _apply_services_filters, by_version, can_get_service_stmt, get_service_history_stmt, @@ -391,6 +392,7 @@ async def list_latest_services( # list args: pagination limit: int | None = None, offset: int | None = None, + filters: ServiceFiltersDB | None = None, ) -> tuple[PositiveInt, list[ServiceWithHistoryDBGet]]: # get page @@ -398,6 +400,7 @@ async def list_latest_services( product_name=product_name, user_id=user_id, access_rights=AccessRightsClauses.can_read, + filters=filters, ) stmt_page = list_latest_services_stmt( product_name=product_name, @@ -405,6 +408,7 @@ async def list_latest_services( access_rights=AccessRightsClauses.can_read, limit=limit, offset=offset, + filters=filters, ) async with self.db_engine.connect() as conn: @@ -480,9 +484,10 @@ async def get_service_history_page( # list args: pagination limit: int | None = None, offset: int | None = None, + filters: ServiceFiltersDB | None = None, ) -> tuple[PositiveInt, list[ReleaseDBGet]]: - base_subquery = ( + base_stmt = ( # Search on service (key, *) for (product_name, user_id w/ access) sql.select( services_meta_data.c.key, @@ -506,11 +511,15 @@ async def get_service_history_page( & (user_to_groups.c.uid == user_id) & AccessRightsClauses.can_read ) - ).subquery() + ) + + if filters: + base_stmt = _apply_services_filters(base_stmt, filters) + + base_subquery = base_stmt.subquery() # Query to count the TOTAL number of rows count_query = sql.select(sql.func.count()).select_from(base_subquery) - _logger.debug("count_query=\n%s", as_postgres_sql_query_str(count_query)) # Query to retrieve page with additional columns, ordering, offset, and limit page_query = ( @@ -541,7 +550,6 @@ async def get_service_history_page( .offset(offset) .limit(limit) ) - _logger.debug("page_query=\n%s", as_postgres_sql_query_str(page_query)) async with pass_or_acquire_connection(self.db_engine) as conn: total_count: PositiveInt = await conn.scalar(count_query) or 0 diff --git a/services/catalog/src/simcore_service_catalog/service/catalog_services.py b/services/catalog/src/simcore_service_catalog/service/catalog_services.py index c192944f11c6..cce17206e259 100644 --- a/services/catalog/src/simcore_service_catalog/service/catalog_services.py +++ b/services/catalog/src/simcore_service_catalog/service/catalog_services.py @@ -32,6 +32,7 @@ from ..clients.director import DirectorClient from ..models.services_db import ( ServiceAccessRightsAtDB, + ServiceFiltersDB, ServiceMetaDataDBPatch, ServiceWithHistoryDBGet, ) @@ -134,11 +135,16 @@ async def list_latest_catalog_services( user_id: UserID, limit: PageLimitInt | None, offset: NonNegativeInt = 0, + filters: ServiceFiltersDB | None = None, ) -> tuple[PageTotalCount, list[LatestServiceGet]]: # defines the order total_count, services = await repo.list_latest_services( - product_name=product_name, user_id=user_id, limit=limit, offset=offset + product_name=product_name, + user_id=user_id, + limit=limit, + offset=offset, + filters=filters, ) if services: @@ -185,6 +191,9 @@ async def list_latest_catalog_services( missing_services=missing_services, user_id=user_id, product_name=product_name, + filters=filters, + limit=limit, + offset=offset, ), tip="This might be due to malfunction of the background-task or that this call was done while the sync was taking place", ) @@ -507,6 +516,8 @@ async def list_user_service_release_history( # pagination limit: PageLimitInt | None = None, offset: NonNegativeInt | None = None, + # filters + filters: ServiceFiltersDB | None = None, # options include_compatibility: bool = False, ) -> tuple[PageTotalCount, list[ServiceRelease]]: @@ -519,6 +530,7 @@ async def list_user_service_release_history( key=service_key, limit=limit, offset=offset, + filters=filters, ) compatibility_map: dict[ServiceVersion, Compatibility | None] = {} diff --git a/services/catalog/tests/unit/with_dbs/test_api_rpc.py b/services/catalog/tests/unit/with_dbs/test_api_rpc.py index 19f7bfeb7da5..692efc846054 100644 --- a/services/catalog/tests/unit/with_dbs/test_api_rpc.py +++ b/services/catalog/tests/unit/with_dbs/test_api_rpc.py @@ -109,7 +109,7 @@ async def background_sync_task_mocked( await services_db_tables_injector(fake_data_for_services) -async def test_rpc_catalog_with_no_services_returns_empty_page( +async def test_rpc_list_services_paginated_with_no_services_returns_empty_page( background_sync_task_mocked: None, mocked_director_rest_api: MockRouter, rpc_client: RabbitMQRPCClient, @@ -128,6 +128,35 @@ async def test_rpc_catalog_with_no_services_returns_empty_page( assert page.meta.total == 0 +async def test_rpc_list_services_paginated_with_filters( + background_sync_task_mocked: None, + mocked_director_rest_api: MockRouter, + rpc_client: RabbitMQRPCClient, + product_name: ProductName, + user_id: UserID, + app: FastAPI, +): + assert app + + # only computational services introduced by the background_sync_task_mocked + page = await list_services_paginated( + rpc_client, + product_name=product_name, + user_id=user_id, + filters={"service_type": "computational"}, + ) + assert page.meta.total == page.meta.count + assert page.meta.total > 0 + + page = await list_services_paginated( + rpc_client, + product_name=product_name, + user_id=user_id, + filters={"service_type": "dynamic"}, + ) + assert page.meta.total == 0 + + async def test_rpc_catalog_client( background_sync_task_mocked: None, mocked_director_rest_api: MockRouter, @@ -484,7 +513,7 @@ async def test_rpc_batch_get_my_services( assert my_services[1].release.version == other_service_version -async def test_rpc_get_my_service_history( +async def test_rpc_list_my_service_history_paginated( background_sync_task_mocked: None, mocked_director_rest_api: MockRouter, rpc_client: RabbitMQRPCClient, diff --git a/services/catalog/tests/unit/with_dbs/test_repositories.py b/services/catalog/tests/unit/with_dbs/test_repositories.py index 402825e703e0..19e4899aa082 100644 --- a/services/catalog/tests/unit/with_dbs/test_repositories.py +++ b/services/catalog/tests/unit/with_dbs/test_repositories.py @@ -13,6 +13,12 @@ import pytest from models_library.products import ProductName +from models_library.services_enums import ServiceType # Import ServiceType enum +from models_library.services_regex import ( + COMPUTATIONAL_SERVICE_KEY_PREFIX, + DYNAMIC_SERVICE_KEY_PREFIX, + SERVICE_TYPE_TO_PREFIX_MAP, +) from models_library.users import UserID from packaging import version from pydantic import EmailStr, HttpUrl, TypeAdapter @@ -21,6 +27,7 @@ from simcore_postgres_database.models.projects import ProjectType, projects from simcore_service_catalog.models.services_db import ( ServiceAccessRightsAtDB, + ServiceFiltersDB, ServiceMetaDataDBCreate, ServiceMetaDataDBGet, ServiceMetaDataDBPatch, @@ -310,7 +317,7 @@ async def test_get_latest_release( assert latest.version == fake_catalog_with_jupyterlab.expected_latest -async def test_list_all_services_and_history( +async def test_list_latest_services( target_product: ProductName, user_id: UserID, services_repo: ServicesRepository, @@ -332,7 +339,7 @@ async def test_list_all_services_and_history( ), "list_latest_service does NOT show history" -async def test_listing_with_no_services( +async def test_list_latest_services_with_no_services( target_product: ProductName, services_repo: ServicesRepository, user_id: UserID, @@ -344,7 +351,7 @@ async def test_listing_with_no_services( assert total_count == 0 -async def test_list_all_services_and_history_with_pagination( +async def test_list_latest_services_with_pagination( target_product: ProductName, create_fake_service_data: Callable, services_db_tables_injector: Callable, @@ -403,6 +410,61 @@ async def test_list_all_services_and_history_with_pagination( ), f"list of latest versions of services cannot have duplicates, found: {duplicates}" +async def test_list_latest_services_with_filters( + target_product: ProductName, + create_fake_service_data: Callable, + services_db_tables_injector: Callable, + services_repo: ServicesRepository, + user_id: UserID, +): + # Setup: Inject services with different service types + await services_db_tables_injector( + [ + create_fake_service_data( + f"{DYNAMIC_SERVICE_KEY_PREFIX}/service-name-a-{i}", + "1.0.0", + team_access=None, + everyone_access=None, + product=target_product, + ) + for i in range(3) + ] + + [ + create_fake_service_data( + f"{COMPUTATIONAL_SERVICE_KEY_PREFIX}/service-name-b-{i}", + "1.0.0", + team_access=None, + everyone_access=None, + product=target_product, + ) + for i in range(2) + ] + ) + + # Test: Apply filter for ServiceType.DYNAMIC + filters = ServiceFiltersDB(service_type=ServiceType.DYNAMIC) + total_count, services_items = await services_repo.list_latest_services( + product_name=target_product, user_id=user_id, filters=filters + ) + assert total_count == 3 + assert len(services_items) == 3 + assert all( + service.key.startswith(DYNAMIC_SERVICE_KEY_PREFIX) for service in services_items + ) + + # Test: Apply filter for ServiceType.COMPUTATIONAL + filters = ServiceFiltersDB(service_type=ServiceType.COMPUTATIONAL) + total_count, services_items = await services_repo.list_latest_services( + product_name=target_product, user_id=user_id, filters=filters + ) + assert total_count == 2 + assert len(services_items) == 2 + assert all( + service.key.startswith(COMPUTATIONAL_SERVICE_KEY_PREFIX) + for service in services_items + ) + + async def test_get_and_update_service_meta_data( target_product: ProductName, create_fake_service_data: Callable, @@ -491,6 +553,15 @@ async def test_can_get_service( ) +def _create_fake_release_versions(num_versions: int) -> set[str]: + release_versions = set() + while len(release_versions) < num_versions: + release_versions.add( + f"{random.randint(0, 2)}.{random.randint(0, 9)}.{random.randint(0, 9)}" # noqa: S311 + ) + return release_versions + + async def test_get_service_history_page( target_product: ProductName, create_fake_service_data: Callable, @@ -502,12 +573,7 @@ async def test_get_service_history_page( service_key = "simcore/services/dynamic/test-some-service" num_versions = 10 - release_versions = set() - while len(release_versions) < num_versions: - release_versions.add( - f"{random.randint(0, 2)}.{random.randint(0, 9)}.{random.randint(0, 9)}" # noqa: S311 - ) - + release_versions = _create_fake_release_versions(num_versions) await services_db_tables_injector( [ create_fake_service_data( @@ -566,6 +632,78 @@ async def test_get_service_history_page( assert paginated_history == history[offset : offset + limit] +@pytest.mark.parametrize( + "expected_service_type,service_prefix", SERVICE_TYPE_TO_PREFIX_MAP.items() +) +async def test_get_service_history_page_with_filters( + target_product: ProductName, + create_fake_service_data: Callable, + services_db_tables_injector: Callable, + services_repo: ServicesRepository, + user_id: UserID, + expected_service_type: ServiceType, + service_prefix: str, +): + # Setup: Inject services with multiple versions and types + service_key = f"{service_prefix}/test-service" + num_versions = 10 + + release_versions = _create_fake_release_versions(num_versions) + + await services_db_tables_injector( + [ + create_fake_service_data( + service_key, + service_version, + team_access=None, + everyone_access=None, + product=target_product, + ) + for _, service_version in enumerate(release_versions) + ] + ) + # Sort versions after injecting + release_versions = sorted(release_versions, key=version.Version, reverse=True) + + # Test: Fetch full history with no filters + total_count, history = await services_repo.get_service_history_page( + product_name=target_product, + user_id=user_id, + key=service_key, + ) + assert total_count == num_versions + assert len(history) == num_versions + assert [release.version for release in history] == release_versions + + # Test: Apply filter for + filters = ServiceFiltersDB(service_type=expected_service_type) + total_count, filtered_history = await services_repo.get_service_history_page( + product_name=target_product, + user_id=user_id, + key=service_key, + filters=filters, + ) + assert total_count == num_versions + assert len(filtered_history) == num_versions + assert [release.version for release in filtered_history] == release_versions + + # Final check: filter by a different service type expecting no results + different_service_type = ( + ServiceType.COMPUTATIONAL + if expected_service_type != ServiceType.COMPUTATIONAL + else ServiceType.DYNAMIC + ) + filters = ServiceFiltersDB(service_type=different_service_type) + total_count, no_history = await services_repo.get_service_history_page( + product_name=target_product, + user_id=user_id, + key=service_key, + filters=filters, + ) + assert total_count == 0 + assert no_history == [] + + async def test_list_services_from_published_templates( user: dict[str, Any], projects_repo: ProjectsRepository, diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_catalog.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_catalog.py index cac54fc484bd..752b1c3e2ee8 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_catalog.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_catalog.py @@ -9,6 +9,10 @@ from aiopg.sa.engine import Engine from models_library.groups import EVERYONE_GROUP_ID from models_library.services import ServiceKey, ServiceVersion +from models_library.services_constants import ( + COMPUTATIONAL_SERVICE_KEY_PREFIX, + DYNAMIC_SERVICE_KEY_PREFIX, +) from pydantic import HttpUrl, PositiveInt, TypeAdapter, ValidationError from servicelib.logging_utils import log_decorator from simcore_postgres_database.models.services import ( @@ -92,8 +96,8 @@ async def iter_latest_product_services( ) .where( ( - services_meta_data.c.key.like("simcore/services/dynamic/%%") - | (services_meta_data.c.key.like("simcore/services/comp/%%")) + services_meta_data.c.key.like(f"{DYNAMIC_SERVICE_KEY_PREFIX}/%") + | services_meta_data.c.key.like(f"{COMPUTATIONAL_SERVICE_KEY_PREFIX}/%") ) & (services_meta_data.c.deprecated.is_(None)) & (services_access_rights.c.gid == EVERYONE_GROUP_ID)