From cbc06aa3741043f10623e07ad4147b49107f6a4f Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Tue, 4 Mar 2025 10:57:50 +0100 Subject: [PATCH 01/30] use service terminology --- .../catalog/_handlers.py | 27 ++++++++------- .../catalog/_models.py | 32 +++++++++++++++++- .../catalog/{_api.py => _service.py} | 33 +++---------------- 3 files changed, 49 insertions(+), 43 deletions(-) rename services/web/server/src/simcore_service_webserver/catalog/{_api.py => _service.py} (89%) diff --git a/services/web/server/src/simcore_service_webserver/catalog/_handlers.py b/services/web/server/src/simcore_service_webserver/catalog/_handlers.py index cdc617c5db34..ef0a92a05b87 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/catalog/_handlers.py @@ -1,4 +1,4 @@ -""" rest api handlers +"""rest api handlers - Take into account that part of the API is also needed in the public API so logic should live in the catalog service in his final version @@ -39,8 +39,8 @@ from ..resource_usage.service import get_default_service_pricing_plan from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response -from . import _api, _handlers_errors, client -from ._api import CatalogRequestContext +from . import _handlers_errors, _service, client +from ._models import CatalogRequestContext from .exceptions import DefaultPricingUnitForServiceNotFoundError _logger = logging.getLogger(__name__) @@ -68,8 +68,7 @@ def ensure_unquoted(cls, v): return v -class ListServiceParams(PageQueryParameters): - ... +class ListServiceParams(PageQueryParameters): ... @routes.get( @@ -85,7 +84,7 @@ async def list_services_latest(request: Request): ListServiceParams, request ) - page_items, page_meta = await _api.list_latest_services( + page_items, page_meta = await _service.list_latest_services( request.app, user_id=request_ctx.user_id, product_name=request_ctx.product_name, @@ -124,7 +123,7 @@ async def get_service(request: Request): assert request_ctx # nosec assert path_params # nosec - service = await _api.get_service_v2( + service = await _service.get_service_v2( request.app, user_id=request_ctx.user_id, product_name=request_ctx.product_name, @@ -154,7 +153,7 @@ async def update_service(request: Request): assert path_params # nosec assert update # nosec - updated = await _api.update_service_v2( + updated = await _service.update_service_v2( request.app, user_id=request_ctx.user_id, product_name=request_ctx.product_name, @@ -178,7 +177,7 @@ async def list_service_inputs(request: Request): path_params = parse_request_path_parameters_as(ServicePathParams, request) # Evaluate and return validated model - response_model = await _api.list_service_inputs( + response_model = await _service.list_service_inputs( path_params.service_key, path_params.service_version, ctx ) @@ -203,7 +202,7 @@ async def get_service_input(request: Request): path_params = parse_request_path_parameters_as(_ServiceInputsPathParams, request) # Evaluate and return validated model - response_model = await _api.get_service_input( + response_model = await _service.get_service_input( path_params.service_key, path_params.service_version, path_params.input_key, @@ -236,7 +235,7 @@ async def get_compatible_inputs_given_source_output(request: Request): ) # Evaluate and return validated model - data = await _api.get_compatible_inputs_given_source_output( + data = await _service.get_compatible_inputs_given_source_output( path_params.service_key, path_params.service_version, query_params.from_service_key, @@ -261,7 +260,7 @@ async def list_service_outputs(request: Request): path_params = parse_request_path_parameters_as(ServicePathParams, request) # Evaluate and return validated model - response_model = await _api.list_service_outputs( + response_model = await _service.list_service_outputs( path_params.service_key, path_params.service_version, ctx ) @@ -286,7 +285,7 @@ async def get_service_output(request: Request): path_params = parse_request_path_parameters_as(_ServiceOutputsPathParams, request) # Evaluate and return validated model - response_model = await _api.get_service_output( + response_model = await _service.get_service_output( path_params.service_key, path_params.service_version, path_params.output_key, @@ -323,7 +322,7 @@ async def get_compatible_outputs_given_target_input(request: Request): _ToServiceInputsParams, request ) - data = await _api.get_compatible_outputs_given_target_input( + data = await _service.get_compatible_outputs_given_target_input( path_params.service_key, path_params.service_version, query_params.to_service_key, diff --git a/services/web/server/src/simcore_service_webserver/catalog/_models.py b/services/web/server/src/simcore_service_webserver/catalog/_models.py index af137ba11d8a..70deed1e6eea 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/_models.py +++ b/services/web/server/src/simcore_service_webserver/catalog/_models.py @@ -3,7 +3,9 @@ from dataclasses import dataclass from typing import Any, Final -from aiocache import cached # type: ignore[import-untyped] +from aiocache import cached +from aiohttp import web +from aiohttp.web import Request from models_library.api_schemas_webserver.catalog import ( ServiceInputGet, ServiceInputKey, @@ -11,7 +13,12 @@ ServiceOutputKey, ) from models_library.services import BaseServiceIOModel +from models_library.users import UserID from pint import PintError, Quantity, UnitRegistry +from pydantic import BaseModel, ConfigDict # type: ignore[import-untyped] +from servicelib.aiohttp.requests_validation import handle_validation_as_http_error + +from ..constants import RQ_PRODUCT_KEY, RQT_USERID_KEY _logger = logging.getLogger(__name__) @@ -131,3 +138,26 @@ async def from_catalog_service_api_model( ) return port + + +class CatalogRequestContext(BaseModel): + app: web.Application + user_id: UserID + product_name: str + unit_registry: UnitRegistry + model_config = ConfigDict(arbitrary_types_allowed=True) + + @classmethod + def create(cls, request: Request) -> "CatalogRequestContext": + with handle_validation_as_http_error( + error_msg_template="Invalid request", + resource_name=request.rel_url.path, + use_error_v1=True, + ): + assert request.app # nosec + return cls( + app=request.app, + user_id=request[RQT_USERID_KEY], + product_name=request[RQ_PRODUCT_KEY], + unit_registry=request.app[UnitRegistry.__name__], + ) diff --git a/services/web/server/src/simcore_service_webserver/catalog/_api.py b/services/web/server/src/simcore_service_webserver/catalog/_service.py similarity index 89% rename from services/web/server/src/simcore_service_webserver/catalog/_api.py rename to services/web/server/src/simcore_service_webserver/catalog/_service.py index 2c85d4007e69..746877692299 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/_api.py +++ b/services/web/server/src/simcore_service_webserver/catalog/_service.py @@ -3,7 +3,6 @@ from typing import Any, cast from aiohttp import web -from aiohttp.web import Request from models_library.api_schemas_catalog.services import ServiceUpdateV2 from models_library.api_schemas_webserver.catalog import ( ServiceInputGet, @@ -22,43 +21,21 @@ from models_library.users import UserID from models_library.utils.fastapi_encoders import jsonable_encoder from pint import UnitRegistry -from pydantic import BaseModel, ConfigDict -from servicelib.aiohttp.requests_validation import handle_validation_as_http_error from servicelib.rabbitmq.rpc_interfaces.catalog import services as catalog_rpc from servicelib.rest_constants import RESPONSE_MODEL_POLICY -from ..constants import RQ_PRODUCT_KEY, RQT_USERID_KEY from ..rabbitmq import get_rabbitmq_rpc_client from . import client from ._api_units import can_connect, replace_service_input_outputs -from ._models import ServiceInputGetFactory, ServiceOutputGetFactory +from ._models import ( + CatalogRequestContext, + ServiceInputGetFactory, + ServiceOutputGetFactory, +) _logger = logging.getLogger(__name__) -class CatalogRequestContext(BaseModel): - app: web.Application - user_id: UserID - product_name: str - unit_registry: UnitRegistry - model_config = ConfigDict(arbitrary_types_allowed=True) - - @classmethod - def create(cls, request: Request) -> "CatalogRequestContext": - with handle_validation_as_http_error( - error_msg_template="Invalid request", - resource_name=request.rel_url.path, - use_error_v1=True, - ): - assert request.app # nosec - return cls( - app=request.app, - user_id=request[RQT_USERID_KEY], - product_name=request[RQ_PRODUCT_KEY], - unit_registry=request.app[UnitRegistry.__name__], - ) - - async def _safe_replace_service_input_outputs( service: dict[str, Any], unit_registry: UnitRegistry ): From 75515cf7310255f242200c8f6afc61c869f58994 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Tue, 4 Mar 2025 11:39:12 +0100 Subject: [PATCH 02/30] further refactoring --- api/specs/web-server/_catalog.py | 32 +++++++------------ .../{_handlers.py => _controller_rest.py} | 27 ++-------------- .../catalog/{exceptions.py => _exceptions.py} | 0 .../catalog/_handlers_errors.py | 2 +- .../catalog/_models.py | 30 +++++++++++++++-- .../catalog/_tags_handlers.py | 2 +- .../catalog/plugin.py | 10 +++--- .../unit/isolated/test_catalog_models.py | 2 +- .../01/test_catalog_handlers__services.py | 6 ++-- 9 files changed, 52 insertions(+), 59 deletions(-) rename services/web/server/src/simcore_service_webserver/catalog/{_handlers.py => _controller_rest.py} (94%) rename services/web/server/src/simcore_service_webserver/catalog/{exceptions.py => _exceptions.py} (100%) diff --git a/api/specs/web-server/_catalog.py b/api/specs/web-server/_catalog.py index 90dd187c55b5..449ecd3711ac 100644 --- a/api/specs/web-server/_catalog.py +++ b/api/specs/web-server/_catalog.py @@ -14,7 +14,7 @@ from models_library.generics import Envelope from models_library.rest_pagination import Page from simcore_service_webserver._meta import API_VTAG -from simcore_service_webserver.catalog._handlers import ( +from simcore_service_webserver.catalog._controller_rest import ( ListServiceParams, ServicePathParams, _FromServiceOutputParams, @@ -48,8 +48,7 @@ def list_services_latest(_query_params: Annotated[ListServiceParams, Depends()]) "/catalog/services/{service_key}/{service_version}", response_model=Envelope[CatalogServiceGet], ) -def get_service(_path_params: Annotated[ServicePathParams, Depends()]): - ... +def get_service(_path_params: Annotated[ServicePathParams, Depends()]): ... @router.patch( @@ -59,8 +58,7 @@ def get_service(_path_params: Annotated[ServicePathParams, Depends()]): def update_service( _path_params: Annotated[ServicePathParams, Depends()], _update: CatalogServiceUpdate, -): - ... +): ... @router.get( @@ -69,8 +67,7 @@ def update_service( ) def list_service_inputs( _path_params: Annotated[ServicePathParams, Depends()], -): - ... +): ... @router.get( @@ -79,8 +76,7 @@ def list_service_inputs( ) def get_service_input( _path_params: Annotated[_ServiceInputsPathParams, Depends()], -): - ... +): ... @router.get( @@ -90,8 +86,7 @@ def get_service_input( def get_compatible_inputs_given_source_output( _path_params: Annotated[ServicePathParams, Depends()], _query_params: Annotated[_FromServiceOutputParams, Depends()], -): - ... +): ... @router.get( @@ -100,8 +95,7 @@ def get_compatible_inputs_given_source_output( ) def list_service_outputs( _path_params: Annotated[ServicePathParams, Depends()], -): - ... +): ... @router.get( @@ -110,8 +104,7 @@ def list_service_outputs( ) def get_service_output( _path_params: Annotated[_ServiceOutputsPathParams, Depends()], -): - ... +): ... @router.get( @@ -121,8 +114,7 @@ def get_service_output( def get_compatible_outputs_given_target_input( _path_params: Annotated[ServicePathParams, Depends()], _query_params: Annotated[_ToServiceInputsParams, Depends()], -): - ... +): ... @router.get( @@ -131,8 +123,7 @@ def get_compatible_outputs_given_target_input( ) def get_service_resources( _params: Annotated[ServicePathParams, Depends()], -): - ... +): ... @router.get( @@ -143,5 +134,4 @@ def get_service_resources( ) async def get_service_pricing_plan( _params: Annotated[ServicePathParams, Depends()], -): - ... +): ... diff --git a/services/web/server/src/simcore_service_webserver/catalog/_handlers.py b/services/web/server/src/simcore_service_webserver/catalog/_controller_rest.py similarity index 94% rename from services/web/server/src/simcore_service_webserver/catalog/_handlers.py rename to services/web/server/src/simcore_service_webserver/catalog/_controller_rest.py index ef0a92a05b87..f6900030682a 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/catalog/_controller_rest.py @@ -7,7 +7,6 @@ import asyncio import logging -import urllib.parse from typing import Final from aiohttp import web @@ -26,7 +25,7 @@ ServiceResourcesDict, ServiceResourcesDictHelpers, ) -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import BaseModel, Field from servicelib.aiohttp.requests_validation import ( parse_request_body_as, parse_request_path_parameters_as, @@ -40,8 +39,8 @@ from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response from . import _handlers_errors, _service, client -from ._models import CatalogRequestContext -from .exceptions import DefaultPricingUnitForServiceNotFoundError +from ._exceptions import DefaultPricingUnitForServiceNotFoundError +from ._models import CatalogRequestContext, ListServiceParams, ServicePathParams _logger = logging.getLogger(__name__) @@ -51,26 +50,6 @@ routes = RouteTableDef() -class ServicePathParams(BaseModel): - service_key: ServiceKey - service_version: ServiceVersion - model_config = ConfigDict( - populate_by_name=True, - extra="forbid", - ) - - @field_validator("service_key", mode="before") - @classmethod - def ensure_unquoted(cls, v): - # NOTE: this is needed as in pytest mode, the aiohttp server does not seem to unquote automatically - if v is not None: - return urllib.parse.unquote(v) - return v - - -class ListServiceParams(PageQueryParameters): ... - - @routes.get( f"{VTAG}/catalog/services/-/latest", name="list_services_latest", diff --git a/services/web/server/src/simcore_service_webserver/catalog/exceptions.py b/services/web/server/src/simcore_service_webserver/catalog/_exceptions.py similarity index 100% rename from services/web/server/src/simcore_service_webserver/catalog/exceptions.py rename to services/web/server/src/simcore_service_webserver/catalog/_exceptions.py diff --git a/services/web/server/src/simcore_service_webserver/catalog/_handlers_errors.py b/services/web/server/src/simcore_service_webserver/catalog/_handlers_errors.py index 4a278cc95dc2..6c93ebcb6092 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/_handlers_errors.py +++ b/services/web/server/src/simcore_service_webserver/catalog/_handlers_errors.py @@ -4,7 +4,7 @@ from servicelib.aiohttp.typing_extension import Handler from ..resource_usage.errors import DefaultPricingPlanNotFoundError -from .exceptions import ( +from ._exceptions import ( CatalogForbiddenError, CatalogItemNotFoundError, DefaultPricingUnitForServiceNotFoundError, diff --git a/services/web/server/src/simcore_service_webserver/catalog/_models.py b/services/web/server/src/simcore_service_webserver/catalog/_models.py index 70deed1e6eea..322837a9691a 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/_models.py +++ b/services/web/server/src/simcore_service_webserver/catalog/_models.py @@ -1,4 +1,5 @@ import logging +import urllib.parse from collections.abc import Callable from dataclasses import dataclass from typing import Any, Final @@ -12,10 +13,15 @@ ServiceOutputGet, ServiceOutputKey, ) -from models_library.services import BaseServiceIOModel +from models_library.rest_pagination import PageQueryParameters +from models_library.services import BaseServiceIOModel, ServiceKey, ServiceVersion from models_library.users import UserID from pint import PintError, Quantity, UnitRegistry -from pydantic import BaseModel, ConfigDict # type: ignore[import-untyped] +from pydantic import ( # type: ignore[import-untyped] + BaseModel, + ConfigDict, + field_validator, +) from servicelib.aiohttp.requests_validation import handle_validation_as_http_error from ..constants import RQ_PRODUCT_KEY, RQT_USERID_KEY @@ -161,3 +167,23 @@ def create(cls, request: Request) -> "CatalogRequestContext": product_name=request[RQ_PRODUCT_KEY], unit_registry=request.app[UnitRegistry.__name__], ) + + +class ServicePathParams(BaseModel): + service_key: ServiceKey + service_version: ServiceVersion + model_config = ConfigDict( + populate_by_name=True, + extra="forbid", + ) + + @field_validator("service_key", mode="before") + @classmethod + def ensure_unquoted(cls, v): + # NOTE: this is needed as in pytest mode, the aiohttp server does not seem to unquote automatically + if v is not None: + return urllib.parse.unquote(v) + return v + + +class ListServiceParams(PageQueryParameters): ... diff --git a/services/web/server/src/simcore_service_webserver/catalog/_tags_handlers.py b/services/web/server/src/simcore_service_webserver/catalog/_tags_handlers.py index dc75617f497f..ca2255cfa71e 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/_tags_handlers.py +++ b/services/web/server/src/simcore_service_webserver/catalog/_tags_handlers.py @@ -7,7 +7,7 @@ from .._meta import API_VTAG from ..login.decorators import login_required from ..security.decorators import permission_required -from ._handlers import ServicePathParams +from ._controller_rest import ServicePathParams _logger = logging.getLogger(__name__) diff --git a/services/web/server/src/simcore_service_webserver/catalog/plugin.py b/services/web/server/src/simcore_service_webserver/catalog/plugin.py index 2af8da917f0e..e0b7be350816 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/plugin.py +++ b/services/web/server/src/simcore_service_webserver/catalog/plugin.py @@ -1,6 +1,4 @@ -""" Subsystem to communicate with catalog service - -""" +"""Subsystem to communicate with catalog service""" import logging @@ -8,7 +6,7 @@ from pint import UnitRegistry from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup -from . import _handlers, _tags_handlers +from . import _controller_rest, _tags_handlers _logger = logging.getLogger(__name__) @@ -24,10 +22,10 @@ def setup_catalog(app: web.Application): # ensures routes are names that corresponds to function names assert all( # nosec route_def.kwargs["name"] == route_def.handler.__name__ # type: ignore[attr-defined] # route_def is a RouteDef not an Abstract - for route_def in _handlers.routes + for route_def in _controller_rest.routes ) - app.add_routes(_handlers.routes) + app.add_routes(_controller_rest.routes) app.add_routes(_tags_handlers.routes) # prepares units registry diff --git a/services/web/server/tests/unit/isolated/test_catalog_models.py b/services/web/server/tests/unit/isolated/test_catalog_models.py index ec82b0ab3671..e891b11b6f27 100644 --- a/services/web/server/tests/unit/isolated/test_catalog_models.py +++ b/services/web/server/tests/unit/isolated/test_catalog_models.py @@ -10,7 +10,7 @@ from pint import UnitRegistry from pytest_benchmark.fixture import BenchmarkFixture from simcore_service_webserver.catalog._api_units import replace_service_input_outputs -from simcore_service_webserver.catalog._handlers import RESPONSE_MODEL_POLICY +from simcore_service_webserver.catalog._controller_rest import RESPONSE_MODEL_POLICY @pytest.fixture(params=["UnitRegistry", None]) diff --git a/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py b/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py index f1024c85af32..8ce6c01d5627 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py +++ b/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py @@ -111,17 +111,17 @@ async def _update( return { "list_services_paginated": mocker.patch( - "simcore_service_webserver.catalog._api.catalog_rpc.list_services_paginated", + "simcore_service_webserver.catalog._service.catalog_rpc.list_services_paginated", autospec=True, side_effect=_list, ), "get_service": mocker.patch( - "simcore_service_webserver.catalog._api.catalog_rpc.get_service", + "simcore_service_webserver.catalog._service.catalog_rpc.get_service", autospec=True, side_effect=_get, ), "update_service": mocker.patch( - "simcore_service_webserver.catalog._api.catalog_rpc.update_service", + "simcore_service_webserver.catalog._service.catalog_rpc.update_service", autospec=True, side_effect=_update, ), From da6debd8d62c6ed33d8adc1a3fcfc56ea3ded534 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Tue, 4 Mar 2025 12:50:21 +0100 Subject: [PATCH 03/30] make the catalog rest client private --- .../{client.py => _catalog_rest_client.py} | 6 +-- .../catalog/_controller_rest.py | 14 ++++--- .../catalog/_service.py | 12 +++--- .../catalog/catalog_service.py | 0 .../diagnostics/_handlers.py | 8 ++-- .../exporter/_formatter/_sds.py | 4 +- .../projects/_crud_api_create.py | 10 ++--- .../projects/_crud_api_read.py | 16 ++++--- .../projects/_crud_handlers.py | 18 ++++---- .../projects/_nodes_handlers.py | 8 ++-- .../projects/projects_service.py | 42 +++++++++---------- .../_pricing_plans_admin_service.py | 12 +++--- .../tests/unit/isolated/test_catalog_setup.py | 6 ++- 13 files changed, 80 insertions(+), 76 deletions(-) rename services/web/server/src/simcore_service_webserver/catalog/{client.py => _catalog_rest_client.py} (97%) create mode 100644 services/web/server/src/simcore_service_webserver/catalog/catalog_service.py diff --git a/services/web/server/src/simcore_service_webserver/catalog/client.py b/services/web/server/src/simcore_service_webserver/catalog/_catalog_rest_client.py similarity index 97% rename from services/web/server/src/simcore_service_webserver/catalog/client.py rename to services/web/server/src/simcore_service_webserver/catalog/_catalog_rest_client.py index 100b4e69318a..58ee6a1f67bc 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/client.py +++ b/services/web/server/src/simcore_service_webserver/catalog/_catalog_rest_client.py @@ -1,7 +1,5 @@ -""" Requests to catalog service API +"""Requests to catalog service API""" -""" -import asyncio import logging import urllib.parse from collections.abc import Iterator @@ -47,7 +45,7 @@ def _handle_client_exceptions(app: web.Application) -> Iterator[ClientSession]: reason=MSG_CATALOG_SERVICE_UNAVAILABLE ) from err - except (asyncio.TimeoutError, ClientConnectionError) as err: + except (TimeoutError, ClientConnectionError) as err: _logger.debug("Request to catalog service failed: %s", err) raise web.HTTPServiceUnavailable( reason=MSG_CATALOG_SERVICE_UNAVAILABLE diff --git a/services/web/server/src/simcore_service_webserver/catalog/_controller_rest.py b/services/web/server/src/simcore_service_webserver/catalog/_controller_rest.py index f6900030682a..d3bb9cf2b0b6 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/_controller_rest.py +++ b/services/web/server/src/simcore_service_webserver/catalog/_controller_rest.py @@ -38,7 +38,7 @@ from ..resource_usage.service import get_default_service_pricing_plan from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response -from . import _handlers_errors, _service, client +from . import _catalog_rest_client, _handlers_errors, _service from ._exceptions import DefaultPricingUnitForServiceNotFoundError from ._models import CatalogRequestContext, ListServiceParams, ServicePathParams @@ -329,11 +329,13 @@ async def get_service_resources(request: Request): """ ctx = CatalogRequestContext.create(request) path_params = parse_request_path_parameters_as(ServicePathParams, request) - service_resources: ServiceResourcesDict = await client.get_service_resources( - request.app, - user_id=ctx.user_id, - service_key=path_params.service_key, - service_version=path_params.service_version, + service_resources: ServiceResourcesDict = ( + await _catalog_rest_client.get_service_resources( + request.app, + user_id=ctx.user_id, + service_key=path_params.service_key, + service_version=path_params.service_version, + ) ) data = ServiceResourcesDictHelpers.create_jsonable(service_resources) diff --git a/services/web/server/src/simcore_service_webserver/catalog/_service.py b/services/web/server/src/simcore_service_webserver/catalog/_service.py index 746877692299..e750fe0cead3 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/_service.py +++ b/services/web/server/src/simcore_service_webserver/catalog/_service.py @@ -25,7 +25,7 @@ from servicelib.rest_constants import RESPONSE_MODEL_POLICY from ..rabbitmq import get_rabbitmq_rpc_client -from . import client +from . import _catalog_rest_client from ._api_units import can_connect, replace_service_input_outputs from ._models import ( CatalogRequestContext, @@ -150,7 +150,7 @@ async def list_services( product_name: str, unit_registry: UnitRegistry, ): - services = await client.get_services_for_user_in_product( + services = await _catalog_rest_client.get_services_for_user_in_product( app, user_id, product_name, only_key_versions=False ) for service in services: @@ -162,7 +162,7 @@ async def list_services( async def list_service_inputs( service_key: ServiceKey, service_version: ServiceVersion, ctx: CatalogRequestContext ) -> list[ServiceInputGet]: - service = await client.get_service( + service = await _catalog_rest_client.get_service( ctx.app, ctx.user_id, service_key, service_version, ctx.product_name ) return [ @@ -179,7 +179,7 @@ async def get_service_input( input_key: ServiceInputKey, ctx: CatalogRequestContext, ) -> ServiceInputGet: - service = await client.get_service( + service = await _catalog_rest_client.get_service( ctx.app, ctx.user_id, service_key, service_version, ctx.product_name ) service_input: ServiceInputGet = ( @@ -238,7 +238,7 @@ async def list_service_outputs( service_version: ServiceVersion, ctx: CatalogRequestContext, ) -> list[ServiceOutputGet]: - service = await client.get_service( + service = await _catalog_rest_client.get_service( ctx.app, ctx.user_id, service_key, service_version, ctx.product_name ) return [ @@ -255,7 +255,7 @@ async def get_service_output( output_key: ServiceOutputKey, ctx: CatalogRequestContext, ) -> ServiceOutputGet: - service = await client.get_service( + service = await _catalog_rest_client.get_service( ctx.app, ctx.user_id, service_key, service_version, ctx.product_name ) return cast( # mypy -> aiocache is not typed. diff --git a/services/web/server/src/simcore_service_webserver/catalog/catalog_service.py b/services/web/server/src/simcore_service_webserver/catalog/catalog_service.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/services/web/server/src/simcore_service_webserver/diagnostics/_handlers.py b/services/web/server/src/simcore_service_webserver/diagnostics/_handlers.py index bed2f77f7f28..214fafff7736 100644 --- a/services/web/server/src/simcore_service_webserver/diagnostics/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/diagnostics/_handlers.py @@ -1,6 +1,4 @@ -""" Handler functions and routing for diagnostics - -""" +"""Handler functions and routing for diagnostics""" import asyncio import logging @@ -15,7 +13,7 @@ from servicelib.utils import logged_gather from .._meta import API_VERSION, APP_NAME, api_version_prefix -from ..catalog.client import is_catalog_service_responsive +from ..catalog import catalog_service from ..db import plugin from ..director_v2 import api as director_v2_api from ..login.decorators import login_required @@ -131,7 +129,7 @@ async def _check_director2(): async def _check_catalog(): check.services["catalog"] = { - "healthy": await is_catalog_service_responsive(request.app) + "healthy": await catalog_service.is_catalog_service_responsive(request.app) } async def _check_resource_usage_tracker(): diff --git a/services/web/server/src/simcore_service_webserver/exporter/_formatter/_sds.py b/services/web/server/src/simcore_service_webserver/exporter/_formatter/_sds.py index 62f02f2b1d18..d605ada5e59a 100644 --- a/services/web/server/src/simcore_service_webserver/exporter/_formatter/_sds.py +++ b/services/web/server/src/simcore_service_webserver/exporter/_formatter/_sds.py @@ -7,7 +7,7 @@ from aiohttp import web from servicelib.pools import non_blocking_process_pool_executor -from ...catalog.client import get_service +from ...catalog import catalog_service from ...projects.exceptions import BaseProjectError from ...projects.models import ProjectDict from ...projects.projects_service import get_project_for_user @@ -183,7 +183,7 @@ async def create_sds_directory( service_version = entry["version"] label = entry["label"] - service_data = await get_service( + service_data = await catalog_service.get_service( app=app, user_id=user_id, service_key=service_key, diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py index 4a93ad9286b5..711b5b0f64cf 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py @@ -27,7 +27,7 @@ from simcore_postgres_database.webserver_models import ProjectType as ProjectTypeDB from ..application_settings import get_application_settings -from ..catalog import client as catalog_client +from ..catalog import catalog_service from ..director_v2 import api as director_v2_api from ..dynamic_scheduler import api as dynamic_scheduler_api from ..folders import _folders_repository as _folders_repository @@ -219,7 +219,7 @@ async def _compose_project_data( NodeID(node_id): ProjectNodeCreate( node_id=NodeID(node_id), required_resources=jsonable_encoder( - await catalog_client.get_service_resources( + await catalog_service.get_service_resources( app, user_id, node_data["key"], node_data["version"] ) ), @@ -419,9 +419,9 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche user_specific_project_data_db = ( await _projects_repository.get_user_specific_project_data_db( project_uuid=new_project["uuid"], - private_workspace_user_id_or_none=user_id - if workspace_id is None - else None, + private_workspace_user_id_or_none=( + user_id if workspace_id is None else None + ), ) ) new_project["folderId"] = user_specific_project_data_db.folder_id diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py index d83d2265bfe2..5965a7e4a2ad 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py @@ -1,4 +1,4 @@ -""" Utils to implement READ operations (from cRud) on the project resource +"""Utils to implement READ operations (from cRud) on the project resource Read operations are list, get @@ -19,7 +19,7 @@ from simcore_postgres_database.models.projects import ProjectType from simcore_postgres_database.webserver_models import ProjectType as ProjectTypeDB -from ..catalog.client import get_services_for_user_in_product +from ..catalog import catalog_service from ..folders import _folders_repository from ..workspaces._workspaces_service import check_user_workspace_access from . import projects_service @@ -104,8 +104,10 @@ async def list_projects( # pylint: disable=too-many-arguments ) -> tuple[list[ProjectDict], int]: db = ProjectDBAPI.get_from_app_context(app) - user_available_services: list[dict] = await get_services_for_user_in_product( - app, user_id, product_name, only_key_versions=True + user_available_services: list[dict] = ( + await catalog_service.get_services_for_user_in_product( + app, user_id, product_name, only_key_versions=True + ) ) workspace_is_private = True @@ -184,8 +186,10 @@ async def list_projects_full_depth( ) -> tuple[list[ProjectDict], int]: db = ProjectDBAPI.get_from_app_context(app) - user_available_services: list[dict] = await get_services_for_user_in_product( - app, user_id, product_name, only_key_versions=True + user_available_services: list[dict] = ( + await catalog_service.get_services_for_user_in_product( + app, user_id, product_name, only_key_versions=True + ) ) db_projects, db_project_types, total_number_projects = await db.list_projects_dicts( diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py index 6c41e94384a8..2868d1a3cf0f 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py @@ -34,7 +34,7 @@ from servicelib.redis import get_project_locked_state from .._meta import API_VTAG as VTAG -from ..catalog.client import get_services_for_user_in_product +from ..catalog import catalog_service from ..login.decorators import login_required from ..redis import get_redis_lock_manager_client_sdk from ..resource_manager.user_sessions import PROJECT_ID_KEY, managed_resource @@ -97,11 +97,11 @@ async def create_project(request: web.Request): predefined_project = None else: # request w/ body (I found cases in which body = {}) - project_create: ( - ProjectCreateNew | ProjectCopyOverride | EmptyModel - ) = await parse_request_body_as( - ProjectCreateNew | ProjectCopyOverride | EmptyModel, # type: ignore[arg-type] # from pydantic v2 --> https://github.com/pydantic/pydantic/discussions/4950 - request, + project_create: ProjectCreateNew | ProjectCopyOverride | EmptyModel = ( + await parse_request_body_as( + ProjectCreateNew | ProjectCopyOverride | EmptyModel, # type: ignore[arg-type] # from pydantic v2 --> https://github.com/pydantic/pydantic/discussions/4950 + request, + ) ) predefined_project = project_create.to_domain_model() or None @@ -280,8 +280,10 @@ async def get_project(request: web.Request): req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(ProjectPathParams, request) - user_available_services: list[dict] = await get_services_for_user_in_product( - request.app, req_ctx.user_id, req_ctx.product_name, only_key_versions=True + user_available_services: list[dict] = ( + await catalog_service.get_services_for_user_in_product( + request.app, req_ctx.user_id, req_ctx.product_name, only_key_versions=True + ) ) project = await projects_service.get_project_for_user( diff --git a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py index 28fcb974511b..6cbb227cefe0 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py @@ -1,6 +1,4 @@ -""" Handlers for CRUD operations on /projects/{*}/nodes/{*} - -""" +"""Handlers for CRUD operations on /projects/{*}/nodes/{*}""" import asyncio import logging @@ -55,7 +53,7 @@ from simcore_postgres_database.models.users import UserRole from .._meta import API_VTAG as VTAG -from ..catalog import client as catalog_client +from ..catalog import catalog_service from ..dynamic_scheduler import api as dynamic_scheduler_api from ..groups.api import get_group_from_gid, list_all_user_groups_ids from ..groups.exceptions import GroupNotFoundError @@ -493,7 +491,7 @@ async def get_project_services_access_for_gid( project_services_access_rights: list[ServiceAccessRightsGet] = await asyncio.gather( *[ - catalog_client.get_service_access_rights( + catalog_service.get_service_access_rights( app=request.app, user_id=req_ctx.user_id, service_key=service.key, diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_service.py b/services/web/server/src/simcore_service_webserver/projects/projects_service.py index 76ba458b3ffe..46fb9b8d0e70 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_service.py @@ -94,7 +94,7 @@ from simcore_postgres_database.webserver_models import ProjectType from ..application_settings import get_application_settings -from ..catalog import client as catalog_client +from ..catalog import catalog_service from ..director_v2 import api as director_v2_api from ..dynamic_scheduler import api as dynamic_scheduler_api from ..products import products_web @@ -412,9 +412,9 @@ async def _get_default_pricing_and_hardware_info( ) -_MACHINE_TOTAL_RAM_SAFE_MARGIN_RATIO: Final[ - float -] = 0.1 # NOTE: machines always have less available RAM than advertised +_MACHINE_TOTAL_RAM_SAFE_MARGIN_RATIO: Final[float] = ( + 0.1 # NOTE: machines always have less available RAM than advertised +) _SIDECARS_OPS_SAFE_RAM_MARGIN: Final[ByteSize] = TypeAdapter(ByteSize).validate_python( "1GiB" ) @@ -437,11 +437,11 @@ async def update_project_node_resources_from_hardware_info( return try: rabbitmq_rpc_client = get_rabbitmq_rpc_client(app) - unordered_list_ec2_instance_types: list[ - EC2InstanceTypeGet - ] = await get_instance_type_details( - rabbitmq_rpc_client, - instance_type_names=set(hardware_info.aws_ec2_instances), + unordered_list_ec2_instance_types: list[EC2InstanceTypeGet] = ( + await get_instance_type_details( + rabbitmq_rpc_client, + instance_type_names=set(hardware_info.aws_ec2_instances), + ) ) assert unordered_list_ec2_instance_types # nosec @@ -817,7 +817,7 @@ async def add_project_node( ) node_uuid = NodeID(service_id if service_id else f"{uuid4()}") - default_resources = await catalog_client.get_service_resources( + default_resources = await catalog_service.get_service_resources( request.app, user_id, service_key, service_version ) db: ProjectDBAPI = ProjectDBAPI.get_from_app_context(request.app) @@ -1366,10 +1366,10 @@ async def _open_project() -> bool: # Assign project_id to current_session current_session: UserSessionID = user_session.get_id() - sessions_with_project: list[ - UserSessionID - ] = await user_session.find_users_of_resource( - app, PROJECT_ID_KEY, f"{project_uuid}" + sessions_with_project: list[UserSessionID] = ( + await user_session.find_users_of_resource( + app, PROJECT_ID_KEY, f"{project_uuid}" + ) ) if not sessions_with_project: # no one has the project so we assign it @@ -1418,10 +1418,10 @@ async def try_close_project_for_user( ): with managed_resource(user_id, client_session_id, app) as user_session: current_session: UserSessionID = user_session.get_id() - all_sessions_with_project: list[ - UserSessionID - ] = await user_session.find_users_of_resource( - app, key=PROJECT_ID_KEY, value=project_uuid + all_sessions_with_project: list[UserSessionID] = ( + await user_session.find_users_of_resource( + app, key=PROJECT_ID_KEY, value=project_uuid + ) ) # first check whether other sessions registered this project @@ -1616,7 +1616,7 @@ async def is_service_deprecated( service_version: str, product_name: str, ) -> bool: - service = await catalog_client.get_service( + service = await catalog_service.get_service( app, user_id, service_key, service_version, product_name ) if deprecation_date := service.get("deprecated"): @@ -1665,7 +1665,7 @@ async def get_project_node_resources( ) if not node_resources: # get default resources - node_resources = await catalog_client.get_service_resources( + node_resources = await catalog_service.get_service_resources( app, user_id, service_key, service_version ) return node_resources @@ -1696,7 +1696,7 @@ async def update_project_node_resources( if not current_resources: # NOTE: this can happen after the migration # get default resources - current_resources = await catalog_client.get_service_resources( + current_resources = await catalog_service.get_service_resources( app, user_id, service_key, service_version ) diff --git a/services/web/server/src/simcore_service_webserver/resource_usage/_pricing_plans_admin_service.py b/services/web/server/src/simcore_service_webserver/resource_usage/_pricing_plans_admin_service.py index f446e089aeff..d74bdee870af 100644 --- a/services/web/server/src/simcore_service_webserver/resource_usage/_pricing_plans_admin_service.py +++ b/services/web/server/src/simcore_service_webserver/resource_usage/_pricing_plans_admin_service.py @@ -24,7 +24,7 @@ pricing_units, ) -from ..catalog import client as catalog_client +from ..catalog import catalog_service from ..rabbitmq import get_rabbitmq_rpc_client ## Pricing Plans @@ -146,10 +146,10 @@ async def list_connected_services_to_pricing_plan( app: web.Application, product_name: ProductName, pricing_plan_id: PricingPlanId ) -> list[PricingPlanToServiceGet]: rpc_client = get_rabbitmq_rpc_client(app) - output: list[ - PricingPlanToServiceGet - ] = await pricing_plans.list_connected_services_to_pricing_plan_by_pricing_plan( - rpc_client, product_name=product_name, pricing_plan_id=pricing_plan_id + output: list[PricingPlanToServiceGet] = ( + await pricing_plans.list_connected_services_to_pricing_plan_by_pricing_plan( + rpc_client, product_name=product_name, pricing_plan_id=pricing_plan_id + ) ) return output @@ -163,7 +163,7 @@ async def connect_service_to_pricing_plan( service_version: ServiceVersion, ) -> PricingPlanToServiceGet: # Check whether service key and version exists - await catalog_client.get_service( + await catalog_service.get_service( app, user_id, service_key, service_version, product_name ) diff --git a/services/web/server/tests/unit/isolated/test_catalog_setup.py b/services/web/server/tests/unit/isolated/test_catalog_setup.py index 2fdd2e336efd..f16efc1695ed 100644 --- a/services/web/server/tests/unit/isolated/test_catalog_setup.py +++ b/services/web/server/tests/unit/isolated/test_catalog_setup.py @@ -9,7 +9,7 @@ from aiohttp.test_utils import TestClient from servicelib.aiohttp.application import create_safe_application from simcore_service_webserver._meta import api_version_prefix -from simcore_service_webserver.catalog.client import to_backend_service +from simcore_service_webserver.catalog import catalog_service from simcore_service_webserver.catalog.plugin import setup_catalog from yarl import URL @@ -35,6 +35,8 @@ def test_url_translation(): assert rel_url.path.startswith(f"/{api_version_prefix}/catalog") api_target_origin = URL("http://catalog:8000") - api_target_url = to_backend_service(rel_url, api_target_origin, "v5") + api_target_url = catalog_service.to_backend_service( + rel_url, api_target_origin, "v5" + ) assert str(api_target_url) == "http://catalog:8000/v5/dags/123?page_size=6" From 8fd6b61865c44e14c6a0c4d5c7ecdd065afc2b26 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Tue, 4 Mar 2025 13:00:15 +0100 Subject: [PATCH 04/30] make the catalog rest client private --- .../catalog/catalog_service.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/services/web/server/src/simcore_service_webserver/catalog/catalog_service.py b/services/web/server/src/simcore_service_webserver/catalog/catalog_service.py index e69de29bb2d1..99bb3d3b93b7 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/catalog_service.py +++ b/services/web/server/src/simcore_service_webserver/catalog/catalog_service.py @@ -0,0 +1,17 @@ +from ._catalog_rest_client import ( + get_service, + get_service_access_rights, + get_service_resources, + get_services_for_user_in_product, + is_catalog_service_responsive, + to_backend_service, +) + +__all__ = ( + "is_catalog_service_responsive", + "to_backend_service", + "get_services_for_user_in_product", + "get_service", + "get_service_resources", + "get_service_access_rights", +) From 4b1bc031e4006f685f44551d99868bfe86148d1d Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Tue, 4 Mar 2025 13:06:32 +0100 Subject: [PATCH 05/30] make the catalog rest client private --- .../catalog/catalog_service.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/catalog/catalog_service.py b/services/web/server/src/simcore_service_webserver/catalog/catalog_service.py index 99bb3d3b93b7..ca741e15a4c5 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/catalog_service.py +++ b/services/web/server/src/simcore_service_webserver/catalog/catalog_service.py @@ -1,13 +1,4 @@ -from ._catalog_rest_client import ( - get_service, - get_service_access_rights, - get_service_resources, - get_services_for_user_in_product, - is_catalog_service_responsive, - to_backend_service, -) - -__all__ = ( +__all__: tuple[str, ...] = ( "is_catalog_service_responsive", "to_backend_service", "get_services_for_user_in_product", From f2ea112786039542b46b88c697cb37ac3818f22b Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Tue, 4 Mar 2025 13:16:29 +0100 Subject: [PATCH 06/30] make the catalog rest client privat --- .../src/simcore_service_webserver/catalog/catalog_service.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/web/server/src/simcore_service_webserver/catalog/catalog_service.py b/services/web/server/src/simcore_service_webserver/catalog/catalog_service.py index ca741e15a4c5..c0bd23ab3ee4 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/catalog_service.py +++ b/services/web/server/src/simcore_service_webserver/catalog/catalog_service.py @@ -1,3 +1,5 @@ +from ._catalog_rest_client import * # noqa + __all__: tuple[str, ...] = ( "is_catalog_service_responsive", "to_backend_service", From 560a2eef0b87d042c998f97bee50641ff8f4c0b6 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Tue, 4 Mar 2025 13:41:22 +0100 Subject: [PATCH 07/30] remove _handler_exceptions file --- .../catalog/_controller_rest.py | 15 +++++---- .../catalog/_exceptions.py | 25 +++++++++++++++ .../catalog/_handlers_errors.py | 31 ------------------- 3 files changed, 34 insertions(+), 37 deletions(-) delete mode 100644 services/web/server/src/simcore_service_webserver/catalog/_handlers_errors.py diff --git a/services/web/server/src/simcore_service_webserver/catalog/_controller_rest.py b/services/web/server/src/simcore_service_webserver/catalog/_controller_rest.py index d3bb9cf2b0b6..e6bf7a5e3d5e 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/_controller_rest.py +++ b/services/web/server/src/simcore_service_webserver/catalog/_controller_rest.py @@ -38,8 +38,11 @@ from ..resource_usage.service import get_default_service_pricing_plan from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response -from . import _catalog_rest_client, _handlers_errors, _service -from ._exceptions import DefaultPricingUnitForServiceNotFoundError +from . import _catalog_rest_client, _service +from ._exceptions import ( + DefaultPricingUnitForServiceNotFoundError, + reraise_catalog_exceptions_as_http_errors, +) from ._models import CatalogRequestContext, ListServiceParams, ServicePathParams _logger = logging.getLogger(__name__) @@ -56,7 +59,7 @@ ) @login_required @permission_required("services.catalog.*") -@_handlers_errors.reraise_catalog_exceptions_as_http_errors +@reraise_catalog_exceptions_as_http_errors async def list_services_latest(request: Request): request_ctx = CatalogRequestContext.create(request) query_params: ListServiceParams = parse_request_query_parameters_as( @@ -94,7 +97,7 @@ async def list_services_latest(request: Request): ) @login_required @permission_required("services.catalog.*") -@_handlers_errors.reraise_catalog_exceptions_as_http_errors +@reraise_catalog_exceptions_as_http_errors async def get_service(request: Request): request_ctx = CatalogRequestContext.create(request) path_params = parse_request_path_parameters_as(ServicePathParams, request) @@ -120,7 +123,7 @@ async def get_service(request: Request): ) @login_required @permission_required("services.catalog.*") -@_handlers_errors.reraise_catalog_exceptions_as_http_errors +@reraise_catalog_exceptions_as_http_errors async def update_service(request: Request): request_ctx = CatalogRequestContext.create(request) path_params = parse_request_path_parameters_as(ServicePathParams, request) @@ -350,7 +353,7 @@ async def get_service_resources(request: Request): ) @login_required @permission_required("services.catalog.*") -@_handlers_errors.reraise_catalog_exceptions_as_http_errors +@reraise_catalog_exceptions_as_http_errors async def get_service_pricing_plan(request: Request): ctx = CatalogRequestContext.create(request) path_params = parse_request_path_parameters_as(ServicePathParams, request) diff --git a/services/web/server/src/simcore_service_webserver/catalog/_exceptions.py b/services/web/server/src/simcore_service_webserver/catalog/_exceptions.py index 11f3794661b3..bf79a120ef53 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/_exceptions.py +++ b/services/web/server/src/simcore_service_webserver/catalog/_exceptions.py @@ -1,11 +1,36 @@ """Defines the different exceptions that may arise in the catalog subpackage""" +import functools + +from aiohttp import web +from servicelib.aiohttp.typing_extension import Handler from servicelib.rabbitmq.rpc_interfaces.catalog.errors import ( CatalogForbiddenError, CatalogItemNotFoundError, ) from ..errors import WebServerBaseError +from ..resource_usage.errors import DefaultPricingPlanNotFoundError + + +def reraise_catalog_exceptions_as_http_errors(handler: Handler): + @functools.wraps(handler) + async def _wrapper(request: web.Request) -> web.StreamResponse: + try: + + return await handler(request) + + except ( + CatalogItemNotFoundError, + DefaultPricingPlanNotFoundError, + DefaultPricingUnitForServiceNotFoundError, + ) as exc: + raise web.HTTPNotFound(reason=f"{exc}") from exc + + except CatalogForbiddenError as exc: + raise web.HTTPForbidden(reason=f"{exc}") from exc + + return _wrapper class BaseCatalogError(WebServerBaseError): diff --git a/services/web/server/src/simcore_service_webserver/catalog/_handlers_errors.py b/services/web/server/src/simcore_service_webserver/catalog/_handlers_errors.py deleted file mode 100644 index 6c93ebcb6092..000000000000 --- a/services/web/server/src/simcore_service_webserver/catalog/_handlers_errors.py +++ /dev/null @@ -1,31 +0,0 @@ -import functools - -from aiohttp import web -from servicelib.aiohttp.typing_extension import Handler - -from ..resource_usage.errors import DefaultPricingPlanNotFoundError -from ._exceptions import ( - CatalogForbiddenError, - CatalogItemNotFoundError, - DefaultPricingUnitForServiceNotFoundError, -) - - -def reraise_catalog_exceptions_as_http_errors(handler: Handler): - @functools.wraps(handler) - async def _wrapper(request: web.Request) -> web.StreamResponse: - try: - - return await handler(request) - - except ( - CatalogItemNotFoundError, - DefaultPricingPlanNotFoundError, - DefaultPricingUnitForServiceNotFoundError, - ) as exc: - raise web.HTTPNotFound(reason=f"{exc}") from exc - - except CatalogForbiddenError as exc: - raise web.HTTPForbidden(reason=f"{exc}") from exc - - return _wrapper From 16f89d7f5650029b673dd08d6e8decc83e704774 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Tue, 4 Mar 2025 16:15:18 +0100 Subject: [PATCH 08/30] start adding test for getting inputs REST endpoint --- .../catalog/_service.py | 16 ------------ .../01/test_catalog_handlers__services.py | 25 +++++++++++++++++++ 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/catalog/_service.py b/services/web/server/src/simcore_service_webserver/catalog/_service.py index e750fe0cead3..7d73e65f13fa 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/_service.py +++ b/services/web/server/src/simcore_service_webserver/catalog/_service.py @@ -143,22 +143,6 @@ async def update_service_v2( return data -async def list_services( - app: web.Application, - *, - user_id: UserID, - product_name: str, - unit_registry: UnitRegistry, -): - services = await _catalog_rest_client.get_services_for_user_in_product( - app, user_id, product_name, only_key_versions=False - ) - for service in services: - await _safe_replace_service_input_outputs(service, unit_registry) - - return services - - async def list_service_inputs( service_key: ServiceKey, service_version: ServiceVersion, ctx: CatalogRequestContext ) -> list[ServiceInputGet]: diff --git a/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py b/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py index 8ce6c01d5627..a8a64df4544f 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py +++ b/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py @@ -8,6 +8,7 @@ import pytest from aiohttp import web from aiohttp.test_utils import TestClient +from aioresponses import aioresponses as AioResponsesMock from faker import Faker from models_library.api_schemas_catalog.services import ServiceGetV2 from models_library.api_schemas_webserver.catalog import ( @@ -156,6 +157,30 @@ async def test_list_services_latest( assert mocked_rpc_catalog_service_api["list_services_paginated"].call_count == 1 +@pytest.mark.parametrize( + "user_role", + [UserRole.USER], +) +async def test_get_inputs( + client: TestClient, logged_user: UserInfoDict, aioresponses_mocker: AioResponsesMock +): + + aioresponses_mocker.get( + r"http://catalog:8000/v0/services/simcore%2Fservices%2Fcomp%2Fitis%2Fsleeper/0.1.0?user_id=1" + ) + + service_key = "simcore/services/comp/itis/sleeper" + service_version = "0.1.0" + assert client.app and client.app.router + url = client.app.router["list_service_inputs"].url_for( + service_key=urllib.parse.quote(service_key, safe=""), + service_version=service_version, + ) + + response = await client.get(f"{url}") + data, error = await assert_status(response, status.HTTP_200_OK) + + @pytest.mark.parametrize( "user_role", [UserRole.USER], From ba066aaf606a5922b6fab49618bc82225ff13a9a Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Tue, 4 Mar 2025 16:22:53 +0100 Subject: [PATCH 09/30] add response data to call to catalog --- .../tests/unit/with_dbs/01/test_catalog_handlers__services.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py b/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py index a8a64df4544f..e1680dc0694c 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py +++ b/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py @@ -166,7 +166,9 @@ async def test_get_inputs( ): aioresponses_mocker.get( - r"http://catalog:8000/v0/services/simcore%2Fservices%2Fcomp%2Fitis%2Fsleeper/0.1.0?user_id=1" + r"http://catalog:8000/v0/services/simcore%2Fservices%2Fcomp%2Fitis%2Fsleeper/0.1.0?user_id=1", + status=status.HTTP_200_OK, + payload=ServiceGetV2.model_json_schema()["examples"][0], ) service_key = "simcore/services/comp/itis/sleeper" From 4a23dce1c2977816ef0ac462d2bcd58dec1a5f99 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 5 Mar 2025 08:51:46 +0100 Subject: [PATCH 10/30] improve unit test --- .../unit/with_dbs/01/test_catalog_handlers__services.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py b/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py index e1680dc0694c..b0fffb3ba0dc 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py +++ b/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py @@ -2,6 +2,7 @@ # pylint:disable=unused-argument # pylint:disable=redefined-outer-name +import re import urllib.parse from unittest.mock import MagicMock @@ -29,6 +30,7 @@ from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.webserver_login import UserInfoDict from servicelib.aiohttp import status +from simcore_service_webserver.catalog._models import ServiceInputGet from simcore_service_webserver.db.models import UserRole @@ -165,10 +167,12 @@ async def test_get_inputs( client: TestClient, logged_user: UserInfoDict, aioresponses_mocker: AioResponsesMock ): + url_pattern = re.compile(r"http://catalog:8000/v0/services/.*") + service_payload = ServiceGetV2.model_json_schema()["examples"][0] aioresponses_mocker.get( - r"http://catalog:8000/v0/services/simcore%2Fservices%2Fcomp%2Fitis%2Fsleeper/0.1.0?user_id=1", + url_pattern, status=status.HTTP_200_OK, - payload=ServiceGetV2.model_json_schema()["examples"][0], + payload=service_payload, ) service_key = "simcore/services/comp/itis/sleeper" @@ -181,6 +185,7 @@ async def test_get_inputs( response = await client.get(f"{url}") data, error = await assert_status(response, status.HTTP_200_OK) + TypeAdapter(list[ServiceInputGet]).validate_python(data) @pytest.mark.parametrize( From 35663caafc4d37c0bf5b7c0283f96dd54e8d75e2 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 5 Mar 2025 12:51:07 +0100 Subject: [PATCH 11/30] add get_inputs_test --- .../01/test_catalog_handlers__services.py | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py b/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py index b0fffb3ba0dc..1f0e6d518a66 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py +++ b/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py @@ -163,7 +163,7 @@ async def test_list_services_latest( "user_role", [UserRole.USER], ) -async def test_get_inputs( +async def test_list_inputs( client: TestClient, logged_user: UserInfoDict, aioresponses_mocker: AioResponsesMock ): @@ -188,6 +188,34 @@ async def test_get_inputs( TypeAdapter(list[ServiceInputGet]).validate_python(data) +@pytest.mark.parametrize( + "user_role", + [UserRole.USER], +) +async def test_get_inputs( + client: TestClient, logged_user: UserInfoDict, aioresponses_mocker: AioResponsesMock +): + url_pattern = re.compile(r"http://catalog:8000/v0/services/.*") + service_payload = ServiceGetV2.model_json_schema()["examples"][0] + aioresponses_mocker.get( + url_pattern, + status=status.HTTP_200_OK, + payload=service_payload, + ) + + service_key = "simcore/services/comp/itis/sleeper" + service_version = "0.1.0" + assert client.app and client.app.router + url = client.app.router["get_service_input"].url_for( + service_key=urllib.parse.quote(service_key, safe=""), + service_version=service_version, + input_key=next(iter(service_payload["inputs"].keys())), + ) + response = await client.get(f"{url}") + data, error = await assert_status(response, status.HTTP_200_OK) + ServiceInputGet.model_validate(data) + + @pytest.mark.parametrize( "user_role", [UserRole.USER], From 73edc8c061bdc7eaeae068c6ab92bbf236447f50 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 5 Mar 2025 13:04:19 +0100 Subject: [PATCH 12/30] add test for checking input-output compatibility --- .../01/test_catalog_handlers__services.py | 41 ++++++++++++++++++- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py b/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py index 1f0e6d518a66..d70fc28e493f 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py +++ b/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py @@ -184,7 +184,7 @@ async def test_list_inputs( ) response = await client.get(f"{url}") - data, error = await assert_status(response, status.HTTP_200_OK) + data, _ = await assert_status(response, status.HTTP_200_OK) TypeAdapter(list[ServiceInputGet]).validate_python(data) @@ -212,10 +212,47 @@ async def test_get_inputs( input_key=next(iter(service_payload["inputs"].keys())), ) response = await client.get(f"{url}") - data, error = await assert_status(response, status.HTTP_200_OK) + data, _ = await assert_status(response, status.HTTP_200_OK) ServiceInputGet.model_validate(data) +@pytest.mark.parametrize( + "user_role", + [UserRole.USER], +) +async def test_get_compatible_inputs_given_source_outputs( + client: TestClient, logged_user: UserInfoDict, aioresponses_mocker: AioResponsesMock +): + url_pattern = re.compile(r"http://catalog:8000/v0/services/.*") + service_payload = ServiceGetV2.model_json_schema()["examples"][0] + for _ in range(2): + aioresponses_mocker.get( + url_pattern, + status=status.HTTP_200_OK, + payload=service_payload, + ) + + service_key = "simcore/services/comp/itis/sleeper" + service_version = "0.1.0" + assert client.app and client.app.router + url = ( + client.app.router["get_compatible_inputs_given_source_output"] + .url_for( + service_key=urllib.parse.quote(service_key, safe=""), + service_version=service_version, + ) + .with_query( + { + "fromService": "simcore/services/comp/itis/sleeper", + "fromVersion": "0.1.0", + "fromOutput": "output_1", + } + ) + ) + response = await client.get(f"{url}") + _, _ = await assert_status(response, status.HTTP_200_OK) + + @pytest.mark.parametrize( "user_role", [UserRole.USER], From 456c638192d446d193e876a7a291239f740b0226 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 5 Mar 2025 13:21:51 +0100 Subject: [PATCH 13/30] add test for matching output endpoints --- .../01/test_catalog_handlers__services.py | 98 ++++++++++++++++++- 1 file changed, 97 insertions(+), 1 deletion(-) diff --git a/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py b/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py index d70fc28e493f..90a8e43bbfcd 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py +++ b/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py @@ -30,7 +30,7 @@ from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.webserver_login import UserInfoDict from servicelib.aiohttp import status -from simcore_service_webserver.catalog._models import ServiceInputGet +from simcore_service_webserver.catalog._models import ServiceInputGet, ServiceOutputGet from simcore_service_webserver.db.models import UserRole @@ -188,6 +188,65 @@ async def test_list_inputs( TypeAdapter(list[ServiceInputGet]).validate_python(data) +@pytest.mark.parametrize( + "user_role", + [UserRole.USER], +) +async def test_list_outputs( + client: TestClient, logged_user: UserInfoDict, aioresponses_mocker: AioResponsesMock +): + + url_pattern = re.compile(r"http://catalog:8000/v0/services/.*") + service_payload = ServiceGetV2.model_json_schema()["examples"][0] + aioresponses_mocker.get( + url_pattern, + status=status.HTTP_200_OK, + payload=service_payload, + ) + + service_key = "simcore/services/comp/itis/sleeper" + service_version = "0.1.0" + assert client.app and client.app.router + url = client.app.router["list_service_outputs"].url_for( + service_key=urllib.parse.quote(service_key, safe=""), + service_version=service_version, + ) + + response = await client.get(f"{url}") + data, _ = await assert_status(response, status.HTTP_200_OK) + TypeAdapter(list[ServiceOutputGet]).validate_python(data) + + +@pytest.mark.parametrize( + "user_role", + [UserRole.USER], +) +async def test_get_outputs( + client: TestClient, logged_user: UserInfoDict, aioresponses_mocker: AioResponsesMock +): + + url_pattern = re.compile(r"http://catalog:8000/v0/services/.*") + service_payload = ServiceGetV2.model_json_schema()["examples"][0] + aioresponses_mocker.get( + url_pattern, + status=status.HTTP_200_OK, + payload=service_payload, + ) + + service_key = "simcore/services/comp/itis/sleeper" + service_version = "0.1.0" + assert client.app and client.app.router + url = client.app.router["get_service_output"].url_for( + service_key=urllib.parse.quote(service_key, safe=""), + service_version=service_version, + output_key=next(iter(service_payload["outputs"].keys())), + ) + + response = await client.get(f"{url}") + data, _ = await assert_status(response, status.HTTP_200_OK) + ServiceOutputGet.model_validate(data) + + @pytest.mark.parametrize( "user_role", [UserRole.USER], @@ -253,6 +312,43 @@ async def test_get_compatible_inputs_given_source_outputs( _, _ = await assert_status(response, status.HTTP_200_OK) +@pytest.mark.parametrize( + "user_role", + [UserRole.USER], +) +async def test_get_compatible_outputs_given_target_inptuts( + client: TestClient, logged_user: UserInfoDict, aioresponses_mocker: AioResponsesMock +): + url_pattern = re.compile(r"http://catalog:8000/v0/services/.*") + service_payload = ServiceGetV2.model_json_schema()["examples"][0] + for _ in range(2): + aioresponses_mocker.get( + url_pattern, + status=status.HTTP_200_OK, + payload=service_payload, + ) + + service_key = "simcore/services/comp/itis/sleeper" + service_version = "0.1.0" + assert client.app and client.app.router + url = ( + client.app.router["get_compatible_outputs_given_target_input"] + .url_for( + service_key=urllib.parse.quote(service_key, safe=""), + service_version=service_version, + ) + .with_query( + { + "toService": "simcore/services/comp/itis/sleeper", + "toVersion": "0.1.0", + "toInput": "input_1", + } + ) + ) + response = await client.get(f"{url}") + _, _ = await assert_status(response, status.HTTP_200_OK) + + @pytest.mark.parametrize( "user_role", [UserRole.USER], From 997b55d004dcfe34af1dd6c0d07ca17cc1b3e380 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 5 Mar 2025 14:32:03 +0100 Subject: [PATCH 14/30] add tests directly for catalog rest client --- .../with_dbs/01/test_catalog_rest_client.py | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 services/web/server/tests/unit/with_dbs/01/test_catalog_rest_client.py diff --git a/services/web/server/tests/unit/with_dbs/01/test_catalog_rest_client.py b/services/web/server/tests/unit/with_dbs/01/test_catalog_rest_client.py new file mode 100644 index 000000000000..9642a4c63ac6 --- /dev/null +++ b/services/web/server/tests/unit/with_dbs/01/test_catalog_rest_client.py @@ -0,0 +1,55 @@ +import re + +import pytest +from aiohttp.test_utils import TestClient +from aioresponses import aioresponses as AioResponsesMock +from common_library.users_enums import UserRole +from pytest_simcore.helpers.webserver_login import UserInfoDict +from servicelib.aiohttp import status +from simcore_service_webserver.catalog.catalog_service import ( + get_services_for_user_in_product, + is_catalog_service_responsive, +) + + +@pytest.mark.parametrize( + "user_role", + [UserRole.USER], +) +async def test_server_responsive( + client: TestClient, logged_user: UserInfoDict, aioresponses_mocker: AioResponsesMock +): + aioresponses_mocker.get( + "http://catalog:8000", + status=status.HTTP_200_OK, + ) + + assert client.app + assert await is_catalog_service_responsive(app=client.app) == True + + +@pytest.mark.parametrize( + "user_role", + [UserRole.USER], +) +@pytest.mark.parametrize( + "backend_status_code", [status.HTTP_200_OK, status.HTTP_404_NOT_FOUND] +) +async def test_get_services_for_user_in_product( + client: TestClient, + logged_user: UserInfoDict, + aioresponses_mocker: AioResponsesMock, + backend_status_code: int, +): + url_pattern = re.compile(r"http://catalog:8000/.*") + aioresponses_mocker.get( + url_pattern, + status=backend_status_code, + ) + assert client.app + _ = await get_services_for_user_in_product( + app=client.app, + user_id=logged_user["id"], + product_name="osparc", + only_key_versions=False, + ) From 039e14f9c17d7ecab03e59ebbaaeca1558a5083d Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 5 Mar 2025 14:47:54 +0100 Subject: [PATCH 15/30] add test for getting access rights --- .../with_dbs/01/test_catalog_rest_client.py | 57 +++++++++++++++++-- 1 file changed, 51 insertions(+), 6 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/01/test_catalog_rest_client.py b/services/web/server/tests/unit/with_dbs/01/test_catalog_rest_client.py index 9642a4c63ac6..eb8ddd2dce8d 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_catalog_rest_client.py +++ b/services/web/server/tests/unit/with_dbs/01/test_catalog_rest_client.py @@ -4,9 +4,13 @@ from aiohttp.test_utils import TestClient from aioresponses import aioresponses as AioResponsesMock from common_library.users_enums import UserRole +from models_library.api_schemas_catalog.service_access_rights import ( + ServiceAccessRightsGet, +) from pytest_simcore.helpers.webserver_login import UserInfoDict from servicelib.aiohttp import status from simcore_service_webserver.catalog.catalog_service import ( + get_service_access_rights, get_services_for_user_in_product, is_catalog_service_responsive, ) @@ -16,16 +20,23 @@ "user_role", [UserRole.USER], ) +@pytest.mark.parametrize( + "backend_status_code", [status.HTTP_200_OK, status.HTTP_500_INTERNAL_SERVER_ERROR] +) async def test_server_responsive( - client: TestClient, logged_user: UserInfoDict, aioresponses_mocker: AioResponsesMock + client: TestClient, + logged_user: UserInfoDict, + aioresponses_mocker: AioResponsesMock, + backend_status_code: int, ): - aioresponses_mocker.get( - "http://catalog:8000", - status=status.HTTP_200_OK, - ) + aioresponses_mocker.get("http://catalog:8000", status=backend_status_code) assert client.app - assert await is_catalog_service_responsive(app=client.app) == True + is_responsive = await is_catalog_service_responsive(app=client.app) + if backend_status_code == status.HTTP_200_OK: + assert is_responsive == True + else: + assert is_responsive == False @pytest.mark.parametrize( @@ -53,3 +64,37 @@ async def test_get_services_for_user_in_product( product_name="osparc", only_key_versions=False, ) + + +@pytest.mark.parametrize( + "user_role", + [UserRole.USER], +) +async def test_get_service_access_rights( + client: TestClient, + logged_user: UserInfoDict, + aioresponses_mocker: AioResponsesMock, +): + url_pattern = re.compile(r"http://catalog:8000/.*") + example = ServiceAccessRightsGet( + service_key="simcore/services/comp/itis/sleeper", + service_version="2.1.4", + gids_with_access_rights={ + 1: {"execute_access": True}, + 5: {"execute_access": True}, + }, + ) + aioresponses_mocker.get( + url_pattern, + status=status.HTTP_200_OK, + payload=example.model_dump(), + ) + assert client.app + access_rights = await get_service_access_rights( + app=client.app, + user_id=logged_user["id"], + service_key="simcore/services/comp/itis/sleeper", + service_version="2.1.4", + product_name="osparc", + ) + assert isinstance(access_rights, ServiceAccessRightsGet) From 476657e87a45c95a7a2d36715752ca68a42ae76b Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Wed, 5 Mar 2025 14:53:11 +0100 Subject: [PATCH 16/30] add test for exceptions --- .../unit/with_dbs/01/test_catalog_rest_client.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/services/web/server/tests/unit/with_dbs/01/test_catalog_rest_client.py b/services/web/server/tests/unit/with_dbs/01/test_catalog_rest_client.py index eb8ddd2dce8d..e937a576ee55 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_catalog_rest_client.py +++ b/services/web/server/tests/unit/with_dbs/01/test_catalog_rest_client.py @@ -9,6 +9,9 @@ ) from pytest_simcore.helpers.webserver_login import UserInfoDict from servicelib.aiohttp import status +from simcore_service_webserver.catalog._exceptions import ( + DefaultPricingUnitForServiceNotFoundError, +) from simcore_service_webserver.catalog.catalog_service import ( get_service_access_rights, get_services_for_user_in_product, @@ -98,3 +101,11 @@ async def test_get_service_access_rights( product_name="osparc", ) assert isinstance(access_rights, ServiceAccessRightsGet) + + +async def test_catalog_exceptions(): + + error = DefaultPricingUnitForServiceNotFoundError( + service_key="key", service_version="version" + ) + assert isinstance(error.debug_message(), str) From 3a1d60fcf125a1fcafa8fa8a867bb7205ace2e7f Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Thu, 6 Mar 2025 06:43:15 +0100 Subject: [PATCH 17/30] @pcrespov models.py -> controller_rest_schemas.py --- .../src/simcore_service_webserver/catalog/_api_units.py | 6 +++++- .../simcore_service_webserver/catalog/_controller_rest.py | 6 +++++- .../src/simcore_service_webserver/catalog/_service.py | 2 +- .../catalog/{_models.py => controller_rest_schemas.py} | 0 .../unit/with_dbs/01/test_catalog_handlers__services.py | 5 ++++- 5 files changed, 15 insertions(+), 4 deletions(-) rename services/web/server/src/simcore_service_webserver/catalog/{_models.py => controller_rest_schemas.py} (100%) diff --git a/services/web/server/src/simcore_service_webserver/catalog/_api_units.py b/services/web/server/src/simcore_service_webserver/catalog/_api_units.py index 65e435f6886e..9cb3c3269b04 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/_api_units.py +++ b/services/web/server/src/simcore_service_webserver/catalog/_api_units.py @@ -3,7 +3,11 @@ from models_library.services import BaseServiceIOModel, ServiceInput, ServiceOutput from pint import PintError, UnitRegistry -from ._models import ServiceInputGetFactory, ServiceOutputGetFactory, get_unit_name +from .controller_rest_schemas import ( + ServiceInputGetFactory, + ServiceOutputGetFactory, + get_unit_name, +) def _get_type_name(port: BaseServiceIOModel) -> str: diff --git a/services/web/server/src/simcore_service_webserver/catalog/_controller_rest.py b/services/web/server/src/simcore_service_webserver/catalog/_controller_rest.py index e6bf7a5e3d5e..c0e81da9acd3 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/_controller_rest.py +++ b/services/web/server/src/simcore_service_webserver/catalog/_controller_rest.py @@ -43,7 +43,11 @@ DefaultPricingUnitForServiceNotFoundError, reraise_catalog_exceptions_as_http_errors, ) -from ._models import CatalogRequestContext, ListServiceParams, ServicePathParams +from .controller_rest_schemas import ( + CatalogRequestContext, + ListServiceParams, + ServicePathParams, +) _logger = logging.getLogger(__name__) diff --git a/services/web/server/src/simcore_service_webserver/catalog/_service.py b/services/web/server/src/simcore_service_webserver/catalog/_service.py index fa52e714589c..a9cc3542cbc2 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/_service.py +++ b/services/web/server/src/simcore_service_webserver/catalog/_service.py @@ -29,7 +29,7 @@ from ..rabbitmq import get_rabbitmq_rpc_client from . import _catalog_rest_client from ._api_units import can_connect, replace_service_input_outputs -from ._models import ( +from .controller_rest_schemas import ( CatalogRequestContext, ServiceInputGetFactory, ServiceOutputGetFactory, diff --git a/services/web/server/src/simcore_service_webserver/catalog/_models.py b/services/web/server/src/simcore_service_webserver/catalog/controller_rest_schemas.py similarity index 100% rename from services/web/server/src/simcore_service_webserver/catalog/_models.py rename to services/web/server/src/simcore_service_webserver/catalog/controller_rest_schemas.py diff --git a/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py b/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py index 90a8e43bbfcd..b5ca8665c37e 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py +++ b/services/web/server/tests/unit/with_dbs/01/test_catalog_handlers__services.py @@ -30,7 +30,10 @@ from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.webserver_login import UserInfoDict from servicelib.aiohttp import status -from simcore_service_webserver.catalog._models import ServiceInputGet, ServiceOutputGet +from simcore_service_webserver.catalog.controller_rest_schemas import ( + ServiceInputGet, + ServiceOutputGet, +) from simcore_service_webserver.db.models import UserRole From 3de057281a501623cdaee8eb8878896459217304 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Thu, 6 Mar 2025 06:45:31 +0100 Subject: [PATCH 18/30] @pcrespov import from controller_rest_schemas --- .../simcore_service_webserver/catalog/_tags_handlers.py | 7 +------ .../catalog/controller_rest_schemas.py | 5 +++++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/catalog/_tags_handlers.py b/services/web/server/src/simcore_service_webserver/catalog/_tags_handlers.py index ca2255cfa71e..199f8b4886fe 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/_tags_handlers.py +++ b/services/web/server/src/simcore_service_webserver/catalog/_tags_handlers.py @@ -1,21 +1,16 @@ import logging from aiohttp import web -from models_library.basic_types import IdInt from servicelib.aiohttp.requests_validation import parse_request_path_parameters_as from .._meta import API_VTAG from ..login.decorators import login_required from ..security.decorators import permission_required -from ._controller_rest import ServicePathParams +from .controller_rest_schemas import ServicePathParams, ServiceTagPathParams _logger = logging.getLogger(__name__) -class ServiceTagPathParams(ServicePathParams): - tag_id: IdInt - - routes = web.RouteTableDef() diff --git a/services/web/server/src/simcore_service_webserver/catalog/controller_rest_schemas.py b/services/web/server/src/simcore_service_webserver/catalog/controller_rest_schemas.py index 322837a9691a..11b184cbac7c 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/controller_rest_schemas.py +++ b/services/web/server/src/simcore_service_webserver/catalog/controller_rest_schemas.py @@ -13,6 +13,7 @@ ServiceOutputGet, ServiceOutputKey, ) +from models_library.basic_types import IdInt from models_library.rest_pagination import PageQueryParameters from models_library.services import BaseServiceIOModel, ServiceKey, ServiceVersion from models_library.users import UserID @@ -187,3 +188,7 @@ def ensure_unquoted(cls, v): class ListServiceParams(PageQueryParameters): ... + + +class ServiceTagPathParams(ServicePathParams): + tag_id: IdInt From 5eec0f57b7ea5caffe3c97bcaa36a2c2e1e35a8a Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Thu, 6 Mar 2025 07:00:30 +0100 Subject: [PATCH 19/30] improve file namings --- api/specs/web-server/_catalog_tags.py | 11 ++++------- .../catalog/_controller_rest.py | 4 ++-- ...handlers.py => _controller_rest_tags_handlers.py} | 0 .../simcore_service_webserver/catalog/_service.py | 12 ++++++------ .../catalog/{_api_units.py => _service_api_units.py} | 0 ...est_client.py => _service_catalog_rest_client.py} | 0 .../catalog/catalog_service.py | 2 +- .../src/simcore_service_webserver/catalog/plugin.py | 4 ++-- .../tests/unit/isolated/test_catalog_api_units.py | 2 +- .../tests/unit/isolated/test_catalog_models.py | 4 +++- 10 files changed, 19 insertions(+), 20 deletions(-) rename services/web/server/src/simcore_service_webserver/catalog/{_tags_handlers.py => _controller_rest_tags_handlers.py} (100%) rename services/web/server/src/simcore_service_webserver/catalog/{_api_units.py => _service_api_units.py} (100%) rename services/web/server/src/simcore_service_webserver/catalog/{_catalog_rest_client.py => _service_catalog_rest_client.py} (100%) diff --git a/api/specs/web-server/_catalog_tags.py b/api/specs/web-server/_catalog_tags.py index 26e90d952a49..cc2882dd26fd 100644 --- a/api/specs/web-server/_catalog_tags.py +++ b/api/specs/web-server/_catalog_tags.py @@ -10,7 +10,7 @@ from models_library.api_schemas_webserver.catalog import CatalogServiceGet from models_library.generics import Envelope from simcore_service_webserver._meta import API_VTAG -from simcore_service_webserver.catalog._tags_handlers import ( +from simcore_service_webserver.catalog._controller_rest_tags_handlers import ( ServicePathParams, ServiceTagPathParams, ) @@ -31,8 +31,7 @@ ) def list_service_tags( _path_params: Annotated[ServicePathParams, Depends()], -): - ... +): ... @router.post( @@ -41,8 +40,7 @@ def list_service_tags( ) def add_service_tag( _path_params: Annotated[ServiceTagPathParams, Depends()], -): - ... +): ... @router.post( @@ -51,5 +49,4 @@ def add_service_tag( ) def remove_service_tag( _path_params: Annotated[ServiceTagPathParams, Depends()], -): - ... +): ... diff --git a/services/web/server/src/simcore_service_webserver/catalog/_controller_rest.py b/services/web/server/src/simcore_service_webserver/catalog/_controller_rest.py index c0e81da9acd3..eaf73e58d17a 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/_controller_rest.py +++ b/services/web/server/src/simcore_service_webserver/catalog/_controller_rest.py @@ -38,7 +38,7 @@ from ..resource_usage.service import get_default_service_pricing_plan from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response -from . import _catalog_rest_client, _service +from . import _service, _service_catalog_rest_client from ._exceptions import ( DefaultPricingUnitForServiceNotFoundError, reraise_catalog_exceptions_as_http_errors, @@ -337,7 +337,7 @@ async def get_service_resources(request: Request): ctx = CatalogRequestContext.create(request) path_params = parse_request_path_parameters_as(ServicePathParams, request) service_resources: ServiceResourcesDict = ( - await _catalog_rest_client.get_service_resources( + await _service_catalog_rest_client.get_service_resources( request.app, user_id=ctx.user_id, service_key=path_params.service_key, diff --git a/services/web/server/src/simcore_service_webserver/catalog/_tags_handlers.py b/services/web/server/src/simcore_service_webserver/catalog/_controller_rest_tags_handlers.py similarity index 100% rename from services/web/server/src/simcore_service_webserver/catalog/_tags_handlers.py rename to services/web/server/src/simcore_service_webserver/catalog/_controller_rest_tags_handlers.py diff --git a/services/web/server/src/simcore_service_webserver/catalog/_service.py b/services/web/server/src/simcore_service_webserver/catalog/_service.py index a9cc3542cbc2..087c72d938d1 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/_service.py +++ b/services/web/server/src/simcore_service_webserver/catalog/_service.py @@ -27,8 +27,8 @@ from servicelib.rest_constants import RESPONSE_MODEL_POLICY from ..rabbitmq import get_rabbitmq_rpc_client -from . import _catalog_rest_client -from ._api_units import can_connect, replace_service_input_outputs +from . import _service_catalog_rest_client +from ._service_api_units import can_connect, replace_service_input_outputs from .controller_rest_schemas import ( CatalogRequestContext, ServiceInputGetFactory, @@ -170,7 +170,7 @@ async def update_service_v2( async def list_service_inputs( service_key: ServiceKey, service_version: ServiceVersion, ctx: CatalogRequestContext ) -> list[ServiceInputGet]: - service = await _catalog_rest_client.get_service( + service = await _service_catalog_rest_client.get_service( ctx.app, ctx.user_id, service_key, service_version, ctx.product_name ) return [ @@ -187,7 +187,7 @@ async def get_service_input( input_key: ServiceInputKey, ctx: CatalogRequestContext, ) -> ServiceInputGet: - service = await _catalog_rest_client.get_service( + service = await _service_catalog_rest_client.get_service( ctx.app, ctx.user_id, service_key, service_version, ctx.product_name ) service_input: ServiceInputGet = ( @@ -246,7 +246,7 @@ async def list_service_outputs( service_version: ServiceVersion, ctx: CatalogRequestContext, ) -> list[ServiceOutputGet]: - service = await _catalog_rest_client.get_service( + service = await _service_catalog_rest_client.get_service( ctx.app, ctx.user_id, service_key, service_version, ctx.product_name ) return [ @@ -263,7 +263,7 @@ async def get_service_output( output_key: ServiceOutputKey, ctx: CatalogRequestContext, ) -> ServiceOutputGet: - service = await _catalog_rest_client.get_service( + service = await _service_catalog_rest_client.get_service( ctx.app, ctx.user_id, service_key, service_version, ctx.product_name ) return cast( # mypy -> aiocache is not typed. diff --git a/services/web/server/src/simcore_service_webserver/catalog/_api_units.py b/services/web/server/src/simcore_service_webserver/catalog/_service_api_units.py similarity index 100% rename from services/web/server/src/simcore_service_webserver/catalog/_api_units.py rename to services/web/server/src/simcore_service_webserver/catalog/_service_api_units.py diff --git a/services/web/server/src/simcore_service_webserver/catalog/_catalog_rest_client.py b/services/web/server/src/simcore_service_webserver/catalog/_service_catalog_rest_client.py similarity index 100% rename from services/web/server/src/simcore_service_webserver/catalog/_catalog_rest_client.py rename to services/web/server/src/simcore_service_webserver/catalog/_service_catalog_rest_client.py diff --git a/services/web/server/src/simcore_service_webserver/catalog/catalog_service.py b/services/web/server/src/simcore_service_webserver/catalog/catalog_service.py index e5a7c191f99f..af51752ec7f5 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/catalog_service.py +++ b/services/web/server/src/simcore_service_webserver/catalog/catalog_service.py @@ -1,4 +1,4 @@ -from ._catalog_rest_client import * # noqa +from ._service_catalog_rest_client import * # noqa __all__: tuple[str, ...] = ( "is_catalog_service_responsive", diff --git a/services/web/server/src/simcore_service_webserver/catalog/plugin.py b/services/web/server/src/simcore_service_webserver/catalog/plugin.py index e0b7be350816..bee27e904f53 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/plugin.py +++ b/services/web/server/src/simcore_service_webserver/catalog/plugin.py @@ -6,7 +6,7 @@ from pint import UnitRegistry from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup -from . import _controller_rest, _tags_handlers +from . import _controller_rest, _controller_rest_tags_handlers _logger = logging.getLogger(__name__) @@ -26,7 +26,7 @@ def setup_catalog(app: web.Application): ) app.add_routes(_controller_rest.routes) - app.add_routes(_tags_handlers.routes) + app.add_routes(_controller_rest_tags_handlers.routes) # prepares units registry app[UnitRegistry.__name__] = UnitRegistry() diff --git a/services/web/server/tests/unit/isolated/test_catalog_api_units.py b/services/web/server/tests/unit/isolated/test_catalog_api_units.py index 39d1824a7755..e6d99768e3bd 100644 --- a/services/web/server/tests/unit/isolated/test_catalog_api_units.py +++ b/services/web/server/tests/unit/isolated/test_catalog_api_units.py @@ -9,7 +9,7 @@ from models_library.function_services_catalog.services import demo_units from models_library.services import ServiceInput, ServiceOutput from pint import UnitRegistry -from simcore_service_webserver.catalog._api_units import can_connect +from simcore_service_webserver.catalog._service_api_units import can_connect def _create_port_data(schema: dict[str, Any]): diff --git a/services/web/server/tests/unit/isolated/test_catalog_models.py b/services/web/server/tests/unit/isolated/test_catalog_models.py index e891b11b6f27..48aee40f600d 100644 --- a/services/web/server/tests/unit/isolated/test_catalog_models.py +++ b/services/web/server/tests/unit/isolated/test_catalog_models.py @@ -9,8 +9,10 @@ import pytest from pint import UnitRegistry from pytest_benchmark.fixture import BenchmarkFixture -from simcore_service_webserver.catalog._api_units import replace_service_input_outputs from simcore_service_webserver.catalog._controller_rest import RESPONSE_MODEL_POLICY +from simcore_service_webserver.catalog._service_api_units import ( + replace_service_input_outputs, +) @pytest.fixture(params=["UnitRegistry", None]) From 2678c71fdcededeaf980fd432f6afc20b9d9a16d Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Thu, 6 Mar 2025 07:01:40 +0100 Subject: [PATCH 20/30] minor correction --- api/specs/web-server/_catalog_tags.py | 2 +- ...troller_rest_tags_handlers.py => _controller_rest_tags.py} | 0 .../server/src/simcore_service_webserver/catalog/plugin.py | 4 ++-- 3 files changed, 3 insertions(+), 3 deletions(-) rename services/web/server/src/simcore_service_webserver/catalog/{_controller_rest_tags_handlers.py => _controller_rest_tags.py} (100%) diff --git a/api/specs/web-server/_catalog_tags.py b/api/specs/web-server/_catalog_tags.py index cc2882dd26fd..9f26071f95ee 100644 --- a/api/specs/web-server/_catalog_tags.py +++ b/api/specs/web-server/_catalog_tags.py @@ -10,7 +10,7 @@ from models_library.api_schemas_webserver.catalog import CatalogServiceGet from models_library.generics import Envelope from simcore_service_webserver._meta import API_VTAG -from simcore_service_webserver.catalog._controller_rest_tags_handlers import ( +from simcore_service_webserver.catalog._controller_rest_tags import ( ServicePathParams, ServiceTagPathParams, ) diff --git a/services/web/server/src/simcore_service_webserver/catalog/_controller_rest_tags_handlers.py b/services/web/server/src/simcore_service_webserver/catalog/_controller_rest_tags.py similarity index 100% rename from services/web/server/src/simcore_service_webserver/catalog/_controller_rest_tags_handlers.py rename to services/web/server/src/simcore_service_webserver/catalog/_controller_rest_tags.py diff --git a/services/web/server/src/simcore_service_webserver/catalog/plugin.py b/services/web/server/src/simcore_service_webserver/catalog/plugin.py index bee27e904f53..c71bcd9754bb 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/plugin.py +++ b/services/web/server/src/simcore_service_webserver/catalog/plugin.py @@ -6,7 +6,7 @@ from pint import UnitRegistry from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup -from . import _controller_rest, _controller_rest_tags_handlers +from . import _controller_rest, _controller_rest_tags _logger = logging.getLogger(__name__) @@ -26,7 +26,7 @@ def setup_catalog(app: web.Application): ) app.add_routes(_controller_rest.routes) - app.add_routes(_controller_rest_tags_handlers.routes) + app.add_routes(_controller_rest_tags.routes) # prepares units registry app[UnitRegistry.__name__] = UnitRegistry() From 5cace5b03410ad7440e0c455c1f233f84b6c6a96 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Thu, 6 Mar 2025 07:32:54 +0100 Subject: [PATCH 21/30] @pcrespov use new exception handling system --- .../catalog/_controller_rest.py | 10 ++-- .../catalog/_exceptions.py | 53 ++++++++++--------- 2 files changed, 34 insertions(+), 29 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/catalog/_controller_rest.py b/services/web/server/src/simcore_service_webserver/catalog/_controller_rest.py index eaf73e58d17a..5a7244e038d2 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/_controller_rest.py +++ b/services/web/server/src/simcore_service_webserver/catalog/_controller_rest.py @@ -41,7 +41,7 @@ from . import _service, _service_catalog_rest_client from ._exceptions import ( DefaultPricingUnitForServiceNotFoundError, - reraise_catalog_exceptions_as_http_errors, + handle_plugin_requests_exceptions, ) from .controller_rest_schemas import ( CatalogRequestContext, @@ -63,7 +63,7 @@ ) @login_required @permission_required("services.catalog.*") -@reraise_catalog_exceptions_as_http_errors +@handle_plugin_requests_exceptions async def list_services_latest(request: Request): request_ctx = CatalogRequestContext.create(request) query_params: ListServiceParams = parse_request_query_parameters_as( @@ -101,7 +101,7 @@ async def list_services_latest(request: Request): ) @login_required @permission_required("services.catalog.*") -@reraise_catalog_exceptions_as_http_errors +@handle_plugin_requests_exceptions async def get_service(request: Request): request_ctx = CatalogRequestContext.create(request) path_params = parse_request_path_parameters_as(ServicePathParams, request) @@ -127,7 +127,7 @@ async def get_service(request: Request): ) @login_required @permission_required("services.catalog.*") -@reraise_catalog_exceptions_as_http_errors +@handle_plugin_requests_exceptions async def update_service(request: Request): request_ctx = CatalogRequestContext.create(request) path_params = parse_request_path_parameters_as(ServicePathParams, request) @@ -357,7 +357,7 @@ async def get_service_resources(request: Request): ) @login_required @permission_required("services.catalog.*") -@reraise_catalog_exceptions_as_http_errors +@handle_plugin_requests_exceptions async def get_service_pricing_plan(request: Request): ctx = CatalogRequestContext.create(request) path_params = parse_request_path_parameters_as(ServicePathParams, request) diff --git a/services/web/server/src/simcore_service_webserver/catalog/_exceptions.py b/services/web/server/src/simcore_service_webserver/catalog/_exceptions.py index bf79a120ef53..292bcb78d5e5 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/_exceptions.py +++ b/services/web/server/src/simcore_service_webserver/catalog/_exceptions.py @@ -1,38 +1,21 @@ """Defines the different exceptions that may arise in the catalog subpackage""" -import functools - -from aiohttp import web -from servicelib.aiohttp.typing_extension import Handler +from servicelib.aiohttp import status from servicelib.rabbitmq.rpc_interfaces.catalog.errors import ( CatalogForbiddenError, CatalogItemNotFoundError, ) from ..errors import WebServerBaseError +from ..exception_handling import ( + ExceptionToHttpErrorMap, + HttpErrorInfo, + exception_handling_decorator, + to_exceptions_handlers_map, +) from ..resource_usage.errors import DefaultPricingPlanNotFoundError -def reraise_catalog_exceptions_as_http_errors(handler: Handler): - @functools.wraps(handler) - async def _wrapper(request: web.Request) -> web.StreamResponse: - try: - - return await handler(request) - - except ( - CatalogItemNotFoundError, - DefaultPricingPlanNotFoundError, - DefaultPricingUnitForServiceNotFoundError, - ) as exc: - raise web.HTTPNotFound(reason=f"{exc}") from exc - - except CatalogForbiddenError as exc: - raise web.HTTPForbidden(reason=f"{exc}") from exc - - return _wrapper - - class BaseCatalogError(WebServerBaseError): msg_template = "Unexpected error occured in catalog submodule" @@ -60,6 +43,28 @@ def __init__(self, *, service_key: str, service_version: str, **ctxs): assert CatalogItemNotFoundError # nosec +_TO_HTTP_ERROR_MAP: ExceptionToHttpErrorMap = { + CatalogItemNotFoundError: HttpErrorInfo( + status.HTTP_404_NOT_FOUND, + "Catalog item not found", + ), + DefaultPricingPlanNotFoundError: HttpErrorInfo( + status.HTTP_404_NOT_FOUND, + "Default pricing plan not found", + ), + DefaultPricingUnitForServiceNotFoundError: HttpErrorInfo( + status.HTTP_404_NOT_FOUND, "Default pricing unit not found" + ), + CatalogForbiddenError: HttpErrorInfo( + status.HTTP_403_FORBIDDEN, "Forbidden catalog access" + ), +} + +handle_plugin_requests_exceptions = exception_handling_decorator( + to_exceptions_handlers_map(_TO_HTTP_ERROR_MAP) +) + + __all__: tuple[str, ...] = ( "CatalogForbiddenError", "CatalogItemNotFoundError", From 16b56ee1cfcacec0453aeb3a0ec1b37d19c1e2d5 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Thu, 6 Mar 2025 07:48:14 +0100 Subject: [PATCH 22/30] make pylint happy --- .../catalog/catalog_service.py | 10 +++++++++- .../tests/unit/with_dbs/01/test_catalog_rest_client.py | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/catalog/catalog_service.py b/services/web/server/src/simcore_service_webserver/catalog/catalog_service.py index af51752ec7f5..2e1b9154efcd 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/catalog_service.py +++ b/services/web/server/src/simcore_service_webserver/catalog/catalog_service.py @@ -1,4 +1,12 @@ -from ._service_catalog_rest_client import * # noqa +from ._service import batch_get_my_services # noqa +from ._service_catalog_rest_client import ( # noqa + get_service, + get_service_access_rights, + get_service_resources, + get_services_for_user_in_product, + is_catalog_service_responsive, + to_backend_service, +) __all__: tuple[str, ...] = ( "is_catalog_service_responsive", diff --git a/services/web/server/tests/unit/with_dbs/01/test_catalog_rest_client.py b/services/web/server/tests/unit/with_dbs/01/test_catalog_rest_client.py index e937a576ee55..386399824ec5 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_catalog_rest_client.py +++ b/services/web/server/tests/unit/with_dbs/01/test_catalog_rest_client.py @@ -1,3 +1,4 @@ +# pylint:disable=unused-argument import re import pytest From 4d53197484d606f1f24b9eb56d652ff3658e7737 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Thu, 6 Mar 2025 09:23:41 +0100 Subject: [PATCH 23/30] @pcrespov _service_api_units.py -> _service_units.py --- .../simcore_service_webserver/catalog/_service.py | 2 +- .../{_service_api_units.py => _service_units.py} | 0 .../tests/unit/isolated/test_catalog_api_units.py | 2 +- .../tests/unit/isolated/test_catalog_models.py | 2 +- ...est_projects_nodes_handlers__services_access.py | 14 +++++++------- 5 files changed, 10 insertions(+), 10 deletions(-) rename services/web/server/src/simcore_service_webserver/catalog/{_service_api_units.py => _service_units.py} (100%) diff --git a/services/web/server/src/simcore_service_webserver/catalog/_service.py b/services/web/server/src/simcore_service_webserver/catalog/_service.py index 087c72d938d1..a7def7db8098 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/_service.py +++ b/services/web/server/src/simcore_service_webserver/catalog/_service.py @@ -28,7 +28,7 @@ from ..rabbitmq import get_rabbitmq_rpc_client from . import _service_catalog_rest_client -from ._service_api_units import can_connect, replace_service_input_outputs +from ._service_units import can_connect, replace_service_input_outputs from .controller_rest_schemas import ( CatalogRequestContext, ServiceInputGetFactory, diff --git a/services/web/server/src/simcore_service_webserver/catalog/_service_api_units.py b/services/web/server/src/simcore_service_webserver/catalog/_service_units.py similarity index 100% rename from services/web/server/src/simcore_service_webserver/catalog/_service_api_units.py rename to services/web/server/src/simcore_service_webserver/catalog/_service_units.py diff --git a/services/web/server/tests/unit/isolated/test_catalog_api_units.py b/services/web/server/tests/unit/isolated/test_catalog_api_units.py index e6d99768e3bd..04c8e909352f 100644 --- a/services/web/server/tests/unit/isolated/test_catalog_api_units.py +++ b/services/web/server/tests/unit/isolated/test_catalog_api_units.py @@ -9,7 +9,7 @@ from models_library.function_services_catalog.services import demo_units from models_library.services import ServiceInput, ServiceOutput from pint import UnitRegistry -from simcore_service_webserver.catalog._service_api_units import can_connect +from simcore_service_webserver.catalog._service_units import can_connect def _create_port_data(schema: dict[str, Any]): diff --git a/services/web/server/tests/unit/isolated/test_catalog_models.py b/services/web/server/tests/unit/isolated/test_catalog_models.py index 48aee40f600d..e882e3b5bbed 100644 --- a/services/web/server/tests/unit/isolated/test_catalog_models.py +++ b/services/web/server/tests/unit/isolated/test_catalog_models.py @@ -10,7 +10,7 @@ from pint import UnitRegistry from pytest_benchmark.fixture import BenchmarkFixture from simcore_service_webserver.catalog._controller_rest import RESPONSE_MODEL_POLICY -from simcore_service_webserver.catalog._service_api_units import ( +from simcore_service_webserver.catalog._service_units import ( replace_service_input_outputs, ) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__services_access.py b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__services_access.py index cedb64451bb0..cd8f8e2f3cbf 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__services_access.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__services_access.py @@ -59,7 +59,7 @@ def fake_project( @pytest.fixture def mock_catalog_api_get_service_access_rights_response(mocker: MockerFixture): mocker.patch( - "simcore_service_webserver.projects._nodes_handlers.catalog_client.get_service_access_rights", + "simcore_service_webserver.projects._nodes_handlers.catalog_service.get_service_access_rights", spec=True, side_effect=[ ServiceAccessRightsGet( @@ -130,7 +130,7 @@ async def test_accessible_thanks_to_everyone_group_id( logged_user: UserInfoDict, ): mocker.patch( - "simcore_service_webserver.projects._nodes_handlers.catalog_client.get_service_access_rights", + "simcore_service_webserver.projects._nodes_handlers.catalog_service.get_service_access_rights", spec=True, side_effect=[ ServiceAccessRightsGet( @@ -185,7 +185,7 @@ async def test_accessible_thanks_to_concrete_group_id( for_gid = logged_user["primary_gid"] mocker.patch( - "simcore_service_webserver.projects._nodes_handlers.catalog_client.get_service_access_rights", + "simcore_service_webserver.projects._nodes_handlers.catalog_service.get_service_access_rights", spec=True, side_effect=[ ServiceAccessRightsGet( @@ -238,7 +238,7 @@ async def test_accessible_through_product_group( for_gid = logged_user["primary_gid"] mocker.patch( - "simcore_service_webserver.projects._nodes_handlers.catalog_client.get_service_access_rights", + "simcore_service_webserver.projects._nodes_handlers.catalog_service.get_service_access_rights", spec=True, side_effect=[ ServiceAccessRightsGet( @@ -297,7 +297,7 @@ async def test_accessible_for_one_service( for_gid = logged_user["primary_gid"] mocker.patch( - "simcore_service_webserver.projects._nodes_handlers.catalog_client.get_service_access_rights", + "simcore_service_webserver.projects._nodes_handlers.catalog_service.get_service_access_rights", spec=True, side_effect=[ ServiceAccessRightsGet( @@ -355,7 +355,7 @@ async def test_not_accessible_for_more_services( logged_user: UserInfoDict, ): mocker.patch( - "simcore_service_webserver.projects._nodes_handlers.catalog_client.get_service_access_rights", + "simcore_service_webserver.projects._nodes_handlers.catalog_service.get_service_access_rights", spec=True, side_effect=[ ServiceAccessRightsGet( @@ -421,7 +421,7 @@ async def test_not_accessible_for_service_because_of_execute_access_false( for_gid = logged_user["primary_gid"] mocker.patch( - "simcore_service_webserver.projects._nodes_handlers.catalog_client.get_service_access_rights", + "simcore_service_webserver.projects._nodes_handlers.catalog_service.get_service_access_rights", spec=True, side_effect=[ ServiceAccessRightsGet( From 9bc18ffd50b1d3c1bc04086da877adde8d3ed6dd Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Thu, 6 Mar 2025 09:46:50 +0100 Subject: [PATCH 24/30] ddd instead of sa @matusdrobuliak66 @pcrespov --- api/specs/web-server/_catalog.py | 2 +- api/specs/web-server/_catalog_tags.py | 2 +- ...est_client.py => _catalog_rest_client_service.py} | 0 .../{_controller_rest.py => _rest_controller.py} | 4 ++-- ...troller_rest_tags.py => _rest_tags_controller.py} | 0 .../simcore_service_webserver/catalog/_service.py | 12 ++++++------ .../catalog/{_service_units.py => _units_service.py} | 0 .../catalog/catalog_service.py | 4 ++-- .../src/simcore_service_webserver/catalog/plugin.py | 8 ++++---- .../tests/unit/isolated/test_catalog_api_units.py | 2 +- .../tests/unit/isolated/test_catalog_models.py | 4 ++-- 11 files changed, 19 insertions(+), 19 deletions(-) rename services/web/server/src/simcore_service_webserver/catalog/{_service_catalog_rest_client.py => _catalog_rest_client_service.py} (100%) rename services/web/server/src/simcore_service_webserver/catalog/{_controller_rest.py => _rest_controller.py} (99%) rename services/web/server/src/simcore_service_webserver/catalog/{_controller_rest_tags.py => _rest_tags_controller.py} (100%) rename services/web/server/src/simcore_service_webserver/catalog/{_service_units.py => _units_service.py} (100%) diff --git a/api/specs/web-server/_catalog.py b/api/specs/web-server/_catalog.py index c087c4077509..d861c44b0c6f 100644 --- a/api/specs/web-server/_catalog.py +++ b/api/specs/web-server/_catalog.py @@ -15,7 +15,7 @@ from models_library.generics import Envelope from models_library.rest_pagination import Page from simcore_service_webserver._meta import API_VTAG -from simcore_service_webserver.catalog._controller_rest import ( +from simcore_service_webserver.catalog._rest_controller import ( ListServiceParams, ServicePathParams, _FromServiceOutputParams, diff --git a/api/specs/web-server/_catalog_tags.py b/api/specs/web-server/_catalog_tags.py index 9f26071f95ee..bfecd6314fc6 100644 --- a/api/specs/web-server/_catalog_tags.py +++ b/api/specs/web-server/_catalog_tags.py @@ -10,7 +10,7 @@ from models_library.api_schemas_webserver.catalog import CatalogServiceGet from models_library.generics import Envelope from simcore_service_webserver._meta import API_VTAG -from simcore_service_webserver.catalog._controller_rest_tags import ( +from simcore_service_webserver.catalog._rest_tags_controller import ( ServicePathParams, ServiceTagPathParams, ) diff --git a/services/web/server/src/simcore_service_webserver/catalog/_service_catalog_rest_client.py b/services/web/server/src/simcore_service_webserver/catalog/_catalog_rest_client_service.py similarity index 100% rename from services/web/server/src/simcore_service_webserver/catalog/_service_catalog_rest_client.py rename to services/web/server/src/simcore_service_webserver/catalog/_catalog_rest_client_service.py diff --git a/services/web/server/src/simcore_service_webserver/catalog/_controller_rest.py b/services/web/server/src/simcore_service_webserver/catalog/_rest_controller.py similarity index 99% rename from services/web/server/src/simcore_service_webserver/catalog/_controller_rest.py rename to services/web/server/src/simcore_service_webserver/catalog/_rest_controller.py index 5a7244e038d2..9a83748fd8cf 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/_controller_rest.py +++ b/services/web/server/src/simcore_service_webserver/catalog/_rest_controller.py @@ -38,7 +38,7 @@ from ..resource_usage.service import get_default_service_pricing_plan from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response -from . import _service, _service_catalog_rest_client +from . import _catalog_rest_client_service, _service from ._exceptions import ( DefaultPricingUnitForServiceNotFoundError, handle_plugin_requests_exceptions, @@ -337,7 +337,7 @@ async def get_service_resources(request: Request): ctx = CatalogRequestContext.create(request) path_params = parse_request_path_parameters_as(ServicePathParams, request) service_resources: ServiceResourcesDict = ( - await _service_catalog_rest_client.get_service_resources( + await _catalog_rest_client_service.get_service_resources( request.app, user_id=ctx.user_id, service_key=path_params.service_key, diff --git a/services/web/server/src/simcore_service_webserver/catalog/_controller_rest_tags.py b/services/web/server/src/simcore_service_webserver/catalog/_rest_tags_controller.py similarity index 100% rename from services/web/server/src/simcore_service_webserver/catalog/_controller_rest_tags.py rename to services/web/server/src/simcore_service_webserver/catalog/_rest_tags_controller.py diff --git a/services/web/server/src/simcore_service_webserver/catalog/_service.py b/services/web/server/src/simcore_service_webserver/catalog/_service.py index a7def7db8098..823bf033b3a7 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/_service.py +++ b/services/web/server/src/simcore_service_webserver/catalog/_service.py @@ -27,8 +27,8 @@ from servicelib.rest_constants import RESPONSE_MODEL_POLICY from ..rabbitmq import get_rabbitmq_rpc_client -from . import _service_catalog_rest_client -from ._service_units import can_connect, replace_service_input_outputs +from . import _catalog_rest_client_service +from ._units_service import can_connect, replace_service_input_outputs from .controller_rest_schemas import ( CatalogRequestContext, ServiceInputGetFactory, @@ -170,7 +170,7 @@ async def update_service_v2( async def list_service_inputs( service_key: ServiceKey, service_version: ServiceVersion, ctx: CatalogRequestContext ) -> list[ServiceInputGet]: - service = await _service_catalog_rest_client.get_service( + service = await _catalog_rest_client_service.get_service( ctx.app, ctx.user_id, service_key, service_version, ctx.product_name ) return [ @@ -187,7 +187,7 @@ async def get_service_input( input_key: ServiceInputKey, ctx: CatalogRequestContext, ) -> ServiceInputGet: - service = await _service_catalog_rest_client.get_service( + service = await _catalog_rest_client_service.get_service( ctx.app, ctx.user_id, service_key, service_version, ctx.product_name ) service_input: ServiceInputGet = ( @@ -246,7 +246,7 @@ async def list_service_outputs( service_version: ServiceVersion, ctx: CatalogRequestContext, ) -> list[ServiceOutputGet]: - service = await _service_catalog_rest_client.get_service( + service = await _catalog_rest_client_service.get_service( ctx.app, ctx.user_id, service_key, service_version, ctx.product_name ) return [ @@ -263,7 +263,7 @@ async def get_service_output( output_key: ServiceOutputKey, ctx: CatalogRequestContext, ) -> ServiceOutputGet: - service = await _service_catalog_rest_client.get_service( + service = await _catalog_rest_client_service.get_service( ctx.app, ctx.user_id, service_key, service_version, ctx.product_name ) return cast( # mypy -> aiocache is not typed. diff --git a/services/web/server/src/simcore_service_webserver/catalog/_service_units.py b/services/web/server/src/simcore_service_webserver/catalog/_units_service.py similarity index 100% rename from services/web/server/src/simcore_service_webserver/catalog/_service_units.py rename to services/web/server/src/simcore_service_webserver/catalog/_units_service.py diff --git a/services/web/server/src/simcore_service_webserver/catalog/catalog_service.py b/services/web/server/src/simcore_service_webserver/catalog/catalog_service.py index 2e1b9154efcd..33b92d864a38 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/catalog_service.py +++ b/services/web/server/src/simcore_service_webserver/catalog/catalog_service.py @@ -1,5 +1,4 @@ -from ._service import batch_get_my_services # noqa -from ._service_catalog_rest_client import ( # noqa +from ._catalog_rest_client_service import ( # noqa get_service, get_service_access_rights, get_service_resources, @@ -7,6 +6,7 @@ is_catalog_service_responsive, to_backend_service, ) +from ._service import batch_get_my_services # noqa __all__: tuple[str, ...] = ( "is_catalog_service_responsive", diff --git a/services/web/server/src/simcore_service_webserver/catalog/plugin.py b/services/web/server/src/simcore_service_webserver/catalog/plugin.py index c71bcd9754bb..250b3810d73a 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/plugin.py +++ b/services/web/server/src/simcore_service_webserver/catalog/plugin.py @@ -6,7 +6,7 @@ from pint import UnitRegistry from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup -from . import _controller_rest, _controller_rest_tags +from . import _rest_controller, _rest_tags_controller _logger = logging.getLogger(__name__) @@ -22,11 +22,11 @@ def setup_catalog(app: web.Application): # ensures routes are names that corresponds to function names assert all( # nosec route_def.kwargs["name"] == route_def.handler.__name__ # type: ignore[attr-defined] # route_def is a RouteDef not an Abstract - for route_def in _controller_rest.routes + for route_def in _rest_controller.routes ) - app.add_routes(_controller_rest.routes) - app.add_routes(_controller_rest_tags.routes) + app.add_routes(_rest_controller.routes) + app.add_routes(_rest_tags_controller.routes) # prepares units registry app[UnitRegistry.__name__] = UnitRegistry() diff --git a/services/web/server/tests/unit/isolated/test_catalog_api_units.py b/services/web/server/tests/unit/isolated/test_catalog_api_units.py index 04c8e909352f..3fa04b3a9335 100644 --- a/services/web/server/tests/unit/isolated/test_catalog_api_units.py +++ b/services/web/server/tests/unit/isolated/test_catalog_api_units.py @@ -9,7 +9,7 @@ from models_library.function_services_catalog.services import demo_units from models_library.services import ServiceInput, ServiceOutput from pint import UnitRegistry -from simcore_service_webserver.catalog._service_units import can_connect +from simcore_service_webserver.catalog._units_service import can_connect def _create_port_data(schema: dict[str, Any]): diff --git a/services/web/server/tests/unit/isolated/test_catalog_models.py b/services/web/server/tests/unit/isolated/test_catalog_models.py index e882e3b5bbed..d42ed45e6034 100644 --- a/services/web/server/tests/unit/isolated/test_catalog_models.py +++ b/services/web/server/tests/unit/isolated/test_catalog_models.py @@ -9,8 +9,8 @@ import pytest from pint import UnitRegistry from pytest_benchmark.fixture import BenchmarkFixture -from simcore_service_webserver.catalog._controller_rest import RESPONSE_MODEL_POLICY -from simcore_service_webserver.catalog._service_units import ( +from simcore_service_webserver.catalog._rest_controller import RESPONSE_MODEL_POLICY +from simcore_service_webserver.catalog._units_service import ( replace_service_input_outputs, ) From 6cb01a19703226021ad6ead4b2b592ccdd5f59c1 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Thu, 6 Mar 2025 10:39:57 +0100 Subject: [PATCH 25/30] fix typecheck --- .../catalog/controller_rest_schemas.py | 4 ++-- .../web/server/tests/unit/with_dbs/04/folders/test_folders.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/catalog/controller_rest_schemas.py b/services/web/server/src/simcore_service_webserver/catalog/controller_rest_schemas.py index 11b184cbac7c..9d594f8954aa 100644 --- a/services/web/server/src/simcore_service_webserver/catalog/controller_rest_schemas.py +++ b/services/web/server/src/simcore_service_webserver/catalog/controller_rest_schemas.py @@ -4,7 +4,7 @@ from dataclasses import dataclass from typing import Any, Final -from aiocache import cached +from aiocache import cached # type: ignore[import-untyped] from aiohttp import web from aiohttp.web import Request from models_library.api_schemas_webserver.catalog import ( @@ -18,7 +18,7 @@ from models_library.services import BaseServiceIOModel, ServiceKey, ServiceVersion from models_library.users import UserID from pint import PintError, Quantity, UnitRegistry -from pydantic import ( # type: ignore[import-untyped] +from pydantic import ( BaseModel, ConfigDict, field_validator, diff --git a/services/web/server/tests/unit/with_dbs/04/folders/test_folders.py b/services/web/server/tests/unit/with_dbs/04/folders/test_folders.py index f4b2df540ae8..7785d0dbaeb0 100644 --- a/services/web/server/tests/unit/with_dbs/04/folders/test_folders.py +++ b/services/web/server/tests/unit/with_dbs/04/folders/test_folders.py @@ -274,7 +274,7 @@ async def test_project_folder_movement_full_workflow( @pytest.fixture def mock_catalog_api_get_services_for_user_in_product(mocker: MockerFixture): mocker.patch( - "simcore_service_webserver.projects._crud_api_read.get_services_for_user_in_product", + "simcore_service_webserver.projects._crud_api_read.catalog_service.get_services_for_user_in_product", spec=True, return_value=[], ) From 19c51f14e3b2587a2ba3b9a67566a9f928214f42 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Thu, 6 Mar 2025 10:46:31 +0100 Subject: [PATCH 26/30] fix further tests --- .../test_workspaces__moving_projects_between_workspaces.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_projects_between_workspaces.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_projects_between_workspaces.py index a81c76012a0d..30cf718a37f0 100644 --- a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_projects_between_workspaces.py +++ b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_projects_between_workspaces.py @@ -28,12 +28,7 @@ @pytest.fixture def mock_catalog_api_get_services_for_user_in_product(mocker: MockerFixture): mocker.patch( - "simcore_service_webserver.projects._crud_api_read.get_services_for_user_in_product", - spec=True, - return_value=[], - ) - mocker.patch( - "simcore_service_webserver.projects._crud_handlers.get_services_for_user_in_product", + "simcore_service_webserver.projects._crud_api_read.catalog_service.get_services_for_user_in_product", spec=True, return_value=[], ) From 28b1544f687f9e95385552ce9c5288f8e38f0890 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Thu, 6 Mar 2025 11:06:08 +0100 Subject: [PATCH 27/30] fix mocks --- services/web/server/tests/unit/with_dbs/conftest.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/conftest.py b/services/web/server/tests/unit/with_dbs/conftest.py index 87d80398658b..10a6e91c5ad4 100644 --- a/services/web/server/tests/unit/with_dbs/conftest.py +++ b/services/web/server/tests/unit/with_dbs/conftest.py @@ -1,9 +1,9 @@ -""" Configuration for unit testing with a postgress fixture +"""Configuration for unit testing with a postgress fixture - - Unit testing of webserver app with a postgress service as fixture - - Starts test session by running a postgres container as a fixture (see postgress_service) +- Unit testing of webserver app with a postgress service as fixture +- Starts test session by running a postgres container as a fixture (see postgress_service) - IMPORTANT: remember that these are still unit-tests! +IMPORTANT: remember that these are still unit-tests! """ # nopycln: file @@ -288,8 +288,7 @@ async def _mocked_get_services_for_user(*args, **kwargs): return services_in_project for namespace in ( - "simcore_service_webserver.projects._crud_api_read.get_services_for_user_in_product", - "simcore_service_webserver.projects._crud_handlers.get_services_for_user_in_product", + "simcore_service_webserver.projects._crud_api_read.catalog_service.get_services_for_user_in_product", ): mock = mocker.patch( namespace, From 7e5684925708a467e2bddb7933ea3f1611f96751 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Thu, 6 Mar 2025 12:09:44 +0100 Subject: [PATCH 28/30] fix tests --- .../web/server/tests/unit/with_dbs/02/conftest.py | 4 ++-- .../02/test_projects_crud_handlers__patch.py | 2 +- .../with_dbs/02/test_projects_groups_handlers.py | 12 +----------- .../test_projects_nodes_handlers__services_access.py | 4 ++-- .../03/resource_usage/test_admin_pricing_plans.py | 2 +- .../test_studies_dispatcher_handlers.py | 2 +- .../test_workspaces__folders_and_projects_crud.py | 7 +------ .../test_workspaces__list_projects_full_search.py | 7 +------ ..._workspaces__moving_folders_between_workspaces.py | 7 +------ 9 files changed, 11 insertions(+), 36 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/02/conftest.py b/services/web/server/tests/unit/with_dbs/02/conftest.py index 2bd29316680d..22d898f2a10c 100644 --- a/services/web/server/tests/unit/with_dbs/02/conftest.py +++ b/services/web/server/tests/unit/with_dbs/02/conftest.py @@ -92,12 +92,12 @@ def mock_catalog_api( ) -> dict[str, mock.Mock]: return { "get_service_resources": mocker.patch( - "simcore_service_webserver.projects.projects_service.catalog_client.get_service_resources", + "simcore_service_webserver.projects.projects_service.catalog_service.get_service_resources", return_value=mock_service_resources, autospec=True, ), "get_service": mocker.patch( - "simcore_service_webserver.projects.projects_service.catalog_client.get_service", + "simcore_service_webserver.projects.projects_service.catalog_service.get_service", return_value=mock_service, autospec=True, ), diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__patch.py b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__patch.py index 04b4db5b7e86..08100ff95f57 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__patch.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__patch.py @@ -25,7 +25,7 @@ @pytest.fixture def mock_catalog_api_get_services_for_user_in_product(mocker: MockerFixture): mocker.patch( - "simcore_service_webserver.projects._crud_handlers.get_services_for_user_in_product", + "simcore_service_webserver.projects._crud_handlers.catalog_service.get_services_for_user_in_product", spec=True, return_value=[], ) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_groups_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_groups_handlers.py index 765691510683..214d284d8f05 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_groups_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_groups_handlers.py @@ -20,7 +20,7 @@ @pytest.fixture def mock_catalog_api_get_services_for_user_in_product(mocker: MockerFixture): mocker.patch( - "simcore_service_webserver.projects._crud_handlers.get_services_for_user_in_product", + "simcore_service_webserver.projects._crud_handlers.catalog_service.get_services_for_user_in_product", spec=True, return_value=[], ) @@ -35,15 +35,6 @@ def mock_project_uses_available_services(mocker: MockerFixture): ) -@pytest.fixture -def mock_catalog_api_get_services_for_user_in_product_2(mocker: MockerFixture): - mocker.patch( - "simcore_service_webserver.projects._crud_api_read.get_services_for_user_in_product", - spec=True, - return_value=[], - ) - - @pytest.mark.acceptance_test( "Driving test for https://github.com/ITISFoundation/osparc-issues/issues/1547" ) @@ -55,7 +46,6 @@ async def test_projects_groups_full_workflow( expected: HTTPStatus, mock_catalog_api_get_services_for_user_in_product, mock_project_uses_available_services, - mock_catalog_api_get_services_for_user_in_product_2, ): assert client.app # check the default project permissions diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__services_access.py b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__services_access.py index cd8f8e2f3cbf..f70587ebe561 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__services_access.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__services_access.py @@ -479,7 +479,7 @@ async def test_get_project_services( ] mocker.patch( - "simcore_service_webserver.catalog._api.catalog_rpc.batch_get_my_services", + "simcore_service_webserver.catalog._service.catalog_rpc.batch_get_my_services", spec=True, return_value=[ MyServiceGet( @@ -561,7 +561,7 @@ async def test_get_project_services_service_unavailable( logged_user: UserInfoDict, ): mocker.patch( - "simcore_service_webserver.catalog._api.catalog_rpc.batch_get_my_services", + "simcore_service_webserver.catalog._service.catalog_rpc.batch_get_my_services", spec=True, side_effect=RPCServerError( exc_message="Service Unavailable", diff --git a/services/web/server/tests/unit/with_dbs/03/resource_usage/test_admin_pricing_plans.py b/services/web/server/tests/unit/with_dbs/03/resource_usage/test_admin_pricing_plans.py index cfb2b06a7899..35f6255a5e1c 100644 --- a/services/web/server/tests/unit/with_dbs/03/resource_usage/test_admin_pricing_plans.py +++ b/services/web/server/tests/unit/with_dbs/03/resource_usage/test_admin_pricing_plans.py @@ -23,7 +23,7 @@ def mock_catalog_client(mocker: MockerFixture, faker: Faker) -> dict[str, MagicMock]: return { "get_service": mocker.patch( - "simcore_service_webserver.resource_usage._pricing_plans_admin_service.catalog_client.get_service", + "simcore_service_webserver.resource_usage._pricing_plans_admin_service.catalog_service.get_service", autospec=True, ) } diff --git a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_handlers.py b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_handlers.py index 941e4b8a098d..aacee1c08870 100644 --- a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_handlers.py +++ b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_handlers.py @@ -283,7 +283,7 @@ def catalog_subsystem_mock(mocker: MockerFixture) -> None: ] mock = mocker.patch( - "simcore_service_webserver.projects._crud_api_read.get_services_for_user_in_product", + "simcore_service_webserver.projects._crud_api_read.catalog_service.get_services_for_user_in_product", autospec=True, ) diff --git a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__folders_and_projects_crud.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__folders_and_projects_crud.py index c301ead5f90d..cb5f211f5333 100644 --- a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__folders_and_projects_crud.py +++ b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__folders_and_projects_crud.py @@ -27,12 +27,7 @@ @pytest.fixture def mock_catalog_api_get_services_for_user_in_product(mocker: MockerFixture): mocker.patch( - "simcore_service_webserver.projects._crud_api_read.get_services_for_user_in_product", - spec=True, - return_value=[], - ) - mocker.patch( - "simcore_service_webserver.projects._crud_handlers.get_services_for_user_in_product", + "simcore_service_webserver.projects._crud_api_read.catalog_service.get_services_for_user_in_product", spec=True, return_value=[], ) diff --git a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__list_projects_full_search.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__list_projects_full_search.py index 99bbaffc4a2e..279bb5916658 100644 --- a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__list_projects_full_search.py +++ b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__list_projects_full_search.py @@ -23,12 +23,7 @@ @pytest.fixture def mock_catalog_api_get_services_for_user_in_product(mocker: MockerFixture): mocker.patch( - "simcore_service_webserver.projects._crud_api_read.get_services_for_user_in_product", - spec=True, - return_value=[], - ) - mocker.patch( - "simcore_service_webserver.projects._crud_handlers.get_services_for_user_in_product", + "simcore_service_webserver.projects._crud_api_read.catalog_service.get_services_for_user_in_product", spec=True, return_value=[], ) diff --git a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py index ea7105a33384..2f1e73ceb84a 100644 --- a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py +++ b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py @@ -28,12 +28,7 @@ def user_role() -> UserRole: @pytest.fixture def mock_catalog_api_get_services_for_user_in_product(mocker: MockerFixture): mocker.patch( - "simcore_service_webserver.projects._crud_api_read.get_services_for_user_in_product", - spec=True, - return_value=[], - ) - mocker.patch( - "simcore_service_webserver.projects._crud_handlers.get_services_for_user_in_product", + "simcore_service_webserver.projects._crud_api_read.catalog_service.get_services_for_user_in_product", spec=True, return_value=[], ) From 79ad133266db08b4ad44a55c91f3e3674ce82de9 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Thu, 6 Mar 2025 12:25:10 +0100 Subject: [PATCH 29/30] further fix to tests --- .../02/test_projects_crud_handlers__list_with_query_params.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__list_with_query_params.py b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__list_with_query_params.py index 8324aff33a2d..b81d8b99dd32 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__list_with_query_params.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__list_with_query_params.py @@ -42,7 +42,7 @@ def standard_user_role() -> tuple[str, tuple[UserRole, ExpectedResponse]]: @pytest.fixture def mock_catalog_api_get_services_for_user_in_product(mocker: MockerFixture): mocker.patch( - "simcore_service_webserver.projects._crud_api_read.get_services_for_user_in_product", + "simcore_service_webserver.projects._crud_api_read.catalog_service.get_services_for_user_in_product", spec=True, return_value=[], ) From 9d55786be4f598b214ea159cb49c50e2328e0689 Mon Sep 17 00:00:00 2001 From: Mads Bisgaard Date: Thu, 6 Mar 2025 12:29:50 +0100 Subject: [PATCH 30/30] fix yet another test --- .../unit/with_dbs/02/test_projects_nodes_handlers__patch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__patch.py b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__patch.py index 77890e530c8d..1819923c9a46 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__patch.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__patch.py @@ -29,7 +29,7 @@ @pytest.fixture def mock_catalog_api_get_services_for_user_in_product(mocker: MockerFixture): mocker.patch( - "simcore_service_webserver.projects._crud_handlers.get_services_for_user_in_product", + "simcore_service_webserver.projects._crud_handlers.catalog_service.get_services_for_user_in_product", spec=True, return_value=[], )