diff --git a/packages/service-library/src/servicelib/fastapi/db_asyncpg_engine.py b/packages/service-library/src/servicelib/fastapi/db_asyncpg_engine.py index 920f68008ae8..cc9932c1dfba 100644 --- a/packages/service-library/src/servicelib/fastapi/db_asyncpg_engine.py +++ b/packages/service-library/src/servicelib/fastapi/db_asyncpg_engine.py @@ -13,13 +13,17 @@ _logger = logging.getLogger(__name__) -async def connect_to_db(app: FastAPI, settings: PostgresSettings) -> None: +async def connect_to_postgres_until_ready(settings: PostgresSettings) -> AsyncEngine: with log_context( _logger, logging.DEBUG, - f"Connecting and migraging {settings.dsn_with_async_sqlalchemy}", + f"Connecting to {settings.dsn_with_async_sqlalchemy}", ): - engine = await create_async_engine_and_pg_database_ready(settings) + return await create_async_engine_and_pg_database_ready(settings) + + +async def connect_to_db(app: FastAPI, settings: PostgresSettings) -> None: + engine = await connect_to_postgres_until_ready(settings) app.state.engine = engine _logger.debug( diff --git a/services/catalog/src/simcore_service_catalog/api/dependencies/__init__.py b/services/catalog/src/simcore_service_catalog/api/_dependencies/__init__.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/api/dependencies/__init__.py rename to services/catalog/src/simcore_service_catalog/api/_dependencies/__init__.py diff --git a/services/catalog/src/simcore_service_catalog/api/dependencies/database.py b/services/catalog/src/simcore_service_catalog/api/_dependencies/database.py similarity index 95% rename from services/catalog/src/simcore_service_catalog/api/dependencies/database.py rename to services/catalog/src/simcore_service_catalog/api/_dependencies/database.py index 2f323db99c52..b0ad2dcacb45 100644 --- a/services/catalog/src/simcore_service_catalog/api/dependencies/database.py +++ b/services/catalog/src/simcore_service_catalog/api/_dependencies/database.py @@ -6,7 +6,7 @@ from fastapi.requests import Request from sqlalchemy.ext.asyncio import AsyncEngine -from ...db.repositories._base import BaseRepository +from ...repository._base import BaseRepository _logger = logging.getLogger(__name__) diff --git a/services/catalog/src/simcore_service_catalog/api/dependencies/director.py b/services/catalog/src/simcore_service_catalog/api/_dependencies/director.py similarity index 74% rename from services/catalog/src/simcore_service_catalog/api/dependencies/director.py rename to services/catalog/src/simcore_service_catalog/api/_dependencies/director.py index 8b6b558428d4..08f808de50f9 100644 --- a/services/catalog/src/simcore_service_catalog/api/dependencies/director.py +++ b/services/catalog/src/simcore_service_catalog/api/_dependencies/director.py @@ -3,7 +3,7 @@ from fastapi import Depends, FastAPI from servicelib.fastapi.dependencies import get_app -from ...services.director import DirectorApi +from ...infrastructure.director import DirectorApi def get_director_api( @@ -11,3 +11,6 @@ def get_director_api( ) -> DirectorApi: director: DirectorApi = app.state.director_api return director + + +__all__: tuple[str, ...] = ("DirectorApi",) diff --git a/services/catalog/src/simcore_service_catalog/api/dependencies/services.py b/services/catalog/src/simcore_service_catalog/api/_dependencies/services.py similarity index 91% rename from services/catalog/src/simcore_service_catalog/api/dependencies/services.py rename to services/catalog/src/simcore_service_catalog/api/_dependencies/services.py index a5d0eba83c2e..eb4cc7a387f6 100644 --- a/services/catalog/src/simcore_service_catalog/api/dependencies/services.py +++ b/services/catalog/src/simcore_service_catalog/api/_dependencies/services.py @@ -13,25 +13,24 @@ from servicelib.fastapi.dependencies import get_app from ...core.settings import ApplicationSettings -from ...db.repositories.groups import GroupsRepository -from ...db.repositories.services import ServicesRepository -from ...services import manifest -from ...services.director import DirectorApi +from ...repository.groups import GroupsRepository +from ...repository.services import ServicesRepository +from ...service import manifest from .database import get_repository -from .director import get_director_api +from .director import DirectorApi, get_director_api _logger = logging.getLogger(__name__) def get_default_service_resources( - app: Annotated[FastAPI, Depends(get_app)] + app: Annotated[FastAPI, Depends(get_app)], ) -> ResourcesDict: app_settings: ApplicationSettings = app.state.settings return app_settings.CATALOG_SERVICES_DEFAULT_RESOURCES def get_default_service_specifications( - app: Annotated[FastAPI, Depends(get_app)] + app: Annotated[FastAPI, Depends(get_app)], ) -> ServiceSpecifications: app_settings: ApplicationSettings = app.state.settings return app_settings.CATALOG_SERVICES_DEFAULT_SPECIFICATIONS diff --git a/services/catalog/src/simcore_service_catalog/api/dependencies/user_groups.py b/services/catalog/src/simcore_service_catalog/api/_dependencies/user_groups.py similarity index 92% rename from services/catalog/src/simcore_service_catalog/api/dependencies/user_groups.py rename to services/catalog/src/simcore_service_catalog/api/_dependencies/user_groups.py index b1b4d23a3de6..f63bc22f45c4 100644 --- a/services/catalog/src/simcore_service_catalog/api/dependencies/user_groups.py +++ b/services/catalog/src/simcore_service_catalog/api/_dependencies/user_groups.py @@ -4,7 +4,7 @@ from models_library.groups import GroupAtDB from models_library.users import UserID -from ...db.repositories.groups import GroupsRepository +from ...repository.groups import GroupsRepository from .database import get_repository diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services.py b/services/catalog/src/simcore_service_catalog/api/rest/_services.py index c97a5545d8c5..1ba3378ef1b5 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services.py @@ -20,13 +20,12 @@ LIST_SERVICES_CACHING_TTL, RESPONSE_MODEL_POLICY, ) -from ...db.repositories.groups import GroupsRepository -from ...db.repositories.services import ServicesRepository from ...models.services_db import ServiceAccessRightsAtDB, ServiceMetaDataDBGet -from ...services.director import DirectorApi -from ..dependencies.database import get_repository -from ..dependencies.director import get_director_api -from ..dependencies.services import get_service_from_manifest +from ...repository.groups import GroupsRepository +from ...repository.services import ServicesRepository +from .._dependencies.database import get_repository +from .._dependencies.director import DirectorApi, get_director_api +from .._dependencies.services import get_service_from_manifest _logger = logging.getLogger(__name__) diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services_access_rights.py b/services/catalog/src/simcore_service_catalog/api/rest/_services_access_rights.py index 6abaf48dd049..2e828996a53c 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services_access_rights.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services_access_rights.py @@ -8,10 +8,10 @@ from models_library.services import ServiceKey, ServiceVersion from ..._constants import RESPONSE_MODEL_POLICY -from ...db.repositories.services import ServicesRepository from ...models.services_db import ServiceAccessRightsAtDB -from ..dependencies.database import get_repository -from ..dependencies.services import AccessInfo, check_service_read_access +from ...repository.services import ServicesRepository +from .._dependencies.database import get_repository +from .._dependencies.services import AccessInfo, check_service_read_access _logger = logging.getLogger(__name__) @@ -33,10 +33,12 @@ async def get_service_access_rights( ], x_simcore_products_name: Annotated[str, Header(...)], ): - service_access_rights: list[ - ServiceAccessRightsAtDB - ] = await services_repo.get_service_access_rights( - key=service_key, version=service_version, product_name=x_simcore_products_name + service_access_rights: list[ServiceAccessRightsAtDB] = ( + await services_repo.get_service_access_rights( + key=service_key, + version=service_version, + product_name=x_simcore_products_name, + ) ) gids_with_access_rights = {} diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services_extras.py b/services/catalog/src/simcore_service_catalog/api/rest/_services_extras.py index 8f1658f57bec..7b7a9418dbb0 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services_extras.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services_extras.py @@ -4,8 +4,7 @@ from models_library.api_schemas_directorv2.services import ServiceExtras from models_library.services import ServiceKey, ServiceVersion -from ...services.director import DirectorApi -from ..dependencies.director import get_director_api +from .._dependencies.director import DirectorApi, get_director_api router = APIRouter() diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services_labels.py b/services/catalog/src/simcore_service_catalog/api/rest/_services_labels.py index bf02bb3b90ff..56af14626e20 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services_labels.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services_labels.py @@ -3,8 +3,7 @@ from fastapi import APIRouter, Depends from models_library.services import ServiceKey, ServiceVersion -from ...services.director import DirectorApi -from ..dependencies.director import get_director_api +from .._dependencies.director import DirectorApi, get_director_api router = APIRouter() diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services_ports.py b/services/catalog/src/simcore_service_catalog/api/rest/_services_ports.py index adc7fd338a9c..ce85bcae37d4 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services_ports.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services_ports.py @@ -6,7 +6,7 @@ from models_library.services_metadata_published import ServiceMetaDataPublished from ..._constants import RESPONSE_MODEL_POLICY -from ..dependencies.services import ( +from .._dependencies.services import ( AccessInfo, check_service_read_access, get_service_from_manifest, diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py b/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py index 72080408663a..2972b1373178 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py @@ -23,17 +23,16 @@ from pydantic import TypeAdapter from ..._constants import RESPONSE_MODEL_POLICY, SIMCORE_SERVICE_SETTINGS_LABELS -from ...db.repositories.services import ServicesRepository -from ...services.director import DirectorApi -from ...services.function_services import is_function_service +from ...repository.services import ServicesRepository +from ...service.function_services import is_function_service from ...utils.service_resources import ( merge_service_resources_with_user_specs, parse_generic_resource, ) -from ..dependencies.database import get_repository -from ..dependencies.director import get_director_api -from ..dependencies.services import get_default_service_resources -from ..dependencies.user_groups import list_user_groups +from .._dependencies.database import get_repository +from .._dependencies.director import DirectorApi, get_director_api +from .._dependencies.services import get_default_service_resources +from .._dependencies.user_groups import list_user_groups router = APIRouter() _logger = logging.getLogger(__name__) @@ -149,7 +148,7 @@ async def _get_service_labels( def _get_service_settings( - labels: dict[str, Any] + labels: dict[str, Any], ) -> list[SimcoreServiceSettingLabelEntry]: service_settings = TypeAdapter(list[SimcoreServiceSettingLabelEntry]).validate_json( labels.get(SIMCORE_SERVICE_SETTINGS_LABELS, "[]"), diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services_specifications.py b/services/catalog/src/simcore_service_catalog/api/rest/_services_specifications.py index 49751a196e80..98649f4be494 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services_specifications.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services_specifications.py @@ -10,11 +10,11 @@ from models_library.users import UserID from ..._constants import RESPONSE_MODEL_POLICY -from ...db.repositories.groups import GroupsRepository -from ...db.repositories.services import ServicesRepository -from ...services.function_services import is_function_service -from ..dependencies.database import get_repository -from ..dependencies.services import get_default_service_specifications +from ...repository.groups import GroupsRepository +from ...repository.services import ServicesRepository +from ...service.function_services import is_function_service +from .._dependencies.database import get_repository +from .._dependencies.services import get_default_service_specifications router = APIRouter() _logger = logging.getLogger(__name__) diff --git a/services/catalog/src/simcore_service_catalog/api/rest/errors.py b/services/catalog/src/simcore_service_catalog/api/rest/errors.py new file mode 100644 index 000000000000..e8e1727c8ce4 --- /dev/null +++ b/services/catalog/src/simcore_service_catalog/api/rest/errors.py @@ -0,0 +1,6 @@ +from fastapi import FastAPI +from servicelib.fastapi.http_error import set_app_default_http_error_handlers + + +def setup_rest_exception_handlers(app: FastAPI) -> None: + set_app_default_http_error_handlers(app) diff --git a/services/catalog/src/simcore_service_catalog/api/rest/routes.py b/services/catalog/src/simcore_service_catalog/api/rest/routes.py index 91e9329d0202..f7e585c9fc76 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/routes.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/routes.py @@ -14,67 +14,67 @@ _services_specifications, ) -v0_router = APIRouter() -# health -health_router = _health.router -v0_router.include_router( - _health.router, - tags=["diagnostics"], -) +def setup_rest_routes(app: FastAPI, vtag: str): + v0_router = APIRouter() -# meta -v0_router.include_router( - _meta.router, - tags=["meta"], - prefix="/meta", -) + # health + health_router = _health.router + v0_router.include_router( + _health.router, + tags=["diagnostics"], + ) -# services -_SERVICE_PREFIX = "/services" -_SERVICE_TAGS: list[str | Enum] = [ - "services", -] -v0_router.include_router( - _services_resources.router, - tags=_SERVICE_TAGS, - prefix=_SERVICE_PREFIX, -) -v0_router.include_router( - _services_labels.router, - tags=_SERVICE_TAGS, - prefix=_SERVICE_PREFIX, -) -v0_router.include_router( - _services_extras.router, - tags=_SERVICE_TAGS, - prefix=_SERVICE_PREFIX, -) -v0_router.include_router( - _services_specifications.router, - tags=_SERVICE_TAGS, - prefix=_SERVICE_PREFIX, -) -v0_router.include_router( - _services_ports.router, - tags=_SERVICE_TAGS, - prefix=_SERVICE_PREFIX, -) -v0_router.include_router( - _services_access_rights.router, - tags=_SERVICE_TAGS, - prefix=_SERVICE_PREFIX, -) + # meta + v0_router.include_router( + _meta.router, + tags=["meta"], + prefix="/meta", + ) -# NOTE: that this router must come after resources/specifications/ports/access_rights -v0_router.include_router( - _services.router, - tags=_SERVICE_TAGS, - prefix=_SERVICE_PREFIX, -) + # services + _SERVICE_PREFIX = "/services" + _SERVICE_TAGS: list[str | Enum] = [ + "services", + ] + v0_router.include_router( + _services_resources.router, + tags=_SERVICE_TAGS, + prefix=_SERVICE_PREFIX, + ) + v0_router.include_router( + _services_labels.router, + tags=_SERVICE_TAGS, + prefix=_SERVICE_PREFIX, + ) + v0_router.include_router( + _services_extras.router, + tags=_SERVICE_TAGS, + prefix=_SERVICE_PREFIX, + ) + v0_router.include_router( + _services_specifications.router, + tags=_SERVICE_TAGS, + prefix=_SERVICE_PREFIX, + ) + v0_router.include_router( + _services_ports.router, + tags=_SERVICE_TAGS, + prefix=_SERVICE_PREFIX, + ) + v0_router.include_router( + _services_access_rights.router, + tags=_SERVICE_TAGS, + prefix=_SERVICE_PREFIX, + ) + # NOTE: that this router must come after resources/specifications/ports/access_rights + v0_router.include_router( + _services.router, + tags=_SERVICE_TAGS, + prefix=_SERVICE_PREFIX, + ) -def setup_rest_api_routes(app: FastAPI, vtag: str): # healthcheck at / and at /v0/ app.include_router(health_router) # api under /v* diff --git a/services/catalog/src/simcore_service_catalog/api/rpc/_services.py b/services/catalog/src/simcore_service_catalog/api/rpc/_services.py index 874ac9bb45fa..63eb5d7d6f5e 100644 --- a/services/catalog/src/simcore_service_catalog/api/rpc/_services.py +++ b/services/catalog/src/simcore_service_catalog/api/rpc/_services.py @@ -23,11 +23,11 @@ CatalogForbiddenError, CatalogItemNotFoundError, ) -from simcore_service_catalog.db.repositories.groups import GroupsRepository +from simcore_service_catalog.repository.groups import GroupsRepository -from ...db.repositories.services import ServicesRepository -from ...services import services_api -from ..dependencies.director import get_director_api +from ...repository.services import ServicesRepository +from ...service import services_api +from .._dependencies.director import get_director_api _logger = logging.getLogger(__name__) diff --git a/services/catalog/src/simcore_service_catalog/api/rpc/routes.py b/services/catalog/src/simcore_service_catalog/api/rpc/routes.py index ce35e7867d17..a1b2270784aa 100644 --- a/services/catalog/src/simcore_service_catalog/api/rpc/routes.py +++ b/services/catalog/src/simcore_service_catalog/api/rpc/routes.py @@ -1,17 +1,18 @@ import logging +from collections.abc import AsyncIterator from fastapi import FastAPI +from fastapi_lifespan_manager import State from models_library.api_schemas_catalog import CATALOG_RPC_NAMESPACE -from ...services.rabbitmq import get_rabbitmq_rpc_server +from ...infrastructure.rabbitmq import get_rabbitmq_rpc_server from . import _services _logger = logging.getLogger(__name__) -def setup_rpc_api_routes(app: FastAPI) -> None: - async def _on_startup() -> None: - rpc_server = get_rabbitmq_rpc_server(app) - await rpc_server.register_router(_services.router, CATALOG_RPC_NAMESPACE, app) +async def setup_rpc_routes(app: FastAPI) -> AsyncIterator[State]: + rpc_server = get_rabbitmq_rpc_server(app) + await rpc_server.register_router(_services.router, CATALOG_RPC_NAMESPACE, app) - app.add_event_handler("startup", _on_startup) + yield {} diff --git a/services/catalog/src/simcore_service_catalog/core/application.py b/services/catalog/src/simcore_service_catalog/core/application.py index 7bedab76a31c..45e911fa5c7d 100644 --- a/services/catalog/src/simcore_service_catalog/core/application.py +++ b/services/catalog/src/simcore_service_catalog/core/application.py @@ -1,7 +1,9 @@ import logging +from collections.abc import AsyncIterator from fastapi import FastAPI from fastapi.middleware.gzip import GZipMiddleware +from fastapi_lifespan_manager import LifespanManager, State from models_library.basic_types import BootModeEnum from servicelib.fastapi import timing_middleware from servicelib.fastapi.openapi import override_fastapi_openapi_method @@ -10,15 +12,26 @@ setup_prometheus_instrumentation, ) from servicelib.fastapi.tracing import initialize_tracing +from simcore_service_catalog.core.background_tasks import setup_background_task from starlette.middleware.base import BaseHTTPMiddleware -from .._meta import API_VERSION, API_VTAG, APP_NAME, PROJECT_NAME, SUMMARY -from ..api.rest.routes import setup_rest_api_routes -from ..api.rpc.routes import setup_rpc_api_routes -from ..exceptions.handlers import setup_exception_handlers -from ..services.function_services import setup_function_services -from ..services.rabbitmq import setup_rabbitmq -from .events import create_on_shutdown, create_on_startup +from .._meta import ( + API_VERSION, + API_VTAG, + APP_FINISHED_BANNER_MSG, + APP_NAME, + APP_STARTED_BANNER_MSG, + PROJECT_NAME, + SUMMARY, +) +from ..api.rest.errors import setup_rest_exception_handlers +from ..api.rest.routes import setup_rest_routes +from ..api.rpc.routes import setup_rpc_routes +from ..infrastructure.director import director_lifespan +from ..infrastructure.postgres import postgres_lifespan +from ..infrastructure.rabbitmq import rabbitmq_lifespan +from ..repository.setup import setup_repository +from ..service.function_services import setup_function_services from .settings import ApplicationSettings _logger = logging.getLogger(__name__) @@ -34,6 +47,40 @@ ) +async def _setup_banner(app: FastAPI) -> AsyncIterator[State]: + # WARNING: this function is spied in the tests + assert app + print(APP_STARTED_BANNER_MSG, flush=True) # noqa: T201 + + yield {} + + print(APP_FINISHED_BANNER_MSG, flush=True) # noqa: T201 + + +def _create_app_lifespan(settings: ApplicationSettings): + assert settings # nosec + + # app lifespan + app_lifespan = LifespanManager() + app_lifespan.add(_setup_banner) + + # - postgres lifespan + postgres_lifespan.add(setup_repository) + app_lifespan.include(postgres_lifespan) + + # - director lifespan + app_lifespan.include(director_lifespan) + + # - rabbitmq lifespan + rabbitmq_lifespan.add(setup_rpc_routes) + app_lifespan.add(rabbitmq_lifespan) + + app_lifespan.add(setup_function_services) + app_lifespan.add(setup_background_task) + + return app_lifespan + + def create_app(settings: ApplicationSettings | None = None) -> FastAPI: # keep mostly quiet noisy loggers quiet_level: int = max( @@ -57,6 +104,7 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI: openapi_url=f"/api/{API_VTAG}/openapi.json", docs_url="/dev/doc", redoc_url=None, # default disabled + lifespan=_create_app_lifespan(settings), ) override_fastapi_openapi_method(app) @@ -66,13 +114,6 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI: if settings.CATALOG_TRACING: initialize_tracing(app, settings.CATALOG_TRACING, APP_NAME) - # STARTUP-EVENT - app.add_event_handler("startup", create_on_startup(app)) - - # PLUGIN SETUP - setup_function_services(app) - setup_rabbitmq(app) - if app.state.settings.CATALOG_PROMETHEUS_INSTRUMENTATION_ENABLED: setup_prometheus_instrumentation(app) @@ -89,13 +130,7 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI: app.add_middleware(GZipMiddleware) # ROUTES - setup_rest_api_routes(app, vtag=API_VTAG) - setup_rpc_api_routes(app) - - # SHUTDOWN-EVENT - app.add_event_handler("shutdown", create_on_shutdown(app)) - - # EXCEPTIONS - setup_exception_handlers(app) + setup_rest_routes(app, vtag=API_VTAG) + setup_rest_exception_handlers(app) return app diff --git a/services/catalog/src/simcore_service_catalog/core/background_tasks.py b/services/catalog/src/simcore_service_catalog/core/background_tasks.py index cb269ee39196..19654dbd2726 100644 --- a/services/catalog/src/simcore_service_catalog/core/background_tasks.py +++ b/services/catalog/src/simcore_service_catalog/core/background_tasks.py @@ -11,25 +11,27 @@ import asyncio import logging +from collections.abc import AsyncIterator from contextlib import suppress from pprint import pformat from typing import Final from fastapi import FastAPI, HTTPException +from fastapi_lifespan_manager import State from models_library.services import ServiceMetaDataPublished from models_library.services_types import ServiceKey, ServiceVersion from packaging.version import Version from pydantic import ValidationError -from simcore_service_catalog.api.dependencies.director import get_director_api -from simcore_service_catalog.services import manifest +from simcore_service_catalog.api._dependencies.director import get_director_api +from simcore_service_catalog.service import manifest from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.asyncio import AsyncEngine -from ..db.repositories.groups import GroupsRepository -from ..db.repositories.projects import ProjectsRepository -from ..db.repositories.services import ServicesRepository from ..models.services_db import ServiceAccessRightsAtDB, ServiceMetaDataDBCreate -from ..services import access_rights +from ..repository.groups import GroupsRepository +from ..repository.projects import ProjectsRepository +from ..repository.services import ServicesRepository +from ..service import access_rights _logger = logging.getLogger(__name__) @@ -115,9 +117,9 @@ async def _ensure_registry_and_database_are_synced(app: FastAPI) -> None: director_api = get_director_api(app) services_in_manifest_map = await manifest.get_services_map(director_api) - services_in_db: set[ - tuple[ServiceKey, ServiceVersion] - ] = await _list_services_in_database(app.state.engine) + services_in_db: set[tuple[ServiceKey, ServiceVersion]] = ( + await _list_services_in_database(app.state.engine) + ) # check that the db has all the services at least once missing_services_in_db = set(services_in_manifest_map.keys()) - services_in_db @@ -213,22 +215,23 @@ async def _sync_services_task(app: FastAPI) -> None: ) -async def start_registry_sync_task(app: FastAPI) -> None: +async def setup_background_task(app: FastAPI) -> AsyncIterator[State]: + # FIXME: check director service is in place and ready. Hand-shake?? + # SEE https://github.com/ITISFoundation/osparc-simcore/issues/1728 + # FIXME: added this variable to overcome the state in which the # task cancelation is ignored and the exceptions enter in a loop # that never stops the background task. This flag is an additional # mechanism to enforce stopping the background task - app.state.registry_syncer_running = True task = asyncio.create_task(_sync_services_task(app)) - app.state.registry_sync_task = task + _logger.info("registry syncing task started") + yield {"registry_syncer_running": True, "registry_sync_task": task} + + with suppress(asyncio.CancelledError): + app.state.registry_syncer_running = False + task.cancel() + await task -async def stop_registry_sync_task(app: FastAPI) -> None: - if task := app.state.registry_sync_task: - with suppress(asyncio.CancelledError): - app.state.registry_syncer_running = False - task.cancel() - await task - app.state.registry_sync_task = None _logger.info("registry syncing task stopped") diff --git a/services/catalog/src/simcore_service_catalog/core/events.py b/services/catalog/src/simcore_service_catalog/core/events.py deleted file mode 100644 index f22adbba4ece..000000000000 --- a/services/catalog/src/simcore_service_catalog/core/events.py +++ /dev/null @@ -1,65 +0,0 @@ -import logging -from collections.abc import Awaitable, Callable -from typing import TypeAlias - -from fastapi import FastAPI -from servicelib.fastapi.db_asyncpg_engine import close_db_connection, connect_to_db -from servicelib.logging_utils import log_context - -from .._meta import APP_FINISHED_BANNER_MSG, APP_STARTED_BANNER_MSG -from ..db.events import setup_default_product -from ..services.director import close_director, setup_director -from .background_tasks import start_registry_sync_task, stop_registry_sync_task - -_logger = logging.getLogger(__name__) - - -EventCallable: TypeAlias = Callable[[], Awaitable[None]] - - -def _flush_started_banner() -> None: - # WARNING: this function is spied in the tests - print(APP_STARTED_BANNER_MSG, flush=True) # noqa: T201 - - -def _flush_finished_banner() -> None: - print(APP_FINISHED_BANNER_MSG, flush=True) # noqa: T201 - - -def create_on_startup(app: FastAPI) -> EventCallable: - async def _() -> None: - _flush_started_banner() - - # setup connection to pg db - if app.state.settings.CATALOG_POSTGRES: - await connect_to_db(app, app.state.settings.CATALOG_POSTGRES) - await setup_default_product(app) - - if app.state.settings.CATALOG_DIRECTOR: - # setup connection to director - await setup_director(app) - - # FIXME: check director service is in place and ready. Hand-shake?? - # SEE https://github.com/ITISFoundation/osparc-simcore/issues/1728 - await start_registry_sync_task(app) - - _logger.info("Application started") - - return _ - - -def create_on_shutdown(app: FastAPI) -> EventCallable: - async def _() -> None: - - with log_context(_logger, logging.INFO, "Application shutdown"): - if app.state.settings.CATALOG_DIRECTOR: - try: - await stop_registry_sync_task(app) - await close_director(app) - await close_db_connection(app) - except Exception: # pylint: disable=broad-except - _logger.exception("Unexpected error while closing application") - - _flush_finished_banner() - - return _ diff --git a/services/catalog/src/simcore_service_catalog/db/events.py b/services/catalog/src/simcore_service_catalog/db/events.py deleted file mode 100644 index 42de4c386201..000000000000 --- a/services/catalog/src/simcore_service_catalog/db/events.py +++ /dev/null @@ -1,12 +0,0 @@ -import logging - -from fastapi import FastAPI - -from .repositories.products import ProductsRepository - -_logger = logging.getLogger(__name__) - - -async def setup_default_product(app: FastAPI): - repo = ProductsRepository(db_engine=app.state.engine) - app.state.default_product_name = await repo.get_default_product_name() diff --git a/services/catalog/src/simcore_service_catalog/db/repositories/__init__.py b/services/catalog/src/simcore_service_catalog/db/repositories/__init__.py deleted file mode 100644 index a5eeffe1ff59..000000000000 --- a/services/catalog/src/simcore_service_catalog/db/repositories/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from ._base import BaseRepository diff --git a/services/catalog/src/simcore_service_catalog/exceptions/errors.py b/services/catalog/src/simcore_service_catalog/errors.py similarity index 69% rename from services/catalog/src/simcore_service_catalog/exceptions/errors.py rename to services/catalog/src/simcore_service_catalog/errors.py index 84010d9a7007..7e33eb08d0d7 100644 --- a/services/catalog/src/simcore_service_catalog/exceptions/errors.py +++ b/services/catalog/src/simcore_service_catalog/errors.py @@ -1,8 +1,7 @@ from common_library.errors_classes import OsparcErrorMixin -class CatalogBaseError(OsparcErrorMixin, Exception): - ... +class CatalogBaseError(OsparcErrorMixin, Exception): ... class RepositoryError(CatalogBaseError): @@ -13,13 +12,11 @@ class UninitializedGroupError(RepositoryError): msg_tempalte = "{group} groups was never initialized" -class BaseDirectorError(CatalogBaseError): - ... +class BaseDirectorError(CatalogBaseError): ... class DirectorUnresponsiveError(BaseDirectorError): msg_template = "Director-v0 is not responsive" -class DirectorStatusError(BaseDirectorError): - ... +class DirectorStatusError(BaseDirectorError): ... diff --git a/services/catalog/src/simcore_service_catalog/exceptions/handlers/__init__.py b/services/catalog/src/simcore_service_catalog/exceptions/handlers/__init__.py deleted file mode 100644 index 49620d73f6ca..000000000000 --- a/services/catalog/src/simcore_service_catalog/exceptions/handlers/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -from fastapi import FastAPI, HTTPException, status -from fastapi.exceptions import RequestValidationError - -from ._http_error import http_error_handler, make_http_error_handler_for_exception -from ._validation_error import http422_error_handler - - -def setup_exception_handlers(app: FastAPI) -> None: - app.add_exception_handler(HTTPException, http_error_handler) - app.add_exception_handler(RequestValidationError, http422_error_handler) - - # SEE https://docs.python.org/3/library/exceptions.html#exception-hierarchy - app.add_exception_handler( - NotImplementedError, - make_http_error_handler_for_exception( - status.HTTP_501_NOT_IMPLEMENTED, NotImplementedError - ), - ) - app.add_exception_handler( - Exception, - make_http_error_handler_for_exception( - status.HTTP_500_INTERNAL_SERVER_ERROR, Exception - ), - ) diff --git a/services/catalog/src/simcore_service_catalog/exceptions/handlers/_http_error.py b/services/catalog/src/simcore_service_catalog/exceptions/handlers/_http_error.py deleted file mode 100644 index 7af4b5d93bd7..000000000000 --- a/services/catalog/src/simcore_service_catalog/exceptions/handlers/_http_error.py +++ /dev/null @@ -1,33 +0,0 @@ -from fastapi import HTTPException -from fastapi.encoders import jsonable_encoder -from starlette.requests import Request -from starlette.responses import JSONResponse -from starlette.types import HTTPExceptionHandler - - -async def http_error_handler(request: Request, exc: Exception) -> JSONResponse: - assert request # nosec - assert isinstance(exc, HTTPException) # nosec - - return JSONResponse( - content=jsonable_encoder({"errors": [exc.detail]}), status_code=exc.status_code - ) - - -def make_http_error_handler_for_exception( - status_code: int, exception_cls: type[BaseException] -) -> HTTPExceptionHandler: - """ - Produces a handler for BaseException-type exceptions which converts them - into an error JSON response with a given status code - - SEE https://docs.python.org/3/library/exceptions.html#concrete-exceptions - """ - - async def _http_error_handler(_: Request, exc: type[BaseException]) -> JSONResponse: - assert isinstance(exc, exception_cls) # nosec - return JSONResponse( - content=jsonable_encoder({"errors": [str(exc)]}), status_code=status_code - ) - - return _http_error_handler # type: ignore[return-value] diff --git a/services/catalog/src/simcore_service_catalog/exceptions/handlers/_validation_error.py b/services/catalog/src/simcore_service_catalog/exceptions/handlers/_validation_error.py deleted file mode 100644 index 8e3ad77f15db..000000000000 --- a/services/catalog/src/simcore_service_catalog/exceptions/handlers/_validation_error.py +++ /dev/null @@ -1,23 +0,0 @@ -from fastapi.encoders import jsonable_encoder -from fastapi.openapi.constants import REF_PREFIX -from fastapi.openapi.utils import validation_error_response_definition -from starlette.requests import Request -from starlette.responses import JSONResponse -from starlette.status import HTTP_422_UNPROCESSABLE_ENTITY - - -async def http422_error_handler(_: Request, exc: Exception) -> JSONResponse: - assert hasattr(exc, "errors") # nosec - return JSONResponse( - content=jsonable_encoder({"errors": exc.errors()}), - status_code=HTTP_422_UNPROCESSABLE_ENTITY, - ) - - -validation_error_response_definition["properties"] = { - "errors": { - "title": "Validation errors", - "type": "array", - "items": {"$ref": f"{REF_PREFIX}ValidationError"}, - }, -} diff --git a/services/catalog/src/simcore_service_catalog/db/__init__.py b/services/catalog/src/simcore_service_catalog/infrastructure/__init__.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/db/__init__.py rename to services/catalog/src/simcore_service_catalog/infrastructure/__init__.py diff --git a/services/catalog/src/simcore_service_catalog/services/director.py b/services/catalog/src/simcore_service_catalog/infrastructure/director.py similarity index 89% rename from services/catalog/src/simcore_service_catalog/services/director.py rename to services/catalog/src/simcore_service_catalog/infrastructure/director.py index 525840621840..167c095ea3e5 100644 --- a/services/catalog/src/simcore_service_catalog/services/director.py +++ b/services/catalog/src/simcore_service_catalog/infrastructure/director.py @@ -3,7 +3,7 @@ import json import logging import urllib.parse -from collections.abc import Awaitable, Callable +from collections.abc import AsyncIterator, Awaitable, Callable from contextlib import suppress from pprint import pformat from typing import Any, Final @@ -11,6 +11,7 @@ import httpx from common_library.json_serialization import json_dumps from fastapi import FastAPI, HTTPException +from fastapi_lifespan_manager import LifespanManager, State from models_library.api_schemas_directorv2.services import ServiceExtras from models_library.services_metadata_published import ServiceMetaDataPublished from models_library.services_types import ServiceKey, ServiceVersion @@ -24,7 +25,7 @@ from tenacity.wait import wait_random from ..core.settings import ApplicationSettings -from ..exceptions.errors import DirectorUnresponsiveError +from ..errors import DirectorUnresponsiveError _logger = logging.getLogger(__name__) @@ -65,7 +66,7 @@ def _validate_kind(entry_to_validate: dict[str, Any], kind_name: str): def _return_data_or_raise_error( - request_func: Callable[..., Awaitable[httpx.Response]] + request_func: Callable[..., Awaitable[httpx.Response]], ) -> Callable[..., Awaitable[list[Any] | dict[str, Any]]]: """ Creates a context for safe inter-process communication (IPC) @@ -288,32 +289,30 @@ async def get_service_extras( return TypeAdapter(ServiceExtras).validate_python(result) -async def setup_director(app: FastAPI) -> None: - if settings := app.state.settings.CATALOG_DIRECTOR: - with log_context( - _logger, logging.DEBUG, "Setup director at %s", f"{settings.base_url=}" - ): - async for attempt in AsyncRetrying(**_director_startup_retry_policy): - client = DirectorApi(base_url=settings.base_url, app=app) - with attempt: - client = DirectorApi(base_url=settings.base_url, app=app) - if not await client.is_responsive(): - with suppress(Exception): - await client.close() - raise DirectorUnresponsiveError +director_lifespan = LifespanManager() + - _logger.info( - "Connection to director-v0 succeded [%s]", - json_dumps(attempt.retry_state.retry_object.statistics), - ) +@director_lifespan.add +async def setup_director(app: FastAPI) -> AsyncIterator[State]: + settings = app.state.settings.CATALOG_DIRECTOR - # set when connected - app.state.director_api = client + with log_context( + _logger, logging.DEBUG, "Setup director at %s", f"{settings.base_url=}" + ): + async for attempt in AsyncRetrying(**_director_startup_retry_policy): + client = DirectorApi(base_url=settings.base_url, app=app) + with attempt: + client = DirectorApi(base_url=settings.base_url, app=app) + if not await client.is_responsive(): + with suppress(Exception): + await client.close() + raise DirectorUnresponsiveError + + _logger.info( + "Connection to director-v0 succeded [%s]", + json_dumps(attempt.retry_state.retry_object.statistics), + ) + yield {"director_api": client} -async def close_director(app: FastAPI) -> None: - client: DirectorApi | None - if client := app.state.director_api: await client.close() - - _logger.debug("Director client closed successfully") diff --git a/services/catalog/src/simcore_service_catalog/infrastructure/postgres.py b/services/catalog/src/simcore_service_catalog/infrastructure/postgres.py new file mode 100644 index 000000000000..5cbd3c89ee70 --- /dev/null +++ b/services/catalog/src/simcore_service_catalog/infrastructure/postgres.py @@ -0,0 +1,29 @@ +import logging +from collections.abc import AsyncIterator + +from fastapi import FastAPI +from fastapi_lifespan_manager import LifespanManager, State +from servicelib.fastapi.db_asyncpg_engine import connect_to_postgres_until_ready +from servicelib.logging_utils import log_catch, log_context +from sqlalchemy.ext.asyncio import AsyncEngine + +_logger = logging.getLogger(__name__) + + +postgres_lifespan = LifespanManager() + + +@postgres_lifespan.add +async def setup_postgres_database(app: FastAPI) -> AsyncIterator[State]: + + with log_context(_logger, logging.INFO, f"{__name__} startup ..."): + engine: AsyncEngine = await connect_to_postgres_until_ready( + app.state.settings.CATALOG_POSTGRES + ) + + yield {"engine": engine} + + with log_context(_logger, logging.INFO, f"{__name__} shutdown ..."), log_catch( + _logger, reraise=False + ): + await engine.dispose() diff --git a/services/catalog/src/simcore_service_catalog/services/rabbitmq.py b/services/catalog/src/simcore_service_catalog/infrastructure/rabbitmq.py similarity index 51% rename from services/catalog/src/simcore_service_catalog/services/rabbitmq.py rename to services/catalog/src/simcore_service_catalog/infrastructure/rabbitmq.py index 8400885efa07..9b717869c95b 100644 --- a/services/catalog/src/simcore_service_catalog/services/rabbitmq.py +++ b/services/catalog/src/simcore_service_catalog/infrastructure/rabbitmq.py @@ -1,7 +1,9 @@ import logging +from collections.abc import AsyncIterator from typing import cast from fastapi import FastAPI +from fastapi_lifespan_manager import LifespanManager, State from servicelib.rabbitmq import RabbitMQRPCClient, wait_till_rabbitmq_responsive from settings_library.rabbit import RabbitSettings @@ -15,24 +17,23 @@ def get_rabbitmq_settings(app: FastAPI) -> RabbitSettings: return settings -def setup_rabbitmq(app: FastAPI) -> None: +rabbitmq_lifespan = LifespanManager() + + +@rabbitmq_lifespan.add +async def setup_rabbitmq(app: FastAPI) -> AsyncIterator[State]: settings: RabbitSettings = get_rabbitmq_settings(app) - app.state.rabbitmq_rpc_server = None - async def _on_startup() -> None: - await wait_till_rabbitmq_responsive(settings.dsn) + await wait_till_rabbitmq_responsive(settings.dsn) - app.state.rabbitmq_rpc_server = await RabbitMQRPCClient.create( - client_name=f"{PROJECT_NAME}_rpc_server", settings=settings - ) + rabbitmq_rpc_server = await RabbitMQRPCClient.create( + client_name=f"{PROJECT_NAME}_rpc_server", + settings=settings, + ) - async def _on_shutdown() -> None: - if app.state.rabbitmq_rpc_server: - await app.state.rabbitmq_rpc_server.close() - app.state.rabbitmq_rpc_server = None + yield {"rabbitmq_rpc_server": rabbitmq_rpc_server} - app.add_event_handler("startup", _on_startup) - app.add_event_handler("shutdown", _on_shutdown) + await rabbitmq_rpc_server.close() def get_rabbitmq_rpc_server(app: FastAPI) -> RabbitMQRPCClient: diff --git a/services/catalog/src/simcore_service_catalog/exceptions/__init__.py b/services/catalog/src/simcore_service_catalog/repository/__init__.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/exceptions/__init__.py rename to services/catalog/src/simcore_service_catalog/repository/__init__.py diff --git a/services/catalog/src/simcore_service_catalog/db/repositories/_base.py b/services/catalog/src/simcore_service_catalog/repository/_base.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/db/repositories/_base.py rename to services/catalog/src/simcore_service_catalog/repository/_base.py diff --git a/services/catalog/src/simcore_service_catalog/db/repositories/_services_sql.py b/services/catalog/src/simcore_service_catalog/repository/_services_sql.py similarity index 99% rename from services/catalog/src/simcore_service_catalog/db/repositories/_services_sql.py rename to services/catalog/src/simcore_service_catalog/repository/_services_sql.py index 935a38349761..58240d56cb12 100644 --- a/services/catalog/src/simcore_service_catalog/db/repositories/_services_sql.py +++ b/services/catalog/src/simcore_service_catalog/repository/_services_sql.py @@ -10,8 +10,8 @@ from sqlalchemy.sql.expression import func from sqlalchemy.sql.selectable import Select -from ...models.services_db import ServiceMetaDataDBGet -from ..tables import ( +from ..models.services_db import ServiceMetaDataDBGet +from ._tables import ( services_access_rights, services_compatibility, services_meta_data, diff --git a/services/catalog/src/simcore_service_catalog/db/tables.py b/services/catalog/src/simcore_service_catalog/repository/_tables.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/db/tables.py rename to services/catalog/src/simcore_service_catalog/repository/_tables.py diff --git a/services/catalog/src/simcore_service_catalog/db/repositories/groups.py b/services/catalog/src/simcore_service_catalog/repository/groups.py similarity index 96% rename from services/catalog/src/simcore_service_catalog/db/repositories/groups.py rename to services/catalog/src/simcore_service_catalog/repository/groups.py index d7061947a104..4df2d96f4b9b 100644 --- a/services/catalog/src/simcore_service_catalog/db/repositories/groups.py +++ b/services/catalog/src/simcore_service_catalog/repository/groups.py @@ -6,9 +6,9 @@ from pydantic import TypeAdapter from pydantic.types import PositiveInt -from ...exceptions.errors import UninitializedGroupError -from ..tables import GroupType, groups, user_to_groups, users +from ..errors import UninitializedGroupError from ._base import BaseRepository +from ._tables import GroupType, groups, user_to_groups, users class GroupsRepository(BaseRepository): diff --git a/services/catalog/src/simcore_service_catalog/db/repositories/products.py b/services/catalog/src/simcore_service_catalog/repository/products.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/db/repositories/products.py rename to services/catalog/src/simcore_service_catalog/repository/products.py diff --git a/services/catalog/src/simcore_service_catalog/db/repositories/projects.py b/services/catalog/src/simcore_service_catalog/repository/projects.py similarity index 97% rename from services/catalog/src/simcore_service_catalog/db/repositories/projects.py rename to services/catalog/src/simcore_service_catalog/repository/projects.py index 5b2b2d1bbfec..791090570c71 100644 --- a/services/catalog/src/simcore_service_catalog/db/repositories/projects.py +++ b/services/catalog/src/simcore_service_catalog/repository/projects.py @@ -4,8 +4,8 @@ from models_library.services import ServiceKeyVersion from pydantic import ValidationError -from ..tables import ProjectType, projects from ._base import BaseRepository +from ._tables import ProjectType, projects _logger = logging.getLogger(__name__) diff --git a/services/catalog/src/simcore_service_catalog/db/repositories/services.py b/services/catalog/src/simcore_service_catalog/repository/services.py similarity index 99% rename from services/catalog/src/simcore_service_catalog/db/repositories/services.py rename to services/catalog/src/simcore_service_catalog/repository/services.py index 06a166659981..bbde9e5409cd 100644 --- a/services/catalog/src/simcore_service_catalog/db/repositories/services.py +++ b/services/catalog/src/simcore_service_catalog/repository/services.py @@ -24,7 +24,7 @@ from sqlalchemy.sql import and_, or_ from sqlalchemy.sql.expression import tuple_ -from ...models.services_db import ( +from ..models.services_db import ( ReleaseDBGet, ServiceAccessRightsAtDB, ServiceMetaDataDBCreate, @@ -32,14 +32,7 @@ ServiceMetaDataDBPatch, ServiceWithHistoryDBGet, ) -from ...models.services_specifications import ServiceSpecificationsAtDB -from ..tables import ( - services_access_rights, - services_compatibility, - services_meta_data, - services_specifications, - user_to_groups, -) +from ..models.services_specifications import ServiceSpecificationsAtDB from ._base import BaseRepository from ._services_sql import ( SERVICES_META_DATA_COLS, @@ -52,6 +45,13 @@ list_latest_services_stmt, list_services_stmt, ) +from ._tables import ( + services_access_rights, + services_compatibility, + services_meta_data, + services_specifications, + user_to_groups, +) _logger = logging.getLogger(__name__) diff --git a/services/catalog/src/simcore_service_catalog/repository/setup.py b/services/catalog/src/simcore_service_catalog/repository/setup.py new file mode 100644 index 000000000000..9c9c8eb80f49 --- /dev/null +++ b/services/catalog/src/simcore_service_catalog/repository/setup.py @@ -0,0 +1,12 @@ +from collections.abc import AsyncIterator + +from fastapi import FastAPI +from fastapi_lifespan_manager import State + +from .products import ProductsRepository + + +async def setup_repository(app: FastAPI) -> AsyncIterator[State]: + repo = ProductsRepository(db_engine=app.state.engine) + + yield {"default_product_name": await repo.get_default_product_name()} diff --git a/services/catalog/src/simcore_service_catalog/services/__init__.py b/services/catalog/src/simcore_service_catalog/service/__init__.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/services/__init__.py rename to services/catalog/src/simcore_service_catalog/service/__init__.py diff --git a/services/catalog/src/simcore_service_catalog/services/access_rights.py b/services/catalog/src/simcore_service_catalog/service/access_rights.py similarity index 97% rename from services/catalog/src/simcore_service_catalog/services/access_rights.py rename to services/catalog/src/simcore_service_catalog/service/access_rights.py index e504fe185841..c8009ed4983a 100644 --- a/services/catalog/src/simcore_service_catalog/services/access_rights.py +++ b/services/catalog/src/simcore_service_catalog/service/access_rights.py @@ -1,6 +1,4 @@ -""" Services Access Rights policies - -""" +"""Services Access Rights policies""" import logging import operator @@ -16,10 +14,10 @@ from pydantic.types import PositiveInt from sqlalchemy.ext.asyncio import AsyncEngine -from ..api.dependencies.director import get_director_api -from ..db.repositories.groups import GroupsRepository -from ..db.repositories.services import ServicesRepository +from ..api._dependencies.director import get_director_api from ..models.services_db import ServiceAccessRightsAtDB +from ..repository.groups import GroupsRepository +from ..repository.services import ServicesRepository from ..utils.versioning import as_version, is_patch_release _logger = logging.getLogger(__name__) diff --git a/services/catalog/src/simcore_service_catalog/services/compatibility.py b/services/catalog/src/simcore_service_catalog/service/compatibility.py similarity index 97% rename from services/catalog/src/simcore_service_catalog/services/compatibility.py rename to services/catalog/src/simcore_service_catalog/service/compatibility.py index db8483e11c99..696726e5ef04 100644 --- a/services/catalog/src/simcore_service_catalog/services/compatibility.py +++ b/services/catalog/src/simcore_service_catalog/service/compatibility.py @@ -1,6 +1,4 @@ -""" Manages service compatibility policies - -""" +"""Manages service compatibility policies""" from models_library.products import ProductName from models_library.services_history import Compatibility, CompatibleService @@ -10,8 +8,8 @@ from packaging.version import Version from simcore_service_catalog.utils.versioning import as_version -from ..db.repositories.services import ServicesRepository from ..models.services_db import ReleaseDBGet +from ..repository.services import ServicesRepository def _get_default_compatibility_specs(target: ServiceVersion | Version) -> SpecifierSet: diff --git a/services/catalog/src/simcore_service_catalog/services/function_services.py b/services/catalog/src/simcore_service_catalog/service/function_services.py similarity index 73% rename from services/catalog/src/simcore_service_catalog/services/function_services.py rename to services/catalog/src/simcore_service_catalog/service/function_services.py index 7ed546f251b0..c0a6ff8e9d42 100644 --- a/services/catalog/src/simcore_service_catalog/services/function_services.py +++ b/services/catalog/src/simcore_service_catalog/service/function_services.py @@ -1,9 +1,12 @@ +from collections.abc import AsyncIterator + # mypy: disable-error-code=truthy-function from typing import Any from fastapi import status from fastapi.applications import FastAPI from fastapi.exceptions import HTTPException +from fastapi_lifespan_manager import State from models_library.function_services_catalog import ( is_function_service, iter_service_docker_data, @@ -31,12 +34,13 @@ def get_function_service(key, version) -> ServiceMetaDataPublished: ) from err -def setup_function_services(app: FastAPI): - def _on_startup() -> None: - catalog = [_as_dict(metadata) for metadata in iter_service_docker_data()] - app.state.frontend_services_catalog = catalog +async def setup_function_services(app: FastAPI) -> AsyncIterator[State]: + assert app # nosec + assert not hasattr(app.state, "frontend_services_catalog") # nosec + + catalog = [_as_dict(metadata) for metadata in iter_service_docker_data()] - app.add_event_handler("startup", _on_startup) + yield {"frontend_services_catalog": catalog} __all__: tuple[str, ...] = ( diff --git a/services/catalog/src/simcore_service_catalog/services/manifest.py b/services/catalog/src/simcore_service_catalog/service/manifest.py similarity index 98% rename from services/catalog/src/simcore_service_catalog/services/manifest.py rename to services/catalog/src/simcore_service_catalog/service/manifest.py index 5cfbb1d961bd..eabee127fad3 100644 --- a/services/catalog/src/simcore_service_catalog/services/manifest.py +++ b/services/catalog/src/simcore_service_catalog/service/manifest.py @@ -35,7 +35,7 @@ from servicelib.utils import limited_gather from .._constants import DIRECTOR_CACHING_TTL -from .director import DirectorApi +from ..infrastructure.director import DirectorApi from .function_services import get_function_service, is_function_service _logger = logging.getLogger(__name__) diff --git a/services/catalog/src/simcore_service_catalog/services/services_api.py b/services/catalog/src/simcore_service_catalog/service/services_api.py similarity index 98% rename from services/catalog/src/simcore_service_catalog/services/services_api.py rename to services/catalog/src/simcore_service_catalog/service/services_api.py index 43a70f45bf27..60c5c8406697 100644 --- a/services/catalog/src/simcore_service_catalog/services/services_api.py +++ b/services/catalog/src/simcore_service_catalog/service/services_api.py @@ -20,16 +20,16 @@ CatalogForbiddenError, CatalogItemNotFoundError, ) -from simcore_service_catalog.db.repositories.groups import GroupsRepository +from simcore_service_catalog.repository.groups import GroupsRepository -from ..db.repositories.services import ServicesRepository +from ..infrastructure.director import DirectorApi from ..models.services_db import ( ServiceAccessRightsAtDB, ServiceMetaDataDBPatch, ServiceWithHistoryDBGet, ) -from ..services import manifest -from ..services.director import DirectorApi +from ..repository.services import ServicesRepository +from ..service import manifest from .compatibility import evaluate_service_compatibility_map from .function_services import is_function_service diff --git a/services/catalog/tests/unit/test_services_director.py b/services/catalog/tests/unit/test_infrastructure_director.py similarity index 93% rename from services/catalog/tests/unit/test_services_director.py rename to services/catalog/tests/unit/test_infrastructure_director.py index eb36988b5198..f1675a570028 100644 --- a/services/catalog/tests/unit/test_services_director.py +++ b/services/catalog/tests/unit/test_infrastructure_director.py @@ -15,8 +15,8 @@ from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from respx.router import MockRouter -from simcore_service_catalog.api.dependencies.director import get_director_api -from simcore_service_catalog.services.director import DirectorApi +from simcore_service_catalog.api._dependencies.director import get_director_api +from simcore_service_catalog.infrastructure.director import DirectorApi @pytest.fixture diff --git a/services/catalog/tests/unit/test_db_repositories_services_sql.py b/services/catalog/tests/unit/test_repository_services_sql.py similarity index 96% rename from services/catalog/tests/unit/test_db_repositories_services_sql.py rename to services/catalog/tests/unit/test_repository_services_sql.py index 27b8db9e770d..7acc31c585fd 100644 --- a/services/catalog/tests/unit/test_db_repositories_services_sql.py +++ b/services/catalog/tests/unit/test_repository_services_sql.py @@ -4,7 +4,7 @@ from simcore_postgres_database.utils import as_postgres_sql_query_str -from simcore_service_catalog.db.repositories._services_sql import ( +from simcore_service_catalog.repository._services_sql import ( AccessRightsClauses, can_get_service_stmt, get_service_history_stmt, diff --git a/services/catalog/tests/unit/test_services_compatibility.py b/services/catalog/tests/unit/test_service_compatibility.py similarity index 98% rename from services/catalog/tests/unit/test_services_compatibility.py rename to services/catalog/tests/unit/test_service_compatibility.py index 1211c25a97be..30cbb6733581 100644 --- a/services/catalog/tests/unit/test_services_compatibility.py +++ b/services/catalog/tests/unit/test_service_compatibility.py @@ -11,9 +11,9 @@ from packaging.specifiers import SpecifierSet from packaging.version import Version from pytest_mock import MockerFixture, MockType -from simcore_service_catalog.db.repositories.services import ServicesRepository from simcore_service_catalog.models.services_db import ReleaseDBGet -from simcore_service_catalog.services.compatibility import ( +from simcore_service_catalog.repository.services import ServicesRepository +from simcore_service_catalog.service.compatibility import ( _get_latest_compatible_version, evaluate_service_compatibility_map, ) diff --git a/services/catalog/tests/unit/test_services_function_services.py b/services/catalog/tests/unit/test_service_function_services.py similarity index 91% rename from services/catalog/tests/unit/test_services_function_services.py rename to services/catalog/tests/unit/test_service_function_services.py index 4c4235ce9aef..06853bcf715b 100644 --- a/services/catalog/tests/unit/test_services_function_services.py +++ b/services/catalog/tests/unit/test_service_function_services.py @@ -8,7 +8,7 @@ import pytest from models_library.api_schemas_catalog.services import ServiceMetaDataPublished -from simcore_service_catalog.services.function_services import ( +from simcore_service_catalog.service.function_services import ( is_function_service, iter_service_docker_data, ) diff --git a/services/catalog/tests/unit/test_services_manifest.py b/services/catalog/tests/unit/test_service_manifest.py similarity index 92% rename from services/catalog/tests/unit/test_services_manifest.py rename to services/catalog/tests/unit/test_service_manifest.py index a43d82d52208..cd40233e589f 100644 --- a/services/catalog/tests/unit/test_services_manifest.py +++ b/services/catalog/tests/unit/test_service_manifest.py @@ -13,9 +13,9 @@ from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from respx.router import MockRouter -from simcore_service_catalog.api.dependencies.director import get_director_api -from simcore_service_catalog.services import manifest -from simcore_service_catalog.services.director import DirectorApi +from simcore_service_catalog.api._dependencies.director import get_director_api +from simcore_service_catalog.infrastructure.director import DirectorApi +from simcore_service_catalog.service import manifest @pytest.fixture diff --git a/services/catalog/tests/unit/with_dbs/conftest.py b/services/catalog/tests/unit/with_dbs/conftest.py index 74daa97804d3..080fb5513559 100644 --- a/services/catalog/tests/unit/with_dbs/conftest.py +++ b/services/catalog/tests/unit/with_dbs/conftest.py @@ -33,7 +33,7 @@ from simcore_postgres_database.models.products import products from simcore_postgres_database.models.users import users from simcore_service_catalog.core.settings import ApplicationSettings -from simcore_service_catalog.db.tables import ( +from simcore_service_catalog.repository._tables import ( groups, services_access_rights, services_meta_data, diff --git a/services/catalog/tests/unit/with_dbs/test_core_background_task__sync.py b/services/catalog/tests/unit/with_dbs/test_core_background_task__sync.py index a2927eefb418..1019383193d5 100644 --- a/services/catalog/tests/unit/with_dbs/test_core_background_task__sync.py +++ b/services/catalog/tests/unit/with_dbs/test_core_background_task__sync.py @@ -14,7 +14,7 @@ from respx.router import MockRouter from simcore_postgres_database.models.services import services_meta_data from simcore_service_catalog.core.background_tasks import _run_sync_services -from simcore_service_catalog.db.repositories.services import ServicesRepository +from simcore_service_catalog.repository.services import ServicesRepository from sqlalchemy.ext.asyncio.engine import AsyncEngine pytest_simcore_core_services_selection = [ diff --git a/services/catalog/tests/unit/with_dbs/test_db_repositories.py b/services/catalog/tests/unit/with_dbs/test_repository.py similarity index 99% rename from services/catalog/tests/unit/with_dbs/test_db_repositories.py rename to services/catalog/tests/unit/with_dbs/test_repository.py index 8618a67c25e4..40c1a34aedaa 100644 --- a/services/catalog/tests/unit/with_dbs/test_db_repositories.py +++ b/services/catalog/tests/unit/with_dbs/test_repository.py @@ -13,13 +13,13 @@ from models_library.users import UserID from packaging import version from pydantic import EmailStr, HttpUrl, TypeAdapter -from simcore_service_catalog.db.repositories.services import ServicesRepository from simcore_service_catalog.models.services_db import ( ServiceAccessRightsAtDB, ServiceMetaDataDBCreate, ServiceMetaDataDBGet, ServiceMetaDataDBPatch, ) +from simcore_service_catalog.repository.services import ServicesRepository from simcore_service_catalog.utils.versioning import is_patch_release from sqlalchemy.ext.asyncio import AsyncEngine diff --git a/services/catalog/tests/unit/with_dbs/test_services_access_rights.py b/services/catalog/tests/unit/with_dbs/test_service_access_rights.py similarity index 97% rename from services/catalog/tests/unit/with_dbs/test_services_access_rights.py rename to services/catalog/tests/unit/with_dbs/test_service_access_rights.py index 47d0dc201aca..887a54aa172b 100644 --- a/services/catalog/tests/unit/with_dbs/test_services_access_rights.py +++ b/services/catalog/tests/unit/with_dbs/test_service_access_rights.py @@ -9,9 +9,9 @@ from models_library.products import ProductName from models_library.services import ServiceMetaDataPublished, ServiceVersion from pydantic import TypeAdapter -from simcore_service_catalog.db.repositories.services import ServicesRepository from simcore_service_catalog.models.services_db import ServiceAccessRightsAtDB -from simcore_service_catalog.services.access_rights import ( +from simcore_service_catalog.repository.services import ServicesRepository +from simcore_service_catalog.service.access_rights import ( evaluate_auto_upgrade_policy, evaluate_default_policy, reduce_access_rights, diff --git a/services/catalog/tests/unit/with_dbs/test_services_services_api.py b/services/catalog/tests/unit/with_dbs/test_service_services_api.py similarity index 95% rename from services/catalog/tests/unit/with_dbs/test_services_services_api.py rename to services/catalog/tests/unit/with_dbs/test_service_services_api.py index 7deccf3be151..b3c6239a2ab1 100644 --- a/services/catalog/tests/unit/with_dbs/test_services_services_api.py +++ b/services/catalog/tests/unit/with_dbs/test_service_services_api.py @@ -16,11 +16,11 @@ from pydantic import TypeAdapter from pytest_simcore.helpers.catalog_services import CreateFakeServiceDataCallable from respx.router import MockRouter -from simcore_service_catalog.api.dependencies.director import get_director_api -from simcore_service_catalog.db.repositories.groups import GroupsRepository -from simcore_service_catalog.db.repositories.services import ServicesRepository -from simcore_service_catalog.services import manifest, services_api -from simcore_service_catalog.services.director import DirectorApi +from simcore_service_catalog.api._dependencies.director import get_director_api +from simcore_service_catalog.infrastructure.director import DirectorApi +from simcore_service_catalog.repository.groups import GroupsRepository +from simcore_service_catalog.repository.services import ServicesRepository +from simcore_service_catalog.service import manifest, services_api from sqlalchemy.ext.asyncio import AsyncEngine pytest_simcore_core_services_selection = [