From b1f249a80fa42045ecd527e8533629159ae42d1a Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 4 Apr 2025 16:25:43 +0200 Subject: [PATCH 01/11] created infrastructure layout --- .../simcore_service_catalog/api/dependencies/director.py | 5 ++++- .../simcore_service_catalog/api/dependencies/services.py | 7 +++---- .../src/simcore_service_catalog/api/rest/_services.py | 3 +-- .../simcore_service_catalog/api/rest/_services_extras.py | 3 +-- .../simcore_service_catalog/api/rest/_services_labels.py | 3 +-- .../api/rest/_services_resources.py | 5 ++--- .../catalog/src/simcore_service_catalog/api/rpc/routes.py | 2 +- .../src/simcore_service_catalog/core/application.py | 2 +- .../catalog/src/simcore_service_catalog/core/events.py | 2 +- .../src/simcore_service_catalog/infrastructure/__init__.py | 0 .../{services => infrastructure}/director.py | 2 +- .../{services => infrastructure}/rabbitmq.py | 0 .../src/simcore_service_catalog/services/manifest.py | 2 +- .../src/simcore_service_catalog/services/services_api.py | 2 +- services/catalog/tests/unit/test_services_director.py | 2 +- services/catalog/tests/unit/test_services_manifest.py | 2 +- .../tests/unit/with_dbs/test_services_services_api.py | 2 +- 17 files changed, 21 insertions(+), 23 deletions(-) create mode 100644 services/catalog/src/simcore_service_catalog/infrastructure/__init__.py rename services/catalog/src/simcore_service_catalog/{services => infrastructure}/director.py (99%) rename services/catalog/src/simcore_service_catalog/{services => infrastructure}/rabbitmq.py (100%) diff --git a/services/catalog/src/simcore_service_catalog/api/dependencies/director.py b/services/catalog/src/simcore_service_catalog/api/dependencies/director.py index 8b6b558428d4..08f808de50f9 100644 --- a/services/catalog/src/simcore_service_catalog/api/dependencies/director.py +++ b/services/catalog/src/simcore_service_catalog/api/dependencies/director.py @@ -3,7 +3,7 @@ from fastapi import Depends, FastAPI from servicelib.fastapi.dependencies import get_app -from ...services.director import DirectorApi +from ...infrastructure.director import DirectorApi def get_director_api( @@ -11,3 +11,6 @@ def get_director_api( ) -> DirectorApi: director: DirectorApi = app.state.director_api return director + + +__all__: tuple[str, ...] = ("DirectorApi",) diff --git a/services/catalog/src/simcore_service_catalog/api/dependencies/services.py b/services/catalog/src/simcore_service_catalog/api/dependencies/services.py index a5d0eba83c2e..5e1c9a02f679 100644 --- a/services/catalog/src/simcore_service_catalog/api/dependencies/services.py +++ b/services/catalog/src/simcore_service_catalog/api/dependencies/services.py @@ -16,22 +16,21 @@ from ...db.repositories.groups import GroupsRepository from ...db.repositories.services import ServicesRepository from ...services import manifest -from ...services.director import DirectorApi from .database import get_repository -from .director import get_director_api +from .director import DirectorApi, get_director_api _logger = logging.getLogger(__name__) def get_default_service_resources( - app: Annotated[FastAPI, Depends(get_app)] + app: Annotated[FastAPI, Depends(get_app)], ) -> ResourcesDict: app_settings: ApplicationSettings = app.state.settings return app_settings.CATALOG_SERVICES_DEFAULT_RESOURCES def get_default_service_specifications( - app: Annotated[FastAPI, Depends(get_app)] + app: Annotated[FastAPI, Depends(get_app)], ) -> ServiceSpecifications: app_settings: ApplicationSettings = app.state.settings return app_settings.CATALOG_SERVICES_DEFAULT_SPECIFICATIONS diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services.py b/services/catalog/src/simcore_service_catalog/api/rest/_services.py index c97a5545d8c5..5452645b8163 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services.py @@ -23,9 +23,8 @@ from ...db.repositories.groups import GroupsRepository from ...db.repositories.services import ServicesRepository from ...models.services_db import ServiceAccessRightsAtDB, ServiceMetaDataDBGet -from ...services.director import DirectorApi from ..dependencies.database import get_repository -from ..dependencies.director import get_director_api +from ..dependencies.director import DirectorApi, get_director_api from ..dependencies.services import get_service_from_manifest _logger = logging.getLogger(__name__) diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services_extras.py b/services/catalog/src/simcore_service_catalog/api/rest/_services_extras.py index 8f1658f57bec..e451dce349b8 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services_extras.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services_extras.py @@ -4,8 +4,7 @@ from models_library.api_schemas_directorv2.services import ServiceExtras from models_library.services import ServiceKey, ServiceVersion -from ...services.director import DirectorApi -from ..dependencies.director import get_director_api +from ..dependencies.director import DirectorApi, get_director_api router = APIRouter() diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services_labels.py b/services/catalog/src/simcore_service_catalog/api/rest/_services_labels.py index bf02bb3b90ff..11b40ad7b55e 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services_labels.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services_labels.py @@ -3,8 +3,7 @@ from fastapi import APIRouter, Depends from models_library.services import ServiceKey, ServiceVersion -from ...services.director import DirectorApi -from ..dependencies.director import get_director_api +from ..dependencies.director import DirectorApi, get_director_api router = APIRouter() diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py b/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py index 72080408663a..b4b2ee0578f2 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py @@ -24,14 +24,13 @@ from ..._constants import RESPONSE_MODEL_POLICY, SIMCORE_SERVICE_SETTINGS_LABELS from ...db.repositories.services import ServicesRepository -from ...services.director import DirectorApi from ...services.function_services import is_function_service from ...utils.service_resources import ( merge_service_resources_with_user_specs, parse_generic_resource, ) from ..dependencies.database import get_repository -from ..dependencies.director import get_director_api +from ..dependencies.director import DirectorApi, get_director_api from ..dependencies.services import get_default_service_resources from ..dependencies.user_groups import list_user_groups @@ -149,7 +148,7 @@ async def _get_service_labels( def _get_service_settings( - labels: dict[str, Any] + labels: dict[str, Any], ) -> list[SimcoreServiceSettingLabelEntry]: service_settings = TypeAdapter(list[SimcoreServiceSettingLabelEntry]).validate_json( labels.get(SIMCORE_SERVICE_SETTINGS_LABELS, "[]"), diff --git a/services/catalog/src/simcore_service_catalog/api/rpc/routes.py b/services/catalog/src/simcore_service_catalog/api/rpc/routes.py index ce35e7867d17..2843db3e5b26 100644 --- a/services/catalog/src/simcore_service_catalog/api/rpc/routes.py +++ b/services/catalog/src/simcore_service_catalog/api/rpc/routes.py @@ -3,7 +3,7 @@ from fastapi import FastAPI from models_library.api_schemas_catalog import CATALOG_RPC_NAMESPACE -from ...services.rabbitmq import get_rabbitmq_rpc_server +from ...infrastructure.rabbitmq import get_rabbitmq_rpc_server from . import _services _logger = logging.getLogger(__name__) diff --git a/services/catalog/src/simcore_service_catalog/core/application.py b/services/catalog/src/simcore_service_catalog/core/application.py index 7bedab76a31c..b457200022b6 100644 --- a/services/catalog/src/simcore_service_catalog/core/application.py +++ b/services/catalog/src/simcore_service_catalog/core/application.py @@ -16,8 +16,8 @@ from ..api.rest.routes import setup_rest_api_routes from ..api.rpc.routes import setup_rpc_api_routes from ..exceptions.handlers import setup_exception_handlers +from ..infrastructure.rabbitmq import setup_rabbitmq from ..services.function_services import setup_function_services -from ..services.rabbitmq import setup_rabbitmq from .events import create_on_shutdown, create_on_startup from .settings import ApplicationSettings diff --git a/services/catalog/src/simcore_service_catalog/core/events.py b/services/catalog/src/simcore_service_catalog/core/events.py index f22adbba4ece..3c7cb6ca3025 100644 --- a/services/catalog/src/simcore_service_catalog/core/events.py +++ b/services/catalog/src/simcore_service_catalog/core/events.py @@ -8,7 +8,7 @@ from .._meta import APP_FINISHED_BANNER_MSG, APP_STARTED_BANNER_MSG from ..db.events import setup_default_product -from ..services.director import close_director, setup_director +from ..infrastructure.director import close_director, setup_director from .background_tasks import start_registry_sync_task, stop_registry_sync_task _logger = logging.getLogger(__name__) diff --git a/services/catalog/src/simcore_service_catalog/infrastructure/__init__.py b/services/catalog/src/simcore_service_catalog/infrastructure/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/services/catalog/src/simcore_service_catalog/services/director.py b/services/catalog/src/simcore_service_catalog/infrastructure/director.py similarity index 99% rename from services/catalog/src/simcore_service_catalog/services/director.py rename to services/catalog/src/simcore_service_catalog/infrastructure/director.py index 525840621840..f35490e9271d 100644 --- a/services/catalog/src/simcore_service_catalog/services/director.py +++ b/services/catalog/src/simcore_service_catalog/infrastructure/director.py @@ -65,7 +65,7 @@ def _validate_kind(entry_to_validate: dict[str, Any], kind_name: str): def _return_data_or_raise_error( - request_func: Callable[..., Awaitable[httpx.Response]] + request_func: Callable[..., Awaitable[httpx.Response]], ) -> Callable[..., Awaitable[list[Any] | dict[str, Any]]]: """ Creates a context for safe inter-process communication (IPC) diff --git a/services/catalog/src/simcore_service_catalog/services/rabbitmq.py b/services/catalog/src/simcore_service_catalog/infrastructure/rabbitmq.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/services/rabbitmq.py rename to services/catalog/src/simcore_service_catalog/infrastructure/rabbitmq.py diff --git a/services/catalog/src/simcore_service_catalog/services/manifest.py b/services/catalog/src/simcore_service_catalog/services/manifest.py index 5cfbb1d961bd..eabee127fad3 100644 --- a/services/catalog/src/simcore_service_catalog/services/manifest.py +++ b/services/catalog/src/simcore_service_catalog/services/manifest.py @@ -35,7 +35,7 @@ from servicelib.utils import limited_gather from .._constants import DIRECTOR_CACHING_TTL -from .director import DirectorApi +from ..infrastructure.director import DirectorApi from .function_services import get_function_service, is_function_service _logger = logging.getLogger(__name__) diff --git a/services/catalog/src/simcore_service_catalog/services/services_api.py b/services/catalog/src/simcore_service_catalog/services/services_api.py index 43a70f45bf27..ce2ba3965c70 100644 --- a/services/catalog/src/simcore_service_catalog/services/services_api.py +++ b/services/catalog/src/simcore_service_catalog/services/services_api.py @@ -23,13 +23,13 @@ from simcore_service_catalog.db.repositories.groups import GroupsRepository from ..db.repositories.services import ServicesRepository +from ..infrastructure.director import DirectorApi from ..models.services_db import ( ServiceAccessRightsAtDB, ServiceMetaDataDBPatch, ServiceWithHistoryDBGet, ) from ..services import manifest -from ..services.director import DirectorApi from .compatibility import evaluate_service_compatibility_map from .function_services import is_function_service diff --git a/services/catalog/tests/unit/test_services_director.py b/services/catalog/tests/unit/test_services_director.py index eb36988b5198..d14c3956f46d 100644 --- a/services/catalog/tests/unit/test_services_director.py +++ b/services/catalog/tests/unit/test_services_director.py @@ -16,7 +16,7 @@ from pytest_simcore.helpers.typing_env import EnvVarsDict from respx.router import MockRouter from simcore_service_catalog.api.dependencies.director import get_director_api -from simcore_service_catalog.services.director import DirectorApi +from simcore_service_catalog.infrastructure.director import DirectorApi @pytest.fixture diff --git a/services/catalog/tests/unit/test_services_manifest.py b/services/catalog/tests/unit/test_services_manifest.py index a43d82d52208..52ccc7001a6d 100644 --- a/services/catalog/tests/unit/test_services_manifest.py +++ b/services/catalog/tests/unit/test_services_manifest.py @@ -14,8 +14,8 @@ from pytest_simcore.helpers.typing_env import EnvVarsDict from respx.router import MockRouter from simcore_service_catalog.api.dependencies.director import get_director_api +from simcore_service_catalog.infrastructure.director import DirectorApi from simcore_service_catalog.services import manifest -from simcore_service_catalog.services.director import DirectorApi @pytest.fixture diff --git a/services/catalog/tests/unit/with_dbs/test_services_services_api.py b/services/catalog/tests/unit/with_dbs/test_services_services_api.py index 7deccf3be151..b867d89402d3 100644 --- a/services/catalog/tests/unit/with_dbs/test_services_services_api.py +++ b/services/catalog/tests/unit/with_dbs/test_services_services_api.py @@ -19,8 +19,8 @@ from simcore_service_catalog.api.dependencies.director import get_director_api from simcore_service_catalog.db.repositories.groups import GroupsRepository from simcore_service_catalog.db.repositories.services import ServicesRepository +from simcore_service_catalog.infrastructure.director import DirectorApi from simcore_service_catalog.services import manifest, services_api -from simcore_service_catalog.services.director import DirectorApi from sqlalchemy.ext.asyncio import AsyncEngine pytest_simcore_core_services_selection = [ From 6bf06f1ae59e66530d6d05ae591b662e4a376452 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 4 Apr 2025 16:27:40 +0200 Subject: [PATCH 02/11] repository --- .../src/simcore_service_catalog/{db => }/repositories/__init__.py | 0 .../src/simcore_service_catalog/{db => }/repositories/_base.py | 0 .../{db => }/repositories/_services_sql.py | 0 .../src/simcore_service_catalog/{db => }/repositories/groups.py | 0 .../src/simcore_service_catalog/{db => }/repositories/products.py | 0 .../src/simcore_service_catalog/{db => }/repositories/projects.py | 0 .../src/simcore_service_catalog/{db => }/repositories/services.py | 0 ...{test_services_director.py => test_infrastructure_director.py} | 0 8 files changed, 0 insertions(+), 0 deletions(-) rename services/catalog/src/simcore_service_catalog/{db => }/repositories/__init__.py (100%) rename services/catalog/src/simcore_service_catalog/{db => }/repositories/_base.py (100%) rename services/catalog/src/simcore_service_catalog/{db => }/repositories/_services_sql.py (100%) rename services/catalog/src/simcore_service_catalog/{db => }/repositories/groups.py (100%) rename services/catalog/src/simcore_service_catalog/{db => }/repositories/products.py (100%) rename services/catalog/src/simcore_service_catalog/{db => }/repositories/projects.py (100%) rename services/catalog/src/simcore_service_catalog/{db => }/repositories/services.py (100%) rename services/catalog/tests/unit/{test_services_director.py => test_infrastructure_director.py} (100%) diff --git a/services/catalog/src/simcore_service_catalog/db/repositories/__init__.py b/services/catalog/src/simcore_service_catalog/repositories/__init__.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/db/repositories/__init__.py rename to services/catalog/src/simcore_service_catalog/repositories/__init__.py diff --git a/services/catalog/src/simcore_service_catalog/db/repositories/_base.py b/services/catalog/src/simcore_service_catalog/repositories/_base.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/db/repositories/_base.py rename to services/catalog/src/simcore_service_catalog/repositories/_base.py diff --git a/services/catalog/src/simcore_service_catalog/db/repositories/_services_sql.py b/services/catalog/src/simcore_service_catalog/repositories/_services_sql.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/db/repositories/_services_sql.py rename to services/catalog/src/simcore_service_catalog/repositories/_services_sql.py diff --git a/services/catalog/src/simcore_service_catalog/db/repositories/groups.py b/services/catalog/src/simcore_service_catalog/repositories/groups.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/db/repositories/groups.py rename to services/catalog/src/simcore_service_catalog/repositories/groups.py diff --git a/services/catalog/src/simcore_service_catalog/db/repositories/products.py b/services/catalog/src/simcore_service_catalog/repositories/products.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/db/repositories/products.py rename to services/catalog/src/simcore_service_catalog/repositories/products.py diff --git a/services/catalog/src/simcore_service_catalog/db/repositories/projects.py b/services/catalog/src/simcore_service_catalog/repositories/projects.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/db/repositories/projects.py rename to services/catalog/src/simcore_service_catalog/repositories/projects.py diff --git a/services/catalog/src/simcore_service_catalog/db/repositories/services.py b/services/catalog/src/simcore_service_catalog/repositories/services.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/db/repositories/services.py rename to services/catalog/src/simcore_service_catalog/repositories/services.py diff --git a/services/catalog/tests/unit/test_services_director.py b/services/catalog/tests/unit/test_infrastructure_director.py similarity index 100% rename from services/catalog/tests/unit/test_services_director.py rename to services/catalog/tests/unit/test_infrastructure_director.py From 1eb69c586cf6aa4cacf86b174f676cbd38e11085 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 4 Apr 2025 16:31:06 +0200 Subject: [PATCH 03/11] =?UTF-8?q?=F0=9F=94=A7=20Refactor:=20Update=20impor?= =?UTF-8?q?t=20paths=20for=20repositories=20to=20improve=20code=20organiza?= =?UTF-8?q?tion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api/dependencies/database.py | 2 +- .../api/dependencies/services.py | 4 ++-- .../api/dependencies/user_groups.py | 2 +- .../api/rest/_services.py | 4 ++-- .../api/rest/_services_access_rights.py | 12 +++++++----- .../api/rest/_services_resources.py | 2 +- .../api/rest/_services_specifications.py | 4 ++-- .../api/rpc/_services.py | 4 ++-- .../core/background_tasks.py | 12 ++++++------ .../src/simcore_service_catalog/core/events.py | 2 +- .../repositories/_services_sql.py | 4 ++-- .../{db/tables.py => repositories/_tables.py} | 0 .../repositories/groups.py | 4 ++-- .../repositories/projects.py | 2 +- .../repositories/services.py | 18 +++++++++--------- .../services/access_rights.py | 8 +++----- .../services/compatibility.py | 6 ++---- .../services/services_api.py | 4 ++-- .../unit/test_db_repositories_services_sql.py | 2 +- .../tests/unit/test_services_compatibility.py | 2 +- .../catalog/tests/unit/with_dbs/conftest.py | 2 +- .../test_core_background_task__sync.py | 2 +- .../unit/with_dbs/test_db_repositories.py | 2 +- .../with_dbs/test_services_access_rights.py | 2 +- .../with_dbs/test_services_services_api.py | 4 ++-- 25 files changed, 54 insertions(+), 56 deletions(-) rename services/catalog/src/simcore_service_catalog/{db/tables.py => repositories/_tables.py} (100%) diff --git a/services/catalog/src/simcore_service_catalog/api/dependencies/database.py b/services/catalog/src/simcore_service_catalog/api/dependencies/database.py index 2f323db99c52..f1c4e2efdf7b 100644 --- a/services/catalog/src/simcore_service_catalog/api/dependencies/database.py +++ b/services/catalog/src/simcore_service_catalog/api/dependencies/database.py @@ -6,7 +6,7 @@ from fastapi.requests import Request from sqlalchemy.ext.asyncio import AsyncEngine -from ...db.repositories._base import BaseRepository +from ...repositories._base import BaseRepository _logger = logging.getLogger(__name__) diff --git a/services/catalog/src/simcore_service_catalog/api/dependencies/services.py b/services/catalog/src/simcore_service_catalog/api/dependencies/services.py index 5e1c9a02f679..70316b07ac25 100644 --- a/services/catalog/src/simcore_service_catalog/api/dependencies/services.py +++ b/services/catalog/src/simcore_service_catalog/api/dependencies/services.py @@ -13,8 +13,8 @@ from servicelib.fastapi.dependencies import get_app from ...core.settings import ApplicationSettings -from ...db.repositories.groups import GroupsRepository -from ...db.repositories.services import ServicesRepository +from ...repositories.groups import GroupsRepository +from ...repositories.services import ServicesRepository from ...services import manifest from .database import get_repository from .director import DirectorApi, get_director_api diff --git a/services/catalog/src/simcore_service_catalog/api/dependencies/user_groups.py b/services/catalog/src/simcore_service_catalog/api/dependencies/user_groups.py index b1b4d23a3de6..d4832fa86b2b 100644 --- a/services/catalog/src/simcore_service_catalog/api/dependencies/user_groups.py +++ b/services/catalog/src/simcore_service_catalog/api/dependencies/user_groups.py @@ -4,7 +4,7 @@ from models_library.groups import GroupAtDB from models_library.users import UserID -from ...db.repositories.groups import GroupsRepository +from ...repositories.groups import GroupsRepository from .database import get_repository diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services.py b/services/catalog/src/simcore_service_catalog/api/rest/_services.py index 5452645b8163..6979a1f0b0df 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services.py @@ -20,9 +20,9 @@ LIST_SERVICES_CACHING_TTL, RESPONSE_MODEL_POLICY, ) -from ...db.repositories.groups import GroupsRepository -from ...db.repositories.services import ServicesRepository from ...models.services_db import ServiceAccessRightsAtDB, ServiceMetaDataDBGet +from ...repositories.groups import GroupsRepository +from ...repositories.services import ServicesRepository from ..dependencies.database import get_repository from ..dependencies.director import DirectorApi, get_director_api from ..dependencies.services import get_service_from_manifest diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services_access_rights.py b/services/catalog/src/simcore_service_catalog/api/rest/_services_access_rights.py index 6abaf48dd049..feb87fb7cfbe 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services_access_rights.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services_access_rights.py @@ -8,8 +8,8 @@ from models_library.services import ServiceKey, ServiceVersion from ..._constants import RESPONSE_MODEL_POLICY -from ...db.repositories.services import ServicesRepository from ...models.services_db import ServiceAccessRightsAtDB +from ...repositories.services import ServicesRepository from ..dependencies.database import get_repository from ..dependencies.services import AccessInfo, check_service_read_access @@ -33,10 +33,12 @@ async def get_service_access_rights( ], x_simcore_products_name: Annotated[str, Header(...)], ): - service_access_rights: list[ - ServiceAccessRightsAtDB - ] = await services_repo.get_service_access_rights( - key=service_key, version=service_version, product_name=x_simcore_products_name + service_access_rights: list[ServiceAccessRightsAtDB] = ( + await services_repo.get_service_access_rights( + key=service_key, + version=service_version, + product_name=x_simcore_products_name, + ) ) gids_with_access_rights = {} diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py b/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py index b4b2ee0578f2..26157f9ea467 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py @@ -23,7 +23,7 @@ from pydantic import TypeAdapter from ..._constants import RESPONSE_MODEL_POLICY, SIMCORE_SERVICE_SETTINGS_LABELS -from ...db.repositories.services import ServicesRepository +from ...repositories.services import ServicesRepository from ...services.function_services import is_function_service from ...utils.service_resources import ( merge_service_resources_with_user_specs, diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services_specifications.py b/services/catalog/src/simcore_service_catalog/api/rest/_services_specifications.py index 49751a196e80..2a57ff597184 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services_specifications.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services_specifications.py @@ -10,8 +10,8 @@ from models_library.users import UserID from ..._constants import RESPONSE_MODEL_POLICY -from ...db.repositories.groups import GroupsRepository -from ...db.repositories.services import ServicesRepository +from ...repositories.groups import GroupsRepository +from ...repositories.services import ServicesRepository from ...services.function_services import is_function_service from ..dependencies.database import get_repository from ..dependencies.services import get_default_service_specifications diff --git a/services/catalog/src/simcore_service_catalog/api/rpc/_services.py b/services/catalog/src/simcore_service_catalog/api/rpc/_services.py index 874ac9bb45fa..c909c472eb4c 100644 --- a/services/catalog/src/simcore_service_catalog/api/rpc/_services.py +++ b/services/catalog/src/simcore_service_catalog/api/rpc/_services.py @@ -23,9 +23,9 @@ CatalogForbiddenError, CatalogItemNotFoundError, ) -from simcore_service_catalog.db.repositories.groups import GroupsRepository +from simcore_service_catalog.repositories.groups import GroupsRepository -from ...db.repositories.services import ServicesRepository +from ...repositories.services import ServicesRepository from ...services import services_api from ..dependencies.director import get_director_api diff --git a/services/catalog/src/simcore_service_catalog/core/background_tasks.py b/services/catalog/src/simcore_service_catalog/core/background_tasks.py index cb269ee39196..657f3c3c6ac5 100644 --- a/services/catalog/src/simcore_service_catalog/core/background_tasks.py +++ b/services/catalog/src/simcore_service_catalog/core/background_tasks.py @@ -25,10 +25,10 @@ from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.asyncio import AsyncEngine -from ..db.repositories.groups import GroupsRepository -from ..db.repositories.projects import ProjectsRepository -from ..db.repositories.services import ServicesRepository from ..models.services_db import ServiceAccessRightsAtDB, ServiceMetaDataDBCreate +from ..repositories.groups import GroupsRepository +from ..repositories.projects import ProjectsRepository +from ..repositories.services import ServicesRepository from ..services import access_rights _logger = logging.getLogger(__name__) @@ -115,9 +115,9 @@ async def _ensure_registry_and_database_are_synced(app: FastAPI) -> None: director_api = get_director_api(app) services_in_manifest_map = await manifest.get_services_map(director_api) - services_in_db: set[ - tuple[ServiceKey, ServiceVersion] - ] = await _list_services_in_database(app.state.engine) + services_in_db: set[tuple[ServiceKey, ServiceVersion]] = ( + await _list_services_in_database(app.state.engine) + ) # check that the db has all the services at least once missing_services_in_db = set(services_in_manifest_map.keys()) - services_in_db diff --git a/services/catalog/src/simcore_service_catalog/core/events.py b/services/catalog/src/simcore_service_catalog/core/events.py index 3c7cb6ca3025..bdc646d89a1d 100644 --- a/services/catalog/src/simcore_service_catalog/core/events.py +++ b/services/catalog/src/simcore_service_catalog/core/events.py @@ -7,7 +7,7 @@ from servicelib.logging_utils import log_context from .._meta import APP_FINISHED_BANNER_MSG, APP_STARTED_BANNER_MSG -from ..db.events import setup_default_product +from ..events import setup_default_product from ..infrastructure.director import close_director, setup_director from .background_tasks import start_registry_sync_task, stop_registry_sync_task diff --git a/services/catalog/src/simcore_service_catalog/repositories/_services_sql.py b/services/catalog/src/simcore_service_catalog/repositories/_services_sql.py index 935a38349761..58240d56cb12 100644 --- a/services/catalog/src/simcore_service_catalog/repositories/_services_sql.py +++ b/services/catalog/src/simcore_service_catalog/repositories/_services_sql.py @@ -10,8 +10,8 @@ from sqlalchemy.sql.expression import func from sqlalchemy.sql.selectable import Select -from ...models.services_db import ServiceMetaDataDBGet -from ..tables import ( +from ..models.services_db import ServiceMetaDataDBGet +from ._tables import ( services_access_rights, services_compatibility, services_meta_data, diff --git a/services/catalog/src/simcore_service_catalog/db/tables.py b/services/catalog/src/simcore_service_catalog/repositories/_tables.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/db/tables.py rename to services/catalog/src/simcore_service_catalog/repositories/_tables.py diff --git a/services/catalog/src/simcore_service_catalog/repositories/groups.py b/services/catalog/src/simcore_service_catalog/repositories/groups.py index d7061947a104..a58e9f9865d5 100644 --- a/services/catalog/src/simcore_service_catalog/repositories/groups.py +++ b/services/catalog/src/simcore_service_catalog/repositories/groups.py @@ -6,9 +6,9 @@ from pydantic import TypeAdapter from pydantic.types import PositiveInt -from ...exceptions.errors import UninitializedGroupError -from ..tables import GroupType, groups, user_to_groups, users +from ..exceptions.errors import UninitializedGroupError from ._base import BaseRepository +from ._tables import GroupType, groups, user_to_groups, users class GroupsRepository(BaseRepository): diff --git a/services/catalog/src/simcore_service_catalog/repositories/projects.py b/services/catalog/src/simcore_service_catalog/repositories/projects.py index 5b2b2d1bbfec..791090570c71 100644 --- a/services/catalog/src/simcore_service_catalog/repositories/projects.py +++ b/services/catalog/src/simcore_service_catalog/repositories/projects.py @@ -4,8 +4,8 @@ from models_library.services import ServiceKeyVersion from pydantic import ValidationError -from ..tables import ProjectType, projects from ._base import BaseRepository +from ._tables import ProjectType, projects _logger = logging.getLogger(__name__) diff --git a/services/catalog/src/simcore_service_catalog/repositories/services.py b/services/catalog/src/simcore_service_catalog/repositories/services.py index 06a166659981..bbde9e5409cd 100644 --- a/services/catalog/src/simcore_service_catalog/repositories/services.py +++ b/services/catalog/src/simcore_service_catalog/repositories/services.py @@ -24,7 +24,7 @@ from sqlalchemy.sql import and_, or_ from sqlalchemy.sql.expression import tuple_ -from ...models.services_db import ( +from ..models.services_db import ( ReleaseDBGet, ServiceAccessRightsAtDB, ServiceMetaDataDBCreate, @@ -32,14 +32,7 @@ ServiceMetaDataDBPatch, ServiceWithHistoryDBGet, ) -from ...models.services_specifications import ServiceSpecificationsAtDB -from ..tables import ( - services_access_rights, - services_compatibility, - services_meta_data, - services_specifications, - user_to_groups, -) +from ..models.services_specifications import ServiceSpecificationsAtDB from ._base import BaseRepository from ._services_sql import ( SERVICES_META_DATA_COLS, @@ -52,6 +45,13 @@ list_latest_services_stmt, list_services_stmt, ) +from ._tables import ( + services_access_rights, + services_compatibility, + services_meta_data, + services_specifications, + user_to_groups, +) _logger = logging.getLogger(__name__) diff --git a/services/catalog/src/simcore_service_catalog/services/access_rights.py b/services/catalog/src/simcore_service_catalog/services/access_rights.py index e504fe185841..554c3602d93c 100644 --- a/services/catalog/src/simcore_service_catalog/services/access_rights.py +++ b/services/catalog/src/simcore_service_catalog/services/access_rights.py @@ -1,6 +1,4 @@ -""" Services Access Rights policies - -""" +"""Services Access Rights policies""" import logging import operator @@ -17,9 +15,9 @@ from sqlalchemy.ext.asyncio import AsyncEngine from ..api.dependencies.director import get_director_api -from ..db.repositories.groups import GroupsRepository -from ..db.repositories.services import ServicesRepository from ..models.services_db import ServiceAccessRightsAtDB +from ..repositories.groups import GroupsRepository +from ..repositories.services import ServicesRepository from ..utils.versioning import as_version, is_patch_release _logger = logging.getLogger(__name__) diff --git a/services/catalog/src/simcore_service_catalog/services/compatibility.py b/services/catalog/src/simcore_service_catalog/services/compatibility.py index db8483e11c99..6f7c5e64257e 100644 --- a/services/catalog/src/simcore_service_catalog/services/compatibility.py +++ b/services/catalog/src/simcore_service_catalog/services/compatibility.py @@ -1,6 +1,4 @@ -""" Manages service compatibility policies - -""" +"""Manages service compatibility policies""" from models_library.products import ProductName from models_library.services_history import Compatibility, CompatibleService @@ -10,8 +8,8 @@ from packaging.version import Version from simcore_service_catalog.utils.versioning import as_version -from ..db.repositories.services import ServicesRepository from ..models.services_db import ReleaseDBGet +from ..repositories.services import ServicesRepository def _get_default_compatibility_specs(target: ServiceVersion | Version) -> SpecifierSet: diff --git a/services/catalog/src/simcore_service_catalog/services/services_api.py b/services/catalog/src/simcore_service_catalog/services/services_api.py index ce2ba3965c70..4d40da8aa35c 100644 --- a/services/catalog/src/simcore_service_catalog/services/services_api.py +++ b/services/catalog/src/simcore_service_catalog/services/services_api.py @@ -20,15 +20,15 @@ CatalogForbiddenError, CatalogItemNotFoundError, ) -from simcore_service_catalog.db.repositories.groups import GroupsRepository +from simcore_service_catalog.repositories.groups import GroupsRepository -from ..db.repositories.services import ServicesRepository from ..infrastructure.director import DirectorApi from ..models.services_db import ( ServiceAccessRightsAtDB, ServiceMetaDataDBPatch, ServiceWithHistoryDBGet, ) +from ..repositories.services import ServicesRepository from ..services import manifest from .compatibility import evaluate_service_compatibility_map from .function_services import is_function_service diff --git a/services/catalog/tests/unit/test_db_repositories_services_sql.py b/services/catalog/tests/unit/test_db_repositories_services_sql.py index 27b8db9e770d..35f771a8c945 100644 --- a/services/catalog/tests/unit/test_db_repositories_services_sql.py +++ b/services/catalog/tests/unit/test_db_repositories_services_sql.py @@ -4,7 +4,7 @@ from simcore_postgres_database.utils import as_postgres_sql_query_str -from simcore_service_catalog.db.repositories._services_sql import ( +from simcore_service_catalog.repositories._services_sql import ( AccessRightsClauses, can_get_service_stmt, get_service_history_stmt, diff --git a/services/catalog/tests/unit/test_services_compatibility.py b/services/catalog/tests/unit/test_services_compatibility.py index 1211c25a97be..fbf6bd17860e 100644 --- a/services/catalog/tests/unit/test_services_compatibility.py +++ b/services/catalog/tests/unit/test_services_compatibility.py @@ -11,8 +11,8 @@ from packaging.specifiers import SpecifierSet from packaging.version import Version from pytest_mock import MockerFixture, MockType -from simcore_service_catalog.db.repositories.services import ServicesRepository from simcore_service_catalog.models.services_db import ReleaseDBGet +from simcore_service_catalog.repositories.services import ServicesRepository from simcore_service_catalog.services.compatibility import ( _get_latest_compatible_version, evaluate_service_compatibility_map, diff --git a/services/catalog/tests/unit/with_dbs/conftest.py b/services/catalog/tests/unit/with_dbs/conftest.py index 74daa97804d3..44fa94e294af 100644 --- a/services/catalog/tests/unit/with_dbs/conftest.py +++ b/services/catalog/tests/unit/with_dbs/conftest.py @@ -33,7 +33,7 @@ from simcore_postgres_database.models.products import products from simcore_postgres_database.models.users import users from simcore_service_catalog.core.settings import ApplicationSettings -from simcore_service_catalog.db.tables import ( +from simcore_service_catalog.repositories._tables import ( groups, services_access_rights, services_meta_data, diff --git a/services/catalog/tests/unit/with_dbs/test_core_background_task__sync.py b/services/catalog/tests/unit/with_dbs/test_core_background_task__sync.py index a2927eefb418..4fcdcaae7912 100644 --- a/services/catalog/tests/unit/with_dbs/test_core_background_task__sync.py +++ b/services/catalog/tests/unit/with_dbs/test_core_background_task__sync.py @@ -14,7 +14,7 @@ from respx.router import MockRouter from simcore_postgres_database.models.services import services_meta_data from simcore_service_catalog.core.background_tasks import _run_sync_services -from simcore_service_catalog.db.repositories.services import ServicesRepository +from simcore_service_catalog.repositories.services import ServicesRepository from sqlalchemy.ext.asyncio.engine import AsyncEngine pytest_simcore_core_services_selection = [ diff --git a/services/catalog/tests/unit/with_dbs/test_db_repositories.py b/services/catalog/tests/unit/with_dbs/test_db_repositories.py index 8618a67c25e4..b310204096b5 100644 --- a/services/catalog/tests/unit/with_dbs/test_db_repositories.py +++ b/services/catalog/tests/unit/with_dbs/test_db_repositories.py @@ -13,13 +13,13 @@ from models_library.users import UserID from packaging import version from pydantic import EmailStr, HttpUrl, TypeAdapter -from simcore_service_catalog.db.repositories.services import ServicesRepository from simcore_service_catalog.models.services_db import ( ServiceAccessRightsAtDB, ServiceMetaDataDBCreate, ServiceMetaDataDBGet, ServiceMetaDataDBPatch, ) +from simcore_service_catalog.repositories.services import ServicesRepository from simcore_service_catalog.utils.versioning import is_patch_release from sqlalchemy.ext.asyncio import AsyncEngine diff --git a/services/catalog/tests/unit/with_dbs/test_services_access_rights.py b/services/catalog/tests/unit/with_dbs/test_services_access_rights.py index 47d0dc201aca..cd8aeacf135d 100644 --- a/services/catalog/tests/unit/with_dbs/test_services_access_rights.py +++ b/services/catalog/tests/unit/with_dbs/test_services_access_rights.py @@ -9,8 +9,8 @@ from models_library.products import ProductName from models_library.services import ServiceMetaDataPublished, ServiceVersion from pydantic import TypeAdapter -from simcore_service_catalog.db.repositories.services import ServicesRepository from simcore_service_catalog.models.services_db import ServiceAccessRightsAtDB +from simcore_service_catalog.repositories.services import ServicesRepository from simcore_service_catalog.services.access_rights import ( evaluate_auto_upgrade_policy, evaluate_default_policy, diff --git a/services/catalog/tests/unit/with_dbs/test_services_services_api.py b/services/catalog/tests/unit/with_dbs/test_services_services_api.py index b867d89402d3..c02f017e2bfe 100644 --- a/services/catalog/tests/unit/with_dbs/test_services_services_api.py +++ b/services/catalog/tests/unit/with_dbs/test_services_services_api.py @@ -17,9 +17,9 @@ from pytest_simcore.helpers.catalog_services import CreateFakeServiceDataCallable from respx.router import MockRouter from simcore_service_catalog.api.dependencies.director import get_director_api -from simcore_service_catalog.db.repositories.groups import GroupsRepository -from simcore_service_catalog.db.repositories.services import ServicesRepository from simcore_service_catalog.infrastructure.director import DirectorApi +from simcore_service_catalog.repositories.groups import GroupsRepository +from simcore_service_catalog.repositories.services import ServicesRepository from simcore_service_catalog.services import manifest, services_api from sqlalchemy.ext.asyncio import AsyncEngine From c54d5df678fa3d97a10582d549835f7b61ee18b2 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 4 Apr 2025 16:57:27 +0200 Subject: [PATCH 04/11] =?UTF-8?q?=F0=9F=94=A7=20Refactor:=20Move=20reposit?= =?UTF-8?q?ory=20files=20to=20a=20new=20directory=20structure=20for=20impr?= =?UTF-8?q?oved=20organization?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api/dependencies/database.py | 2 +- .../api/dependencies/services.py | 4 +-- .../api/dependencies/user_groups.py | 2 +- .../api/rest/_services.py | 4 +-- .../api/rest/_services_access_rights.py | 2 +- .../api/rest/_services_resources.py | 2 +- .../api/rest/_services_specifications.py | 4 +-- .../api/rpc/_services.py | 4 +-- .../core/application.py | 32 ++++++++++++++++--- .../core/background_tasks.py | 6 ++-- .../simcore_service_catalog/core/events.py | 2 +- .../infrastructure/director.py | 10 ++++-- .../infrastructure/postgres.py | 30 +++++++++++++++++ .../repositories/__init__.py | 1 - .../{db => repository}/__init__.py | 0 .../{repositories => repository}/_base.py | 0 .../_services_sql.py | 0 .../{repositories => repository}/_tables.py | 0 .../{repositories => repository}/groups.py | 0 .../{repositories => repository}/products.py | 0 .../{repositories => repository}/projects.py | 0 .../{repositories => repository}/services.py | 0 .../{db/events.py => repository/setup.py} | 8 ++--- .../services/access_rights.py | 4 +-- .../services/compatibility.py | 2 +- .../services/services_api.py | 4 +-- .../unit/test_db_repositories_services_sql.py | 2 +- .../tests/unit/test_services_compatibility.py | 2 +- .../catalog/tests/unit/with_dbs/conftest.py | 2 +- .../test_core_background_task__sync.py | 2 +- .../unit/with_dbs/test_db_repositories.py | 2 +- .../with_dbs/test_services_access_rights.py | 2 +- .../with_dbs/test_services_services_api.py | 4 +-- 33 files changed, 96 insertions(+), 43 deletions(-) create mode 100644 services/catalog/src/simcore_service_catalog/infrastructure/postgres.py delete mode 100644 services/catalog/src/simcore_service_catalog/repositories/__init__.py rename services/catalog/src/simcore_service_catalog/{db => repository}/__init__.py (100%) rename services/catalog/src/simcore_service_catalog/{repositories => repository}/_base.py (100%) rename services/catalog/src/simcore_service_catalog/{repositories => repository}/_services_sql.py (100%) rename services/catalog/src/simcore_service_catalog/{repositories => repository}/_tables.py (100%) rename services/catalog/src/simcore_service_catalog/{repositories => repository}/groups.py (100%) rename services/catalog/src/simcore_service_catalog/{repositories => repository}/products.py (100%) rename services/catalog/src/simcore_service_catalog/{repositories => repository}/projects.py (100%) rename services/catalog/src/simcore_service_catalog/{repositories => repository}/services.py (100%) rename services/catalog/src/simcore_service_catalog/{db/events.py => repository/setup.py} (51%) diff --git a/services/catalog/src/simcore_service_catalog/api/dependencies/database.py b/services/catalog/src/simcore_service_catalog/api/dependencies/database.py index f1c4e2efdf7b..b0ad2dcacb45 100644 --- a/services/catalog/src/simcore_service_catalog/api/dependencies/database.py +++ b/services/catalog/src/simcore_service_catalog/api/dependencies/database.py @@ -6,7 +6,7 @@ from fastapi.requests import Request from sqlalchemy.ext.asyncio import AsyncEngine -from ...repositories._base import BaseRepository +from ...repository._base import BaseRepository _logger = logging.getLogger(__name__) diff --git a/services/catalog/src/simcore_service_catalog/api/dependencies/services.py b/services/catalog/src/simcore_service_catalog/api/dependencies/services.py index 70316b07ac25..1ed6bc7be988 100644 --- a/services/catalog/src/simcore_service_catalog/api/dependencies/services.py +++ b/services/catalog/src/simcore_service_catalog/api/dependencies/services.py @@ -13,8 +13,8 @@ from servicelib.fastapi.dependencies import get_app from ...core.settings import ApplicationSettings -from ...repositories.groups import GroupsRepository -from ...repositories.services import ServicesRepository +from ...repository.groups import GroupsRepository +from ...repository.services import ServicesRepository from ...services import manifest from .database import get_repository from .director import DirectorApi, get_director_api diff --git a/services/catalog/src/simcore_service_catalog/api/dependencies/user_groups.py b/services/catalog/src/simcore_service_catalog/api/dependencies/user_groups.py index d4832fa86b2b..f63bc22f45c4 100644 --- a/services/catalog/src/simcore_service_catalog/api/dependencies/user_groups.py +++ b/services/catalog/src/simcore_service_catalog/api/dependencies/user_groups.py @@ -4,7 +4,7 @@ from models_library.groups import GroupAtDB from models_library.users import UserID -from ...repositories.groups import GroupsRepository +from ...repository.groups import GroupsRepository from .database import get_repository diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services.py b/services/catalog/src/simcore_service_catalog/api/rest/_services.py index 6979a1f0b0df..d9d408083401 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services.py @@ -21,8 +21,8 @@ RESPONSE_MODEL_POLICY, ) from ...models.services_db import ServiceAccessRightsAtDB, ServiceMetaDataDBGet -from ...repositories.groups import GroupsRepository -from ...repositories.services import ServicesRepository +from ...repository.groups import GroupsRepository +from ...repository.services import ServicesRepository from ..dependencies.database import get_repository from ..dependencies.director import DirectorApi, get_director_api from ..dependencies.services import get_service_from_manifest diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services_access_rights.py b/services/catalog/src/simcore_service_catalog/api/rest/_services_access_rights.py index feb87fb7cfbe..d2be0206f63e 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services_access_rights.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services_access_rights.py @@ -9,7 +9,7 @@ from ..._constants import RESPONSE_MODEL_POLICY from ...models.services_db import ServiceAccessRightsAtDB -from ...repositories.services import ServicesRepository +from ...repository.services import ServicesRepository from ..dependencies.database import get_repository from ..dependencies.services import AccessInfo, check_service_read_access diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py b/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py index 26157f9ea467..13985227a6f4 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py @@ -23,7 +23,7 @@ from pydantic import TypeAdapter from ..._constants import RESPONSE_MODEL_POLICY, SIMCORE_SERVICE_SETTINGS_LABELS -from ...repositories.services import ServicesRepository +from ...repository.services import ServicesRepository from ...services.function_services import is_function_service from ...utils.service_resources import ( merge_service_resources_with_user_specs, diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services_specifications.py b/services/catalog/src/simcore_service_catalog/api/rest/_services_specifications.py index 2a57ff597184..87153120139e 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services_specifications.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services_specifications.py @@ -10,8 +10,8 @@ from models_library.users import UserID from ..._constants import RESPONSE_MODEL_POLICY -from ...repositories.groups import GroupsRepository -from ...repositories.services import ServicesRepository +from ...repository.groups import GroupsRepository +from ...repository.services import ServicesRepository from ...services.function_services import is_function_service from ..dependencies.database import get_repository from ..dependencies.services import get_default_service_specifications diff --git a/services/catalog/src/simcore_service_catalog/api/rpc/_services.py b/services/catalog/src/simcore_service_catalog/api/rpc/_services.py index c909c472eb4c..0c10ab5406a5 100644 --- a/services/catalog/src/simcore_service_catalog/api/rpc/_services.py +++ b/services/catalog/src/simcore_service_catalog/api/rpc/_services.py @@ -23,9 +23,9 @@ CatalogForbiddenError, CatalogItemNotFoundError, ) -from simcore_service_catalog.repositories.groups import GroupsRepository +from simcore_service_catalog.repository.groups import GroupsRepository -from ...repositories.services import ServicesRepository +from ...repository.services import ServicesRepository from ...services import services_api from ..dependencies.director import get_director_api diff --git a/services/catalog/src/simcore_service_catalog/core/application.py b/services/catalog/src/simcore_service_catalog/core/application.py index b457200022b6..fe6c098c4dfb 100644 --- a/services/catalog/src/simcore_service_catalog/core/application.py +++ b/services/catalog/src/simcore_service_catalog/core/application.py @@ -10,15 +10,25 @@ setup_prometheus_instrumentation, ) from servicelib.fastapi.tracing import initialize_tracing +from simcore_service_catalog.core.background_tasks import start_registry_sync_task +from simcore_service_catalog.infrastructure.director import setup_director +from simcore_service_catalog.infrastructure.postgres import setup_postgres_database from starlette.middleware.base import BaseHTTPMiddleware -from .._meta import API_VERSION, API_VTAG, APP_NAME, PROJECT_NAME, SUMMARY +from .._meta import ( + API_VERSION, + API_VTAG, + APP_FINISHED_BANNER_MSG, + APP_NAME, + APP_STARTED_BANNER_MSG, + PROJECT_NAME, + SUMMARY, +) from ..api.rest.routes import setup_rest_api_routes from ..api.rpc.routes import setup_rpc_api_routes from ..exceptions.handlers import setup_exception_handlers from ..infrastructure.rabbitmq import setup_rabbitmq from ..services.function_services import setup_function_services -from .events import create_on_shutdown, create_on_startup from .settings import ApplicationSettings _logger = logging.getLogger(__name__) @@ -34,6 +44,15 @@ ) +def _flush_started_banner() -> None: + # WARNING: this function is spied in the tests + print(APP_STARTED_BANNER_MSG, flush=True) # noqa: T201 + + +def _flush_finished_banner() -> None: + print(APP_FINISHED_BANNER_MSG, flush=True) # noqa: T201 + + def create_app(settings: ApplicationSettings | None = None) -> FastAPI: # keep mostly quiet noisy loggers quiet_level: int = max( @@ -67,12 +86,15 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI: initialize_tracing(app, settings.CATALOG_TRACING, APP_NAME) # STARTUP-EVENT - app.add_event_handler("startup", create_on_startup(app)) + app.add_event_handler("startup", _flush_started_banner) - # PLUGIN SETUP + setup_postgres_database(app) + setup_director(app) setup_function_services(app) setup_rabbitmq(app) + app.add_event_handler("startup", start_registry_sync_task) + if app.state.settings.CATALOG_PROMETHEUS_INSTRUMENTATION_ENABLED: setup_prometheus_instrumentation(app) @@ -93,7 +115,7 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI: setup_rpc_api_routes(app) # SHUTDOWN-EVENT - app.add_event_handler("shutdown", create_on_shutdown(app)) + app.add_event_handler("shutdown", _flush_finished_banner) # EXCEPTIONS setup_exception_handlers(app) diff --git a/services/catalog/src/simcore_service_catalog/core/background_tasks.py b/services/catalog/src/simcore_service_catalog/core/background_tasks.py index 657f3c3c6ac5..7d1ef490ed84 100644 --- a/services/catalog/src/simcore_service_catalog/core/background_tasks.py +++ b/services/catalog/src/simcore_service_catalog/core/background_tasks.py @@ -26,9 +26,9 @@ from sqlalchemy.ext.asyncio import AsyncEngine from ..models.services_db import ServiceAccessRightsAtDB, ServiceMetaDataDBCreate -from ..repositories.groups import GroupsRepository -from ..repositories.projects import ProjectsRepository -from ..repositories.services import ServicesRepository +from ..repository.groups import GroupsRepository +from ..repository.projects import ProjectsRepository +from ..repository.services import ServicesRepository from ..services import access_rights _logger = logging.getLogger(__name__) diff --git a/services/catalog/src/simcore_service_catalog/core/events.py b/services/catalog/src/simcore_service_catalog/core/events.py index bdc646d89a1d..3c7cb6ca3025 100644 --- a/services/catalog/src/simcore_service_catalog/core/events.py +++ b/services/catalog/src/simcore_service_catalog/core/events.py @@ -7,7 +7,7 @@ from servicelib.logging_utils import log_context from .._meta import APP_FINISHED_BANNER_MSG, APP_STARTED_BANNER_MSG -from ..events import setup_default_product +from ..db.events import setup_default_product from ..infrastructure.director import close_director, setup_director from .background_tasks import start_registry_sync_task, stop_registry_sync_task diff --git a/services/catalog/src/simcore_service_catalog/infrastructure/director.py b/services/catalog/src/simcore_service_catalog/infrastructure/director.py index f35490e9271d..e6bf72502d9e 100644 --- a/services/catalog/src/simcore_service_catalog/infrastructure/director.py +++ b/services/catalog/src/simcore_service_catalog/infrastructure/director.py @@ -288,7 +288,7 @@ async def get_service_extras( return TypeAdapter(ServiceExtras).validate_python(result) -async def setup_director(app: FastAPI) -> None: +async def _initialize_director_client(app: FastAPI) -> None: if settings := app.state.settings.CATALOG_DIRECTOR: with log_context( _logger, logging.DEBUG, "Setup director at %s", f"{settings.base_url=}" @@ -311,9 +311,15 @@ async def setup_director(app: FastAPI) -> None: app.state.director_api = client -async def close_director(app: FastAPI) -> None: +async def _shutdown_director_client(app: FastAPI) -> None: client: DirectorApi | None if client := app.state.director_api: await client.close() _logger.debug("Director client closed successfully") + + +def setup_director(app: FastAPI): + + app.add_event_handler("startup", _initialize_director_client) + app.add_event_handler("shutdown", _shutdown_director_client) diff --git a/services/catalog/src/simcore_service_catalog/infrastructure/postgres.py b/services/catalog/src/simcore_service_catalog/infrastructure/postgres.py new file mode 100644 index 000000000000..a42b52e08a4f --- /dev/null +++ b/services/catalog/src/simcore_service_catalog/infrastructure/postgres.py @@ -0,0 +1,30 @@ +import contextlib +import logging + +from fastapi import FastAPI +from servicelib.fastapi.db_asyncpg_engine import close_db_connection, connect_to_db +from servicelib.logging_utils import log_catch, log_context + +from ..repository.products import setup_default_product + +_logger = logging.getLogger(__name__) + + +def setup_postgres_database(app: FastAPI): + + async def _(): + with log_context(_logger, logging.INFO, f"{__name__} startup ..."): + # connection + await connect_to_db(app, app.state.settings.CATALOG_POSTGRES) + + # configuring default product + await setup_default_product(app) + + yield + + with log_context( + _logger, logging.INFO, f"{__name__} shutdown ..." + ), contextlib.suppress(Exception), log_catch(_logger): + await close_db_connection(app) + + return _ diff --git a/services/catalog/src/simcore_service_catalog/repositories/__init__.py b/services/catalog/src/simcore_service_catalog/repositories/__init__.py deleted file mode 100644 index a5eeffe1ff59..000000000000 --- a/services/catalog/src/simcore_service_catalog/repositories/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from ._base import BaseRepository diff --git a/services/catalog/src/simcore_service_catalog/db/__init__.py b/services/catalog/src/simcore_service_catalog/repository/__init__.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/db/__init__.py rename to services/catalog/src/simcore_service_catalog/repository/__init__.py diff --git a/services/catalog/src/simcore_service_catalog/repositories/_base.py b/services/catalog/src/simcore_service_catalog/repository/_base.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/repositories/_base.py rename to services/catalog/src/simcore_service_catalog/repository/_base.py diff --git a/services/catalog/src/simcore_service_catalog/repositories/_services_sql.py b/services/catalog/src/simcore_service_catalog/repository/_services_sql.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/repositories/_services_sql.py rename to services/catalog/src/simcore_service_catalog/repository/_services_sql.py diff --git a/services/catalog/src/simcore_service_catalog/repositories/_tables.py b/services/catalog/src/simcore_service_catalog/repository/_tables.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/repositories/_tables.py rename to services/catalog/src/simcore_service_catalog/repository/_tables.py diff --git a/services/catalog/src/simcore_service_catalog/repositories/groups.py b/services/catalog/src/simcore_service_catalog/repository/groups.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/repositories/groups.py rename to services/catalog/src/simcore_service_catalog/repository/groups.py diff --git a/services/catalog/src/simcore_service_catalog/repositories/products.py b/services/catalog/src/simcore_service_catalog/repository/products.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/repositories/products.py rename to services/catalog/src/simcore_service_catalog/repository/products.py diff --git a/services/catalog/src/simcore_service_catalog/repositories/projects.py b/services/catalog/src/simcore_service_catalog/repository/projects.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/repositories/projects.py rename to services/catalog/src/simcore_service_catalog/repository/projects.py diff --git a/services/catalog/src/simcore_service_catalog/repositories/services.py b/services/catalog/src/simcore_service_catalog/repository/services.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/repositories/services.py rename to services/catalog/src/simcore_service_catalog/repository/services.py diff --git a/services/catalog/src/simcore_service_catalog/db/events.py b/services/catalog/src/simcore_service_catalog/repository/setup.py similarity index 51% rename from services/catalog/src/simcore_service_catalog/db/events.py rename to services/catalog/src/simcore_service_catalog/repository/setup.py index 42de4c386201..f6e5b0d68cda 100644 --- a/services/catalog/src/simcore_service_catalog/db/events.py +++ b/services/catalog/src/simcore_service_catalog/repository/setup.py @@ -1,12 +1,8 @@ -import logging - from fastapi import FastAPI -from .repositories.products import ProductsRepository - -_logger = logging.getLogger(__name__) +from .products import ProductsRepository -async def setup_default_product(app: FastAPI): +async def setup_repository(app: FastAPI): repo = ProductsRepository(db_engine=app.state.engine) app.state.default_product_name = await repo.get_default_product_name() diff --git a/services/catalog/src/simcore_service_catalog/services/access_rights.py b/services/catalog/src/simcore_service_catalog/services/access_rights.py index 554c3602d93c..b6049d93521f 100644 --- a/services/catalog/src/simcore_service_catalog/services/access_rights.py +++ b/services/catalog/src/simcore_service_catalog/services/access_rights.py @@ -16,8 +16,8 @@ from ..api.dependencies.director import get_director_api from ..models.services_db import ServiceAccessRightsAtDB -from ..repositories.groups import GroupsRepository -from ..repositories.services import ServicesRepository +from ..repository.groups import GroupsRepository +from ..repository.services import ServicesRepository from ..utils.versioning import as_version, is_patch_release _logger = logging.getLogger(__name__) diff --git a/services/catalog/src/simcore_service_catalog/services/compatibility.py b/services/catalog/src/simcore_service_catalog/services/compatibility.py index 6f7c5e64257e..696726e5ef04 100644 --- a/services/catalog/src/simcore_service_catalog/services/compatibility.py +++ b/services/catalog/src/simcore_service_catalog/services/compatibility.py @@ -9,7 +9,7 @@ from simcore_service_catalog.utils.versioning import as_version from ..models.services_db import ReleaseDBGet -from ..repositories.services import ServicesRepository +from ..repository.services import ServicesRepository def _get_default_compatibility_specs(target: ServiceVersion | Version) -> SpecifierSet: diff --git a/services/catalog/src/simcore_service_catalog/services/services_api.py b/services/catalog/src/simcore_service_catalog/services/services_api.py index 4d40da8aa35c..60e1ee58f60f 100644 --- a/services/catalog/src/simcore_service_catalog/services/services_api.py +++ b/services/catalog/src/simcore_service_catalog/services/services_api.py @@ -20,7 +20,7 @@ CatalogForbiddenError, CatalogItemNotFoundError, ) -from simcore_service_catalog.repositories.groups import GroupsRepository +from simcore_service_catalog.repository.groups import GroupsRepository from ..infrastructure.director import DirectorApi from ..models.services_db import ( @@ -28,7 +28,7 @@ ServiceMetaDataDBPatch, ServiceWithHistoryDBGet, ) -from ..repositories.services import ServicesRepository +from ..repository.services import ServicesRepository from ..services import manifest from .compatibility import evaluate_service_compatibility_map from .function_services import is_function_service diff --git a/services/catalog/tests/unit/test_db_repositories_services_sql.py b/services/catalog/tests/unit/test_db_repositories_services_sql.py index 35f771a8c945..7acc31c585fd 100644 --- a/services/catalog/tests/unit/test_db_repositories_services_sql.py +++ b/services/catalog/tests/unit/test_db_repositories_services_sql.py @@ -4,7 +4,7 @@ from simcore_postgres_database.utils import as_postgres_sql_query_str -from simcore_service_catalog.repositories._services_sql import ( +from simcore_service_catalog.repository._services_sql import ( AccessRightsClauses, can_get_service_stmt, get_service_history_stmt, diff --git a/services/catalog/tests/unit/test_services_compatibility.py b/services/catalog/tests/unit/test_services_compatibility.py index fbf6bd17860e..9182d3f1c761 100644 --- a/services/catalog/tests/unit/test_services_compatibility.py +++ b/services/catalog/tests/unit/test_services_compatibility.py @@ -12,7 +12,7 @@ from packaging.version import Version from pytest_mock import MockerFixture, MockType from simcore_service_catalog.models.services_db import ReleaseDBGet -from simcore_service_catalog.repositories.services import ServicesRepository +from simcore_service_catalog.repository.services import ServicesRepository from simcore_service_catalog.services.compatibility import ( _get_latest_compatible_version, evaluate_service_compatibility_map, diff --git a/services/catalog/tests/unit/with_dbs/conftest.py b/services/catalog/tests/unit/with_dbs/conftest.py index 44fa94e294af..080fb5513559 100644 --- a/services/catalog/tests/unit/with_dbs/conftest.py +++ b/services/catalog/tests/unit/with_dbs/conftest.py @@ -33,7 +33,7 @@ from simcore_postgres_database.models.products import products from simcore_postgres_database.models.users import users from simcore_service_catalog.core.settings import ApplicationSettings -from simcore_service_catalog.repositories._tables import ( +from simcore_service_catalog.repository._tables import ( groups, services_access_rights, services_meta_data, diff --git a/services/catalog/tests/unit/with_dbs/test_core_background_task__sync.py b/services/catalog/tests/unit/with_dbs/test_core_background_task__sync.py index 4fcdcaae7912..1019383193d5 100644 --- a/services/catalog/tests/unit/with_dbs/test_core_background_task__sync.py +++ b/services/catalog/tests/unit/with_dbs/test_core_background_task__sync.py @@ -14,7 +14,7 @@ from respx.router import MockRouter from simcore_postgres_database.models.services import services_meta_data from simcore_service_catalog.core.background_tasks import _run_sync_services -from simcore_service_catalog.repositories.services import ServicesRepository +from simcore_service_catalog.repository.services import ServicesRepository from sqlalchemy.ext.asyncio.engine import AsyncEngine pytest_simcore_core_services_selection = [ diff --git a/services/catalog/tests/unit/with_dbs/test_db_repositories.py b/services/catalog/tests/unit/with_dbs/test_db_repositories.py index b310204096b5..40c1a34aedaa 100644 --- a/services/catalog/tests/unit/with_dbs/test_db_repositories.py +++ b/services/catalog/tests/unit/with_dbs/test_db_repositories.py @@ -19,7 +19,7 @@ ServiceMetaDataDBGet, ServiceMetaDataDBPatch, ) -from simcore_service_catalog.repositories.services import ServicesRepository +from simcore_service_catalog.repository.services import ServicesRepository from simcore_service_catalog.utils.versioning import is_patch_release from sqlalchemy.ext.asyncio import AsyncEngine diff --git a/services/catalog/tests/unit/with_dbs/test_services_access_rights.py b/services/catalog/tests/unit/with_dbs/test_services_access_rights.py index cd8aeacf135d..4c4c1e96775f 100644 --- a/services/catalog/tests/unit/with_dbs/test_services_access_rights.py +++ b/services/catalog/tests/unit/with_dbs/test_services_access_rights.py @@ -10,7 +10,7 @@ from models_library.services import ServiceMetaDataPublished, ServiceVersion from pydantic import TypeAdapter from simcore_service_catalog.models.services_db import ServiceAccessRightsAtDB -from simcore_service_catalog.repositories.services import ServicesRepository +from simcore_service_catalog.repository.services import ServicesRepository from simcore_service_catalog.services.access_rights import ( evaluate_auto_upgrade_policy, evaluate_default_policy, diff --git a/services/catalog/tests/unit/with_dbs/test_services_services_api.py b/services/catalog/tests/unit/with_dbs/test_services_services_api.py index c02f017e2bfe..001d6732c9f4 100644 --- a/services/catalog/tests/unit/with_dbs/test_services_services_api.py +++ b/services/catalog/tests/unit/with_dbs/test_services_services_api.py @@ -18,8 +18,8 @@ from respx.router import MockRouter from simcore_service_catalog.api.dependencies.director import get_director_api from simcore_service_catalog.infrastructure.director import DirectorApi -from simcore_service_catalog.repositories.groups import GroupsRepository -from simcore_service_catalog.repositories.services import ServicesRepository +from simcore_service_catalog.repository.groups import GroupsRepository +from simcore_service_catalog.repository.services import ServicesRepository from simcore_service_catalog.services import manifest, services_api from sqlalchemy.ext.asyncio import AsyncEngine From f308f534903e594fde5cbacf3d52631e18c54228 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 4 Apr 2025 16:58:36 +0200 Subject: [PATCH 05/11] =?UTF-8?q?=F0=9F=94=A7=20Refactor:=20Update=20impor?= =?UTF-8?q?t=20paths=20to=20new=20service=20module=20structure=20and=20add?= =?UTF-8?q?=20function=5Fservices=20implementation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/simcore_service_catalog/api/dependencies/services.py | 2 +- .../simcore_service_catalog/api/rest/_services_resources.py | 2 +- .../api/rest/_services_specifications.py | 2 +- .../catalog/src/simcore_service_catalog/api/rpc/_services.py | 2 +- .../catalog/src/simcore_service_catalog/core/application.py | 2 +- .../src/simcore_service_catalog/core/background_tasks.py | 4 ++-- .../simcore_service_catalog/{services => service}/__init__.py | 0 .../{services => service}/access_rights.py | 0 .../{services => service}/compatibility.py | 0 .../{services => service}/function_services.py | 0 .../simcore_service_catalog/{services => service}/manifest.py | 0 .../{services => service}/services_api.py | 2 +- services/catalog/tests/unit/test_services_compatibility.py | 2 +- .../catalog/tests/unit/test_services_function_services.py | 2 +- services/catalog/tests/unit/test_services_manifest.py | 2 +- .../tests/unit/with_dbs/test_services_access_rights.py | 2 +- .../catalog/tests/unit/with_dbs/test_services_services_api.py | 2 +- 17 files changed, 13 insertions(+), 13 deletions(-) rename services/catalog/src/simcore_service_catalog/{services => service}/__init__.py (100%) rename services/catalog/src/simcore_service_catalog/{services => service}/access_rights.py (100%) rename services/catalog/src/simcore_service_catalog/{services => service}/compatibility.py (100%) rename services/catalog/src/simcore_service_catalog/{services => service}/function_services.py (100%) rename services/catalog/src/simcore_service_catalog/{services => service}/manifest.py (100%) rename services/catalog/src/simcore_service_catalog/{services => service}/services_api.py (99%) diff --git a/services/catalog/src/simcore_service_catalog/api/dependencies/services.py b/services/catalog/src/simcore_service_catalog/api/dependencies/services.py index 1ed6bc7be988..eb4cc7a387f6 100644 --- a/services/catalog/src/simcore_service_catalog/api/dependencies/services.py +++ b/services/catalog/src/simcore_service_catalog/api/dependencies/services.py @@ -15,7 +15,7 @@ from ...core.settings import ApplicationSettings from ...repository.groups import GroupsRepository from ...repository.services import ServicesRepository -from ...services import manifest +from ...service import manifest from .database import get_repository from .director import DirectorApi, get_director_api diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py b/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py index 13985227a6f4..09d950d12499 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py @@ -24,7 +24,7 @@ from ..._constants import RESPONSE_MODEL_POLICY, SIMCORE_SERVICE_SETTINGS_LABELS from ...repository.services import ServicesRepository -from ...services.function_services import is_function_service +from ...service.function_services import is_function_service from ...utils.service_resources import ( merge_service_resources_with_user_specs, parse_generic_resource, diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services_specifications.py b/services/catalog/src/simcore_service_catalog/api/rest/_services_specifications.py index 87153120139e..b0e2036d83b5 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services_specifications.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services_specifications.py @@ -12,7 +12,7 @@ from ..._constants import RESPONSE_MODEL_POLICY from ...repository.groups import GroupsRepository from ...repository.services import ServicesRepository -from ...services.function_services import is_function_service +from ...service.function_services import is_function_service from ..dependencies.database import get_repository from ..dependencies.services import get_default_service_specifications diff --git a/services/catalog/src/simcore_service_catalog/api/rpc/_services.py b/services/catalog/src/simcore_service_catalog/api/rpc/_services.py index 0c10ab5406a5..b053c185d7e9 100644 --- a/services/catalog/src/simcore_service_catalog/api/rpc/_services.py +++ b/services/catalog/src/simcore_service_catalog/api/rpc/_services.py @@ -26,7 +26,7 @@ from simcore_service_catalog.repository.groups import GroupsRepository from ...repository.services import ServicesRepository -from ...services import services_api +from ...service import services_api from ..dependencies.director import get_director_api _logger = logging.getLogger(__name__) diff --git a/services/catalog/src/simcore_service_catalog/core/application.py b/services/catalog/src/simcore_service_catalog/core/application.py index fe6c098c4dfb..2dc9ae0886f8 100644 --- a/services/catalog/src/simcore_service_catalog/core/application.py +++ b/services/catalog/src/simcore_service_catalog/core/application.py @@ -28,7 +28,7 @@ from ..api.rpc.routes import setup_rpc_api_routes from ..exceptions.handlers import setup_exception_handlers from ..infrastructure.rabbitmq import setup_rabbitmq -from ..services.function_services import setup_function_services +from ..service.function_services import setup_function_services from .settings import ApplicationSettings _logger = logging.getLogger(__name__) diff --git a/services/catalog/src/simcore_service_catalog/core/background_tasks.py b/services/catalog/src/simcore_service_catalog/core/background_tasks.py index 7d1ef490ed84..7f772ae1a87e 100644 --- a/services/catalog/src/simcore_service_catalog/core/background_tasks.py +++ b/services/catalog/src/simcore_service_catalog/core/background_tasks.py @@ -21,7 +21,7 @@ from packaging.version import Version from pydantic import ValidationError from simcore_service_catalog.api.dependencies.director import get_director_api -from simcore_service_catalog.services import manifest +from simcore_service_catalog.service import manifest from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.asyncio import AsyncEngine @@ -29,7 +29,7 @@ from ..repository.groups import GroupsRepository from ..repository.projects import ProjectsRepository from ..repository.services import ServicesRepository -from ..services import access_rights +from ..service import access_rights _logger = logging.getLogger(__name__) diff --git a/services/catalog/src/simcore_service_catalog/services/__init__.py b/services/catalog/src/simcore_service_catalog/service/__init__.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/services/__init__.py rename to services/catalog/src/simcore_service_catalog/service/__init__.py diff --git a/services/catalog/src/simcore_service_catalog/services/access_rights.py b/services/catalog/src/simcore_service_catalog/service/access_rights.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/services/access_rights.py rename to services/catalog/src/simcore_service_catalog/service/access_rights.py diff --git a/services/catalog/src/simcore_service_catalog/services/compatibility.py b/services/catalog/src/simcore_service_catalog/service/compatibility.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/services/compatibility.py rename to services/catalog/src/simcore_service_catalog/service/compatibility.py diff --git a/services/catalog/src/simcore_service_catalog/services/function_services.py b/services/catalog/src/simcore_service_catalog/service/function_services.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/services/function_services.py rename to services/catalog/src/simcore_service_catalog/service/function_services.py diff --git a/services/catalog/src/simcore_service_catalog/services/manifest.py b/services/catalog/src/simcore_service_catalog/service/manifest.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/services/manifest.py rename to services/catalog/src/simcore_service_catalog/service/manifest.py diff --git a/services/catalog/src/simcore_service_catalog/services/services_api.py b/services/catalog/src/simcore_service_catalog/service/services_api.py similarity index 99% rename from services/catalog/src/simcore_service_catalog/services/services_api.py rename to services/catalog/src/simcore_service_catalog/service/services_api.py index 60e1ee58f60f..60c5c8406697 100644 --- a/services/catalog/src/simcore_service_catalog/services/services_api.py +++ b/services/catalog/src/simcore_service_catalog/service/services_api.py @@ -29,7 +29,7 @@ ServiceWithHistoryDBGet, ) from ..repository.services import ServicesRepository -from ..services import manifest +from ..service import manifest from .compatibility import evaluate_service_compatibility_map from .function_services import is_function_service diff --git a/services/catalog/tests/unit/test_services_compatibility.py b/services/catalog/tests/unit/test_services_compatibility.py index 9182d3f1c761..30cbb6733581 100644 --- a/services/catalog/tests/unit/test_services_compatibility.py +++ b/services/catalog/tests/unit/test_services_compatibility.py @@ -13,7 +13,7 @@ from pytest_mock import MockerFixture, MockType from simcore_service_catalog.models.services_db import ReleaseDBGet from simcore_service_catalog.repository.services import ServicesRepository -from simcore_service_catalog.services.compatibility import ( +from simcore_service_catalog.service.compatibility import ( _get_latest_compatible_version, evaluate_service_compatibility_map, ) diff --git a/services/catalog/tests/unit/test_services_function_services.py b/services/catalog/tests/unit/test_services_function_services.py index 4c4235ce9aef..06853bcf715b 100644 --- a/services/catalog/tests/unit/test_services_function_services.py +++ b/services/catalog/tests/unit/test_services_function_services.py @@ -8,7 +8,7 @@ import pytest from models_library.api_schemas_catalog.services import ServiceMetaDataPublished -from simcore_service_catalog.services.function_services import ( +from simcore_service_catalog.service.function_services import ( is_function_service, iter_service_docker_data, ) diff --git a/services/catalog/tests/unit/test_services_manifest.py b/services/catalog/tests/unit/test_services_manifest.py index 52ccc7001a6d..ad4e0dae9d8d 100644 --- a/services/catalog/tests/unit/test_services_manifest.py +++ b/services/catalog/tests/unit/test_services_manifest.py @@ -15,7 +15,7 @@ from respx.router import MockRouter from simcore_service_catalog.api.dependencies.director import get_director_api from simcore_service_catalog.infrastructure.director import DirectorApi -from simcore_service_catalog.services import manifest +from simcore_service_catalog.service import manifest @pytest.fixture diff --git a/services/catalog/tests/unit/with_dbs/test_services_access_rights.py b/services/catalog/tests/unit/with_dbs/test_services_access_rights.py index 4c4c1e96775f..887a54aa172b 100644 --- a/services/catalog/tests/unit/with_dbs/test_services_access_rights.py +++ b/services/catalog/tests/unit/with_dbs/test_services_access_rights.py @@ -11,7 +11,7 @@ from pydantic import TypeAdapter from simcore_service_catalog.models.services_db import ServiceAccessRightsAtDB from simcore_service_catalog.repository.services import ServicesRepository -from simcore_service_catalog.services.access_rights import ( +from simcore_service_catalog.service.access_rights import ( evaluate_auto_upgrade_policy, evaluate_default_policy, reduce_access_rights, diff --git a/services/catalog/tests/unit/with_dbs/test_services_services_api.py b/services/catalog/tests/unit/with_dbs/test_services_services_api.py index 001d6732c9f4..507e0f6b6697 100644 --- a/services/catalog/tests/unit/with_dbs/test_services_services_api.py +++ b/services/catalog/tests/unit/with_dbs/test_services_services_api.py @@ -20,7 +20,7 @@ from simcore_service_catalog.infrastructure.director import DirectorApi from simcore_service_catalog.repository.groups import GroupsRepository from simcore_service_catalog.repository.services import ServicesRepository -from simcore_service_catalog.services import manifest, services_api +from simcore_service_catalog.service import manifest, services_api from sqlalchemy.ext.asyncio import AsyncEngine pytest_simcore_core_services_selection = [ From b229fbfcc13de08feb17f471f49fa959e53ef3b9 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 4 Apr 2025 16:59:55 +0200 Subject: [PATCH 06/11] =?UTF-8?q?=F0=9F=94=A7=20Test:=20Add=20unit=20tests?= =?UTF-8?q?=20for=20function=20services=20and=20SQL=20statements=20in=20th?= =?UTF-8?q?e=20catalog=20service?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...positories_services_sql.py => test_repository_services_sql.py} | 0 ...st_services_compatibility.py => test_service_compatibility.py} | 0 ...ces_function_services.py => test_service_function_services.py} | 0 .../unit/{test_services_manifest.py => test_service_manifest.py} | 0 .../unit/with_dbs/{test_db_repositories.py => test_repository.py} | 0 ...st_services_access_rights.py => test_service_access_rights.py} | 0 ...test_services_services_api.py => test_service_services_api.py} | 0 7 files changed, 0 insertions(+), 0 deletions(-) rename services/catalog/tests/unit/{test_db_repositories_services_sql.py => test_repository_services_sql.py} (100%) rename services/catalog/tests/unit/{test_services_compatibility.py => test_service_compatibility.py} (100%) rename services/catalog/tests/unit/{test_services_function_services.py => test_service_function_services.py} (100%) rename services/catalog/tests/unit/{test_services_manifest.py => test_service_manifest.py} (100%) rename services/catalog/tests/unit/with_dbs/{test_db_repositories.py => test_repository.py} (100%) rename services/catalog/tests/unit/with_dbs/{test_services_access_rights.py => test_service_access_rights.py} (100%) rename services/catalog/tests/unit/with_dbs/{test_services_services_api.py => test_service_services_api.py} (100%) diff --git a/services/catalog/tests/unit/test_db_repositories_services_sql.py b/services/catalog/tests/unit/test_repository_services_sql.py similarity index 100% rename from services/catalog/tests/unit/test_db_repositories_services_sql.py rename to services/catalog/tests/unit/test_repository_services_sql.py diff --git a/services/catalog/tests/unit/test_services_compatibility.py b/services/catalog/tests/unit/test_service_compatibility.py similarity index 100% rename from services/catalog/tests/unit/test_services_compatibility.py rename to services/catalog/tests/unit/test_service_compatibility.py diff --git a/services/catalog/tests/unit/test_services_function_services.py b/services/catalog/tests/unit/test_service_function_services.py similarity index 100% rename from services/catalog/tests/unit/test_services_function_services.py rename to services/catalog/tests/unit/test_service_function_services.py diff --git a/services/catalog/tests/unit/test_services_manifest.py b/services/catalog/tests/unit/test_service_manifest.py similarity index 100% rename from services/catalog/tests/unit/test_services_manifest.py rename to services/catalog/tests/unit/test_service_manifest.py diff --git a/services/catalog/tests/unit/with_dbs/test_db_repositories.py b/services/catalog/tests/unit/with_dbs/test_repository.py similarity index 100% rename from services/catalog/tests/unit/with_dbs/test_db_repositories.py rename to services/catalog/tests/unit/with_dbs/test_repository.py diff --git a/services/catalog/tests/unit/with_dbs/test_services_access_rights.py b/services/catalog/tests/unit/with_dbs/test_service_access_rights.py similarity index 100% rename from services/catalog/tests/unit/with_dbs/test_services_access_rights.py rename to services/catalog/tests/unit/with_dbs/test_service_access_rights.py diff --git a/services/catalog/tests/unit/with_dbs/test_services_services_api.py b/services/catalog/tests/unit/with_dbs/test_service_services_api.py similarity index 100% rename from services/catalog/tests/unit/with_dbs/test_services_services_api.py rename to services/catalog/tests/unit/with_dbs/test_service_services_api.py From c85a7c937c4b4029ca432b223e844b54abf2ca94 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 4 Apr 2025 17:01:01 +0200 Subject: [PATCH 07/11] =?UTF-8?q?=F0=9F=94=A7=20Refactor:=20Update=20impor?= =?UTF-8?q?t=20paths=20to=20use=20new=20=5Fdependencies=20structure=20and?= =?UTF-8?q?=20add=20user=5Fgroups=20functionality?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api/{dependencies => _dependencies}/__init__.py | 0 .../api/{dependencies => _dependencies}/database.py | 0 .../api/{dependencies => _dependencies}/director.py | 0 .../api/{dependencies => _dependencies}/services.py | 0 .../api/{dependencies => _dependencies}/user_groups.py | 0 .../src/simcore_service_catalog/api/rest/_services.py | 6 +++--- .../api/rest/_services_access_rights.py | 4 ++-- .../simcore_service_catalog/api/rest/_services_extras.py | 2 +- .../simcore_service_catalog/api/rest/_services_labels.py | 2 +- .../simcore_service_catalog/api/rest/_services_ports.py | 2 +- .../api/rest/_services_resources.py | 8 ++++---- .../api/rest/_services_specifications.py | 4 ++-- .../src/simcore_service_catalog/api/rpc/_services.py | 2 +- .../src/simcore_service_catalog/core/background_tasks.py | 2 +- .../src/simcore_service_catalog/service/access_rights.py | 2 +- .../catalog/tests/unit/test_infrastructure_director.py | 2 +- services/catalog/tests/unit/test_service_manifest.py | 2 +- .../tests/unit/with_dbs/test_service_services_api.py | 2 +- 18 files changed, 20 insertions(+), 20 deletions(-) rename services/catalog/src/simcore_service_catalog/api/{dependencies => _dependencies}/__init__.py (100%) rename services/catalog/src/simcore_service_catalog/api/{dependencies => _dependencies}/database.py (100%) rename services/catalog/src/simcore_service_catalog/api/{dependencies => _dependencies}/director.py (100%) rename services/catalog/src/simcore_service_catalog/api/{dependencies => _dependencies}/services.py (100%) rename services/catalog/src/simcore_service_catalog/api/{dependencies => _dependencies}/user_groups.py (100%) diff --git a/services/catalog/src/simcore_service_catalog/api/dependencies/__init__.py b/services/catalog/src/simcore_service_catalog/api/_dependencies/__init__.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/api/dependencies/__init__.py rename to services/catalog/src/simcore_service_catalog/api/_dependencies/__init__.py diff --git a/services/catalog/src/simcore_service_catalog/api/dependencies/database.py b/services/catalog/src/simcore_service_catalog/api/_dependencies/database.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/api/dependencies/database.py rename to services/catalog/src/simcore_service_catalog/api/_dependencies/database.py diff --git a/services/catalog/src/simcore_service_catalog/api/dependencies/director.py b/services/catalog/src/simcore_service_catalog/api/_dependencies/director.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/api/dependencies/director.py rename to services/catalog/src/simcore_service_catalog/api/_dependencies/director.py diff --git a/services/catalog/src/simcore_service_catalog/api/dependencies/services.py b/services/catalog/src/simcore_service_catalog/api/_dependencies/services.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/api/dependencies/services.py rename to services/catalog/src/simcore_service_catalog/api/_dependencies/services.py diff --git a/services/catalog/src/simcore_service_catalog/api/dependencies/user_groups.py b/services/catalog/src/simcore_service_catalog/api/_dependencies/user_groups.py similarity index 100% rename from services/catalog/src/simcore_service_catalog/api/dependencies/user_groups.py rename to services/catalog/src/simcore_service_catalog/api/_dependencies/user_groups.py diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services.py b/services/catalog/src/simcore_service_catalog/api/rest/_services.py index d9d408083401..1ba3378ef1b5 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services.py @@ -23,9 +23,9 @@ from ...models.services_db import ServiceAccessRightsAtDB, ServiceMetaDataDBGet from ...repository.groups import GroupsRepository from ...repository.services import ServicesRepository -from ..dependencies.database import get_repository -from ..dependencies.director import DirectorApi, get_director_api -from ..dependencies.services import get_service_from_manifest +from .._dependencies.database import get_repository +from .._dependencies.director import DirectorApi, get_director_api +from .._dependencies.services import get_service_from_manifest _logger = logging.getLogger(__name__) diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services_access_rights.py b/services/catalog/src/simcore_service_catalog/api/rest/_services_access_rights.py index d2be0206f63e..2e828996a53c 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services_access_rights.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services_access_rights.py @@ -10,8 +10,8 @@ from ..._constants import RESPONSE_MODEL_POLICY from ...models.services_db import ServiceAccessRightsAtDB from ...repository.services import ServicesRepository -from ..dependencies.database import get_repository -from ..dependencies.services import AccessInfo, check_service_read_access +from .._dependencies.database import get_repository +from .._dependencies.services import AccessInfo, check_service_read_access _logger = logging.getLogger(__name__) diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services_extras.py b/services/catalog/src/simcore_service_catalog/api/rest/_services_extras.py index e451dce349b8..7b7a9418dbb0 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services_extras.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services_extras.py @@ -4,7 +4,7 @@ from models_library.api_schemas_directorv2.services import ServiceExtras from models_library.services import ServiceKey, ServiceVersion -from ..dependencies.director import DirectorApi, get_director_api +from .._dependencies.director import DirectorApi, get_director_api router = APIRouter() diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services_labels.py b/services/catalog/src/simcore_service_catalog/api/rest/_services_labels.py index 11b40ad7b55e..56af14626e20 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services_labels.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services_labels.py @@ -3,7 +3,7 @@ from fastapi import APIRouter, Depends from models_library.services import ServiceKey, ServiceVersion -from ..dependencies.director import DirectorApi, get_director_api +from .._dependencies.director import DirectorApi, get_director_api router = APIRouter() diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services_ports.py b/services/catalog/src/simcore_service_catalog/api/rest/_services_ports.py index adc7fd338a9c..ce85bcae37d4 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services_ports.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services_ports.py @@ -6,7 +6,7 @@ from models_library.services_metadata_published import ServiceMetaDataPublished from ..._constants import RESPONSE_MODEL_POLICY -from ..dependencies.services import ( +from .._dependencies.services import ( AccessInfo, check_service_read_access, get_service_from_manifest, diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py b/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py index 09d950d12499..2972b1373178 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services_resources.py @@ -29,10 +29,10 @@ merge_service_resources_with_user_specs, parse_generic_resource, ) -from ..dependencies.database import get_repository -from ..dependencies.director import DirectorApi, get_director_api -from ..dependencies.services import get_default_service_resources -from ..dependencies.user_groups import list_user_groups +from .._dependencies.database import get_repository +from .._dependencies.director import DirectorApi, get_director_api +from .._dependencies.services import get_default_service_resources +from .._dependencies.user_groups import list_user_groups router = APIRouter() _logger = logging.getLogger(__name__) diff --git a/services/catalog/src/simcore_service_catalog/api/rest/_services_specifications.py b/services/catalog/src/simcore_service_catalog/api/rest/_services_specifications.py index b0e2036d83b5..98649f4be494 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/_services_specifications.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/_services_specifications.py @@ -13,8 +13,8 @@ from ...repository.groups import GroupsRepository from ...repository.services import ServicesRepository from ...service.function_services import is_function_service -from ..dependencies.database import get_repository -from ..dependencies.services import get_default_service_specifications +from .._dependencies.database import get_repository +from .._dependencies.services import get_default_service_specifications router = APIRouter() _logger = logging.getLogger(__name__) diff --git a/services/catalog/src/simcore_service_catalog/api/rpc/_services.py b/services/catalog/src/simcore_service_catalog/api/rpc/_services.py index b053c185d7e9..63eb5d7d6f5e 100644 --- a/services/catalog/src/simcore_service_catalog/api/rpc/_services.py +++ b/services/catalog/src/simcore_service_catalog/api/rpc/_services.py @@ -27,7 +27,7 @@ from ...repository.services import ServicesRepository from ...service import services_api -from ..dependencies.director import get_director_api +from .._dependencies.director import get_director_api _logger = logging.getLogger(__name__) diff --git a/services/catalog/src/simcore_service_catalog/core/background_tasks.py b/services/catalog/src/simcore_service_catalog/core/background_tasks.py index 7f772ae1a87e..05c9d54d68c8 100644 --- a/services/catalog/src/simcore_service_catalog/core/background_tasks.py +++ b/services/catalog/src/simcore_service_catalog/core/background_tasks.py @@ -20,7 +20,7 @@ from models_library.services_types import ServiceKey, ServiceVersion from packaging.version import Version from pydantic import ValidationError -from simcore_service_catalog.api.dependencies.director import get_director_api +from simcore_service_catalog.api._dependencies.director import get_director_api from simcore_service_catalog.service import manifest from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.asyncio import AsyncEngine diff --git a/services/catalog/src/simcore_service_catalog/service/access_rights.py b/services/catalog/src/simcore_service_catalog/service/access_rights.py index b6049d93521f..c8009ed4983a 100644 --- a/services/catalog/src/simcore_service_catalog/service/access_rights.py +++ b/services/catalog/src/simcore_service_catalog/service/access_rights.py @@ -14,7 +14,7 @@ from pydantic.types import PositiveInt from sqlalchemy.ext.asyncio import AsyncEngine -from ..api.dependencies.director import get_director_api +from ..api._dependencies.director import get_director_api from ..models.services_db import ServiceAccessRightsAtDB from ..repository.groups import GroupsRepository from ..repository.services import ServicesRepository diff --git a/services/catalog/tests/unit/test_infrastructure_director.py b/services/catalog/tests/unit/test_infrastructure_director.py index d14c3956f46d..f1675a570028 100644 --- a/services/catalog/tests/unit/test_infrastructure_director.py +++ b/services/catalog/tests/unit/test_infrastructure_director.py @@ -15,7 +15,7 @@ from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from respx.router import MockRouter -from simcore_service_catalog.api.dependencies.director import get_director_api +from simcore_service_catalog.api._dependencies.director import get_director_api from simcore_service_catalog.infrastructure.director import DirectorApi diff --git a/services/catalog/tests/unit/test_service_manifest.py b/services/catalog/tests/unit/test_service_manifest.py index ad4e0dae9d8d..cd40233e589f 100644 --- a/services/catalog/tests/unit/test_service_manifest.py +++ b/services/catalog/tests/unit/test_service_manifest.py @@ -13,7 +13,7 @@ from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from respx.router import MockRouter -from simcore_service_catalog.api.dependencies.director import get_director_api +from simcore_service_catalog.api._dependencies.director import get_director_api from simcore_service_catalog.infrastructure.director import DirectorApi from simcore_service_catalog.service import manifest diff --git a/services/catalog/tests/unit/with_dbs/test_service_services_api.py b/services/catalog/tests/unit/with_dbs/test_service_services_api.py index 507e0f6b6697..b3c6239a2ab1 100644 --- a/services/catalog/tests/unit/with_dbs/test_service_services_api.py +++ b/services/catalog/tests/unit/with_dbs/test_service_services_api.py @@ -16,7 +16,7 @@ from pydantic import TypeAdapter from pytest_simcore.helpers.catalog_services import CreateFakeServiceDataCallable from respx.router import MockRouter -from simcore_service_catalog.api.dependencies.director import get_director_api +from simcore_service_catalog.api._dependencies.director import get_director_api from simcore_service_catalog.infrastructure.director import DirectorApi from simcore_service_catalog.repository.groups import GroupsRepository from simcore_service_catalog.repository.services import ServicesRepository From 9c677c02459fb31617cc2a119687488fd04b1f89 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 4 Apr 2025 17:08:24 +0200 Subject: [PATCH 08/11] =?UTF-8?q?=F0=9F=94=A7=20Refactor:=20Rename=20regis?= =?UTF-8?q?try=20sync=20task=20functions=20and=20update=20background=20tas?= =?UTF-8?q?k=20setup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/application.py | 5 +- .../core/background_tasks.py | 11 +++- .../simcore_service_catalog/core/events.py | 65 ------------------- 3 files changed, 11 insertions(+), 70 deletions(-) delete mode 100644 services/catalog/src/simcore_service_catalog/core/events.py diff --git a/services/catalog/src/simcore_service_catalog/core/application.py b/services/catalog/src/simcore_service_catalog/core/application.py index 2dc9ae0886f8..23dff35dee18 100644 --- a/services/catalog/src/simcore_service_catalog/core/application.py +++ b/services/catalog/src/simcore_service_catalog/core/application.py @@ -10,7 +10,7 @@ setup_prometheus_instrumentation, ) from servicelib.fastapi.tracing import initialize_tracing -from simcore_service_catalog.core.background_tasks import start_registry_sync_task +from simcore_service_catalog.core.background_tasks import setup_background_task from simcore_service_catalog.infrastructure.director import setup_director from simcore_service_catalog.infrastructure.postgres import setup_postgres_database from starlette.middleware.base import BaseHTTPMiddleware @@ -92,8 +92,7 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI: setup_director(app) setup_function_services(app) setup_rabbitmq(app) - - app.add_event_handler("startup", start_registry_sync_task) + setup_background_task(app) if app.state.settings.CATALOG_PROMETHEUS_INSTRUMENTATION_ENABLED: setup_prometheus_instrumentation(app) diff --git a/services/catalog/src/simcore_service_catalog/core/background_tasks.py b/services/catalog/src/simcore_service_catalog/core/background_tasks.py index 05c9d54d68c8..fdf8dde77496 100644 --- a/services/catalog/src/simcore_service_catalog/core/background_tasks.py +++ b/services/catalog/src/simcore_service_catalog/core/background_tasks.py @@ -213,7 +213,7 @@ async def _sync_services_task(app: FastAPI) -> None: ) -async def start_registry_sync_task(app: FastAPI) -> None: +async def _start_registry_sync_task(app: FastAPI) -> None: # FIXME: added this variable to overcome the state in which the # task cancelation is ignored and the exceptions enter in a loop # that never stops the background task. This flag is an additional @@ -224,7 +224,7 @@ async def start_registry_sync_task(app: FastAPI) -> None: _logger.info("registry syncing task started") -async def stop_registry_sync_task(app: FastAPI) -> None: +async def _stop_registry_sync_task(app: FastAPI) -> None: if task := app.state.registry_sync_task: with suppress(asyncio.CancelledError): app.state.registry_syncer_running = False @@ -232,3 +232,10 @@ async def stop_registry_sync_task(app: FastAPI) -> None: await task app.state.registry_sync_task = None _logger.info("registry syncing task stopped") + + +def setup_background_task(app: FastAPI): + # FIXME: check director service is in place and ready. Hand-shake?? + # SEE https://github.com/ITISFoundation/osparc-simcore/issues/1728 + app.add_event_handler("startup", _start_registry_sync_task) + app.add_event_handler("shutdown", _stop_registry_sync_task) diff --git a/services/catalog/src/simcore_service_catalog/core/events.py b/services/catalog/src/simcore_service_catalog/core/events.py deleted file mode 100644 index 3c7cb6ca3025..000000000000 --- a/services/catalog/src/simcore_service_catalog/core/events.py +++ /dev/null @@ -1,65 +0,0 @@ -import logging -from collections.abc import Awaitable, Callable -from typing import TypeAlias - -from fastapi import FastAPI -from servicelib.fastapi.db_asyncpg_engine import close_db_connection, connect_to_db -from servicelib.logging_utils import log_context - -from .._meta import APP_FINISHED_BANNER_MSG, APP_STARTED_BANNER_MSG -from ..db.events import setup_default_product -from ..infrastructure.director import close_director, setup_director -from .background_tasks import start_registry_sync_task, stop_registry_sync_task - -_logger = logging.getLogger(__name__) - - -EventCallable: TypeAlias = Callable[[], Awaitable[None]] - - -def _flush_started_banner() -> None: - # WARNING: this function is spied in the tests - print(APP_STARTED_BANNER_MSG, flush=True) # noqa: T201 - - -def _flush_finished_banner() -> None: - print(APP_FINISHED_BANNER_MSG, flush=True) # noqa: T201 - - -def create_on_startup(app: FastAPI) -> EventCallable: - async def _() -> None: - _flush_started_banner() - - # setup connection to pg db - if app.state.settings.CATALOG_POSTGRES: - await connect_to_db(app, app.state.settings.CATALOG_POSTGRES) - await setup_default_product(app) - - if app.state.settings.CATALOG_DIRECTOR: - # setup connection to director - await setup_director(app) - - # FIXME: check director service is in place and ready. Hand-shake?? - # SEE https://github.com/ITISFoundation/osparc-simcore/issues/1728 - await start_registry_sync_task(app) - - _logger.info("Application started") - - return _ - - -def create_on_shutdown(app: FastAPI) -> EventCallable: - async def _() -> None: - - with log_context(_logger, logging.INFO, "Application shutdown"): - if app.state.settings.CATALOG_DIRECTOR: - try: - await stop_registry_sync_task(app) - await close_director(app) - await close_db_connection(app) - except Exception: # pylint: disable=broad-except - _logger.exception("Unexpected error while closing application") - - _flush_finished_banner() - - return _ From 8b8d89d7aa3d014f0f8146852f712c1469a004ee Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 4 Apr 2025 18:44:18 +0200 Subject: [PATCH 09/11] =?UTF-8?q?=F0=9F=94=A7=20Refactor:=20Update=20setup?= =?UTF-8?q?=20functions=20to=20use=20AsyncIterator=20for=20improved=20life?= =?UTF-8?q?cycle=20management?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../servicelib/fastapi/db_asyncpg_engine.py | 10 +++- .../core/application.py | 44 +++++++++------ .../core/background_tasks.py | 30 +++++----- .../infrastructure/director.py | 55 ++++++++----------- .../infrastructure/postgres.py | 33 ++++++----- .../infrastructure/rabbitmq.py | 27 ++++----- .../repository/setup.py | 8 ++- .../service/function_services.py | 14 +++-- 8 files changed, 116 insertions(+), 105 deletions(-) diff --git a/packages/service-library/src/servicelib/fastapi/db_asyncpg_engine.py b/packages/service-library/src/servicelib/fastapi/db_asyncpg_engine.py index 920f68008ae8..cc9932c1dfba 100644 --- a/packages/service-library/src/servicelib/fastapi/db_asyncpg_engine.py +++ b/packages/service-library/src/servicelib/fastapi/db_asyncpg_engine.py @@ -13,13 +13,17 @@ _logger = logging.getLogger(__name__) -async def connect_to_db(app: FastAPI, settings: PostgresSettings) -> None: +async def connect_to_postgres_until_ready(settings: PostgresSettings) -> AsyncEngine: with log_context( _logger, logging.DEBUG, - f"Connecting and migraging {settings.dsn_with_async_sqlalchemy}", + f"Connecting to {settings.dsn_with_async_sqlalchemy}", ): - engine = await create_async_engine_and_pg_database_ready(settings) + return await create_async_engine_and_pg_database_ready(settings) + + +async def connect_to_db(app: FastAPI, settings: PostgresSettings) -> None: + engine = await connect_to_postgres_until_ready(settings) app.state.engine = engine _logger.debug( diff --git a/services/catalog/src/simcore_service_catalog/core/application.py b/services/catalog/src/simcore_service_catalog/core/application.py index 23dff35dee18..81fa5e48dbba 100644 --- a/services/catalog/src/simcore_service_catalog/core/application.py +++ b/services/catalog/src/simcore_service_catalog/core/application.py @@ -1,7 +1,9 @@ import logging +from collections.abc import AsyncIterator from fastapi import FastAPI from fastapi.middleware.gzip import GZipMiddleware +from fastapi_lifespan_manager import LifespanManager, State from models_library.basic_types import BootModeEnum from servicelib.fastapi import timing_middleware from servicelib.fastapi.openapi import override_fastapi_openapi_method @@ -11,8 +13,6 @@ ) from servicelib.fastapi.tracing import initialize_tracing from simcore_service_catalog.core.background_tasks import setup_background_task -from simcore_service_catalog.infrastructure.director import setup_director -from simcore_service_catalog.infrastructure.postgres import setup_postgres_database from starlette.middleware.base import BaseHTTPMiddleware from .._meta import ( @@ -27,7 +27,10 @@ from ..api.rest.routes import setup_rest_api_routes from ..api.rpc.routes import setup_rpc_api_routes from ..exceptions.handlers import setup_exception_handlers -from ..infrastructure.rabbitmq import setup_rabbitmq +from ..infrastructure.director import director_lifespan +from ..infrastructure.postgres import postgres_lifespan +from ..infrastructure.rabbitmq import rabbitmq_lifespan +from ..repository.setup import setup_repository from ..service.function_services import setup_function_services from .settings import ApplicationSettings @@ -44,15 +47,33 @@ ) -def _flush_started_banner() -> None: +async def _setup_banner(app: FastAPI) -> AsyncIterator[State]: # WARNING: this function is spied in the tests + assert app print(APP_STARTED_BANNER_MSG, flush=True) # noqa: T201 + yield {} -def _flush_finished_banner() -> None: print(APP_FINISHED_BANNER_MSG, flush=True) # noqa: T201 +def _create_app_lifespan(settings: ApplicationSettings): + app_lifespan = LifespanManager() + + app_lifespan.add(_setup_banner) + + postgres_lifespan.add(setup_repository) + app_lifespan.include(postgres_lifespan) + + app_lifespan.include(director_lifespan) + app_lifespan.add(rabbitmq_lifespan) + + app_lifespan.add(setup_function_services) + app_lifespan.add(setup_background_task) + + return app_lifespan + + def create_app(settings: ApplicationSettings | None = None) -> FastAPI: # keep mostly quiet noisy loggers quiet_level: int = max( @@ -76,6 +97,7 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI: openapi_url=f"/api/{API_VTAG}/openapi.json", docs_url="/dev/doc", redoc_url=None, # default disabled + lifespan=_create_app_lifespan(settings), ) override_fastapi_openapi_method(app) @@ -85,15 +107,6 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI: if settings.CATALOG_TRACING: initialize_tracing(app, settings.CATALOG_TRACING, APP_NAME) - # STARTUP-EVENT - app.add_event_handler("startup", _flush_started_banner) - - setup_postgres_database(app) - setup_director(app) - setup_function_services(app) - setup_rabbitmq(app) - setup_background_task(app) - if app.state.settings.CATALOG_PROMETHEUS_INSTRUMENTATION_ENABLED: setup_prometheus_instrumentation(app) @@ -113,9 +126,6 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI: setup_rest_api_routes(app, vtag=API_VTAG) setup_rpc_api_routes(app) - # SHUTDOWN-EVENT - app.add_event_handler("shutdown", _flush_finished_banner) - # EXCEPTIONS setup_exception_handlers(app) diff --git a/services/catalog/src/simcore_service_catalog/core/background_tasks.py b/services/catalog/src/simcore_service_catalog/core/background_tasks.py index fdf8dde77496..19654dbd2726 100644 --- a/services/catalog/src/simcore_service_catalog/core/background_tasks.py +++ b/services/catalog/src/simcore_service_catalog/core/background_tasks.py @@ -11,11 +11,13 @@ import asyncio import logging +from collections.abc import AsyncIterator from contextlib import suppress from pprint import pformat from typing import Final from fastapi import FastAPI, HTTPException +from fastapi_lifespan_manager import State from models_library.services import ServiceMetaDataPublished from models_library.services_types import ServiceKey, ServiceVersion from packaging.version import Version @@ -213,29 +215,23 @@ async def _sync_services_task(app: FastAPI) -> None: ) -async def _start_registry_sync_task(app: FastAPI) -> None: +async def setup_background_task(app: FastAPI) -> AsyncIterator[State]: + # FIXME: check director service is in place and ready. Hand-shake?? + # SEE https://github.com/ITISFoundation/osparc-simcore/issues/1728 + # FIXME: added this variable to overcome the state in which the # task cancelation is ignored and the exceptions enter in a loop # that never stops the background task. This flag is an additional # mechanism to enforce stopping the background task - app.state.registry_syncer_running = True task = asyncio.create_task(_sync_services_task(app)) - app.state.registry_sync_task = task + _logger.info("registry syncing task started") + yield {"registry_syncer_running": True, "registry_sync_task": task} -async def _stop_registry_sync_task(app: FastAPI) -> None: - if task := app.state.registry_sync_task: - with suppress(asyncio.CancelledError): - app.state.registry_syncer_running = False - task.cancel() - await task - app.state.registry_sync_task = None - _logger.info("registry syncing task stopped") + with suppress(asyncio.CancelledError): + app.state.registry_syncer_running = False + task.cancel() + await task - -def setup_background_task(app: FastAPI): - # FIXME: check director service is in place and ready. Hand-shake?? - # SEE https://github.com/ITISFoundation/osparc-simcore/issues/1728 - app.add_event_handler("startup", _start_registry_sync_task) - app.add_event_handler("shutdown", _stop_registry_sync_task) + _logger.info("registry syncing task stopped") diff --git a/services/catalog/src/simcore_service_catalog/infrastructure/director.py b/services/catalog/src/simcore_service_catalog/infrastructure/director.py index e6bf72502d9e..8ac9f8af6c05 100644 --- a/services/catalog/src/simcore_service_catalog/infrastructure/director.py +++ b/services/catalog/src/simcore_service_catalog/infrastructure/director.py @@ -3,7 +3,7 @@ import json import logging import urllib.parse -from collections.abc import Awaitable, Callable +from collections.abc import AsyncIterator, Awaitable, Callable from contextlib import suppress from pprint import pformat from typing import Any, Final @@ -11,6 +11,7 @@ import httpx from common_library.json_serialization import json_dumps from fastapi import FastAPI, HTTPException +from fastapi_lifespan_manager import LifespanManager, State from models_library.api_schemas_directorv2.services import ServiceExtras from models_library.services_metadata_published import ServiceMetaDataPublished from models_library.services_types import ServiceKey, ServiceVersion @@ -288,38 +289,30 @@ async def get_service_extras( return TypeAdapter(ServiceExtras).validate_python(result) -async def _initialize_director_client(app: FastAPI) -> None: - if settings := app.state.settings.CATALOG_DIRECTOR: - with log_context( - _logger, logging.DEBUG, "Setup director at %s", f"{settings.base_url=}" - ): - async for attempt in AsyncRetrying(**_director_startup_retry_policy): - client = DirectorApi(base_url=settings.base_url, app=app) - with attempt: - client = DirectorApi(base_url=settings.base_url, app=app) - if not await client.is_responsive(): - with suppress(Exception): - await client.close() - raise DirectorUnresponsiveError - - _logger.info( - "Connection to director-v0 succeded [%s]", - json_dumps(attempt.retry_state.retry_object.statistics), - ) - - # set when connected - app.state.director_api = client +director_lifespan = LifespanManager() -async def _shutdown_director_client(app: FastAPI) -> None: - client: DirectorApi | None - if client := app.state.director_api: - await client.close() - - _logger.debug("Director client closed successfully") +@director_lifespan.add +async def _setup_director(app: FastAPI) -> AsyncIterator[State]: + settings = app.state.settings.CATALOG_DIRECTOR + with log_context( + _logger, logging.DEBUG, "Setup director at %s", f"{settings.base_url=}" + ): + async for attempt in AsyncRetrying(**_director_startup_retry_policy): + client = DirectorApi(base_url=settings.base_url, app=app) + with attempt: + client = DirectorApi(base_url=settings.base_url, app=app) + if not await client.is_responsive(): + with suppress(Exception): + await client.close() + raise DirectorUnresponsiveError + + _logger.info( + "Connection to director-v0 succeded [%s]", + json_dumps(attempt.retry_state.retry_object.statistics), + ) -def setup_director(app: FastAPI): + yield {"director_api": client} - app.add_event_handler("startup", _initialize_director_client) - app.add_event_handler("shutdown", _shutdown_director_client) + await client.close() diff --git a/services/catalog/src/simcore_service_catalog/infrastructure/postgres.py b/services/catalog/src/simcore_service_catalog/infrastructure/postgres.py index a42b52e08a4f..5cbd3c89ee70 100644 --- a/services/catalog/src/simcore_service_catalog/infrastructure/postgres.py +++ b/services/catalog/src/simcore_service_catalog/infrastructure/postgres.py @@ -1,30 +1,29 @@ -import contextlib import logging +from collections.abc import AsyncIterator from fastapi import FastAPI -from servicelib.fastapi.db_asyncpg_engine import close_db_connection, connect_to_db +from fastapi_lifespan_manager import LifespanManager, State +from servicelib.fastapi.db_asyncpg_engine import connect_to_postgres_until_ready from servicelib.logging_utils import log_catch, log_context - -from ..repository.products import setup_default_product +from sqlalchemy.ext.asyncio import AsyncEngine _logger = logging.getLogger(__name__) -def setup_postgres_database(app: FastAPI): +postgres_lifespan = LifespanManager() - async def _(): - with log_context(_logger, logging.INFO, f"{__name__} startup ..."): - # connection - await connect_to_db(app, app.state.settings.CATALOG_POSTGRES) - # configuring default product - await setup_default_product(app) +@postgres_lifespan.add +async def setup_postgres_database(app: FastAPI) -> AsyncIterator[State]: - yield + with log_context(_logger, logging.INFO, f"{__name__} startup ..."): + engine: AsyncEngine = await connect_to_postgres_until_ready( + app.state.settings.CATALOG_POSTGRES + ) - with log_context( - _logger, logging.INFO, f"{__name__} shutdown ..." - ), contextlib.suppress(Exception), log_catch(_logger): - await close_db_connection(app) + yield {"engine": engine} - return _ + with log_context(_logger, logging.INFO, f"{__name__} shutdown ..."), log_catch( + _logger, reraise=False + ): + await engine.dispose() diff --git a/services/catalog/src/simcore_service_catalog/infrastructure/rabbitmq.py b/services/catalog/src/simcore_service_catalog/infrastructure/rabbitmq.py index 8400885efa07..9b717869c95b 100644 --- a/services/catalog/src/simcore_service_catalog/infrastructure/rabbitmq.py +++ b/services/catalog/src/simcore_service_catalog/infrastructure/rabbitmq.py @@ -1,7 +1,9 @@ import logging +from collections.abc import AsyncIterator from typing import cast from fastapi import FastAPI +from fastapi_lifespan_manager import LifespanManager, State from servicelib.rabbitmq import RabbitMQRPCClient, wait_till_rabbitmq_responsive from settings_library.rabbit import RabbitSettings @@ -15,24 +17,23 @@ def get_rabbitmq_settings(app: FastAPI) -> RabbitSettings: return settings -def setup_rabbitmq(app: FastAPI) -> None: +rabbitmq_lifespan = LifespanManager() + + +@rabbitmq_lifespan.add +async def setup_rabbitmq(app: FastAPI) -> AsyncIterator[State]: settings: RabbitSettings = get_rabbitmq_settings(app) - app.state.rabbitmq_rpc_server = None - async def _on_startup() -> None: - await wait_till_rabbitmq_responsive(settings.dsn) + await wait_till_rabbitmq_responsive(settings.dsn) - app.state.rabbitmq_rpc_server = await RabbitMQRPCClient.create( - client_name=f"{PROJECT_NAME}_rpc_server", settings=settings - ) + rabbitmq_rpc_server = await RabbitMQRPCClient.create( + client_name=f"{PROJECT_NAME}_rpc_server", + settings=settings, + ) - async def _on_shutdown() -> None: - if app.state.rabbitmq_rpc_server: - await app.state.rabbitmq_rpc_server.close() - app.state.rabbitmq_rpc_server = None + yield {"rabbitmq_rpc_server": rabbitmq_rpc_server} - app.add_event_handler("startup", _on_startup) - app.add_event_handler("shutdown", _on_shutdown) + await rabbitmq_rpc_server.close() def get_rabbitmq_rpc_server(app: FastAPI) -> RabbitMQRPCClient: diff --git a/services/catalog/src/simcore_service_catalog/repository/setup.py b/services/catalog/src/simcore_service_catalog/repository/setup.py index f6e5b0d68cda..9c9c8eb80f49 100644 --- a/services/catalog/src/simcore_service_catalog/repository/setup.py +++ b/services/catalog/src/simcore_service_catalog/repository/setup.py @@ -1,8 +1,12 @@ +from collections.abc import AsyncIterator + from fastapi import FastAPI +from fastapi_lifespan_manager import State from .products import ProductsRepository -async def setup_repository(app: FastAPI): +async def setup_repository(app: FastAPI) -> AsyncIterator[State]: repo = ProductsRepository(db_engine=app.state.engine) - app.state.default_product_name = await repo.get_default_product_name() + + yield {"default_product_name": await repo.get_default_product_name()} diff --git a/services/catalog/src/simcore_service_catalog/service/function_services.py b/services/catalog/src/simcore_service_catalog/service/function_services.py index 7ed546f251b0..c0a6ff8e9d42 100644 --- a/services/catalog/src/simcore_service_catalog/service/function_services.py +++ b/services/catalog/src/simcore_service_catalog/service/function_services.py @@ -1,9 +1,12 @@ +from collections.abc import AsyncIterator + # mypy: disable-error-code=truthy-function from typing import Any from fastapi import status from fastapi.applications import FastAPI from fastapi.exceptions import HTTPException +from fastapi_lifespan_manager import State from models_library.function_services_catalog import ( is_function_service, iter_service_docker_data, @@ -31,12 +34,13 @@ def get_function_service(key, version) -> ServiceMetaDataPublished: ) from err -def setup_function_services(app: FastAPI): - def _on_startup() -> None: - catalog = [_as_dict(metadata) for metadata in iter_service_docker_data()] - app.state.frontend_services_catalog = catalog +async def setup_function_services(app: FastAPI) -> AsyncIterator[State]: + assert app # nosec + assert not hasattr(app.state, "frontend_services_catalog") # nosec + + catalog = [_as_dict(metadata) for metadata in iter_service_docker_data()] - app.add_event_handler("startup", _on_startup) + yield {"frontend_services_catalog": catalog} __all__: tuple[str, ...] = ( From 12089190a1283abd5d2104acf64eae2ed53bd214 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 4 Apr 2025 18:55:05 +0200 Subject: [PATCH 10/11] =?UTF-8?q?=F0=9F=94=A7=20Refactor:=20Update=20RPC?= =?UTF-8?q?=20API=20route=20setup=20to=20use=20AsyncIterator=20for=20impro?= =?UTF-8?q?ved=20lifecycle=20management?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/simcore_service_catalog/api/rpc/routes.py | 11 ++++++----- .../src/simcore_service_catalog/core/application.py | 10 ++++++++-- .../infrastructure/director.py | 2 +- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/services/catalog/src/simcore_service_catalog/api/rpc/routes.py b/services/catalog/src/simcore_service_catalog/api/rpc/routes.py index 2843db3e5b26..14d5963dabbb 100644 --- a/services/catalog/src/simcore_service_catalog/api/rpc/routes.py +++ b/services/catalog/src/simcore_service_catalog/api/rpc/routes.py @@ -1,6 +1,8 @@ import logging +from collections.abc import AsyncIterator from fastapi import FastAPI +from fastapi_lifespan_manager import State from models_library.api_schemas_catalog import CATALOG_RPC_NAMESPACE from ...infrastructure.rabbitmq import get_rabbitmq_rpc_server @@ -9,9 +11,8 @@ _logger = logging.getLogger(__name__) -def setup_rpc_api_routes(app: FastAPI) -> None: - async def _on_startup() -> None: - rpc_server = get_rabbitmq_rpc_server(app) - await rpc_server.register_router(_services.router, CATALOG_RPC_NAMESPACE, app) +async def setup_rpc_api_routes(app: FastAPI) -> AsyncIterator[State]: + rpc_server = get_rabbitmq_rpc_server(app) + await rpc_server.register_router(_services.router, CATALOG_RPC_NAMESPACE, app) - app.add_event_handler("startup", _on_startup) + yield {} diff --git a/services/catalog/src/simcore_service_catalog/core/application.py b/services/catalog/src/simcore_service_catalog/core/application.py index 81fa5e48dbba..dc3905c8e007 100644 --- a/services/catalog/src/simcore_service_catalog/core/application.py +++ b/services/catalog/src/simcore_service_catalog/core/application.py @@ -58,14 +58,21 @@ async def _setup_banner(app: FastAPI) -> AsyncIterator[State]: def _create_app_lifespan(settings: ApplicationSettings): - app_lifespan = LifespanManager() + assert settings # nosec + # app lifespan + app_lifespan = LifespanManager() app_lifespan.add(_setup_banner) + # - postgres lifespan postgres_lifespan.add(setup_repository) app_lifespan.include(postgres_lifespan) + # - director lifespan app_lifespan.include(director_lifespan) + + # - rabbitmq lifespan + rabbitmq_lifespan.add(setup_rpc_api_routes) app_lifespan.add(rabbitmq_lifespan) app_lifespan.add(setup_function_services) @@ -124,7 +131,6 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI: # ROUTES setup_rest_api_routes(app, vtag=API_VTAG) - setup_rpc_api_routes(app) # EXCEPTIONS setup_exception_handlers(app) diff --git a/services/catalog/src/simcore_service_catalog/infrastructure/director.py b/services/catalog/src/simcore_service_catalog/infrastructure/director.py index 8ac9f8af6c05..b87d838158fc 100644 --- a/services/catalog/src/simcore_service_catalog/infrastructure/director.py +++ b/services/catalog/src/simcore_service_catalog/infrastructure/director.py @@ -293,7 +293,7 @@ async def get_service_extras( @director_lifespan.add -async def _setup_director(app: FastAPI) -> AsyncIterator[State]: +async def setup_director(app: FastAPI) -> AsyncIterator[State]: settings = app.state.settings.CATALOG_DIRECTOR with log_context( From bff6724acbb497e02e578b162ff164bcf94b4957 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 4 Apr 2025 19:17:59 +0200 Subject: [PATCH 11/11] =?UTF-8?q?=F0=9F=94=A7=20Refactor:=20Introduce=20ce?= =?UTF-8?q?ntralized=20error=20handling=20and=20update=20route=20setup=20f?= =?UTF-8?q?or=20improved=20clarity=20and=20maintainability?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api/rest/errors.py | 6 + .../api/rest/routes.py | 110 +++++++++--------- .../simcore_service_catalog/api/rpc/routes.py | 2 +- .../core/application.py | 14 +-- .../{exceptions => }/errors.py | 9 +- .../exceptions/__init__.py | 0 .../exceptions/handlers/__init__.py | 24 ---- .../exceptions/handlers/_http_error.py | 33 ------ .../exceptions/handlers/_validation_error.py | 23 ---- .../infrastructure/director.py | 2 +- .../repository/groups.py | 2 +- 11 files changed, 73 insertions(+), 152 deletions(-) create mode 100644 services/catalog/src/simcore_service_catalog/api/rest/errors.py rename services/catalog/src/simcore_service_catalog/{exceptions => }/errors.py (69%) delete mode 100644 services/catalog/src/simcore_service_catalog/exceptions/__init__.py delete mode 100644 services/catalog/src/simcore_service_catalog/exceptions/handlers/__init__.py delete mode 100644 services/catalog/src/simcore_service_catalog/exceptions/handlers/_http_error.py delete mode 100644 services/catalog/src/simcore_service_catalog/exceptions/handlers/_validation_error.py diff --git a/services/catalog/src/simcore_service_catalog/api/rest/errors.py b/services/catalog/src/simcore_service_catalog/api/rest/errors.py new file mode 100644 index 000000000000..e8e1727c8ce4 --- /dev/null +++ b/services/catalog/src/simcore_service_catalog/api/rest/errors.py @@ -0,0 +1,6 @@ +from fastapi import FastAPI +from servicelib.fastapi.http_error import set_app_default_http_error_handlers + + +def setup_rest_exception_handlers(app: FastAPI) -> None: + set_app_default_http_error_handlers(app) diff --git a/services/catalog/src/simcore_service_catalog/api/rest/routes.py b/services/catalog/src/simcore_service_catalog/api/rest/routes.py index 91e9329d0202..f7e585c9fc76 100644 --- a/services/catalog/src/simcore_service_catalog/api/rest/routes.py +++ b/services/catalog/src/simcore_service_catalog/api/rest/routes.py @@ -14,67 +14,67 @@ _services_specifications, ) -v0_router = APIRouter() -# health -health_router = _health.router -v0_router.include_router( - _health.router, - tags=["diagnostics"], -) +def setup_rest_routes(app: FastAPI, vtag: str): + v0_router = APIRouter() -# meta -v0_router.include_router( - _meta.router, - tags=["meta"], - prefix="/meta", -) + # health + health_router = _health.router + v0_router.include_router( + _health.router, + tags=["diagnostics"], + ) -# services -_SERVICE_PREFIX = "/services" -_SERVICE_TAGS: list[str | Enum] = [ - "services", -] -v0_router.include_router( - _services_resources.router, - tags=_SERVICE_TAGS, - prefix=_SERVICE_PREFIX, -) -v0_router.include_router( - _services_labels.router, - tags=_SERVICE_TAGS, - prefix=_SERVICE_PREFIX, -) -v0_router.include_router( - _services_extras.router, - tags=_SERVICE_TAGS, - prefix=_SERVICE_PREFIX, -) -v0_router.include_router( - _services_specifications.router, - tags=_SERVICE_TAGS, - prefix=_SERVICE_PREFIX, -) -v0_router.include_router( - _services_ports.router, - tags=_SERVICE_TAGS, - prefix=_SERVICE_PREFIX, -) -v0_router.include_router( - _services_access_rights.router, - tags=_SERVICE_TAGS, - prefix=_SERVICE_PREFIX, -) + # meta + v0_router.include_router( + _meta.router, + tags=["meta"], + prefix="/meta", + ) -# NOTE: that this router must come after resources/specifications/ports/access_rights -v0_router.include_router( - _services.router, - tags=_SERVICE_TAGS, - prefix=_SERVICE_PREFIX, -) + # services + _SERVICE_PREFIX = "/services" + _SERVICE_TAGS: list[str | Enum] = [ + "services", + ] + v0_router.include_router( + _services_resources.router, + tags=_SERVICE_TAGS, + prefix=_SERVICE_PREFIX, + ) + v0_router.include_router( + _services_labels.router, + tags=_SERVICE_TAGS, + prefix=_SERVICE_PREFIX, + ) + v0_router.include_router( + _services_extras.router, + tags=_SERVICE_TAGS, + prefix=_SERVICE_PREFIX, + ) + v0_router.include_router( + _services_specifications.router, + tags=_SERVICE_TAGS, + prefix=_SERVICE_PREFIX, + ) + v0_router.include_router( + _services_ports.router, + tags=_SERVICE_TAGS, + prefix=_SERVICE_PREFIX, + ) + v0_router.include_router( + _services_access_rights.router, + tags=_SERVICE_TAGS, + prefix=_SERVICE_PREFIX, + ) + # NOTE: that this router must come after resources/specifications/ports/access_rights + v0_router.include_router( + _services.router, + tags=_SERVICE_TAGS, + prefix=_SERVICE_PREFIX, + ) -def setup_rest_api_routes(app: FastAPI, vtag: str): # healthcheck at / and at /v0/ app.include_router(health_router) # api under /v* diff --git a/services/catalog/src/simcore_service_catalog/api/rpc/routes.py b/services/catalog/src/simcore_service_catalog/api/rpc/routes.py index 14d5963dabbb..a1b2270784aa 100644 --- a/services/catalog/src/simcore_service_catalog/api/rpc/routes.py +++ b/services/catalog/src/simcore_service_catalog/api/rpc/routes.py @@ -11,7 +11,7 @@ _logger = logging.getLogger(__name__) -async def setup_rpc_api_routes(app: FastAPI) -> AsyncIterator[State]: +async def setup_rpc_routes(app: FastAPI) -> AsyncIterator[State]: rpc_server = get_rabbitmq_rpc_server(app) await rpc_server.register_router(_services.router, CATALOG_RPC_NAMESPACE, app) diff --git a/services/catalog/src/simcore_service_catalog/core/application.py b/services/catalog/src/simcore_service_catalog/core/application.py index dc3905c8e007..45e911fa5c7d 100644 --- a/services/catalog/src/simcore_service_catalog/core/application.py +++ b/services/catalog/src/simcore_service_catalog/core/application.py @@ -24,9 +24,9 @@ PROJECT_NAME, SUMMARY, ) -from ..api.rest.routes import setup_rest_api_routes -from ..api.rpc.routes import setup_rpc_api_routes -from ..exceptions.handlers import setup_exception_handlers +from ..api.rest.errors import setup_rest_exception_handlers +from ..api.rest.routes import setup_rest_routes +from ..api.rpc.routes import setup_rpc_routes from ..infrastructure.director import director_lifespan from ..infrastructure.postgres import postgres_lifespan from ..infrastructure.rabbitmq import rabbitmq_lifespan @@ -72,7 +72,7 @@ def _create_app_lifespan(settings: ApplicationSettings): app_lifespan.include(director_lifespan) # - rabbitmq lifespan - rabbitmq_lifespan.add(setup_rpc_api_routes) + rabbitmq_lifespan.add(setup_rpc_routes) app_lifespan.add(rabbitmq_lifespan) app_lifespan.add(setup_function_services) @@ -130,9 +130,7 @@ def create_app(settings: ApplicationSettings | None = None) -> FastAPI: app.add_middleware(GZipMiddleware) # ROUTES - setup_rest_api_routes(app, vtag=API_VTAG) - - # EXCEPTIONS - setup_exception_handlers(app) + setup_rest_routes(app, vtag=API_VTAG) + setup_rest_exception_handlers(app) return app diff --git a/services/catalog/src/simcore_service_catalog/exceptions/errors.py b/services/catalog/src/simcore_service_catalog/errors.py similarity index 69% rename from services/catalog/src/simcore_service_catalog/exceptions/errors.py rename to services/catalog/src/simcore_service_catalog/errors.py index 84010d9a7007..7e33eb08d0d7 100644 --- a/services/catalog/src/simcore_service_catalog/exceptions/errors.py +++ b/services/catalog/src/simcore_service_catalog/errors.py @@ -1,8 +1,7 @@ from common_library.errors_classes import OsparcErrorMixin -class CatalogBaseError(OsparcErrorMixin, Exception): - ... +class CatalogBaseError(OsparcErrorMixin, Exception): ... class RepositoryError(CatalogBaseError): @@ -13,13 +12,11 @@ class UninitializedGroupError(RepositoryError): msg_tempalte = "{group} groups was never initialized" -class BaseDirectorError(CatalogBaseError): - ... +class BaseDirectorError(CatalogBaseError): ... class DirectorUnresponsiveError(BaseDirectorError): msg_template = "Director-v0 is not responsive" -class DirectorStatusError(BaseDirectorError): - ... +class DirectorStatusError(BaseDirectorError): ... diff --git a/services/catalog/src/simcore_service_catalog/exceptions/__init__.py b/services/catalog/src/simcore_service_catalog/exceptions/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/services/catalog/src/simcore_service_catalog/exceptions/handlers/__init__.py b/services/catalog/src/simcore_service_catalog/exceptions/handlers/__init__.py deleted file mode 100644 index 49620d73f6ca..000000000000 --- a/services/catalog/src/simcore_service_catalog/exceptions/handlers/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -from fastapi import FastAPI, HTTPException, status -from fastapi.exceptions import RequestValidationError - -from ._http_error import http_error_handler, make_http_error_handler_for_exception -from ._validation_error import http422_error_handler - - -def setup_exception_handlers(app: FastAPI) -> None: - app.add_exception_handler(HTTPException, http_error_handler) - app.add_exception_handler(RequestValidationError, http422_error_handler) - - # SEE https://docs.python.org/3/library/exceptions.html#exception-hierarchy - app.add_exception_handler( - NotImplementedError, - make_http_error_handler_for_exception( - status.HTTP_501_NOT_IMPLEMENTED, NotImplementedError - ), - ) - app.add_exception_handler( - Exception, - make_http_error_handler_for_exception( - status.HTTP_500_INTERNAL_SERVER_ERROR, Exception - ), - ) diff --git a/services/catalog/src/simcore_service_catalog/exceptions/handlers/_http_error.py b/services/catalog/src/simcore_service_catalog/exceptions/handlers/_http_error.py deleted file mode 100644 index 7af4b5d93bd7..000000000000 --- a/services/catalog/src/simcore_service_catalog/exceptions/handlers/_http_error.py +++ /dev/null @@ -1,33 +0,0 @@ -from fastapi import HTTPException -from fastapi.encoders import jsonable_encoder -from starlette.requests import Request -from starlette.responses import JSONResponse -from starlette.types import HTTPExceptionHandler - - -async def http_error_handler(request: Request, exc: Exception) -> JSONResponse: - assert request # nosec - assert isinstance(exc, HTTPException) # nosec - - return JSONResponse( - content=jsonable_encoder({"errors": [exc.detail]}), status_code=exc.status_code - ) - - -def make_http_error_handler_for_exception( - status_code: int, exception_cls: type[BaseException] -) -> HTTPExceptionHandler: - """ - Produces a handler for BaseException-type exceptions which converts them - into an error JSON response with a given status code - - SEE https://docs.python.org/3/library/exceptions.html#concrete-exceptions - """ - - async def _http_error_handler(_: Request, exc: type[BaseException]) -> JSONResponse: - assert isinstance(exc, exception_cls) # nosec - return JSONResponse( - content=jsonable_encoder({"errors": [str(exc)]}), status_code=status_code - ) - - return _http_error_handler # type: ignore[return-value] diff --git a/services/catalog/src/simcore_service_catalog/exceptions/handlers/_validation_error.py b/services/catalog/src/simcore_service_catalog/exceptions/handlers/_validation_error.py deleted file mode 100644 index 8e3ad77f15db..000000000000 --- a/services/catalog/src/simcore_service_catalog/exceptions/handlers/_validation_error.py +++ /dev/null @@ -1,23 +0,0 @@ -from fastapi.encoders import jsonable_encoder -from fastapi.openapi.constants import REF_PREFIX -from fastapi.openapi.utils import validation_error_response_definition -from starlette.requests import Request -from starlette.responses import JSONResponse -from starlette.status import HTTP_422_UNPROCESSABLE_ENTITY - - -async def http422_error_handler(_: Request, exc: Exception) -> JSONResponse: - assert hasattr(exc, "errors") # nosec - return JSONResponse( - content=jsonable_encoder({"errors": exc.errors()}), - status_code=HTTP_422_UNPROCESSABLE_ENTITY, - ) - - -validation_error_response_definition["properties"] = { - "errors": { - "title": "Validation errors", - "type": "array", - "items": {"$ref": f"{REF_PREFIX}ValidationError"}, - }, -} diff --git a/services/catalog/src/simcore_service_catalog/infrastructure/director.py b/services/catalog/src/simcore_service_catalog/infrastructure/director.py index b87d838158fc..167c095ea3e5 100644 --- a/services/catalog/src/simcore_service_catalog/infrastructure/director.py +++ b/services/catalog/src/simcore_service_catalog/infrastructure/director.py @@ -25,7 +25,7 @@ from tenacity.wait import wait_random from ..core.settings import ApplicationSettings -from ..exceptions.errors import DirectorUnresponsiveError +from ..errors import DirectorUnresponsiveError _logger = logging.getLogger(__name__) diff --git a/services/catalog/src/simcore_service_catalog/repository/groups.py b/services/catalog/src/simcore_service_catalog/repository/groups.py index a58e9f9865d5..4df2d96f4b9b 100644 --- a/services/catalog/src/simcore_service_catalog/repository/groups.py +++ b/services/catalog/src/simcore_service_catalog/repository/groups.py @@ -6,7 +6,7 @@ from pydantic import TypeAdapter from pydantic.types import PositiveInt -from ..exceptions.errors import UninitializedGroupError +from ..errors import UninitializedGroupError from ._base import BaseRepository from ._tables import GroupType, groups, user_to_groups, users