♻️ Refactor simcore_service_catalog: Apply Layered Architecture and Initialization Cleanup (#7491)

Author: pcrespov

packages/models-library/src/models_library/api_schemas_directorv2/services.py

@@ -1,6 +1,7 @@
 from typing import Final
 
-from pydantic import BaseModel, ConfigDict, Field, field_validator
+from pydantic import BaseModel, ConfigDict, Field, JsonValue, field_validator
+from pydantic.config import JsonDict
 from pydantic.types import ByteSize, NonNegativeInt
 
 from ..service_settings_labels import ContainerSpec
@@ -63,36 +64,43 @@ class ServiceExtras(BaseModel):
     service_build_details: ServiceBuildDetails | None = None
     container_spec: ContainerSpec | None = None
 
+    @staticmethod
+    def _update_json_schema_extra(schema: JsonDict) -> None:
+
+        node_requirements_examples = NodeRequirements.model_json_schema()["examples"]
+
+        examples: list[JsonValue] = [
+            {"node_requirements": node_example}
+            for node_example in node_requirements_examples
+        ]
+        examples += [
+            {
+                "node_requirements": node_example,
+                "service_build_details": {
+                    "build_date": "2021-08-13T12:56:28Z",
+                    "vcs_ref": "8251ade",
+                    "vcs_url": "[email protected]:ITISFoundation/osparc-simcore.git",
+                },
+            }
+            for node_example in node_requirements_examples
+        ]
+        examples += [
+            {
+                "node_requirements": node_example,
+                "service_build_details": {
+                    "build_date": "2021-08-13T12:56:28Z",
+                    "vcs_ref": "8251ade",
+                    "vcs_url": "[email protected]:ITISFoundation/osparc-simcore.git",
+                },
+                "container_spec": {"Command": ["run", "subcommand"]},
+            }
+            for node_example in node_requirements_examples
+        ]
+
+        schema.update({"examples": examples})
+
     model_config = ConfigDict(
-        json_schema_extra={
-            "examples": [
-                {"node_requirements": node_example}
-                for node_example in NodeRequirements.model_json_schema()["examples"]
-            ]
-            + [
-                {
-                    "node_requirements": node_example,
-                    "service_build_details": {
-                        "build_date": "2021-08-13T12:56:28Z",
-                        "vcs_ref": "8251ade",
-                        "vcs_url": "[email protected]:ITISFoundation/osparc-simcore.git",
-                    },
-                }
-                for node_example in NodeRequirements.model_json_schema()["examples"]
-            ]
-            + [
-                {
-                    "node_requirements": node_example,
-                    "service_build_details": {
-                        "build_date": "2021-08-13T12:56:28Z",
-                        "vcs_ref": "8251ade",
-                        "vcs_url": "[email protected]:ITISFoundation/osparc-simcore.git",
-                    },
-                    "container_spec": {"Command": ["run", "subcommand"]},
-                }
-                for node_example in NodeRequirements.model_json_schema()["examples"]
-            ]
-        }
+        json_schema_extra=_update_json_schema_extra,
     )
 
 

packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/errors.py

@@ -5,6 +5,10 @@ class CatalogApiBaseError(OsparcErrorMixin, Exception):
     pass
 
 
+class CatalogInconsistentError(CatalogApiBaseError):
+    msg_template = "Catalog is inconsistent: The following  services are in the database but missing in the registry manifest {missing_services}"
+
+
 class CatalogItemNotFoundError(CatalogApiBaseError):
     msg_template = "{name} was not found"
 

services/catalog/src/simcore_service_catalog/api/dependencies/__init__.py renamed to services/catalog/src/simcore_service_catalog/api/_dependencies/__init__.py

@@ -3,11 +3,11 @@
 from fastapi import Depends, FastAPI
 from servicelib.fastapi.dependencies import get_app
 
-from ...services.director import DirectorApi
+from ...clients.director import DirectorClient
 
 
-def get_director_api(
+def get_director_client(
     app: Annotated[FastAPI, Depends(get_app)],
-) -> DirectorApi:
-    director: DirectorApi = app.state.director_api
+) -> DirectorClient:
+    director: DirectorClient = app.state.director_api
     return director

services/catalog/src/simcore_service_catalog/api/dependencies/director.py renamed to services/catalog/src/simcore_service_catalog/api/_dependencies/director.py

@@ -6,7 +6,7 @@
 from fastapi.requests import Request
 from sqlalchemy.ext.asyncio import AsyncEngine
 
-from ...db.repositories._base import BaseRepository
+from ...repository._base import BaseRepository
 
 _logger = logging.getLogger(__name__)
 

services/catalog/src/simcore_service_catalog/api/dependencies/database.py renamed to services/catalog/src/simcore_service_catalog/api/_dependencies/repository.py

@@ -12,26 +12,26 @@
 from pydantic import ValidationError
 from servicelib.fastapi.dependencies import get_app
 
+from ...clients.director import DirectorClient
 from ...core.settings import ApplicationSettings
-from ...db.repositories.groups import GroupsRepository
-from ...db.repositories.services import ServicesRepository
-from ...services import manifest
-from ...services.director import DirectorApi
-from .database import get_repository
-from .director import get_director_api
+from ...repository.groups import GroupsRepository
+from ...repository.services import ServicesRepository
+from ...service import manifest
+from .director import get_director_client
+from .repository import get_repository
 
 _logger = logging.getLogger(__name__)
 
 
 def get_default_service_resources(
-    app: Annotated[FastAPI, Depends(get_app)]
+    app: Annotated[FastAPI, Depends(get_app)],
 ) -> ResourcesDict:
     app_settings: ApplicationSettings = app.state.settings
     return app_settings.CATALOG_SERVICES_DEFAULT_RESOURCES
 
 
 def get_default_service_specifications(
-    app: Annotated[FastAPI, Depends(get_app)]
+    app: Annotated[FastAPI, Depends(get_app)],
 ) -> ServiceSpecifications:
     app_settings: ApplicationSettings = app.state.settings
     return app_settings.CATALOG_SERVICES_DEFAULT_SPECIFICATIONS
@@ -86,7 +86,7 @@ async def check_service_read_access(
 async def get_service_from_manifest(
     service_key: ServiceKey,
     service_version: ServiceVersion,
-    director_client: Annotated[DirectorApi, Depends(get_director_api)],
+    director_client: Annotated[DirectorClient, Depends(get_director_client)],
 ) -> ServiceMetaDataPublished:
     """
     Retrieves service metadata from the docker registry via the director

services/catalog/src/simcore_service_catalog/api/dependencies/services.py renamed to services/catalog/src/simcore_service_catalog/api/_dependencies/services.py

@@ -4,8 +4,8 @@
 from models_library.groups import GroupAtDB
 from models_library.users import UserID
 
-from ...db.repositories.groups import GroupsRepository
-from .database import get_repository
+from ...repository.groups import GroupsRepository
+from .repository import get_repository
 
 
 async def list_user_groups(

services/catalog/src/simcore_service_catalog/api/dependencies/user_groups.py renamed to services/catalog/src/simcore_service_catalog/api/_dependencies/user_groups.py

@@ -0,0 +1,10 @@
+from fastapi import FastAPI
+
+from .errors import setup_rest_api_error_handlers
+from .routes import setup_rest_api_routes
+
+
+def initialize_rest_api(app: FastAPI):
+
+    setup_rest_api_routes(app)
+    setup_rest_api_error_handlers(app)

services/catalog/src/simcore_service_catalog/api/rest/__init__.py

@@ -20,13 +20,13 @@
     LIST_SERVICES_CACHING_TTL,
     RESPONSE_MODEL_POLICY,
 )
-from ...db.repositories.groups import GroupsRepository
-from ...db.repositories.services import ServicesRepository
+from ...clients.director import DirectorClient
 from ...models.services_db import ServiceAccessRightsAtDB, ServiceMetaDataDBGet
-from ...services.director import DirectorApi
-from ..dependencies.database import get_repository
-from ..dependencies.director import get_director_api
-from ..dependencies.services import get_service_from_manifest
+from ...repository.groups import GroupsRepository
+from ...repository.services import ServicesRepository
+from .._dependencies.director import get_director_client
+from .._dependencies.repository import get_repository
+from .._dependencies.services import get_service_from_manifest
 
 _logger = logging.getLogger(__name__)
 
@@ -86,7 +86,7 @@ async def list_services(
     request: Request,  # pylint:disable=unused-argument
     *,
     user_id: PositiveInt,
-    director_client: Annotated[DirectorApi, Depends(get_director_api)],
+    director_client: Annotated[DirectorClient, Depends(get_director_client)],
     groups_repository: Annotated[
         GroupsRepository, Depends(get_repository(GroupsRepository))
     ],

services/catalog/src/simcore_service_catalog/api/rest/_services.py

@@ -8,10 +8,9 @@
 from models_library.services import ServiceKey, ServiceVersion
 
 from ..._constants import RESPONSE_MODEL_POLICY
-from ...db.repositories.services import ServicesRepository
-from ...models.services_db import ServiceAccessRightsAtDB
-from ..dependencies.database import get_repository
-from ..dependencies.services import AccessInfo, check_service_read_access
+from ...repository.services import ServicesRepository
+from .._dependencies.repository import get_repository
+from .._dependencies.services import AccessInfo, check_service_read_access
 
 _logger = logging.getLogger(__name__)
 
@@ -33,10 +32,10 @@ async def get_service_access_rights(
     ],
     x_simcore_products_name: Annotated[str, Header(...)],
 ):
-    service_access_rights: list[
-        ServiceAccessRightsAtDB
-    ] = await services_repo.get_service_access_rights(
-        key=service_key, version=service_version, product_name=x_simcore_products_name
+    service_access_rights = await services_repo.get_service_access_rights(
+        key=service_key,
+        version=service_version,
+        product_name=x_simcore_products_name,
     )
 
     gids_with_access_rights = {}