🎨 add Postgres connection to dynamic-scheduler (#7600)

Co-authored-by: Andrei Neagu <[email protected]>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

Author: 3 people

packages/service-library/src/servicelib/logging_utils.py

@@ -415,15 +415,17 @@ def log_context(
     if extra:
         kwargs["extra"] = extra
     log_msg = f"Starting {msg} ..."
-    logger.log(level, log_msg, *args, **kwargs)
+
+    stackelvel = 3  # NOTE: 1 => log_context, 2 => contextlib, 3 => caller
+    logger.log(level, log_msg, *args, **kwargs, stacklevel=stackelvel)
     yield
     duration = (
         f" in {(datetime.now() - start ).total_seconds()}s"  # noqa: DTZ005
         if log_duration
         else ""
     )
     log_msg = f"Finished {msg}{duration}"
-    logger.log(level, log_msg, *args, **kwargs)
+    logger.log(level, log_msg, *args, **kwargs, stacklevel=stackelvel)
 
 
 def guess_message_log_level(message: str) -> LogLevelInt:

packages/service-library/tests/test_logging_utils.py

@@ -1,7 +1,10 @@
 # pylint:disable=redefined-outer-name
+# pylint:disable=unused-argument
 
 import logging
+from collections.abc import Iterable
 from contextlib import suppress
+from pathlib import Path
 from typing import Any
 
 import pytest
@@ -243,6 +246,36 @@ def test_log_context(
     assert len(caplog.messages) == 2
 
 
+@pytest.fixture
+def log_format_with_module_name() -> Iterable[None]:
+    for handler in logging.root.handlers:
+        original_formatter = handler.formatter
+        handler.setFormatter(
+            logging.Formatter(
+                "%(asctime)s %(levelname)s %(module)s:%(filename)s:%(lineno)d %(message)s",
+                datefmt="%Y-%m-%d %H:%M:%S",
+            )
+        )
+
+    yield
+
+    for handler in logging.root.handlers:
+        handler.formatter = original_formatter
+
+
+def test_log_context_caller_is_included_in_log(
+    caplog: pytest.LogCaptureFixture,
+    log_format_with_module_name: None,
+):
+    caplog.clear()
+
+    with log_context(_logger, logging.ERROR, "a test message"):
+        ...
+
+    # Verify file name is in the log
+    assert Path(__file__).name in caplog.text
+
+
 @pytest.mark.parametrize("level", _ALL_LOGGING_LEVELS, ids=_to_level_name)
 def test_logs_no_exceptions(caplog: pytest.LogCaptureFixture, level: int):
     caplog.set_level(level)

services/docker-compose.yml

@@ -552,36 +552,48 @@ services:
       - default
       - docker-api-network
     environment:
-      LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED}
+      CATALOG_HOST: ${CATALOG_HOST}
+      CATALOG_PORT: ${CATALOG_PORT}
+      DIRECTOR_V2_HOST: ${DIRECTOR_V2_HOST}
+      DIRECTOR_V2_PORT: ${DIRECTOR_V2_PORT}
+
+      DOCKER_API_PROXY_HOST: ${DOCKER_API_PROXY_HOST}
+      DOCKER_API_PROXY_PASSWORD: ${DOCKER_API_PROXY_PASSWORD}
+      DOCKER_API_PROXY_PORT: ${DOCKER_API_PROXY_PORT}
+      DOCKER_API_PROXY_SECURE: ${DOCKER_API_PROXY_SECURE}
+      DOCKER_API_PROXY_USER: ${DOCKER_API_PROXY_USER}
+
+      DYNAMIC_SCHEDULER_LOGLEVEL: ${DYNAMIC_SCHEDULER_LOGLEVEL}
+      DYNAMIC_SCHEDULER_PROFILING: ${DYNAMIC_SCHEDULER_PROFILING}
+      DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT: ${DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT}
+      DYNAMIC_SCHEDULER_TRACING: ${DYNAMIC_SCHEDULER_TRACING}
+      DYNAMIC_SCHEDULER_UI_STORAGE_SECRET: ${DYNAMIC_SCHEDULER_UI_STORAGE_SECRET}
+      DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER: ${DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER}
+      DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT: ${DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT}
+
       LOG_FILTER_MAPPING : ${LOG_FILTER_MAPPING}
+      LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED}
+
+      POSTGRES_DB: ${POSTGRES_DB}
+      POSTGRES_HOST: ${POSTGRES_HOST}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      POSTGRES_PORT: ${POSTGRES_PORT}
+      POSTGRES_USER: ${POSTGRES_USER}
+
       RABBIT_HOST: ${RABBIT_HOST}
       RABBIT_PASSWORD: ${RABBIT_PASSWORD}
       RABBIT_PORT: ${RABBIT_PORT}
       RABBIT_SECURE: ${RABBIT_SECURE}
       RABBIT_USER: ${RABBIT_USER}
+
       REDIS_HOST: ${REDIS_HOST}
+      REDIS_PASSWORD: ${REDIS_PASSWORD}
       REDIS_PORT: ${REDIS_PORT}
       REDIS_SECURE: ${REDIS_SECURE}
       REDIS_USER: ${REDIS_USER}
-      REDIS_PASSWORD: ${REDIS_PASSWORD}
-      CATALOG_HOST: ${CATALOG_HOST}
-      CATALOG_PORT: ${CATALOG_PORT}
-      DIRECTOR_V2_HOST: ${DIRECTOR_V2_HOST}
-      DIRECTOR_V2_PORT: ${DIRECTOR_V2_PORT}
-      DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER: ${DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER}
-      DYNAMIC_SCHEDULER_LOGLEVEL: ${DYNAMIC_SCHEDULER_LOGLEVEL}
-      DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT: ${DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT}
-      DYNAMIC_SCHEDULER_PROFILING: ${DYNAMIC_SCHEDULER_PROFILING}
-      DYNAMIC_SCHEDULER_TRACING: ${DYNAMIC_SCHEDULER_TRACING}
-      DYNAMIC_SCHEDULER_UI_STORAGE_SECRET: ${DYNAMIC_SCHEDULER_UI_STORAGE_SECRET}
-      DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT: ${DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT}
+
       TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT: ${TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT}
       TRACING_OPENTELEMETRY_COLLECTOR_PORT: ${TRACING_OPENTELEMETRY_COLLECTOR_PORT}
-      DOCKER_API_PROXY_HOST: ${DOCKER_API_PROXY_HOST}
-      DOCKER_API_PROXY_PASSWORD: ${DOCKER_API_PROXY_PASSWORD}
-      DOCKER_API_PROXY_PORT: ${DOCKER_API_PROXY_PORT}
-      DOCKER_API_PROXY_SECURE: ${DOCKER_API_PROXY_SECURE}
-      DOCKER_API_PROXY_USER: ${DOCKER_API_PROXY_USER}
   docker-api-proxy:
     image: ${DOCKER_REGISTRY:-itisfoundation}/docker-api-proxy:${DOCKER_IMAGE_TAG:-latest}
     init: true

services/dynamic-scheduler/requirements/_test.in

@@ -26,3 +26,5 @@ pytest-runner
 pytest-sugar
 python-dotenv
 respx
+sqlalchemy[mypy]
+types-psycopg2

services/dynamic-scheduler/requirements/_test.txt

@@ -27,6 +27,7 @@ greenlet==3.1.1
     # via
     #   -c requirements/_base.txt
     #   playwright
+    #   sqlalchemy
 h11==0.14.0
     # via
     #   -c requirements/_base.txt
@@ -60,6 +61,10 @@ idna==3.10
     #   requests
 iniconfig==2.0.0
     # via pytest
+mypy==1.15.0
+    # via sqlalchemy
+mypy-extensions==1.1.0
+    # via mypy
 packaging==24.2
     # via
     #   -c requirements/_base.txt
@@ -112,13 +117,24 @@ sniffio==1.3.1
     #   -c requirements/_base.txt
     #   anyio
     #   asgi-lifespan
+sqlalchemy==1.4.54
+    # via
+    #   -c requirements/../../../requirements/constraints.txt
+    #   -c requirements/_base.txt
+    #   -r requirements/_test.in
+sqlalchemy2-stubs==0.0.2a38
+    # via sqlalchemy
 termcolor==2.5.0
     # via pytest-sugar
+types-psycopg2==2.9.21.20250318
+    # via -r requirements/_test.in
 typing-extensions==4.12.2
     # via
     #   -c requirements/_base.txt
     #   anyio
+    #   mypy
     #   pyee
+    #   sqlalchemy2-stubs
 tzdata==2025.1
     # via faker
 urllib3==2.3.0

services/dynamic-scheduler/requirements/_tools.txt

@@ -28,9 +28,12 @@ isort==6.0.1
 mccabe==0.7.0
     # via pylint
 mypy==1.15.0
-    # via -r requirements/../../../requirements/devenv.txt
-mypy-extensions==1.0.0
     # via
+    #   -c requirements/_test.txt
+    #   -r requirements/../../../requirements/devenv.txt
+mypy-extensions==1.1.0
+    # via
+    #   -c requirements/_test.txt
     #   black
     #   mypy
 nodeenv==1.9.1

services/dynamic-scheduler/setup.cfg

@@ -9,9 +9,10 @@ commit_args = --no-verify
 
 [tool:pytest]
 asyncio_mode = auto
-markers = 
+markers =
 	testit: "marks test to run during development"
 
 [mypy]
-plugins = 
+plugins =
 	pydantic.mypy
+  sqlalchemy.ext.mypy.plugin

services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/cli.py

@@ -3,6 +3,7 @@
 
 import typer
 from settings_library.docker_api_proxy import DockerApiProxysettings
+from settings_library.postgres import PostgresSettings
 from settings_library.rabbit import RabbitSettings
 from settings_library.utils_cli import (
     create_settings_command,
@@ -49,14 +50,29 @@ def echo_dotenv(ctx: typer.Context, *, minimal: bool = True):
                 RABBIT_SECURE=os.environ.get("RABBIT_SECURE", "0"),
                 RABBIT_USER=os.environ.get("RABBIT_USER", "replace-with-rabbit-user"),
                 RABBIT_PASSWORD=os.environ.get(
-                    "RABBIT_PASSWORD", "replace-with-rabbit-user"
+                    "RABBIT_PASSWORD", "replace-with-rabbit-password"
                 ),
             ),
         ),
         DYNAMIC_SCHEDULER_UI_STORAGE_SECRET=os.environ.get(
             "DYNAMIC_SCHEDULER_UI_STORAGE_SECRET",
             "replace-with-ui-storage-secret",
         ),
+        DYNAMIC_SCHEDULER_POSTGRES=os.environ.get(
+            "DYNAMIC_SCHEDULER_POSTGRES",
+            PostgresSettings.create_from_envs(
+                POSTGRES_HOST=os.environ.get(
+                    "POSTGRES_HOST", "replace-with-postgres-host"
+                ),
+                POSTGRES_USER=os.environ.get(
+                    "POSTGRES_USER", "replace-with-postgres-user"
+                ),
+                POSTGRES_PASSWORD=os.environ.get(
+                    "POSTGRES_PASSWORD", "replace-with-postgres-password"
+                ),
+                POSTGRES_DB=os.environ.get("POSTGRES_DB", "replace-with-postgres-db"),
+            ),
+        ),
         DYNAMIC_SCHEDULER_DOCKER_API_PROXY=os.environ.get(
             "DYNAMIC_SCHEDULER_DOCKER_API_PROXY",
             DockerApiProxysettings.create_from_envs(

services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/events.py

@@ -6,13 +6,17 @@
     create_remote_docker_client_input_state,
     remote_docker_client_lifespan,
 )
+from servicelib.fastapi.postgres_lifespan import (
+    create_postgres_database_input_state,
+)
 from servicelib.fastapi.prometheus_instrumentation import (
     create_prometheus_instrumentationmain_input_state,
     prometheus_instrumentation_lifespan,
 )
 
 from .._meta import APP_FINISHED_BANNER_MSG, APP_STARTED_BANNER_MSG
 from ..api.rpc.routes import rpc_api_routes_lifespan
+from ..repository.events import repository_lifespan_manager
 from ..services.catalog import catalog_lifespan
 from ..services.deferred_manager import deferred_manager_lifespan
 from ..services.director_v0 import director_v0_lifespan
@@ -36,6 +40,7 @@ async def _settings_lifespan(app: FastAPI) -> AsyncIterator[State]:
     settings: ApplicationSettings = app.state.settings
 
     yield {
+        **create_postgres_database_input_state(settings.DYNAMIC_SCHEDULER_POSTGRES),
         **create_prometheus_instrumentationmain_input_state(
             enabled=settings.DYNAMIC_SCHEDULER_PROMETHEUS_INSTRUMENTATION_ENABLED
         ),
@@ -49,6 +54,7 @@ def create_app_lifespan() -> LifespanManager:
     app_lifespan = LifespanManager()
     app_lifespan.add(_settings_lifespan)
 
+    app_lifespan.include(repository_lifespan_manager)
     app_lifespan.add(director_v2_lifespan)
     app_lifespan.add(director_v0_lifespan)
     app_lifespan.add(catalog_lifespan)