From fefdae2b18c9117a9298c4fc647b4d6c5f1b0e84 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 10 Oct 2025 15:48:27 +0200 Subject: [PATCH 01/58] prefork experiments --- .../src/celery_library/signals.py | 11 +++- .../celery-library/src/celery_library/task.py | 2 + .../src/celery_library/utils.py | 8 ++- .../servicelib/fastapi/celery/app_server.py | 2 +- services/docker-compose.yml | 3 +- services/storage/docker/boot.sh | 8 +-- .../modules/celery/worker_main_prefork.py | 57 +++++++++++++++++++ ...{worker_main.py => worker_main_threads.py} | 6 ++ 8 files changed, 87 insertions(+), 10 deletions(-) create mode 100644 services/storage/src/simcore_service_storage/modules/celery/worker_main_prefork.py rename services/storage/src/simcore_service_storage/modules/celery/{worker_main.py => worker_main_threads.py} (89%) diff --git a/packages/celery-library/src/celery_library/signals.py b/packages/celery-library/src/celery_library/signals.py index 02f1a56f0ec2..12e901dc508c 100644 --- a/packages/celery-library/src/celery_library/signals.py +++ b/packages/celery-library/src/celery_library/signals.py @@ -1,9 +1,9 @@ import asyncio import logging import threading +from typing import Any from celery import Celery # type: ignore[import-untyped] -from celery.worker.worker import WorkController # type: ignore[import-untyped] from servicelib.celery.app_server import BaseAppServer from servicelib.logging_utils import log_context @@ -13,12 +13,17 @@ def on_worker_init( - sender: WorkController, + sender: Any, app_server: BaseAppServer, **_kwargs, ) -> None: startup_complete_event = threading.Event() + _logger.info("Worker init") + _logger.info("Sender: %s", sender) + _logger.info("App server: %s", app_server) + _logger.info("App server _kwargs: %s", _kwargs) + def _init(startup_complete_event: threading.Event) -> None: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) @@ -26,6 +31,8 @@ def _init(startup_complete_event: threading.Event) -> None: assert sender.app # nosec assert isinstance(sender.app, Celery) # nosec + _logger.info("App inside on_worker_init: %s", sender.app) + set_app_server(sender.app, app_server) app_server.event_loop = loop diff --git a/packages/celery-library/src/celery_library/task.py b/packages/celery-library/src/celery_library/task.py index f01bf301347c..b7fcb95f911a 100644 --- a/packages/celery-library/src/celery_library/task.py +++ b/packages/celery-library/src/celery_library/task.py @@ -43,6 +43,8 @@ def decorator( ) -> Callable[Concatenate[Task, P], R]: @wraps(coro) def wrapper(task: Task, *args: P.args, **kwargs: P.kwargs) -> R: + _logger.info("App inside task: %s", app) + app_server = get_app_server(app) # NOTE: task.request is a thread local object, so we need to pass the id explicitly assert task.request.id is not None # nosec diff --git a/packages/celery-library/src/celery_library/utils.py b/packages/celery-library/src/celery_library/utils.py index 4d8bad73b96d..856f191ad10c 100644 --- a/packages/celery-library/src/celery_library/utils.py +++ b/packages/celery-library/src/celery_library/utils.py @@ -1,3 +1,6 @@ +import os +from collections import defaultdict + from celery import Celery # type: ignore[import-untyped] from servicelib.celery.app_server import BaseAppServer @@ -5,10 +8,11 @@ def get_app_server(app: Celery) -> BaseAppServer: - app_server = app.conf[_APP_SERVER_KEY] + app_server = app.conf[_APP_SERVER_KEY][os.getpid()] assert isinstance(app_server, BaseAppServer) return app_server def set_app_server(app: Celery, app_server: BaseAppServer) -> None: - app.conf[_APP_SERVER_KEY] = app_server + app.conf.setdefault(_APP_SERVER_KEY, defaultdict(dict)) + app.conf[_APP_SERVER_KEY][os.getpid()] = app_server diff --git a/packages/service-library/src/servicelib/fastapi/celery/app_server.py b/packages/service-library/src/servicelib/fastapi/celery/app_server.py index 3c42aa9144d0..57e47a3441b3 100644 --- a/packages/service-library/src/servicelib/fastapi/celery/app_server.py +++ b/packages/service-library/src/servicelib/fastapi/celery/app_server.py @@ -30,6 +30,6 @@ async def run_until_shutdown( startup_timeout=None, # waits for full app initialization (DB migrations, etc.) shutdown_timeout=_SHUTDOWN_TIMEOUT, ): - _logger.info("fastapi app initialized") + _logger.info("FastAPI initialized: %s", self.app) startup_completed_event.set() await self.shutdown_event.wait() # NOTE: wait here until shutdown is requested diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 44ef1e437fd7..5c39e9cff566 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -1355,6 +1355,7 @@ services: STORAGE_WORKER_NAME: "sto-worker-{{.Node.Hostname}}-{{.Task.Slot}}-{{.Task.ID}}" STORAGE_WORKER_MODE: "true" CELERY_CONCURRENCY: 100 + CELERY_POOL: "threads" networks: *storage_networks sto-worker-cpu-bound: @@ -1366,7 +1367,7 @@ services: STORAGE_TRACING: "null" STORAGE_WORKER_NAME: "sto-worker-cpu-bound-{{.Node.Hostname}}-{{.Task.Slot}}-{{.Task.ID}}" STORAGE_WORKER_MODE: "true" - CELERY_CONCURRENCY: 1 + CELERY_CONCURRENCY: 2 CELERY_QUEUES: "cpu_bound" networks: *storage_networks diff --git a/services/storage/docker/boot.sh b/services/storage/docker/boot.sh index 6dd4e72f8e6c..9284e1104210 100755 --- a/services/storage/docker/boot.sh +++ b/services/storage/docker/boot.sh @@ -55,16 +55,16 @@ if [ "${STORAGE_WORKER_MODE}" = "true" ]; then --recursive \ -- \ celery \ - --app=simcore_service_storage.modules.celery.worker_main:app \ - worker --pool=threads \ + --app=simcore_service_storage.modules.celery.worker_main_"${CELERY_POOL:-prefork}":app \ + worker --pool="${CELERY_POOL:-prefork}" \ --loglevel="${SERVER_LOG_LEVEL}" \ --concurrency="${CELERY_CONCURRENCY}" \ --hostname="${STORAGE_WORKER_NAME}" \ --queues="${CELERY_QUEUES:-default}" else exec celery \ - --app=simcore_service_storage.modules.celery.worker_main:app \ - worker --pool=threads \ + --app=simcore_service_storage.modules.celery.worker_main_"${CELERY_POOL:-prefork}":app \ + worker --pool="${CELERY_POOL:-prefork}" \ --loglevel="${SERVER_LOG_LEVEL}" \ --concurrency="${CELERY_CONCURRENCY}" \ --hostname="${STORAGE_WORKER_NAME}" \ diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker_main_prefork.py b/services/storage/src/simcore_service_storage/modules/celery/worker_main_prefork.py new file mode 100644 index 000000000000..1d31dd5b808c --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/celery/worker_main_prefork.py @@ -0,0 +1,57 @@ +"""Main application to be deployed in for example uvicorn.""" + +import logging +from dataclasses import dataclass + +from celery import Celery +from celery.signals import ( # type: ignore[import-untyped] + worker_process_init, + worker_process_shutdown, +) +from celery_library.common import create_app as create_celery_app +from celery_library.signals import ( + on_worker_init, + on_worker_shutdown, +) +from servicelib.fastapi.celery.app_server import FastAPIAppServer +from servicelib.logging_utils import setup_loggers + +from ...api._worker_tasks.tasks import setup_worker_tasks +from ...core.application import create_app +from ...core.settings import ApplicationSettings + +_settings = ApplicationSettings.create_from_envs() + +setup_loggers( + log_format_local_dev_enabled=_settings.STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED, + logger_filter_mapping=_settings.STORAGE_LOG_FILTER_MAPPING, + tracing_settings=_settings.STORAGE_TRACING, + log_base_level=_settings.log_level, + noisy_loggers=None, +) + + +_logger = logging.getLogger(__name__) + +assert _settings.STORAGE_CELERY # nosec +app = create_celery_app(_settings.STORAGE_CELERY) + + +@dataclass +class AppWrapper: + app: Celery + + +def worker_init_wrapper(**kwargs): + kwargs.pop("sender", None) # remove sender + fastapi_instance = create_app(_settings) + app_server = FastAPIAppServer(app=fastapi_instance) + assert _settings.STORAGE_CELERY # nosec + + return on_worker_init(AppWrapper(app), app_server, **kwargs) + + +worker_process_init.connect(worker_init_wrapper) +worker_process_shutdown.connect(on_worker_shutdown) + +setup_worker_tasks(app) diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker_main.py b/services/storage/src/simcore_service_storage/modules/celery/worker_main_threads.py similarity index 89% rename from services/storage/src/simcore_service_storage/modules/celery/worker_main.py rename to services/storage/src/simcore_service_storage/modules/celery/worker_main_threads.py index f2e90e900244..8c9dc2aaad47 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker_main.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker_main_threads.py @@ -1,5 +1,8 @@ """Main application to be deployed in for example uvicorn.""" +import logging +import os + from celery.signals import worker_init, worker_shutdown # type: ignore[import-untyped] from celery_library.common import create_app as create_celery_app from celery_library.signals import ( @@ -24,8 +27,11 @@ ) +_logger = logging.getLogger(__name__) + assert _settings.STORAGE_CELERY # nosec app = create_celery_app(_settings.STORAGE_CELERY) +_logger.info("Starting worker with pool=%s", os.environ.get("CELERY_POOL")) app_server = FastAPIAppServer(app=create_app(_settings)) From 3ebb954c37071b1b3cb3202ff6a81ac20e922d6a Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 10 Oct 2025 15:56:00 +0200 Subject: [PATCH 02/58] fix shutdown --- .../modules/celery/worker_main_prefork.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker_main_prefork.py b/services/storage/src/simcore_service_storage/modules/celery/worker_main_prefork.py index 1d31dd5b808c..09e5d6cb87d8 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker_main_prefork.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker_main_prefork.py @@ -51,7 +51,12 @@ def worker_init_wrapper(**kwargs): return on_worker_init(AppWrapper(app), app_server, **kwargs) +def worker_shutdown_wrapper(**kwargs): + kwargs.pop("sender", None) # remove sender + return on_worker_shutdown(AppWrapper(app), **kwargs) + + worker_process_init.connect(worker_init_wrapper) -worker_process_shutdown.connect(on_worker_shutdown) +worker_process_shutdown.connect(worker_shutdown_wrapper) setup_worker_tasks(app) From 97e50d45708ab45a28d00a4061c665942692ba43 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 13 Oct 2025 10:14:05 +0200 Subject: [PATCH 03/58] remove logs --- .../celery-library/src/celery_library/signals.py | 7 ------- packages/celery-library/src/celery_library/utils.py | 8 ++------ .../modules/celery/worker_main_prefork.py | 13 +++++++------ 3 files changed, 9 insertions(+), 19 deletions(-) diff --git a/packages/celery-library/src/celery_library/signals.py b/packages/celery-library/src/celery_library/signals.py index 12e901dc508c..c66d959ee31a 100644 --- a/packages/celery-library/src/celery_library/signals.py +++ b/packages/celery-library/src/celery_library/signals.py @@ -19,11 +19,6 @@ def on_worker_init( ) -> None: startup_complete_event = threading.Event() - _logger.info("Worker init") - _logger.info("Sender: %s", sender) - _logger.info("App server: %s", app_server) - _logger.info("App server _kwargs: %s", _kwargs) - def _init(startup_complete_event: threading.Event) -> None: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) @@ -31,8 +26,6 @@ def _init(startup_complete_event: threading.Event) -> None: assert sender.app # nosec assert isinstance(sender.app, Celery) # nosec - _logger.info("App inside on_worker_init: %s", sender.app) - set_app_server(sender.app, app_server) app_server.event_loop = loop diff --git a/packages/celery-library/src/celery_library/utils.py b/packages/celery-library/src/celery_library/utils.py index 856f191ad10c..4d8bad73b96d 100644 --- a/packages/celery-library/src/celery_library/utils.py +++ b/packages/celery-library/src/celery_library/utils.py @@ -1,6 +1,3 @@ -import os -from collections import defaultdict - from celery import Celery # type: ignore[import-untyped] from servicelib.celery.app_server import BaseAppServer @@ -8,11 +5,10 @@ def get_app_server(app: Celery) -> BaseAppServer: - app_server = app.conf[_APP_SERVER_KEY][os.getpid()] + app_server = app.conf[_APP_SERVER_KEY] assert isinstance(app_server, BaseAppServer) return app_server def set_app_server(app: Celery, app_server: BaseAppServer) -> None: - app.conf.setdefault(_APP_SERVER_KEY, defaultdict(dict)) - app.conf[_APP_SERVER_KEY][os.getpid()] = app_server + app.conf[_APP_SERVER_KEY] = app_server diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker_main_prefork.py b/services/storage/src/simcore_service_storage/modules/celery/worker_main_prefork.py index 09e5d6cb87d8..9f117f8cff4a 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker_main_prefork.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker_main_prefork.py @@ -1,6 +1,5 @@ """Main application to be deployed in for example uvicorn.""" -import logging from dataclasses import dataclass from celery import Celery @@ -15,24 +14,26 @@ ) from servicelib.fastapi.celery.app_server import FastAPIAppServer from servicelib.logging_utils import setup_loggers +from servicelib.tracing import TracingConfig from ...api._worker_tasks.tasks import setup_worker_tasks from ...core.application import create_app from ...core.settings import ApplicationSettings _settings = ApplicationSettings.create_from_envs() +_tracing_config = TracingConfig.create( + tracing_settings=_settings.STORAGE_TRACING, + service_name="storage-celery-worker", +) setup_loggers( log_format_local_dev_enabled=_settings.STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=_settings.STORAGE_LOG_FILTER_MAPPING, - tracing_settings=_settings.STORAGE_TRACING, + tracing_config=_tracing_config, log_base_level=_settings.log_level, noisy_loggers=None, ) - -_logger = logging.getLogger(__name__) - assert _settings.STORAGE_CELERY # nosec app = create_celery_app(_settings.STORAGE_CELERY) @@ -44,7 +45,7 @@ class AppWrapper: def worker_init_wrapper(**kwargs): kwargs.pop("sender", None) # remove sender - fastapi_instance = create_app(_settings) + fastapi_instance = create_app(_settings, tracing_config=_tracing_config) app_server = FastAPIAppServer(app=fastapi_instance) assert _settings.STORAGE_CELERY # nosec From 29083b95512817e0fa00c387edb2a150aab34c8e Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 13 Oct 2025 11:56:20 +0200 Subject: [PATCH 04/58] fix typecheck --- .../modules/celery/worker_main_prefork.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker_main_prefork.py b/services/storage/src/simcore_service_storage/modules/celery/worker_main_prefork.py index 9f117f8cff4a..d5323b813dc9 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker_main_prefork.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker_main_prefork.py @@ -2,8 +2,8 @@ from dataclasses import dataclass -from celery import Celery -from celery.signals import ( # type: ignore[import-untyped] +from celery import Celery # pylint: disable=no-name-in-module +from celery.signals import ( # pylint: disable=no-name-in-module worker_process_init, worker_process_shutdown, ) From b3636fb2899274a78dabcbba8ea808b65513f0f0 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 13 Oct 2025 13:50:05 +0200 Subject: [PATCH 05/58] typecheck --- .../modules/celery/worker_main_prefork.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker_main_prefork.py b/services/storage/src/simcore_service_storage/modules/celery/worker_main_prefork.py index d5323b813dc9..6766c61002cf 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker_main_prefork.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker_main_prefork.py @@ -2,8 +2,8 @@ from dataclasses import dataclass -from celery import Celery # pylint: disable=no-name-in-module -from celery.signals import ( # pylint: disable=no-name-in-module +from celery import Celery # type: ignore[import-untyped] +from celery.signals import ( # type: ignore[import-untyped] worker_process_init, worker_process_shutdown, ) From 76e886fe1624aa482481f0b9fe70ef327545ef0c Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 13 Oct 2025 14:08:05 +0200 Subject: [PATCH 06/58] add settings --- packages/celery-library/src/celery_library/settings.py | 9 +++++++++ packages/settings-library/src/settings_library/celery.py | 9 ++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 packages/celery-library/src/celery_library/settings.py diff --git a/packages/celery-library/src/celery_library/settings.py b/packages/celery-library/src/celery_library/settings.py new file mode 100644 index 000000000000..c94c40d3180a --- /dev/null +++ b/packages/celery-library/src/celery_library/settings.py @@ -0,0 +1,9 @@ +from enum import StrEnum + + +class CeleryPoolType(StrEnum): + PREFORK = "prefork" + EVENTLET = "eventlet" + GEVENT = "gevent" + SOLO = "solo" + THREADS = "threads" diff --git a/packages/settings-library/src/settings_library/celery.py b/packages/settings-library/src/settings_library/celery.py index 168b86c6745c..6c767976b42f 100644 --- a/packages/settings-library/src/settings_library/celery.py +++ b/packages/settings-library/src/settings_library/celery.py @@ -1,5 +1,5 @@ from datetime import timedelta -from typing import Annotated +from typing import Annotated, Literal from pydantic import Field from pydantic_settings import SettingsConfigDict @@ -35,6 +35,13 @@ class CelerySettings(BaseCustomSettings): ), ] = True + CELERY_POOL: Annotated[ + Literal["prefork", "eventlet", "gevent", "solo", "threads"], + Field( + description="Type of pool to use. One of: prefork, eventlet, gevent, solo, threads. See https://docs.celeryq.dev/en/stable/userguide/workers.html#choosing-a-concurrency-implementation for details.", + ), + ] = "prefork" + model_config = SettingsConfigDict( json_schema_extra={ "examples": [ From fe05e66e42fb5e20959f011fb5313fd22274ac1e Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 13 Oct 2025 14:52:01 +0200 Subject: [PATCH 07/58] unify worker setup --- .../src/celery_library/signals.py | 20 +------ services/docker-compose.yml | 2 +- services/storage/docker/boot.sh | 4 +- .../modules/celery/worker/__init__.py | 0 .../main.py} | 40 ++++++-------- .../modules/celery/worker_main_threads.py | 52 ------------------- 6 files changed, 21 insertions(+), 97 deletions(-) create mode 100644 services/storage/src/simcore_service_storage/modules/celery/worker/__init__.py rename services/storage/src/simcore_service_storage/modules/celery/{worker_main_prefork.py => worker/main.py} (55%) delete mode 100644 services/storage/src/simcore_service_storage/modules/celery/worker_main_threads.py diff --git a/packages/celery-library/src/celery_library/signals.py b/packages/celery-library/src/celery_library/signals.py index c66d959ee31a..e2ed1b7f7518 100644 --- a/packages/celery-library/src/celery_library/signals.py +++ b/packages/celery-library/src/celery_library/signals.py @@ -1,33 +1,20 @@ import asyncio import logging import threading -from typing import Any -from celery import Celery # type: ignore[import-untyped] from servicelib.celery.app_server import BaseAppServer from servicelib.logging_utils import log_context -from .utils import get_app_server, set_app_server - _logger = logging.getLogger(__name__) -def on_worker_init( - sender: Any, - app_server: BaseAppServer, - **_kwargs, -) -> None: +def on_worker_init(app_server: BaseAppServer, **_kwargs) -> None: startup_complete_event = threading.Event() def _init(startup_complete_event: threading.Event) -> None: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) - assert sender.app # nosec - assert isinstance(sender.app, Celery) # nosec - - set_app_server(sender.app, app_server) - app_server.event_loop = loop loop.run_until_complete(app_server.run_until_shutdown(startup_complete_event)) @@ -44,9 +31,6 @@ def _init(startup_complete_event: threading.Event) -> None: startup_complete_event.wait() -def on_worker_shutdown(sender, **_kwargs) -> None: +def on_worker_shutdown(app_server: BaseAppServer, **_kwargs) -> None: with log_context(_logger, logging.INFO, "Worker shutdown"): - assert isinstance(sender.app, Celery) - app_server = get_app_server(sender.app) - app_server.shutdown_event.set() diff --git a/services/docker-compose.yml b/services/docker-compose.yml index fb711d1c8690..7e6b97d7ba03 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -1367,7 +1367,7 @@ services: STORAGE_TRACING: "null" STORAGE_WORKER_NAME: "sto-worker-cpu-bound-{{.Node.Hostname}}-{{.Task.Slot}}-{{.Task.ID}}" STORAGE_WORKER_MODE: "true" - CELERY_CONCURRENCY: 2 + CELERY_CONCURRENCY: 1 CELERY_QUEUES: "cpu_bound" networks: *storage_networks diff --git a/services/storage/docker/boot.sh b/services/storage/docker/boot.sh index 9284e1104210..627333dc35c6 100755 --- a/services/storage/docker/boot.sh +++ b/services/storage/docker/boot.sh @@ -55,7 +55,7 @@ if [ "${STORAGE_WORKER_MODE}" = "true" ]; then --recursive \ -- \ celery \ - --app=simcore_service_storage.modules.celery.worker_main_"${CELERY_POOL:-prefork}":app \ + --app=simcore_service_storage.modules.celery.worker.main:app \ worker --pool="${CELERY_POOL:-prefork}" \ --loglevel="${SERVER_LOG_LEVEL}" \ --concurrency="${CELERY_CONCURRENCY}" \ @@ -63,7 +63,7 @@ if [ "${STORAGE_WORKER_MODE}" = "true" ]; then --queues="${CELERY_QUEUES:-default}" else exec celery \ - --app=simcore_service_storage.modules.celery.worker_main_"${CELERY_POOL:-prefork}":app \ + --app=simcore_service_storage.modules.celery.worker.main:app \ worker --pool="${CELERY_POOL:-prefork}" \ --loglevel="${SERVER_LOG_LEVEL}" \ --concurrency="${CELERY_CONCURRENCY}" \ diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker/__init__.py b/services/storage/src/simcore_service_storage/modules/celery/worker/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker_main_prefork.py b/services/storage/src/simcore_service_storage/modules/celery/worker/main.py similarity index 55% rename from services/storage/src/simcore_service_storage/modules/celery/worker_main_prefork.py rename to services/storage/src/simcore_service_storage/modules/celery/worker/main.py index 6766c61002cf..b09c47097fe9 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker_main_prefork.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker/main.py @@ -1,24 +1,22 @@ -"""Main application to be deployed in for example uvicorn.""" - -from dataclasses import dataclass - -from celery import Celery # type: ignore[import-untyped] from celery.signals import ( # type: ignore[import-untyped] + worker_init, worker_process_init, worker_process_shutdown, + worker_shutdown, ) from celery_library.common import create_app as create_celery_app from celery_library.signals import ( on_worker_init, on_worker_shutdown, ) +from celery_library.utils import get_app_server, set_app_server from servicelib.fastapi.celery.app_server import FastAPIAppServer from servicelib.logging_utils import setup_loggers from servicelib.tracing import TracingConfig -from ...api._worker_tasks.tasks import setup_worker_tasks -from ...core.application import create_app -from ...core.settings import ApplicationSettings +from ....api._worker_tasks.tasks import setup_worker_tasks +from ....core.application import create_app +from ....core.settings import ApplicationSettings _settings = ApplicationSettings.create_from_envs() _tracing_config = TracingConfig.create( @@ -38,26 +36,20 @@ app = create_celery_app(_settings.STORAGE_CELERY) -@dataclass -class AppWrapper: - app: Celery - - +@worker_init.connect +@worker_process_init.connect def worker_init_wrapper(**kwargs): - kwargs.pop("sender", None) # remove sender - fastapi_instance = create_app(_settings, tracing_config=_tracing_config) - app_server = FastAPIAppServer(app=fastapi_instance) - assert _settings.STORAGE_CELERY # nosec - - return on_worker_init(AppWrapper(app), app_server, **kwargs) + fastapi_app = create_app(_settings, tracing_config=_tracing_config) + app_server = FastAPIAppServer(app=fastapi_app) + set_app_server(app, app_server) + return on_worker_init(app_server, **kwargs) +@worker_shutdown.connect +@worker_process_shutdown.connect def worker_shutdown_wrapper(**kwargs): - kwargs.pop("sender", None) # remove sender - return on_worker_shutdown(AppWrapper(app), **kwargs) - + app_server = get_app_server(app) + return on_worker_shutdown(app_server, **kwargs) -worker_process_init.connect(worker_init_wrapper) -worker_process_shutdown.connect(worker_shutdown_wrapper) setup_worker_tasks(app) diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker_main_threads.py b/services/storage/src/simcore_service_storage/modules/celery/worker_main_threads.py deleted file mode 100644 index bb5e5896cc57..000000000000 --- a/services/storage/src/simcore_service_storage/modules/celery/worker_main_threads.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Main application to be deployed in for example uvicorn.""" - -import logging -import os - -from celery.signals import worker_init, worker_shutdown # type: ignore[import-untyped] -from celery_library.common import create_app as create_celery_app -from celery_library.signals import ( - on_worker_init, - on_worker_shutdown, -) -from servicelib.fastapi.celery.app_server import FastAPIAppServer -from servicelib.logging_utils import setup_loggers -from servicelib.tracing import TracingConfig - -from ...api._worker_tasks.tasks import setup_worker_tasks -from ...core.application import create_app -from ...core.settings import ApplicationSettings - -_settings = ApplicationSettings.create_from_envs() -_tracing_config = TracingConfig.create( - tracing_settings=_settings.STORAGE_TRACING, - service_name="storage-celery-worker", -) - -setup_loggers( - log_format_local_dev_enabled=_settings.STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED, - logger_filter_mapping=_settings.STORAGE_LOG_FILTER_MAPPING, - tracing_config=_tracing_config, - log_base_level=_settings.log_level, - noisy_loggers=None, -) - - -_logger = logging.getLogger(__name__) - -assert _settings.STORAGE_CELERY # nosec -app = create_celery_app(_settings.STORAGE_CELERY) -_logger.info("Starting worker with pool=%s", os.environ.get("CELERY_POOL")) - -app_server = FastAPIAppServer(app=create_app(_settings, tracing_config=_tracing_config)) - - -def worker_init_wrapper(sender, **kwargs): - return on_worker_init(sender, app_server, **kwargs) - - -worker_init.connect(worker_init_wrapper) -worker_shutdown.connect(on_worker_shutdown) - - -setup_worker_tasks(app) From 33d23bc96640f1ea116f01709acfac2f58948f47 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 13 Oct 2025 14:53:14 +0200 Subject: [PATCH 08/58] remove log --- packages/celery-library/src/celery_library/task.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/celery-library/src/celery_library/task.py b/packages/celery-library/src/celery_library/task.py index b7fcb95f911a..f01bf301347c 100644 --- a/packages/celery-library/src/celery_library/task.py +++ b/packages/celery-library/src/celery_library/task.py @@ -43,8 +43,6 @@ def decorator( ) -> Callable[Concatenate[Task, P], R]: @wraps(coro) def wrapper(task: Task, *args: P.args, **kwargs: P.kwargs) -> R: - _logger.info("App inside task: %s", app) - app_server = get_app_server(app) # NOTE: task.request is a thread local object, so we need to pass the id explicitly assert task.request.id is not None # nosec From 4ab42d81ed28f196d18446188980f4ee51d0e20e Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 13 Oct 2025 14:54:15 +0200 Subject: [PATCH 09/58] remove unused --- packages/celery-library/src/celery_library/settings.py | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 packages/celery-library/src/celery_library/settings.py diff --git a/packages/celery-library/src/celery_library/settings.py b/packages/celery-library/src/celery_library/settings.py deleted file mode 100644 index c94c40d3180a..000000000000 --- a/packages/celery-library/src/celery_library/settings.py +++ /dev/null @@ -1,9 +0,0 @@ -from enum import StrEnum - - -class CeleryPoolType(StrEnum): - PREFORK = "prefork" - EVENTLET = "eventlet" - GEVENT = "gevent" - SOLO = "solo" - THREADS = "threads" From c8e957b5051ce7896183e0ce7c539af89c42e3aa Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 13 Oct 2025 15:46:48 +0200 Subject: [PATCH 10/58] fix api-server --- services/api-server/docker/boot.sh | 10 ++--- .../api-server/docker/boot_celery_worker.py | 13 ------ .../{celery_worker => modules}/__init__.py | 0 .../celery}/__init__.py | 0 .../modules/celery/worker/__init__.py | 0 .../celery/worker/_functions_tasks.py} | 23 ++++++----- .../celery/worker/main.py} | 41 +++++++++++++------ .../celery/worker}/tasks.py | 4 +- .../unit/api_functions/celery/conftest.py | 2 +- .../celery/test_functions_celery.py | 2 +- .../test_api_routers_functions.py | 2 +- services/docker-compose.yml | 1 + services/storage/docker/boot.sh | 4 +- .../modules/celery/worker/main.py | 33 ++++++++------- 14 files changed, 72 insertions(+), 63 deletions(-) delete mode 100644 services/api-server/docker/boot_celery_worker.py rename services/api-server/src/simcore_service_api_server/{celery_worker => modules}/__init__.py (100%) rename services/api-server/src/simcore_service_api_server/{celery_worker/worker_tasks => modules/celery}/__init__.py (100%) create mode 100644 services/api-server/src/simcore_service_api_server/modules/celery/worker/__init__.py rename services/api-server/src/simcore_service_api_server/{celery_worker/worker_tasks/functions_tasks.py => modules/celery/worker/_functions_tasks.py} (87%) rename services/api-server/src/simcore_service_api_server/{celery_worker/worker_main.py => modules/celery/worker/main.py} (52%) rename services/api-server/src/simcore_service_api_server/{celery_worker/worker_tasks => modules/celery/worker}/tasks.py (83%) diff --git a/services/api-server/docker/boot.sh b/services/api-server/docker/boot.sh index 227be9c56b96..4cf40ee470c8 100755 --- a/services/api-server/docker/boot.sh +++ b/services/api-server/docker/boot.sh @@ -48,18 +48,16 @@ if [ "${API_SERVER_WORKER_MODE}" = "true" ]; then --recursive \ -- \ celery \ - --app=boot_celery_worker:app \ - --workdir=services/api-server/docker \ - worker --pool=threads \ + --app=simcore_service_api_server.modules.celery.worker.main:the_app \ + worker --pool="${CELERY_POOL:-prefork}" \ --loglevel="${API_SERVER_LOGLEVEL}" \ --concurrency="${CELERY_CONCURRENCY}" \ --hostname="${API_SERVER_WORKER_NAME}" \ --queues="${CELERY_QUEUES:-default}" else exec celery \ - --app=boot_celery_worker:app \ - --workdir=services/api-server/docker \ - worker --pool=threads \ + --app=simcore_service_api_server.modules.celery.worker.main:the_app \ + worker --pool="${CELERY_POOL:-prefork}" \ --loglevel="${API_SERVER_LOGLEVEL}" \ --concurrency="${CELERY_CONCURRENCY}" \ --hostname="${API_SERVER_WORKER_NAME}" \ diff --git a/services/api-server/docker/boot_celery_worker.py b/services/api-server/docker/boot_celery_worker.py deleted file mode 100644 index e0c7e119ced8..000000000000 --- a/services/api-server/docker/boot_celery_worker.py +++ /dev/null @@ -1,13 +0,0 @@ -from celery.signals import worker_init, worker_shutdown # type: ignore[import-untyped] -from celery_library.signals import ( - on_worker_shutdown, -) -from simcore_service_api_server.celery_worker.worker_main import ( - get_app, - worker_init_wrapper, -) - -app = get_app() - -worker_init.connect(worker_init_wrapper) -worker_shutdown.connect(on_worker_shutdown) diff --git a/services/api-server/src/simcore_service_api_server/celery_worker/__init__.py b/services/api-server/src/simcore_service_api_server/modules/__init__.py similarity index 100% rename from services/api-server/src/simcore_service_api_server/celery_worker/__init__.py rename to services/api-server/src/simcore_service_api_server/modules/__init__.py diff --git a/services/api-server/src/simcore_service_api_server/celery_worker/worker_tasks/__init__.py b/services/api-server/src/simcore_service_api_server/modules/celery/__init__.py similarity index 100% rename from services/api-server/src/simcore_service_api_server/celery_worker/worker_tasks/__init__.py rename to services/api-server/src/simcore_service_api_server/modules/celery/__init__.py diff --git a/services/api-server/src/simcore_service_api_server/modules/celery/worker/__init__.py b/services/api-server/src/simcore_service_api_server/modules/celery/worker/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/services/api-server/src/simcore_service_api_server/celery_worker/worker_tasks/functions_tasks.py b/services/api-server/src/simcore_service_api_server/modules/celery/worker/_functions_tasks.py similarity index 87% rename from services/api-server/src/simcore_service_api_server/celery_worker/worker_tasks/functions_tasks.py rename to services/api-server/src/simcore_service_api_server/modules/celery/worker/_functions_tasks.py index e0687f6ab89a..1a0bcfc33b5f 100644 --- a/services/api-server/src/simcore_service_api_server/celery_worker/worker_tasks/functions_tasks.py +++ b/services/api-server/src/simcore_service_api_server/modules/celery/worker/_functions_tasks.py @@ -9,9 +9,9 @@ from servicelib.celery.models import TaskKey from simcore_service_api_server._service_function_jobs import FunctionJobService -from ...api.dependencies.authentication import Identity -from ...api.dependencies.rabbitmq import get_rabbitmq_rpc_client -from ...api.dependencies.services import ( +from ....api.dependencies.authentication import Identity +from ....api.dependencies.rabbitmq import get_rabbitmq_rpc_client +from ....api.dependencies.services import ( get_catalog_service, get_directorv2_service, get_function_job_service, @@ -20,13 +20,16 @@ get_solver_service, get_storage_service, ) -from ...api.dependencies.webserver_http import get_session_cookie, get_webserver_session -from ...api.dependencies.webserver_rpc import get_wb_api_rpc_client -from ...models.api_resources import JobLinks -from ...models.domain.functions import PreRegisteredFunctionJobData -from ...models.schemas.jobs import JobPricingSpecification -from ...services_http.director_v2 import DirectorV2Api -from ...services_http.storage import StorageApi +from ....api.dependencies.webserver_http import ( + get_session_cookie, + get_webserver_session, +) +from ....api.dependencies.webserver_rpc import get_wb_api_rpc_client +from ....models.api_resources import JobLinks +from ....models.domain.functions import PreRegisteredFunctionJobData +from ....models.schemas.jobs import JobPricingSpecification +from ....services_http.director_v2 import DirectorV2Api +from ....services_http.storage import StorageApi async def _assemble_function_job_service( diff --git a/services/api-server/src/simcore_service_api_server/celery_worker/worker_main.py b/services/api-server/src/simcore_service_api_server/modules/celery/worker/main.py similarity index 52% rename from services/api-server/src/simcore_service_api_server/celery_worker/worker_main.py rename to services/api-server/src/simcore_service_api_server/modules/celery/worker/main.py index c9e99cda269f..c872ca4bba2e 100644 --- a/services/api-server/src/simcore_service_api_server/celery_worker/worker_main.py +++ b/services/api-server/src/simcore_service_api_server/modules/celery/worker/main.py @@ -1,27 +1,32 @@ """Main application to be deployed in for example uvicorn.""" -from functools import partial - +from celery.signals import ( # type: ignore[import-untyped] + worker_init, + worker_shutdown, +) from celery_library.common import create_app as create_celery_app from celery_library.signals import ( on_worker_init, + on_worker_shutdown, ) +from celery_library.utils import get_app_server, set_app_server from servicelib.fastapi.celery.app_server import FastAPIAppServer from servicelib.logging_utils import setup_loggers from servicelib.tracing import TracingConfig -from ..core.application import create_app -from ..core.settings import ApplicationSettings -from .worker_tasks.tasks import setup_worker_tasks +from ....core.application import create_app +from ....core.settings import ApplicationSettings +from .tasks import setup_worker_tasks + +_settings = ApplicationSettings.create_from_envs() +_tracing_settings = _settings.API_SERVER_TRACING +_tracing_config = TracingConfig.create( + tracing_settings=_tracing_settings, + service_name="api-server-celery-worker", +) def get_app(): - _settings = ApplicationSettings.create_from_envs() - _tracing_settings = _settings.API_SERVER_TRACING - _tracing_config = TracingConfig.create( - tracing_settings=_tracing_settings, - service_name="api-server-celery-worker", - ) setup_loggers( log_format_local_dev_enabled=_settings.API_SERVER_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=_settings.API_SERVER_LOG_FILTER_MAPPING, @@ -37,9 +42,19 @@ def get_app(): return app -def worker_init_wrapper(sender, **_kwargs): +the_app = get_app() + + +@worker_init.connect +def worker_init_wrapper(**kwargs): _settings = ApplicationSettings.create_from_envs() assert _settings.API_SERVER_CELERY # nosec app_server = FastAPIAppServer(app=create_app(_settings)) + set_app_server(the_app, app_server) + return on_worker_init(app_server, **kwargs) + - return partial(on_worker_init, app_server=app_server)(sender, **_kwargs) +@worker_shutdown.connect +def worker_shutdown_wrapper(**kwargs): + app_server = get_app_server(the_app) + return on_worker_shutdown(app_server, **kwargs) diff --git a/services/api-server/src/simcore_service_api_server/celery_worker/worker_tasks/tasks.py b/services/api-server/src/simcore_service_api_server/modules/celery/worker/tasks.py similarity index 83% rename from services/api-server/src/simcore_service_api_server/celery_worker/worker_tasks/tasks.py rename to services/api-server/src/simcore_service_api_server/modules/celery/worker/tasks.py index cef6ad06d18c..4a0321c92a34 100644 --- a/services/api-server/src/simcore_service_api_server/celery_worker/worker_tasks/tasks.py +++ b/services/api-server/src/simcore_service_api_server/modules/celery/worker/tasks.py @@ -7,8 +7,8 @@ from celery_library.types import register_celery_types, register_pydantic_types from servicelib.logging_utils import log_context -from ...models.domain.celery_models import pydantic_types_to_register -from .functions_tasks import run_function +from ....models.domain.celery_models import pydantic_types_to_register +from ._functions_tasks import run_function _logger = logging.getLogger(__name__) diff --git a/services/api-server/tests/unit/api_functions/celery/conftest.py b/services/api-server/tests/unit/api_functions/celery/conftest.py index 75c3a32123a9..7b39ce60ce86 100644 --- a/services/api-server/tests/unit/api_functions/celery/conftest.py +++ b/services/api-server/tests/unit/api_functions/celery/conftest.py @@ -25,7 +25,7 @@ from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.fastapi.celery.app_server import FastAPIAppServer from settings_library.redis import RedisSettings -from simcore_service_api_server.celery_worker.worker_main import setup_worker_tasks +from simcore_service_api_server.celery.main import setup_worker_tasks from simcore_service_api_server.clients import celery_task_manager from simcore_service_api_server.core.application import create_app from simcore_service_api_server.core.settings import ApplicationSettings diff --git a/services/api-server/tests/unit/api_functions/celery/test_functions_celery.py b/services/api-server/tests/unit/api_functions/celery/test_functions_celery.py index 75088e96df30..0eb828743247 100644 --- a/services/api-server/tests/unit/api_functions/celery/test_functions_celery.py +++ b/services/api-server/tests/unit/api_functions/celery/test_functions_celery.py @@ -52,7 +52,7 @@ from simcore_service_api_server.api.dependencies.celery import ( get_task_manager, ) -from simcore_service_api_server.celery_worker.worker_tasks.functions_tasks import ( +from simcore_service_api_server.celery.worker_tasks.functions_tasks import ( run_function as run_function_task, ) from simcore_service_api_server.exceptions.backend_errors import BaseBackEndError diff --git a/services/api-server/tests/unit/api_functions/test_api_routers_functions.py b/services/api-server/tests/unit/api_functions/test_api_routers_functions.py index fcc8b8e4fafc..a0ed663010fa 100644 --- a/services/api-server/tests/unit/api_functions/test_api_routers_functions.py +++ b/services/api-server/tests/unit/api_functions/test_api_routers_functions.py @@ -46,7 +46,7 @@ from servicelib.rabbitmq import RabbitMQRPCClient from simcore_service_api_server._meta import API_VTAG from simcore_service_api_server.api.dependencies.authentication import Identity -from simcore_service_api_server.celery_worker.worker_tasks import functions_tasks +from simcore_service_api_server.celery.worker_tasks import functions_tasks from simcore_service_api_server.models.api_resources import JobLinks from simcore_service_api_server.models.domain.functions import ( PreRegisteredFunctionJobData, diff --git a/services/docker-compose.yml b/services/docker-compose.yml index b9d935488d7b..da5eb344ba1f 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -168,6 +168,7 @@ services: API_SERVER_WORKER_NAME: "api-worker-{{.Node.Hostname}}-{{.Task.Slot}}-{{.Task.ID}}" API_SERVER_WORKER_MODE: "true" CELERY_CONCURRENCY: ${API_SERVER_CELERY_CONCURRENCY} + CELERY_POOL: threads CELERY_QUEUES: "api_worker_queue" networks: *api_server_networks diff --git a/services/storage/docker/boot.sh b/services/storage/docker/boot.sh index 627333dc35c6..acaf9555fff0 100755 --- a/services/storage/docker/boot.sh +++ b/services/storage/docker/boot.sh @@ -55,7 +55,7 @@ if [ "${STORAGE_WORKER_MODE}" = "true" ]; then --recursive \ -- \ celery \ - --app=simcore_service_storage.modules.celery.worker.main:app \ + --app=simcore_service_storage.modules.celery.worker.main:the_app \ worker --pool="${CELERY_POOL:-prefork}" \ --loglevel="${SERVER_LOG_LEVEL}" \ --concurrency="${CELERY_CONCURRENCY}" \ @@ -63,7 +63,7 @@ if [ "${STORAGE_WORKER_MODE}" = "true" ]; then --queues="${CELERY_QUEUES:-default}" else exec celery \ - --app=simcore_service_storage.modules.celery.worker.main:app \ + --app=simcore_service_storage.modules.celery.worker.main:the_app \ worker --pool="${CELERY_POOL:-prefork}" \ --loglevel="${SERVER_LOG_LEVEL}" \ --concurrency="${CELERY_CONCURRENCY}" \ diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker/main.py b/services/storage/src/simcore_service_storage/modules/celery/worker/main.py index b09c47097fe9..63a4ec27591a 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker/main.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker/main.py @@ -24,16 +24,24 @@ service_name="storage-celery-worker", ) -setup_loggers( - log_format_local_dev_enabled=_settings.STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED, - logger_filter_mapping=_settings.STORAGE_LOG_FILTER_MAPPING, - tracing_config=_tracing_config, - log_base_level=_settings.log_level, - noisy_loggers=None, -) -assert _settings.STORAGE_CELERY # nosec -app = create_celery_app(_settings.STORAGE_CELERY) +def get_app(): + setup_loggers( + log_format_local_dev_enabled=_settings.STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED, + logger_filter_mapping=_settings.STORAGE_LOG_FILTER_MAPPING, + tracing_config=_tracing_config, + log_base_level=_settings.log_level, + noisy_loggers=None, + ) + + assert _settings.STORAGE_CELERY # nosec + app = create_celery_app(_settings.STORAGE_CELERY) + setup_worker_tasks(app) + + return app + + +the_app = get_app() @worker_init.connect @@ -41,15 +49,12 @@ def worker_init_wrapper(**kwargs): fastapi_app = create_app(_settings, tracing_config=_tracing_config) app_server = FastAPIAppServer(app=fastapi_app) - set_app_server(app, app_server) + set_app_server(the_app, app_server) return on_worker_init(app_server, **kwargs) @worker_shutdown.connect @worker_process_shutdown.connect def worker_shutdown_wrapper(**kwargs): - app_server = get_app_server(app) + app_server = get_app_server(the_app) return on_worker_shutdown(app_server, **kwargs) - - -setup_worker_tasks(app) From 1a8d1350275a69167b13b3986731f819a05be9c1 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 13 Oct 2025 15:55:08 +0200 Subject: [PATCH 11/58] update docker-compose --- services/docker-compose.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/services/docker-compose.yml b/services/docker-compose.yml index da5eb344ba1f..fee1a2f55ed8 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -169,7 +169,7 @@ services: API_SERVER_WORKER_MODE: "true" CELERY_CONCURRENCY: ${API_SERVER_CELERY_CONCURRENCY} CELERY_POOL: threads - CELERY_QUEUES: "api_worker_queue" + CELERY_QUEUES: api_worker_queue networks: *api_server_networks @@ -1255,7 +1255,7 @@ services: STORAGE_WORKER_NAME: "sto-worker-{{.Node.Hostname}}-{{.Task.Slot}}-{{.Task.ID}}" STORAGE_WORKER_MODE: "true" CELERY_CONCURRENCY: 100 - CELERY_POOL: "threads" + CELERY_POOL: threads networks: *storage_networks sto-worker-cpu-bound: @@ -1268,7 +1268,8 @@ services: STORAGE_WORKER_NAME: "sto-worker-cpu-bound-{{.Node.Hostname}}-{{.Task.Slot}}-{{.Task.ID}}" STORAGE_WORKER_MODE: "true" CELERY_CONCURRENCY: 1 - CELERY_QUEUES: "cpu_bound" + CELERY_QUEUES: cpu_bound + CELERY_POOL: prefork networks: *storage_networks rabbit: From d624966f649c88d23be0b5a6aec20df7631649c4 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 13 Oct 2025 16:01:46 +0200 Subject: [PATCH 12/58] fix test --- services/api-server/tests/unit/api_functions/celery/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/api-server/tests/unit/api_functions/celery/conftest.py b/services/api-server/tests/unit/api_functions/celery/conftest.py index 7b39ce60ce86..3bdacdd04333 100644 --- a/services/api-server/tests/unit/api_functions/celery/conftest.py +++ b/services/api-server/tests/unit/api_functions/celery/conftest.py @@ -25,10 +25,10 @@ from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.fastapi.celery.app_server import FastAPIAppServer from settings_library.redis import RedisSettings -from simcore_service_api_server.celery.main import setup_worker_tasks from simcore_service_api_server.clients import celery_task_manager from simcore_service_api_server.core.application import create_app from simcore_service_api_server.core.settings import ApplicationSettings +from simcore_service_api_server.modules.celery.worker.tasks import setup_worker_tasks @pytest.fixture(scope="session") From c97c75ebc561daa9c95fc95458f89e452c05d7c7 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 13 Oct 2025 16:04:20 +0200 Subject: [PATCH 13/58] fix test --- packages/celery-library/tests/conftest.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/celery-library/tests/conftest.py b/packages/celery-library/tests/conftest.py index e37f7d003f1e..78f97973e14d 100644 --- a/packages/celery-library/tests/conftest.py +++ b/packages/celery-library/tests/conftest.py @@ -14,7 +14,6 @@ start_worker, ) from celery.signals import worker_init, worker_shutdown -from celery.worker.worker import WorkController from celery_library.backends.redis import RedisTaskStore from celery_library.signals import on_worker_init, on_worker_shutdown from celery_library.task_manager import CeleryTaskManager @@ -131,8 +130,8 @@ async def with_celery_worker( app_server: BaseAppServer, register_celery_tasks: Callable[[Celery], None], ) -> AsyncIterator[TestWorkController]: - def _on_worker_init_wrapper(sender: WorkController, **_kwargs): - return on_worker_init(sender, app_server, **_kwargs) + def _on_worker_init_wrapper(**kwargs): + return on_worker_init(app_server, **kwargs) worker_init.connect(_on_worker_init_wrapper) worker_shutdown.connect(on_worker_shutdown) From d66004188cc4d8f5207c10556bae20bcb0c102e1 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 13 Oct 2025 16:08:05 +0200 Subject: [PATCH 14/58] fix test --- .../unit/api_functions/celery/test_functions_celery.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/api-server/tests/unit/api_functions/celery/test_functions_celery.py b/services/api-server/tests/unit/api_functions/celery/test_functions_celery.py index 0eb828743247..7d5d197d1774 100644 --- a/services/api-server/tests/unit/api_functions/celery/test_functions_celery.py +++ b/services/api-server/tests/unit/api_functions/celery/test_functions_celery.py @@ -52,9 +52,6 @@ from simcore_service_api_server.api.dependencies.celery import ( get_task_manager, ) -from simcore_service_api_server.celery.worker_tasks.functions_tasks import ( - run_function as run_function_task, -) from simcore_service_api_server.exceptions.backend_errors import BaseBackEndError from simcore_service_api_server.models.api_resources import JobLinks from simcore_service_api_server.models.domain.celery_models import ( @@ -67,6 +64,9 @@ JobPricingSpecification, NodeID, ) +from simcore_service_api_server.modules.celery.worker._functions_tasks import ( + run_function as run_function_task, +) from tenacity import ( AsyncRetrying, retry_if_exception_type, From c4d33c4ea097eb679692bc76b5366a0e41fb9622 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 13 Oct 2025 16:51:06 +0200 Subject: [PATCH 15/58] fix test --- .../unit/api_functions/test_api_routers_functions.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/services/api-server/tests/unit/api_functions/test_api_routers_functions.py b/services/api-server/tests/unit/api_functions/test_api_routers_functions.py index a0ed663010fa..a3ae47a0fcd2 100644 --- a/services/api-server/tests/unit/api_functions/test_api_routers_functions.py +++ b/services/api-server/tests/unit/api_functions/test_api_routers_functions.py @@ -46,12 +46,12 @@ from servicelib.rabbitmq import RabbitMQRPCClient from simcore_service_api_server._meta import API_VTAG from simcore_service_api_server.api.dependencies.authentication import Identity -from simcore_service_api_server.celery.worker_tasks import functions_tasks from simcore_service_api_server.models.api_resources import JobLinks from simcore_service_api_server.models.domain.functions import ( PreRegisteredFunctionJobData, ) from simcore_service_api_server.models.schemas.jobs import JobInputs +from simcore_service_api_server.modules.celery.worker import _functions_tasks from simcore_service_api_server.services_rpc.wb_api_server import WbApiRpcClient _faker = Faker() @@ -426,13 +426,13 @@ def _get_app_server(celery_app: Any) -> FastAPI: app_server.app = app return app_server - mocker.patch.object(functions_tasks, "get_app_server", _get_app_server) + mocker.patch.object(_functions_tasks, "get_app_server", _get_app_server) def _get_rabbitmq_rpc_client(app: FastAPI) -> RabbitMQRPCClient: return mocker.MagicMock(spec=RabbitMQRPCClient) mocker.patch.object( - functions_tasks, "get_rabbitmq_rpc_client", _get_rabbitmq_rpc_client + _functions_tasks, "get_rabbitmq_rpc_client", _get_rabbitmq_rpc_client ) async def _get_wb_api_rpc_client(app: FastAPI) -> WbApiRpcClient: @@ -442,7 +442,7 @@ async def _get_wb_api_rpc_client(app: FastAPI) -> WbApiRpcClient: return WbApiRpcClient.get_from_app_state(app) mocker.patch.object( - functions_tasks, "get_wb_api_rpc_client", _get_wb_api_rpc_client + _functions_tasks, "get_wb_api_rpc_client", _get_wb_api_rpc_client ) def _default_side_effect( @@ -492,7 +492,7 @@ def _default_side_effect( function_job_id=fake_registered_project_function.uid, ) - job = await functions_tasks.run_function( + job = await _functions_tasks.run_function( task=MagicMock(spec=Task), task_key=TaskKey(_faker.uuid4()), user_identity=user_identity, From 012ea990b71b7efd5b5e258e86f84d2329927c43 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 13 Oct 2025 16:52:00 +0200 Subject: [PATCH 16/58] fix test --- services/api-server/tests/unit/api_functions/celery/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/api-server/tests/unit/api_functions/celery/conftest.py b/services/api-server/tests/unit/api_functions/celery/conftest.py index 3bdacdd04333..677ff6013b9e 100644 --- a/services/api-server/tests/unit/api_functions/celery/conftest.py +++ b/services/api-server/tests/unit/api_functions/celery/conftest.py @@ -125,7 +125,7 @@ async def with_api_server_celery_worker( app_server = FastAPIAppServer(app=create_app(app_settings)) def _on_worker_init_wrapper(sender: WorkController, **kwargs): - return on_worker_init(sender, app_server=app_server, **kwargs) + return on_worker_init(app_server, **kwargs) worker_init.connect(_on_worker_init_wrapper) worker_shutdown.connect(on_worker_shutdown) From fb1ac0547342ad924187112814c03d29344a0a3a Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 13 Oct 2025 19:55:49 +0200 Subject: [PATCH 17/58] fix --- services/api-server/docker/boot.sh | 4 ++-- .../simcore_service_api_server/modules/celery/worker/main.py | 4 ++-- services/storage/docker/boot.sh | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/services/api-server/docker/boot.sh b/services/api-server/docker/boot.sh index 4cf40ee470c8..163595817e82 100755 --- a/services/api-server/docker/boot.sh +++ b/services/api-server/docker/boot.sh @@ -49,7 +49,7 @@ if [ "${API_SERVER_WORKER_MODE}" = "true" ]; then -- \ celery \ --app=simcore_service_api_server.modules.celery.worker.main:the_app \ - worker --pool="${CELERY_POOL:-prefork}" \ + worker --pool="${CELERY_POOL}" \ --loglevel="${API_SERVER_LOGLEVEL}" \ --concurrency="${CELERY_CONCURRENCY}" \ --hostname="${API_SERVER_WORKER_NAME}" \ @@ -57,7 +57,7 @@ if [ "${API_SERVER_WORKER_MODE}" = "true" ]; then else exec celery \ --app=simcore_service_api_server.modules.celery.worker.main:the_app \ - worker --pool="${CELERY_POOL:-prefork}" \ + worker --pool="${CELERY_POOL}" \ --loglevel="${API_SERVER_LOGLEVEL}" \ --concurrency="${CELERY_CONCURRENCY}" \ --hostname="${API_SERVER_WORKER_NAME}" \ diff --git a/services/api-server/src/simcore_service_api_server/modules/celery/worker/main.py b/services/api-server/src/simcore_service_api_server/modules/celery/worker/main.py index c872ca4bba2e..24c5ba15420f 100644 --- a/services/api-server/src/simcore_service_api_server/modules/celery/worker/main.py +++ b/services/api-server/src/simcore_service_api_server/modules/celery/worker/main.py @@ -46,7 +46,7 @@ def get_app(): @worker_init.connect -def worker_init_wrapper(**kwargs): +def _worker_init_wrapper(**kwargs): _settings = ApplicationSettings.create_from_envs() assert _settings.API_SERVER_CELERY # nosec app_server = FastAPIAppServer(app=create_app(_settings)) @@ -55,6 +55,6 @@ def worker_init_wrapper(**kwargs): @worker_shutdown.connect -def worker_shutdown_wrapper(**kwargs): +def _worker_shutdown_wrapper(**kwargs): app_server = get_app_server(the_app) return on_worker_shutdown(app_server, **kwargs) diff --git a/services/storage/docker/boot.sh b/services/storage/docker/boot.sh index acaf9555fff0..09153e7f6bbe 100755 --- a/services/storage/docker/boot.sh +++ b/services/storage/docker/boot.sh @@ -56,7 +56,7 @@ if [ "${STORAGE_WORKER_MODE}" = "true" ]; then -- \ celery \ --app=simcore_service_storage.modules.celery.worker.main:the_app \ - worker --pool="${CELERY_POOL:-prefork}" \ + worker --pool="${CELERY_POOL}" \ --loglevel="${SERVER_LOG_LEVEL}" \ --concurrency="${CELERY_CONCURRENCY}" \ --hostname="${STORAGE_WORKER_NAME}" \ @@ -64,7 +64,7 @@ if [ "${STORAGE_WORKER_MODE}" = "true" ]; then else exec celery \ --app=simcore_service_storage.modules.celery.worker.main:the_app \ - worker --pool="${CELERY_POOL:-prefork}" \ + worker --pool="${CELERY_POOL}" \ --loglevel="${SERVER_LOG_LEVEL}" \ --concurrency="${CELERY_CONCURRENCY}" \ --hostname="${STORAGE_WORKER_NAME}" \ From e6d01f304233474b2b0a581d7b233529816b48fa Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 13 Oct 2025 21:28:42 +0200 Subject: [PATCH 18/58] fix tests --- packages/celery-library/tests/conftest.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/celery-library/tests/conftest.py b/packages/celery-library/tests/conftest.py index 78f97973e14d..a61dd890e086 100644 --- a/packages/celery-library/tests/conftest.py +++ b/packages/celery-library/tests/conftest.py @@ -18,6 +18,7 @@ from celery_library.signals import on_worker_init, on_worker_shutdown from celery_library.task_manager import CeleryTaskManager from celery_library.types import register_celery_types +from celery_library.utils import set_app_server from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.celery.app_server import BaseAppServer @@ -131,10 +132,14 @@ async def with_celery_worker( register_celery_tasks: Callable[[Celery], None], ) -> AsyncIterator[TestWorkController]: def _on_worker_init_wrapper(**kwargs): + set_app_server(celery_app, app_server) return on_worker_init(app_server, **kwargs) + def _on_worker_shutdown_wrapper(**kwargs): + return on_worker_shutdown(app_server, **kwargs) + worker_init.connect(_on_worker_init_wrapper) - worker_shutdown.connect(on_worker_shutdown) + worker_shutdown.connect(_on_worker_shutdown_wrapper) register_celery_tasks(celery_app) From 08e6c1a018c195ea6cc85919cd2b47e781da57b3 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 13 Oct 2025 21:30:41 +0200 Subject: [PATCH 19/58] fix tests --- .../tests/unit/api_functions/celery/conftest.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/services/api-server/tests/unit/api_functions/celery/conftest.py b/services/api-server/tests/unit/api_functions/celery/conftest.py index 677ff6013b9e..da9b074f8098 100644 --- a/services/api-server/tests/unit/api_functions/celery/conftest.py +++ b/services/api-server/tests/unit/api_functions/celery/conftest.py @@ -18,7 +18,6 @@ worker_init, worker_shutdown, ) -from celery.worker.worker import WorkController # pylint: disable=no-name-in-module from celery_library.signals import on_worker_init, on_worker_shutdown from pytest_mock import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import delenvs_from_dict, setenvs_from_dict @@ -124,11 +123,14 @@ async def with_api_server_celery_worker( app_server = FastAPIAppServer(app=create_app(app_settings)) - def _on_worker_init_wrapper(sender: WorkController, **kwargs): + def _on_worker_init_wrapper(**kwargs): return on_worker_init(app_server, **kwargs) + def _on_worker_shutdown_wrapper(**kwargs): + return on_worker_shutdown(app_server, **kwargs) + worker_init.connect(_on_worker_init_wrapper) - worker_shutdown.connect(on_worker_shutdown) + worker_shutdown.connect(_on_worker_shutdown_wrapper) if add_worker_tasks: setup_worker_tasks(celery_app) From 4f839b8d6aec7df09a3993b71854f2cfe536e32e Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 13 Oct 2025 21:35:05 +0200 Subject: [PATCH 20/58] fix tests --- services/api-server/tests/unit/api_functions/celery/conftest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/api-server/tests/unit/api_functions/celery/conftest.py b/services/api-server/tests/unit/api_functions/celery/conftest.py index da9b074f8098..83e93e452455 100644 --- a/services/api-server/tests/unit/api_functions/celery/conftest.py +++ b/services/api-server/tests/unit/api_functions/celery/conftest.py @@ -19,6 +19,7 @@ worker_shutdown, ) from celery_library.signals import on_worker_init, on_worker_shutdown +from celery_library.utils import set_app_server from pytest_mock import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import delenvs_from_dict, setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict @@ -124,6 +125,7 @@ async def with_api_server_celery_worker( app_server = FastAPIAppServer(app=create_app(app_settings)) def _on_worker_init_wrapper(**kwargs): + set_app_server(celery_app, app_server) return on_worker_init(app_server, **kwargs) def _on_worker_shutdown_wrapper(**kwargs): From 1a86d714e4e471f478aa03d38ea3519cdcfdd974 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 13 Oct 2025 21:36:11 +0200 Subject: [PATCH 21/58] typecheck --- .../simcore_service_api_server/modules/celery/worker/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/api-server/src/simcore_service_api_server/modules/celery/worker/main.py b/services/api-server/src/simcore_service_api_server/modules/celery/worker/main.py index 24c5ba15420f..4eddb8c47840 100644 --- a/services/api-server/src/simcore_service_api_server/modules/celery/worker/main.py +++ b/services/api-server/src/simcore_service_api_server/modules/celery/worker/main.py @@ -1,6 +1,6 @@ """Main application to be deployed in for example uvicorn.""" -from celery.signals import ( # type: ignore[import-untyped] +from celery.signals import ( # type: ignore[import-untyped] # pylint: disable=no-name-in-module worker_init, worker_shutdown, ) From fb4a8474331f42edb87ef1a825cbd88c0d975384 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 13 Oct 2025 23:27:51 +0200 Subject: [PATCH 22/58] fix tests --- services/storage/tests/conftest.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index 1831fab085a9..0e12ba750d68 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -24,8 +24,8 @@ from celery import Celery from celery.contrib.testing.worker import TestWorkController, start_worker from celery.signals import worker_init, worker_shutdown -from celery.worker.worker import WorkController from celery_library.signals import on_worker_init, on_worker_shutdown +from celery_library.utils import set_app_server from faker import Faker from fakeredis.aioredis import FakeRedis from fastapi import FastAPI @@ -1027,8 +1027,9 @@ async def with_storage_celery_worker( app=create_app(app_settings, tracing_config=tracing_config) ) - def _on_worker_init_wrapper(sender: WorkController, **_kwargs): - return on_worker_init(sender, app_server, **_kwargs) + def _on_worker_init_wrapper(**kwargs): + set_app_server(celery_app, app_server) + return on_worker_init(app_server, **kwargs) worker_init.connect(_on_worker_init_wrapper) worker_shutdown.connect(on_worker_shutdown) From da302cba2358881a6f8de9b8906e9f7e4bfd1b73 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 14 Oct 2025 12:46:06 +0200 Subject: [PATCH 23/58] update description --- packages/settings-library/src/settings_library/celery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/settings-library/src/settings_library/celery.py b/packages/settings-library/src/settings_library/celery.py index 6c767976b42f..3152eb203160 100644 --- a/packages/settings-library/src/settings_library/celery.py +++ b/packages/settings-library/src/settings_library/celery.py @@ -38,7 +38,7 @@ class CelerySettings(BaseCustomSettings): CELERY_POOL: Annotated[ Literal["prefork", "eventlet", "gevent", "solo", "threads"], Field( - description="Type of pool to use. One of: prefork, eventlet, gevent, solo, threads. See https://docs.celeryq.dev/en/stable/userguide/workers.html#choosing-a-concurrency-implementation for details.", + description="Type of pool to use. One of: prefork, eventlet, gevent, solo, threads. See https://docs.celeryq.dev/en/stable/userguide/concurrency/index.html for details.", ), ] = "prefork" From d5aa656b7fa4dd24fd1752a71f6e67cc23c772ae Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 14 Oct 2025 16:41:30 +0200 Subject: [PATCH 24/58] execute only once init in prefork --- .../modules/celery/worker/main.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker/main.py b/services/storage/src/simcore_service_storage/modules/celery/worker/main.py index 63a4ec27591a..3bd9dff4f8d9 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker/main.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker/main.py @@ -44,8 +44,6 @@ def get_app(): the_app = get_app() -@worker_init.connect -@worker_process_init.connect def worker_init_wrapper(**kwargs): fastapi_app = create_app(_settings, tracing_config=_tracing_config) app_server = FastAPIAppServer(app=fastapi_app) @@ -53,8 +51,15 @@ def worker_init_wrapper(**kwargs): return on_worker_init(app_server, **kwargs) -@worker_shutdown.connect -@worker_process_shutdown.connect def worker_shutdown_wrapper(**kwargs): app_server = get_app_server(the_app) return on_worker_shutdown(app_server, **kwargs) + + +assert _settings.STORAGE_CELERY # nosec +if _settings.STORAGE_CELERY.CELERY_POOL == "prefork": + worker_process_init.connect(worker_init_wrapper) + worker_process_shutdown.connect(worker_shutdown_wrapper) +else: + worker_init.connect(worker_init_wrapper) + worker_shutdown.connect(worker_shutdown_wrapper) From 88ad3192be6d341dae8e3b5ee36daa69d1a97472 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 14 Oct 2025 22:54:49 +0200 Subject: [PATCH 25/58] refactoring --- .../src/celery_library/{common.py => app.py} | 0 .../celery-library/src/celery_library/task.py | 2 +- .../src/celery_library/worker/__init__.py | 0 .../src/celery_library/worker/app.py | 20 +++++++ .../{utils.py => worker/app_server.py} | 2 +- .../src/celery_library/worker/signals.py | 59 +++++++++++++++++++ packages/celery-library/tests/conftest.py | 2 +- .../tests/unit/test_task_manager.py | 2 +- services/api-server/docker/boot.sh | 4 +- .../clients/celery_task_manager.py | 2 +- .../modules/celery/worker/_functions_tasks.py | 2 +- .../modules/celery/worker/main.py | 43 ++++---------- .../modules/celery/worker/tasks.py | 2 +- .../unit/api_functions/celery/conftest.py | 4 +- services/storage/docker/boot.sh | 4 +- .../api/_worker_tasks/_files.py | 2 +- .../api/_worker_tasks/_paths.py | 2 +- .../api/_worker_tasks/_simcore_s3.py | 2 +- .../api/_worker_tasks/tasks.py | 2 +- .../modules/celery/__init__.py | 2 +- .../modules/celery/worker/main.py | 51 ++++------------ services/storage/tests/conftest.py | 4 +- .../celery/_task_manager.py | 2 +- 23 files changed, 123 insertions(+), 92 deletions(-) rename packages/celery-library/src/celery_library/{common.py => app.py} (100%) create mode 100644 packages/celery-library/src/celery_library/worker/__init__.py create mode 100644 packages/celery-library/src/celery_library/worker/app.py rename packages/celery-library/src/celery_library/{utils.py => worker/app_server.py} (86%) create mode 100644 packages/celery-library/src/celery_library/worker/signals.py diff --git a/packages/celery-library/src/celery_library/common.py b/packages/celery-library/src/celery_library/app.py similarity index 100% rename from packages/celery-library/src/celery_library/common.py rename to packages/celery-library/src/celery_library/app.py diff --git a/packages/celery-library/src/celery_library/task.py b/packages/celery-library/src/celery_library/task.py index f01bf301347c..dfcc9f889a0b 100644 --- a/packages/celery-library/src/celery_library/task.py +++ b/packages/celery-library/src/celery_library/task.py @@ -13,7 +13,7 @@ from servicelib.celery.models import TaskKey from .errors import encode_celery_transferrable_error -from .utils import get_app_server +from .worker.app_server import get_app_server _logger = logging.getLogger(__name__) diff --git a/packages/celery-library/src/celery_library/worker/__init__.py b/packages/celery-library/src/celery_library/worker/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/packages/celery-library/src/celery_library/worker/app.py b/packages/celery-library/src/celery_library/worker/app.py new file mode 100644 index 000000000000..2695cbe0e0a4 --- /dev/null +++ b/packages/celery-library/src/celery_library/worker/app.py @@ -0,0 +1,20 @@ +from collections.abc import Callable + +from celery import Celery +from servicelib.celery.app_server import BaseAppServer +from settings_library.celery import CelerySettings + +from ..app import create_app +from .signals import register_worker_signals + + +def create_worker_app( + settings: CelerySettings, + register_worker_tasks_cb: Callable[[Celery], None], + app_server_factory_cb: Callable[[], BaseAppServer], +) -> Celery: + app = create_app(settings) + register_worker_tasks_cb(app) + register_worker_signals(app, settings, app_server_factory_cb) + + return app diff --git a/packages/celery-library/src/celery_library/utils.py b/packages/celery-library/src/celery_library/worker/app_server.py similarity index 86% rename from packages/celery-library/src/celery_library/utils.py rename to packages/celery-library/src/celery_library/worker/app_server.py index 4d8bad73b96d..cf5864891bf3 100644 --- a/packages/celery-library/src/celery_library/utils.py +++ b/packages/celery-library/src/celery_library/worker/app_server.py @@ -1,4 +1,4 @@ -from celery import Celery # type: ignore[import-untyped] +from celery import Celery from servicelib.celery.app_server import BaseAppServer _APP_SERVER_KEY = "app_server" diff --git a/packages/celery-library/src/celery_library/worker/signals.py b/packages/celery-library/src/celery_library/worker/signals.py new file mode 100644 index 000000000000..3c1a57d8811d --- /dev/null +++ b/packages/celery-library/src/celery_library/worker/signals.py @@ -0,0 +1,59 @@ +import asyncio +import threading +from collections.abc import Callable + +from celery import Celery # type: ignore[import-untyped] +from celery.signals import ( + worker_init, + worker_process_init, + worker_process_shutdown, + worker_shutdown, +) +from servicelib.celery.app_server import BaseAppServer +from settings_library.celery import CelerySettings + +from .app_server import get_app_server, set_app_server + + +def register_worker_signals( + app: Celery, + settings: CelerySettings, + app_server_factory: Callable[[], BaseAppServer], +) -> None: + def _worker_init_wrapper(**_kwargs) -> None: + startup_complete_event = threading.Event() + + def _init(startup_complete_event: threading.Event) -> None: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + app_server = app_server_factory() + app_server.event_loop = loop + + set_app_server(app, app_server) + + loop.run_until_complete( + app_server.run_until_shutdown(startup_complete_event) + ) + + thread = threading.Thread( + group=None, + target=_init, + name="app_server_init", + args=(startup_complete_event,), + daemon=True, + ) + thread.start() + + startup_complete_event.wait() + + def _worker_shutdown_wrapper(**_kwargs) -> None: + get_app_server(app).shutdown_event.set() + + match settings.CELERY_POOL: + case "prefork": + worker_process_init.connect(_worker_init_wrapper, weak=False) + worker_process_shutdown.connect(_worker_shutdown_wrapper, weak=False) + case _: + worker_init.connect(_worker_init_wrapper, weak=False) + worker_shutdown.connect(_worker_shutdown_wrapper, weak=False) diff --git a/packages/celery-library/tests/conftest.py b/packages/celery-library/tests/conftest.py index a61dd890e086..e5adbcc2172f 100644 --- a/packages/celery-library/tests/conftest.py +++ b/packages/celery-library/tests/conftest.py @@ -18,7 +18,7 @@ from celery_library.signals import on_worker_init, on_worker_shutdown from celery_library.task_manager import CeleryTaskManager from celery_library.types import register_celery_types -from celery_library.utils import set_app_server +from celery_library.worker.app_server import set_app_server from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.celery.app_server import BaseAppServer diff --git a/packages/celery-library/tests/unit/test_task_manager.py b/packages/celery-library/tests/unit/test_task_manager.py index 040c0541ed53..8a79adf9d7a1 100644 --- a/packages/celery-library/tests/unit/test_task_manager.py +++ b/packages/celery-library/tests/unit/test_task_manager.py @@ -16,7 +16,7 @@ from celery_library.errors import TaskNotFoundError, TransferrableCeleryError from celery_library.task import register_task from celery_library.task_manager import CeleryTaskManager -from celery_library.utils import get_app_server +from celery_library.worker.app_server import get_app_server from common_library.errors_classes import OsparcErrorMixin from faker import Faker from models_library.progress_bar import ProgressReport diff --git a/services/api-server/docker/boot.sh b/services/api-server/docker/boot.sh index 163595817e82..ef5bf4fd42e2 100755 --- a/services/api-server/docker/boot.sh +++ b/services/api-server/docker/boot.sh @@ -48,7 +48,7 @@ if [ "${API_SERVER_WORKER_MODE}" = "true" ]; then --recursive \ -- \ celery \ - --app=simcore_service_api_server.modules.celery.worker.main:the_app \ + --app=simcore_service_api_server.modules.celery.worker.main:app \ worker --pool="${CELERY_POOL}" \ --loglevel="${API_SERVER_LOGLEVEL}" \ --concurrency="${CELERY_CONCURRENCY}" \ @@ -56,7 +56,7 @@ if [ "${API_SERVER_WORKER_MODE}" = "true" ]; then --queues="${CELERY_QUEUES:-default}" else exec celery \ - --app=simcore_service_api_server.modules.celery.worker.main:the_app \ + --app=simcore_service_api_server.modules.celery.worker.main:app \ worker --pool="${CELERY_POOL}" \ --loglevel="${API_SERVER_LOGLEVEL}" \ --concurrency="${CELERY_CONCURRENCY}" \ diff --git a/services/api-server/src/simcore_service_api_server/clients/celery_task_manager.py b/services/api-server/src/simcore_service_api_server/clients/celery_task_manager.py index 13829cfdd303..fcfbbb35678f 100644 --- a/services/api-server/src/simcore_service_api_server/clients/celery_task_manager.py +++ b/services/api-server/src/simcore_service_api_server/clients/celery_task_manager.py @@ -1,7 +1,7 @@ import logging +from celery_library.app import create_app from celery_library.backends.redis import RedisTaskStore -from celery_library.common import create_app from celery_library.task_manager import CeleryTaskManager from celery_library.types import register_celery_types, register_pydantic_types from fastapi import FastAPI diff --git a/services/api-server/src/simcore_service_api_server/modules/celery/worker/_functions_tasks.py b/services/api-server/src/simcore_service_api_server/modules/celery/worker/_functions_tasks.py index 1a0bcfc33b5f..bfe4eb8e63cb 100644 --- a/services/api-server/src/simcore_service_api_server/modules/celery/worker/_functions_tasks.py +++ b/services/api-server/src/simcore_service_api_server/modules/celery/worker/_functions_tasks.py @@ -1,7 +1,7 @@ from celery import ( # type: ignore[import-untyped] # pylint: disable=no-name-in-module Task, ) -from celery_library.utils import get_app_server # pylint: disable=no-name-in-module +from celery_library.worker.app_server import get_app_server from fastapi import FastAPI from models_library.functions import RegisteredFunction, RegisteredFunctionJob from models_library.projects import ProjectID diff --git a/services/api-server/src/simcore_service_api_server/modules/celery/worker/main.py b/services/api-server/src/simcore_service_api_server/modules/celery/worker/main.py index 4eddb8c47840..826802f4d34b 100644 --- a/services/api-server/src/simcore_service_api_server/modules/celery/worker/main.py +++ b/services/api-server/src/simcore_service_api_server/modules/celery/worker/main.py @@ -1,22 +1,11 @@ -"""Main application to be deployed in for example uvicorn.""" - -from celery.signals import ( # type: ignore[import-untyped] # pylint: disable=no-name-in-module - worker_init, - worker_shutdown, -) -from celery_library.common import create_app as create_celery_app -from celery_library.signals import ( - on_worker_init, - on_worker_shutdown, -) -from celery_library.utils import get_app_server, set_app_server +from celery_library.worker.app import create_worker_app from servicelib.fastapi.celery.app_server import FastAPIAppServer from servicelib.logging_utils import setup_loggers from servicelib.tracing import TracingConfig from ....core.application import create_app from ....core.settings import ApplicationSettings -from .tasks import setup_worker_tasks +from .tasks import register_worker_tasks _settings = ApplicationSettings.create_from_envs() _tracing_settings = _settings.API_SERVER_TRACING @@ -35,26 +24,16 @@ def get_app(): noisy_loggers=None, ) - assert _settings.API_SERVER_CELERY # nosec - app = create_celery_app(_settings.API_SERVER_CELERY) - setup_worker_tasks(app) - - return app + def _app_server_factory() -> FastAPIAppServer: + fastapi_app = create_app(_settings, tracing_config=_tracing_config) + return FastAPIAppServer(app=fastapi_app) - -the_app = get_app() - - -@worker_init.connect -def _worker_init_wrapper(**kwargs): - _settings = ApplicationSettings.create_from_envs() assert _settings.API_SERVER_CELERY # nosec - app_server = FastAPIAppServer(app=create_app(_settings)) - set_app_server(the_app, app_server) - return on_worker_init(app_server, **kwargs) + return create_worker_app( + _settings.API_SERVER_CELERY, + register_worker_tasks, + _app_server_factory, + ) -@worker_shutdown.connect -def _worker_shutdown_wrapper(**kwargs): - app_server = get_app_server(the_app) - return on_worker_shutdown(app_server, **kwargs) +app = get_app() diff --git a/services/api-server/src/simcore_service_api_server/modules/celery/worker/tasks.py b/services/api-server/src/simcore_service_api_server/modules/celery/worker/tasks.py index 4a0321c92a34..b567d1cc63dc 100644 --- a/services/api-server/src/simcore_service_api_server/modules/celery/worker/tasks.py +++ b/services/api-server/src/simcore_service_api_server/modules/celery/worker/tasks.py @@ -13,7 +13,7 @@ _logger = logging.getLogger(__name__) -def setup_worker_tasks(app: Celery) -> None: +def register_worker_tasks(app: Celery) -> None: register_celery_types() register_pydantic_types(*pydantic_types_to_register) diff --git a/services/api-server/tests/unit/api_functions/celery/conftest.py b/services/api-server/tests/unit/api_functions/celery/conftest.py index 83e93e452455..8e004aa35df3 100644 --- a/services/api-server/tests/unit/api_functions/celery/conftest.py +++ b/services/api-server/tests/unit/api_functions/celery/conftest.py @@ -28,7 +28,7 @@ from simcore_service_api_server.clients import celery_task_manager from simcore_service_api_server.core.application import create_app from simcore_service_api_server.core.settings import ApplicationSettings -from simcore_service_api_server.modules.celery.worker.tasks import setup_worker_tasks +from simcore_service_api_server.modules.celery.worker.tasks import register_worker_tasks @pytest.fixture(scope="session") @@ -135,7 +135,7 @@ def _on_worker_shutdown_wrapper(**kwargs): worker_shutdown.connect(_on_worker_shutdown_wrapper) if add_worker_tasks: - setup_worker_tasks(celery_app) + register_worker_tasks(celery_app) register_celery_tasks(celery_app) with start_worker( diff --git a/services/storage/docker/boot.sh b/services/storage/docker/boot.sh index 09153e7f6bbe..2c9803f1b0ac 100755 --- a/services/storage/docker/boot.sh +++ b/services/storage/docker/boot.sh @@ -55,7 +55,7 @@ if [ "${STORAGE_WORKER_MODE}" = "true" ]; then --recursive \ -- \ celery \ - --app=simcore_service_storage.modules.celery.worker.main:the_app \ + --app=simcore_service_storage.modules.celery.worker.main:app \ worker --pool="${CELERY_POOL}" \ --loglevel="${SERVER_LOG_LEVEL}" \ --concurrency="${CELERY_CONCURRENCY}" \ @@ -63,7 +63,7 @@ if [ "${STORAGE_WORKER_MODE}" = "true" ]; then --queues="${CELERY_QUEUES:-default}" else exec celery \ - --app=simcore_service_storage.modules.celery.worker.main:the_app \ + --app=simcore_service_storage.modules.celery.worker.main:app \ worker --pool="${CELERY_POOL}" \ --loglevel="${SERVER_LOG_LEVEL}" \ --concurrency="${CELERY_CONCURRENCY}" \ diff --git a/services/storage/src/simcore_service_storage/api/_worker_tasks/_files.py b/services/storage/src/simcore_service_storage/api/_worker_tasks/_files.py index 29c4cb72857a..4bf12b3dca25 100644 --- a/services/storage/src/simcore_service_storage/api/_worker_tasks/_files.py +++ b/services/storage/src/simcore_service_storage/api/_worker_tasks/_files.py @@ -1,7 +1,7 @@ import logging from celery import Task # type: ignore[import-untyped] -from celery_library.utils import get_app_server +from celery_library.worker.app_server import get_app_server from models_library.api_schemas_storage.storage_schemas import ( FileUploadCompletionBody, ) diff --git a/services/storage/src/simcore_service_storage/api/_worker_tasks/_paths.py b/services/storage/src/simcore_service_storage/api/_worker_tasks/_paths.py index 0e6ea4c3749f..0401d2400c89 100644 --- a/services/storage/src/simcore_service_storage/api/_worker_tasks/_paths.py +++ b/services/storage/src/simcore_service_storage/api/_worker_tasks/_paths.py @@ -2,7 +2,7 @@ from pathlib import Path from celery import Task # type: ignore[import-untyped] -from celery_library.utils import get_app_server +from celery_library.worker.app_server import get_app_server from models_library.projects_nodes_io import LocationID, StorageFileID from models_library.users import UserID from pydantic import ByteSize, TypeAdapter diff --git a/services/storage/src/simcore_service_storage/api/_worker_tasks/_simcore_s3.py b/services/storage/src/simcore_service_storage/api/_worker_tasks/_simcore_s3.py index d40a90b084c8..85b3912c0221 100644 --- a/services/storage/src/simcore_service_storage/api/_worker_tasks/_simcore_s3.py +++ b/services/storage/src/simcore_service_storage/api/_worker_tasks/_simcore_s3.py @@ -5,7 +5,7 @@ from aws_library.s3._models import S3ObjectKey from celery import Task # type: ignore[import-untyped] -from celery_library.utils import get_app_server +from celery_library.worker.app_server import get_app_server from models_library.api_schemas_storage.search_async_jobs import SearchResultItem from models_library.api_schemas_storage.storage_schemas import ( FoldersBody, diff --git a/services/storage/src/simcore_service_storage/api/_worker_tasks/tasks.py b/services/storage/src/simcore_service_storage/api/_worker_tasks/tasks.py index 5475b8eed8a4..52c81bc96213 100644 --- a/services/storage/src/simcore_service_storage/api/_worker_tasks/tasks.py +++ b/services/storage/src/simcore_service_storage/api/_worker_tasks/tasks.py @@ -25,7 +25,7 @@ _logger = logging.getLogger(__name__) -def setup_worker_tasks(app: Celery) -> None: +def register_worker_tasks(app: Celery) -> None: register_celery_types() register_pydantic_types( FileUploadCompletionBody, diff --git a/services/storage/src/simcore_service_storage/modules/celery/__init__.py b/services/storage/src/simcore_service_storage/modules/celery/__init__.py index 48e30e60a4f3..bdb434580b85 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/__init__.py +++ b/services/storage/src/simcore_service_storage/modules/celery/__init__.py @@ -1,7 +1,7 @@ import logging +from celery_library.app import create_app from celery_library.backends.redis import RedisTaskStore -from celery_library.common import create_app from celery_library.task_manager import CeleryTaskManager from celery_library.types import register_celery_types, register_pydantic_types from fastapi import FastAPI diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker/main.py b/services/storage/src/simcore_service_storage/modules/celery/worker/main.py index 3bd9dff4f8d9..20cd3c2b51f4 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker/main.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker/main.py @@ -1,20 +1,9 @@ -from celery.signals import ( # type: ignore[import-untyped] - worker_init, - worker_process_init, - worker_process_shutdown, - worker_shutdown, -) -from celery_library.common import create_app as create_celery_app -from celery_library.signals import ( - on_worker_init, - on_worker_shutdown, -) -from celery_library.utils import get_app_server, set_app_server +from celery_library.worker.app import create_worker_app from servicelib.fastapi.celery.app_server import FastAPIAppServer from servicelib.logging_utils import setup_loggers from servicelib.tracing import TracingConfig -from ....api._worker_tasks.tasks import setup_worker_tasks +from ....api._worker_tasks.tasks import register_worker_tasks from ....core.application import create_app from ....core.settings import ApplicationSettings @@ -34,32 +23,16 @@ def get_app(): noisy_loggers=None, ) - assert _settings.STORAGE_CELERY # nosec - app = create_celery_app(_settings.STORAGE_CELERY) - setup_worker_tasks(app) - - return app - - -the_app = get_app() + def _app_server_factory() -> FastAPIAppServer: + fastapi_app = create_app(_settings, tracing_config=_tracing_config) + return FastAPIAppServer(app=fastapi_app) - -def worker_init_wrapper(**kwargs): - fastapi_app = create_app(_settings, tracing_config=_tracing_config) - app_server = FastAPIAppServer(app=fastapi_app) - set_app_server(the_app, app_server) - return on_worker_init(app_server, **kwargs) - - -def worker_shutdown_wrapper(**kwargs): - app_server = get_app_server(the_app) - return on_worker_shutdown(app_server, **kwargs) + assert _settings.STORAGE_CELERY # nosec + return create_worker_app( + _settings.STORAGE_CELERY, + register_worker_tasks_cb=register_worker_tasks, + app_server_factory_cb=_app_server_factory, + ) -assert _settings.STORAGE_CELERY # nosec -if _settings.STORAGE_CELERY.CELERY_POOL == "prefork": - worker_process_init.connect(worker_init_wrapper) - worker_process_shutdown.connect(worker_shutdown_wrapper) -else: - worker_init.connect(worker_init_wrapper) - worker_shutdown.connect(worker_shutdown_wrapper) +app = get_app() diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index 0e12ba750d68..8d865cd54776 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -70,7 +70,7 @@ from settings_library.rabbit import RabbitSettings from simcore_postgres_database.models.tokens import tokens from simcore_postgres_database.storage_models import file_meta_data, projects, users -from simcore_service_storage.api._worker_tasks.tasks import setup_worker_tasks +from simcore_service_storage.api._worker_tasks.tasks import register_worker_tasks from simcore_service_storage.core.application import create_app from simcore_service_storage.core.settings import ApplicationSettings from simcore_service_storage.datcore_dsm import DatCoreDataManager @@ -1034,7 +1034,7 @@ def _on_worker_init_wrapper(**kwargs): worker_init.connect(_on_worker_init_wrapper) worker_shutdown.connect(on_worker_shutdown) - setup_worker_tasks(celery_app) + register_worker_tasks(celery_app) register_celery_tasks(celery_app) with start_worker( diff --git a/services/web/server/src/simcore_service_webserver/celery/_task_manager.py b/services/web/server/src/simcore_service_webserver/celery/_task_manager.py index adc06a8e219a..ef136098826c 100644 --- a/services/web/server/src/simcore_service_webserver/celery/_task_manager.py +++ b/services/web/server/src/simcore_service_webserver/celery/_task_manager.py @@ -2,8 +2,8 @@ from typing import Final from aiohttp import web +from celery_library.app import create_app from celery_library.backends.redis import RedisTaskStore -from celery_library.common import create_app from celery_library.task_manager import CeleryTaskManager from celery_library.types import register_celery_types from servicelib.celery.task_manager import TaskManager From ca12f4ba78d6b25fc64b1c46030166246e87f443 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 14 Oct 2025 22:59:32 +0200 Subject: [PATCH 26/58] fix import --- services/api-server/tests/unit/api_functions/celery/conftest.py | 2 +- services/storage/tests/conftest.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/api-server/tests/unit/api_functions/celery/conftest.py b/services/api-server/tests/unit/api_functions/celery/conftest.py index 8e004aa35df3..dd9b78fdb3af 100644 --- a/services/api-server/tests/unit/api_functions/celery/conftest.py +++ b/services/api-server/tests/unit/api_functions/celery/conftest.py @@ -19,7 +19,7 @@ worker_shutdown, ) from celery_library.signals import on_worker_init, on_worker_shutdown -from celery_library.utils import set_app_server +from celery_library.worker.app_server import set_app_server from pytest_mock import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import delenvs_from_dict, setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index 8d865cd54776..0e0da9e6a42f 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -25,7 +25,7 @@ from celery.contrib.testing.worker import TestWorkController, start_worker from celery.signals import worker_init, worker_shutdown from celery_library.signals import on_worker_init, on_worker_shutdown -from celery_library.utils import set_app_server +from celery_library.worker.app_server import set_app_server from faker import Faker from fakeredis.aioredis import FakeRedis from fastapi import FastAPI From 377d3345399ef358a12c6e624727a14793aadb23 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 14 Oct 2025 23:05:03 +0200 Subject: [PATCH 27/58] typecheck --- packages/celery-library/src/celery_library/worker/app.py | 2 +- packages/celery-library/src/celery_library/worker/app_server.py | 2 +- packages/celery-library/src/celery_library/worker/signals.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/celery-library/src/celery_library/worker/app.py b/packages/celery-library/src/celery_library/worker/app.py index 2695cbe0e0a4..9c243f0e3961 100644 --- a/packages/celery-library/src/celery_library/worker/app.py +++ b/packages/celery-library/src/celery_library/worker/app.py @@ -1,6 +1,6 @@ from collections.abc import Callable -from celery import Celery +from celery import Celery # type: ignore[import-untyped] from servicelib.celery.app_server import BaseAppServer from settings_library.celery import CelerySettings diff --git a/packages/celery-library/src/celery_library/worker/app_server.py b/packages/celery-library/src/celery_library/worker/app_server.py index cf5864891bf3..4d8bad73b96d 100644 --- a/packages/celery-library/src/celery_library/worker/app_server.py +++ b/packages/celery-library/src/celery_library/worker/app_server.py @@ -1,4 +1,4 @@ -from celery import Celery +from celery import Celery # type: ignore[import-untyped] from servicelib.celery.app_server import BaseAppServer _APP_SERVER_KEY = "app_server" diff --git a/packages/celery-library/src/celery_library/worker/signals.py b/packages/celery-library/src/celery_library/worker/signals.py index 3c1a57d8811d..56347aa6b618 100644 --- a/packages/celery-library/src/celery_library/worker/signals.py +++ b/packages/celery-library/src/celery_library/worker/signals.py @@ -3,7 +3,7 @@ from collections.abc import Callable from celery import Celery # type: ignore[import-untyped] -from celery.signals import ( +from celery.signals import ( # type: ignore[import-untyped] worker_init, worker_process_init, worker_process_shutdown, From d766256859850b55eb4d6b74123bc06964e30e05 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 15 Oct 2025 11:49:35 +0200 Subject: [PATCH 28/58] use new signal handlers --- .../src/celery_library/signals.py | 36 ----------------- .../src/celery_library/worker/signals.py | 40 +++++++++++++------ packages/celery-library/tests/conftest.py | 28 +++++-------- .../src/settings_library/celery.py | 16 ++++++-- .../unit/api_functions/celery/conftest.py | 22 +++------- services/storage/tests/conftest.py | 29 +++++++------- 6 files changed, 70 insertions(+), 101 deletions(-) delete mode 100644 packages/celery-library/src/celery_library/signals.py diff --git a/packages/celery-library/src/celery_library/signals.py b/packages/celery-library/src/celery_library/signals.py deleted file mode 100644 index e2ed1b7f7518..000000000000 --- a/packages/celery-library/src/celery_library/signals.py +++ /dev/null @@ -1,36 +0,0 @@ -import asyncio -import logging -import threading - -from servicelib.celery.app_server import BaseAppServer -from servicelib.logging_utils import log_context - -_logger = logging.getLogger(__name__) - - -def on_worker_init(app_server: BaseAppServer, **_kwargs) -> None: - startup_complete_event = threading.Event() - - def _init(startup_complete_event: threading.Event) -> None: - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - - app_server.event_loop = loop - - loop.run_until_complete(app_server.run_until_shutdown(startup_complete_event)) - - thread = threading.Thread( - group=None, - target=_init, - name="app_server_init", - args=(startup_complete_event,), - daemon=True, - ) - thread.start() - - startup_complete_event.wait() - - -def on_worker_shutdown(app_server: BaseAppServer, **_kwargs) -> None: - with log_context(_logger, logging.INFO, "Worker shutdown"): - app_server.shutdown_event.set() diff --git a/packages/celery-library/src/celery_library/worker/signals.py b/packages/celery-library/src/celery_library/worker/signals.py index 56347aa6b618..6bef1278de7e 100644 --- a/packages/celery-library/src/celery_library/worker/signals.py +++ b/packages/celery-library/src/celery_library/worker/signals.py @@ -10,17 +10,15 @@ worker_shutdown, ) from servicelib.celery.app_server import BaseAppServer -from settings_library.celery import CelerySettings +from settings_library.celery import CeleryPoolType, CelerySettings from .app_server import get_app_server, set_app_server -def register_worker_signals( - app: Celery, - settings: CelerySettings, - app_server_factory: Callable[[], BaseAppServer], -) -> None: - def _worker_init_wrapper(**_kwargs) -> None: +def _worker_init_wrapper( + app: Celery, app_server_factory: Callable[[], BaseAppServer] +) -> Callable[..., None]: + def _worker_init_handler(**_kwargs) -> None: startup_complete_event = threading.Event() def _init(startup_complete_event: threading.Event) -> None: @@ -47,13 +45,29 @@ def _init(startup_complete_event: threading.Event) -> None: startup_complete_event.wait() - def _worker_shutdown_wrapper(**_kwargs) -> None: + return _worker_init_handler + + +def _worker_shutdown_wrapper(app: Celery) -> Callable[..., None]: + def _worker_shutdown_handler(**_kwargs) -> None: get_app_server(app).shutdown_event.set() + return _worker_shutdown_handler + + +def register_worker_signals( + app: Celery, + settings: CelerySettings, + app_server_factory: Callable[[], BaseAppServer], +) -> None: match settings.CELERY_POOL: - case "prefork": - worker_process_init.connect(_worker_init_wrapper, weak=False) - worker_process_shutdown.connect(_worker_shutdown_wrapper, weak=False) + case CeleryPoolType.PREFORK: + worker_process_init.connect( + _worker_init_wrapper(app, app_server_factory), weak=False + ) + worker_process_shutdown.connect(_worker_shutdown_wrapper(app), weak=False) case _: - worker_init.connect(_worker_init_wrapper, weak=False) - worker_shutdown.connect(_worker_shutdown_wrapper, weak=False) + worker_init.connect( + _worker_init_wrapper(app, app_server_factory), weak=False + ) + worker_shutdown.connect(_worker_shutdown_wrapper(app), weak=False) diff --git a/packages/celery-library/tests/conftest.py b/packages/celery-library/tests/conftest.py index e5adbcc2172f..9fd16d217cc2 100644 --- a/packages/celery-library/tests/conftest.py +++ b/packages/celery-library/tests/conftest.py @@ -15,16 +15,15 @@ ) from celery.signals import worker_init, worker_shutdown from celery_library.backends.redis import RedisTaskStore -from celery_library.signals import on_worker_init, on_worker_shutdown from celery_library.task_manager import CeleryTaskManager from celery_library.types import register_celery_types -from celery_library.worker.app_server import set_app_server +from celery_library.worker.signals import _worker_init_wrapper, _worker_shutdown_wrapper from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.celery.app_server import BaseAppServer from servicelib.celery.task_manager import TaskManager from servicelib.redis import RedisClientSDK -from settings_library.celery import CelerySettings +from settings_library.celery import CeleryPoolType, CelerySettings from settings_library.redis import RedisDatabase, RedisSettings pytest_plugins = [ @@ -104,11 +103,6 @@ def celery_settings( return CelerySettings.create_from_envs() -@pytest.fixture -def app_server(celery_app: Celery, celery_settings: CelerySettings) -> BaseAppServer: - return FakeAppServer(app=celery_app, settings=celery_settings) - - @pytest.fixture(scope="session") def celery_config() -> dict[str, Any]: return { @@ -128,25 +122,25 @@ def celery_config() -> dict[str, Any]: @pytest.fixture async def with_celery_worker( celery_app: Celery, - app_server: BaseAppServer, + celery_settings: CelerySettings, register_celery_tasks: Callable[[Celery], None], ) -> AsyncIterator[TestWorkController]: - def _on_worker_init_wrapper(**kwargs): - set_app_server(celery_app, app_server) - return on_worker_init(app_server, **kwargs) - def _on_worker_shutdown_wrapper(**kwargs): - return on_worker_shutdown(app_server, **kwargs) + def _app_server_factory() -> BaseAppServer: + return FakeAppServer(app=celery_app, settings=celery_settings) - worker_init.connect(_on_worker_init_wrapper) - worker_shutdown.connect(_on_worker_shutdown_wrapper) + # NOTE: explicitly connect the signals in tests + worker_init.connect( + _worker_init_wrapper(celery_app, _app_server_factory), weak=False + ) + worker_shutdown.connect(_worker_shutdown_wrapper(celery_app), weak=False) register_celery_tasks(celery_app) with start_worker( celery_app, concurrency=1, - pool="threads", + pool=CeleryPoolType.THREADS, loglevel="info", perform_ping_check=False, queues="default", diff --git a/packages/settings-library/src/settings_library/celery.py b/packages/settings-library/src/settings_library/celery.py index 3152eb203160..d6c9e837d157 100644 --- a/packages/settings-library/src/settings_library/celery.py +++ b/packages/settings-library/src/settings_library/celery.py @@ -1,5 +1,6 @@ from datetime import timedelta -from typing import Annotated, Literal +from enum import StrEnum +from typing import Annotated from pydantic import Field from pydantic_settings import SettingsConfigDict @@ -9,6 +10,13 @@ from .base import BaseCustomSettings +class CeleryPoolType(StrEnum): + PREFORK = "prefork" + EVENTLET = "eventlet" + GEVENT = "gevent" + THREADS = "threads" + + class CelerySettings(BaseCustomSettings): CELERY_RABBIT_BROKER: Annotated[ RabbitSettings, Field(json_schema_extra={"auto_default_from_env": True}) @@ -36,11 +44,11 @@ class CelerySettings(BaseCustomSettings): ] = True CELERY_POOL: Annotated[ - Literal["prefork", "eventlet", "gevent", "solo", "threads"], + CeleryPoolType, Field( - description="Type of pool to use. One of: prefork, eventlet, gevent, solo, threads. See https://docs.celeryq.dev/en/stable/userguide/concurrency/index.html for details.", + description="Type of pool to use. One of: prefork, eventlet, gevent, threads. See https://docs.celeryq.dev/en/stable/userguide/concurrency/index.html for details.", ), - ] = "prefork" + ] = CeleryPoolType.PREFORK model_config = SettingsConfigDict( json_schema_extra={ diff --git a/services/api-server/tests/unit/api_functions/celery/conftest.py b/services/api-server/tests/unit/api_functions/celery/conftest.py index dd9b78fdb3af..b622b2e5d597 100644 --- a/services/api-server/tests/unit/api_functions/celery/conftest.py +++ b/services/api-server/tests/unit/api_functions/celery/conftest.py @@ -14,12 +14,7 @@ TestWorkController, start_worker, ) -from celery.signals import ( # pylint: disable=no-name-in-module - worker_init, - worker_shutdown, -) -from celery_library.signals import on_worker_init, on_worker_shutdown -from celery_library.worker.app_server import set_app_server +from celery_library.worker.signals import register_worker_signals from pytest_mock import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import delenvs_from_dict, setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict @@ -122,17 +117,12 @@ async def with_api_server_celery_worker( monkeypatch.setenv("API_SERVER_WORKER_MODE", "true") app_settings = ApplicationSettings.create_from_envs() - app_server = FastAPIAppServer(app=create_app(app_settings)) - - def _on_worker_init_wrapper(**kwargs): - set_app_server(celery_app, app_server) - return on_worker_init(app_server, **kwargs) - - def _on_worker_shutdown_wrapper(**kwargs): - return on_worker_shutdown(app_server, **kwargs) + def _app_server_factory() -> FastAPIAppServer: + return FastAPIAppServer(app=create_app(app_settings)) - worker_init.connect(_on_worker_init_wrapper) - worker_shutdown.connect(_on_worker_shutdown_wrapper) + celery_settings = app_settings.API_SERVER_CELERY + assert celery_settings + register_worker_signals(celery_app, celery_settings, _app_server_factory) if add_worker_tasks: register_worker_tasks(celery_app) diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index 0e0da9e6a42f..8fde467c3dec 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -23,9 +23,7 @@ from aws_library.s3 import SimcoreS3API from celery import Celery from celery.contrib.testing.worker import TestWorkController, start_worker -from celery.signals import worker_init, worker_shutdown -from celery_library.signals import on_worker_init, on_worker_shutdown -from celery_library.worker.app_server import set_app_server +from celery_library.worker.signals import register_worker_signals from faker import Faker from fakeredis.aioredis import FakeRedis from fastapi import FastAPI @@ -67,6 +65,7 @@ from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient from servicelib.tracing import TracingConfig from servicelib.utils import limited_gather +from settings_library.celery import CeleryPoolType from settings_library.rabbit import RabbitSettings from simcore_postgres_database.models.tokens import tokens from simcore_postgres_database.storage_models import file_meta_data, projects, users @@ -1000,7 +999,7 @@ def mock_celery_app(mocker: MockerFixture, celery_config: dict[str, Any]) -> Cel @pytest.fixture -def register_celery_tasks() -> Callable[[Celery], None]: +def register_test_tasks() -> Callable[[Celery], None]: """override if tasks are needed""" def _(celery_app: Celery) -> None: ... @@ -1013,7 +1012,7 @@ async def with_storage_celery_worker( app_environment: EnvVarsDict, celery_app: Celery, monkeypatch: pytest.MonkeyPatch, - register_celery_tasks: Callable[[Celery], None], + register_test_tasks: Callable[[Celery], None], ) -> AsyncIterator[TestWorkController]: # Signals must be explicitily connected monkeypatch.setenv("STORAGE_WORKER_MODE", "true") @@ -1023,23 +1022,23 @@ async def with_storage_celery_worker( service_name="storage-api", ) - app_server = FastAPIAppServer( - app=create_app(app_settings, tracing_config=tracing_config) - ) + def _app_server_factory() -> FastAPIAppServer: + return FastAPIAppServer( + app=create_app(app_settings, tracing_config=tracing_config) + ) - def _on_worker_init_wrapper(**kwargs): - set_app_server(celery_app, app_server) - return on_worker_init(app_server, **kwargs) + assert app_settings.STORAGE_CELERY - worker_init.connect(_on_worker_init_wrapper) - worker_shutdown.connect(on_worker_shutdown) + register_worker_signals( + celery_app, app_settings.STORAGE_CELERY, _app_server_factory + ) register_worker_tasks(celery_app) - register_celery_tasks(celery_app) + register_test_tasks(celery_app) with start_worker( celery_app, - pool="threads", + pool=CeleryPoolType.THREADS, concurrency=1, loglevel="info", perform_ping_check=False, From ade0d43a0b3fc5aedaa8c9e09b8b53cf2cc75532 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 15 Oct 2025 13:26:49 +0200 Subject: [PATCH 29/58] update name --- .../src/models_library/api_schemas_storage/search_async_jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/models-library/src/models_library/api_schemas_storage/search_async_jobs.py b/packages/models-library/src/models_library/api_schemas_storage/search_async_jobs.py index 5de7cf9d1c0e..5d4a14877421 100644 --- a/packages/models-library/src/models_library/api_schemas_storage/search_async_jobs.py +++ b/packages/models-library/src/models_library/api_schemas_storage/search_async_jobs.py @@ -5,7 +5,7 @@ from pydantic import BaseModel, ByteSize, ConfigDict from pydantic.alias_generators import to_camel -SEARCH_TASK_NAME: Final[str] = "files.search" +SEARCH_TASK_NAME: Final[str] = "files_search" class SearchResultItem(BaseModel): From 41ad9464f8e64cbb699abfc2a56575d6be525dfc Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 15 Oct 2025 13:30:19 +0200 Subject: [PATCH 30/58] fix signals --- .../tests/unit/api_functions/celery/conftest.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/services/api-server/tests/unit/api_functions/celery/conftest.py b/services/api-server/tests/unit/api_functions/celery/conftest.py index b622b2e5d597..4826279412c8 100644 --- a/services/api-server/tests/unit/api_functions/celery/conftest.py +++ b/services/api-server/tests/unit/api_functions/celery/conftest.py @@ -14,11 +14,13 @@ TestWorkController, start_worker, ) -from celery_library.worker.signals import register_worker_signals +from celery.signals import worker_init, worker_shutdown +from celery_library.worker.signals import _worker_init_wrapper, _worker_shutdown_wrapper from pytest_mock import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import delenvs_from_dict, setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.fastapi.celery.app_server import FastAPIAppServer +from settings_library.celery import CeleryPoolType from settings_library.redis import RedisSettings from simcore_service_api_server.clients import celery_task_manager from simcore_service_api_server.core.application import create_app @@ -120,9 +122,11 @@ async def with_api_server_celery_worker( def _app_server_factory() -> FastAPIAppServer: return FastAPIAppServer(app=create_app(app_settings)) - celery_settings = app_settings.API_SERVER_CELERY - assert celery_settings - register_worker_signals(celery_app, celery_settings, _app_server_factory) + # NOTE: explicitly connect the signals in tests + worker_init.connect( + _worker_init_wrapper(celery_app, _app_server_factory), weak=False + ) + worker_shutdown.connect(_worker_shutdown_wrapper(celery_app), weak=False) if add_worker_tasks: register_worker_tasks(celery_app) @@ -130,7 +134,7 @@ def _app_server_factory() -> FastAPIAppServer: with start_worker( celery_app, - pool="threads", + pool=CeleryPoolType.THREADS, concurrency=1, loglevel="info", perform_ping_check=False, From 1d8bb478af7417bccf25a9a9c04fc2ea412c795b Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 15 Oct 2025 13:35:29 +0200 Subject: [PATCH 31/58] fix signals --- services/storage/tests/conftest.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index 8fde467c3dec..5fce7e3929f7 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -23,7 +23,8 @@ from aws_library.s3 import SimcoreS3API from celery import Celery from celery.contrib.testing.worker import TestWorkController, start_worker -from celery_library.worker.signals import register_worker_signals +from celery.signals import worker_init, worker_shutdown +from celery_library.worker.signals import _worker_init_wrapper, _worker_shutdown_wrapper from faker import Faker from fakeredis.aioredis import FakeRedis from fastapi import FastAPI @@ -1027,11 +1028,11 @@ def _app_server_factory() -> FastAPIAppServer: app=create_app(app_settings, tracing_config=tracing_config) ) - assert app_settings.STORAGE_CELERY - - register_worker_signals( - celery_app, app_settings.STORAGE_CELERY, _app_server_factory + # NOTE: explicitly connect the signals in tests + worker_init.connect( + _worker_init_wrapper(celery_app, _app_server_factory), weak=False ) + worker_shutdown.connect(_worker_shutdown_wrapper(celery_app), weak=False) register_worker_tasks(celery_app) register_test_tasks(celery_app) From f20badd59d672cdbfac97013ed2d5aeafaee6e9e Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 15 Oct 2025 20:35:29 +0200 Subject: [PATCH 32/58] add rabbit --- services/storage/tests/unit/test_handlers_files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/storage/tests/unit/test_handlers_files.py b/services/storage/tests/unit/test_handlers_files.py index f07b63cdbe92..6cced97eb689 100644 --- a/services/storage/tests/unit/test_handlers_files.py +++ b/services/storage/tests/unit/test_handlers_files.py @@ -63,7 +63,7 @@ from types_aiobotocore_s3 import S3Client from yarl import URL -pytest_simcore_core_services_selection = ["postgres"] +pytest_simcore_core_services_selection = ["postgres", "rabbit"] pytest_simcore_ops_services_selection = ["adminer"] From 056954c4de69a3e255396d1203ff65a3146640ef Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 15 Oct 2025 20:58:15 +0200 Subject: [PATCH 33/58] fix service selection --- services/storage/tests/unit/test_handlers_datasets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/storage/tests/unit/test_handlers_datasets.py b/services/storage/tests/unit/test_handlers_datasets.py index 5808a63f1f1b..47d83a98fb5b 100644 --- a/services/storage/tests/unit/test_handlers_datasets.py +++ b/services/storage/tests/unit/test_handlers_datasets.py @@ -29,7 +29,7 @@ from servicelib.aiohttp import status from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager -pytest_simcore_core_services_selection = ["postgres"] +pytest_simcore_core_services_selection = ["postgres", "rabbit"] pytest_simcore_ops_services_selection = ["adminer"] From c95d706e7034ba6c4b5e6df4e310b1645eb17fcd Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 15 Oct 2025 21:29:13 +0200 Subject: [PATCH 34/58] fix test --- services/storage/tests/conftest.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index 5fce7e3929f7..8caf1bd78efd 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -1017,6 +1017,9 @@ async def with_storage_celery_worker( ) -> AsyncIterator[TestWorkController]: # Signals must be explicitily connected monkeypatch.setenv("STORAGE_WORKER_MODE", "true") + monkeypatch.setenv("STORAGE_RABBIT", "null") + monkeypatch.setenv("STORAGE_REDIS", "null") + app_settings = ApplicationSettings.create_from_envs() tracing_config = TracingConfig.create( tracing_settings=None, # disable tracing in tests @@ -1024,9 +1027,7 @@ async def with_storage_celery_worker( ) def _app_server_factory() -> FastAPIAppServer: - return FastAPIAppServer( - app=create_app(app_settings, tracing_config=tracing_config) - ) + return FastAPIAppServer(app=create_app(app_settings, tracing_config)) # NOTE: explicitly connect the signals in tests worker_init.connect( From 4ea087efd76f5e48b5147891e048d34cd3949b7d Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 15 Oct 2025 21:36:29 +0200 Subject: [PATCH 35/58] set worker mode --- services/storage/tests/conftest.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index 8caf1bd78efd..bf56e36968cb 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -1015,12 +1015,9 @@ async def with_storage_celery_worker( monkeypatch: pytest.MonkeyPatch, register_test_tasks: Callable[[Celery], None], ) -> AsyncIterator[TestWorkController]: - # Signals must be explicitily connected - monkeypatch.setenv("STORAGE_WORKER_MODE", "true") - monkeypatch.setenv("STORAGE_RABBIT", "null") - monkeypatch.setenv("STORAGE_REDIS", "null") - app_settings = ApplicationSettings.create_from_envs() + app_settings.STORAGE_WORKER_MODE = True + tracing_config = TracingConfig.create( tracing_settings=None, # disable tracing in tests service_name="storage-api", From 935478fd11c7ecd0f67eaa22ea2c85843606d87e Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 15 Oct 2025 21:44:25 +0200 Subject: [PATCH 36/58] set worker mode --- services/storage/tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index bf56e36968cb..0c79fdb48c22 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -1015,8 +1015,8 @@ async def with_storage_celery_worker( monkeypatch: pytest.MonkeyPatch, register_test_tasks: Callable[[Celery], None], ) -> AsyncIterator[TestWorkController]: + monkeypatch.setenv("STORAGE_WORKER_MODE", "true") # disable rabbit in celery worker app_settings = ApplicationSettings.create_from_envs() - app_settings.STORAGE_WORKER_MODE = True tracing_config = TracingConfig.create( tracing_settings=None, # disable tracing in tests From 83fb4fe8f816bb728849903f8ee4fb54cbe4dde1 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 15 Oct 2025 21:44:55 +0200 Subject: [PATCH 37/58] fix --- services/storage/tests/conftest.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index 0c79fdb48c22..6147868c4294 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -1015,9 +1015,8 @@ async def with_storage_celery_worker( monkeypatch: pytest.MonkeyPatch, register_test_tasks: Callable[[Celery], None], ) -> AsyncIterator[TestWorkController]: - monkeypatch.setenv("STORAGE_WORKER_MODE", "true") # disable rabbit in celery worker + monkeypatch.setenv("STORAGE_WORKER_MODE", "true") app_settings = ApplicationSettings.create_from_envs() - tracing_config = TracingConfig.create( tracing_settings=None, # disable tracing in tests service_name="storage-api", From 80dfc300424887bcad56febca941ca1bad31a0d9 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 15 Oct 2025 22:11:54 +0200 Subject: [PATCH 38/58] add redis --- services/storage/tests/conftest.py | 55 ++++++++++--------- .../tests/unit/test_handlers_datasets.py | 2 +- 2 files changed, 30 insertions(+), 27 deletions(-) diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index 6147868c4294..1787985052c8 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -1015,34 +1015,37 @@ async def with_storage_celery_worker( monkeypatch: pytest.MonkeyPatch, register_test_tasks: Callable[[Celery], None], ) -> AsyncIterator[TestWorkController]: - monkeypatch.setenv("STORAGE_WORKER_MODE", "true") - app_settings = ApplicationSettings.create_from_envs() - tracing_config = TracingConfig.create( - tracing_settings=None, # disable tracing in tests - service_name="storage-api", - ) + with monkeypatch.context() as patch: + patch.setenv("STORAGE_WORKER_MODE", "true") + patch.setenv("CELERY_POOL", "threads") + app_settings = ApplicationSettings.create_from_envs() + + tracing_config = TracingConfig.create( + tracing_settings=None, # disable tracing in tests + service_name="storage-api", + ) - def _app_server_factory() -> FastAPIAppServer: - return FastAPIAppServer(app=create_app(app_settings, tracing_config)) + def _app_server_factory() -> FastAPIAppServer: + return FastAPIAppServer(app=create_app(app_settings, tracing_config)) - # NOTE: explicitly connect the signals in tests - worker_init.connect( - _worker_init_wrapper(celery_app, _app_server_factory), weak=False - ) - worker_shutdown.connect(_worker_shutdown_wrapper(celery_app), weak=False) - - register_worker_tasks(celery_app) - register_test_tasks(celery_app) - - with start_worker( - celery_app, - pool=CeleryPoolType.THREADS, - concurrency=1, - loglevel="info", - perform_ping_check=False, - queues="default,cpu_bound", - ) as worker: - yield worker + # NOTE: explicitly connect the signals in tests + worker_init.connect( + _worker_init_wrapper(celery_app, _app_server_factory), weak=False + ) + worker_shutdown.connect(_worker_shutdown_wrapper(celery_app), weak=False) + + register_worker_tasks(celery_app) + register_test_tasks(celery_app) + + with start_worker( + celery_app, + pool=CeleryPoolType.THREADS, + concurrency=1, + loglevel="info", + perform_ping_check=False, + queues="default,cpu_bound", + ) as worker: + yield worker @pytest.fixture diff --git a/services/storage/tests/unit/test_handlers_datasets.py b/services/storage/tests/unit/test_handlers_datasets.py index 47d83a98fb5b..0d8e12f2bc56 100644 --- a/services/storage/tests/unit/test_handlers_datasets.py +++ b/services/storage/tests/unit/test_handlers_datasets.py @@ -29,7 +29,7 @@ from servicelib.aiohttp import status from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager -pytest_simcore_core_services_selection = ["postgres", "rabbit"] +pytest_simcore_core_services_selection = ["postgres", "rabbit", "redis"] pytest_simcore_ops_services_selection = ["adminer"] From 927fa7bc53b2a827509b9e10630eb4115785462f Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 15 Oct 2025 22:38:11 +0200 Subject: [PATCH 39/58] fix --- services/storage/tests/conftest.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index 1787985052c8..d2fc5437214c 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -1015,9 +1015,12 @@ async def with_storage_celery_worker( monkeypatch: pytest.MonkeyPatch, register_test_tasks: Callable[[Celery], None], ) -> AsyncIterator[TestWorkController]: + with monkeypatch.context() as patch: + pool = f"{CeleryPoolType.THREADS.value}" + patch.setenv("STORAGE_WORKER_MODE", "true") - patch.setenv("CELERY_POOL", "threads") + patch.setenv("CELERY_POOL", pool) app_settings = ApplicationSettings.create_from_envs() tracing_config = TracingConfig.create( @@ -1039,7 +1042,7 @@ def _app_server_factory() -> FastAPIAppServer: with start_worker( celery_app, - pool=CeleryPoolType.THREADS, + pool=pool, concurrency=1, loglevel="info", perform_ping_check=False, From 901df8c2e0d6a6bae2f8848b950cda7c66c007f1 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 15 Oct 2025 22:45:18 +0200 Subject: [PATCH 40/58] fix pool name --- services/storage/tests/conftest.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index d2fc5437214c..e9336a8bb3f2 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -66,7 +66,6 @@ from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient from servicelib.tracing import TracingConfig from servicelib.utils import limited_gather -from settings_library.celery import CeleryPoolType from settings_library.rabbit import RabbitSettings from simcore_postgres_database.models.tokens import tokens from simcore_postgres_database.storage_models import file_meta_data, projects, users @@ -1017,10 +1016,8 @@ async def with_storage_celery_worker( ) -> AsyncIterator[TestWorkController]: with monkeypatch.context() as patch: - pool = f"{CeleryPoolType.THREADS.value}" - patch.setenv("STORAGE_WORKER_MODE", "true") - patch.setenv("CELERY_POOL", pool) + patch.setenv("CELERY_POOL", "threads") app_settings = ApplicationSettings.create_from_envs() tracing_config = TracingConfig.create( @@ -1042,7 +1039,7 @@ def _app_server_factory() -> FastAPIAppServer: with start_worker( celery_app, - pool=pool, + pool="threads", concurrency=1, loglevel="info", perform_ping_check=False, From 2409bd34c8d13acf05f957a1ef632732f2dacc45 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 15 Oct 2025 23:17:49 +0200 Subject: [PATCH 41/58] use wprker app settings --- services/storage/tests/conftest.py | 77 ++++++++++++++++++------------ 1 file changed, 47 insertions(+), 30 deletions(-) diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index e9336a8bb3f2..51a9ed1bd6ea 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -1008,44 +1008,61 @@ def _(celery_app: Celery) -> None: ... @pytest.fixture -async def with_storage_celery_worker( +def storage_worker_mode(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("STORAGE_WORKER_MODE", "true") + + +@pytest.fixture +def worker_app_settings( + enable_tracing, app_environment: EnvVarsDict, + enabled_rabbitmq: RabbitSettings, + sqlalchemy_async_engine: AsyncEngine, + postgres_host_config: dict[str, str], + mocked_s3_server_envs: EnvVarsDict, + datcore_adapter_service_mock: respx.MockRouter, + mocked_redis_server: None, + storage_worker_mode: None, +) -> ApplicationSettings: + test_app_settings = ApplicationSettings.create_from_envs() + print(f"{test_app_settings.model_dump_json(indent=2)=}") + return test_app_settings + + +@pytest.fixture +async def with_storage_celery_worker( + worker_app_settings: ApplicationSettings, celery_app: Celery, monkeypatch: pytest.MonkeyPatch, register_test_tasks: Callable[[Celery], None], ) -> AsyncIterator[TestWorkController]: - with monkeypatch.context() as patch: - patch.setenv("STORAGE_WORKER_MODE", "true") - patch.setenv("CELERY_POOL", "threads") - app_settings = ApplicationSettings.create_from_envs() - - tracing_config = TracingConfig.create( - tracing_settings=None, # disable tracing in tests - service_name="storage-api", - ) + tracing_config = TracingConfig.create( + tracing_settings=None, # disable tracing in tests + service_name="storage-api", + ) - def _app_server_factory() -> FastAPIAppServer: - return FastAPIAppServer(app=create_app(app_settings, tracing_config)) + def _app_server_factory() -> FastAPIAppServer: + return FastAPIAppServer(app=create_app(worker_app_settings, tracing_config)) - # NOTE: explicitly connect the signals in tests - worker_init.connect( - _worker_init_wrapper(celery_app, _app_server_factory), weak=False - ) - worker_shutdown.connect(_worker_shutdown_wrapper(celery_app), weak=False) - - register_worker_tasks(celery_app) - register_test_tasks(celery_app) - - with start_worker( - celery_app, - pool="threads", - concurrency=1, - loglevel="info", - perform_ping_check=False, - queues="default,cpu_bound", - ) as worker: - yield worker + # NOTE: explicitly connect the signals in tests + worker_init.connect( + _worker_init_wrapper(celery_app, _app_server_factory), weak=False + ) + worker_shutdown.connect(_worker_shutdown_wrapper(celery_app), weak=False) + + register_worker_tasks(celery_app) + register_test_tasks(celery_app) + + with start_worker( + celery_app, + pool="threads", + concurrency=1, + loglevel="info", + perform_ping_check=False, + queues="default,cpu_bound", + ) as worker: + yield worker @pytest.fixture From 47fb61c91a1171ac9fe0a87fe9b77f6a14e9eb4b Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 15 Oct 2025 23:39:01 +0200 Subject: [PATCH 42/58] try --- services/storage/tests/conftest.py | 23 +++---------------- .../tests/unit/test_handlers_datasets.py | 2 +- .../storage/tests/unit/test_handlers_files.py | 2 +- 3 files changed, 5 insertions(+), 22 deletions(-) diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index 51a9ed1bd6ea..2da8f6376342 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -1012,28 +1012,11 @@ def storage_worker_mode(app_environment: EnvVarsDict, monkeypatch: pytest.Monkey monkeypatch.setenv("STORAGE_WORKER_MODE", "true") -@pytest.fixture -def worker_app_settings( - enable_tracing, - app_environment: EnvVarsDict, - enabled_rabbitmq: RabbitSettings, - sqlalchemy_async_engine: AsyncEngine, - postgres_host_config: dict[str, str], - mocked_s3_server_envs: EnvVarsDict, - datcore_adapter_service_mock: respx.MockRouter, - mocked_redis_server: None, - storage_worker_mode: None, -) -> ApplicationSettings: - test_app_settings = ApplicationSettings.create_from_envs() - print(f"{test_app_settings.model_dump_json(indent=2)=}") - return test_app_settings - - @pytest.fixture async def with_storage_celery_worker( - worker_app_settings: ApplicationSettings, celery_app: Celery, - monkeypatch: pytest.MonkeyPatch, + storage_worker_mode: None, + app_settings: ApplicationSettings, register_test_tasks: Callable[[Celery], None], ) -> AsyncIterator[TestWorkController]: @@ -1043,7 +1026,7 @@ async def with_storage_celery_worker( ) def _app_server_factory() -> FastAPIAppServer: - return FastAPIAppServer(app=create_app(worker_app_settings, tracing_config)) + return FastAPIAppServer(app=create_app(app_settings, tracing_config)) # NOTE: explicitly connect the signals in tests worker_init.connect( diff --git a/services/storage/tests/unit/test_handlers_datasets.py b/services/storage/tests/unit/test_handlers_datasets.py index 0d8e12f2bc56..5808a63f1f1b 100644 --- a/services/storage/tests/unit/test_handlers_datasets.py +++ b/services/storage/tests/unit/test_handlers_datasets.py @@ -29,7 +29,7 @@ from servicelib.aiohttp import status from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager -pytest_simcore_core_services_selection = ["postgres", "rabbit", "redis"] +pytest_simcore_core_services_selection = ["postgres"] pytest_simcore_ops_services_selection = ["adminer"] diff --git a/services/storage/tests/unit/test_handlers_files.py b/services/storage/tests/unit/test_handlers_files.py index 6cced97eb689..f07b63cdbe92 100644 --- a/services/storage/tests/unit/test_handlers_files.py +++ b/services/storage/tests/unit/test_handlers_files.py @@ -63,7 +63,7 @@ from types_aiobotocore_s3 import S3Client from yarl import URL -pytest_simcore_core_services_selection = ["postgres", "rabbit"] +pytest_simcore_core_services_selection = ["postgres"] pytest_simcore_ops_services_selection = ["adminer"] From 91a3f824a3219c95d4b069f5718f5527364f75fb Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 16 Oct 2025 00:00:03 +0200 Subject: [PATCH 43/58] set pool --- services/storage/tests/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index 2da8f6376342..69ed2b414aee 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -1010,6 +1010,7 @@ def _(celery_app: Celery) -> None: ... @pytest.fixture def storage_worker_mode(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch): monkeypatch.setenv("STORAGE_WORKER_MODE", "true") + monkeypatch.setenv("CELERY_POOL", "threads") @pytest.fixture From b31d2f0c9f4f32dbc2f8ed840aaf5b081b9e14d6 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 16 Oct 2025 07:40:43 +0200 Subject: [PATCH 44/58] fix --- services/storage/tests/conftest.py | 45 ++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index 69ed2b414aee..965002e22439 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -1008,30 +1008,47 @@ def _(celery_app: Celery) -> None: ... @pytest.fixture -def storage_worker_mode(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch): - monkeypatch.setenv("STORAGE_WORKER_MODE", "true") - monkeypatch.setenv("CELERY_POOL", "threads") +def app_server_factory_with_worker_mode( + enable_tracing, + app_environment: EnvVarsDict, + enabled_rabbitmq: RabbitSettings, + sqlalchemy_async_engine: AsyncEngine, + postgres_host_config: dict[str, str], + mocked_s3_server_envs: EnvVarsDict, + datcore_adapter_service_mock: respx.MockRouter, + mocked_redis_server: None, + monkeypatch: pytest.MonkeyPatch, +) -> Callable[[], FastAPIAppServer]: + with monkeypatch.context() as patch: + patch.setenv("STORAGE_WORKER_MODE", "true") + patch.setenv("CELERY_POOL", "threads") + + # Create settings with worker mode enabled + worker_mode_app_settings = ApplicationSettings.create_from_envs() + + def _app_server_factory() -> FastAPIAppServer: + tracing_config = TracingConfig.create( + tracing_settings=None, # disable tracing in tests + service_name="storage-api", + ) + return FastAPIAppServer( + app=create_app(worker_mode_app_settings, tracing_config) + ) + + return _app_server_factory @pytest.fixture async def with_storage_celery_worker( celery_app: Celery, - storage_worker_mode: None, - app_settings: ApplicationSettings, + app_server_factory_with_worker_mode: Callable[[], FastAPIAppServer], register_test_tasks: Callable[[Celery], None], ) -> AsyncIterator[TestWorkController]: - tracing_config = TracingConfig.create( - tracing_settings=None, # disable tracing in tests - service_name="storage-api", - ) - - def _app_server_factory() -> FastAPIAppServer: - return FastAPIAppServer(app=create_app(app_settings, tracing_config)) - # NOTE: explicitly connect the signals in tests worker_init.connect( - _worker_init_wrapper(celery_app, _app_server_factory), weak=False + _worker_init_wrapper(celery_app, app_server_factory_with_worker_mode), + weak=False, ) worker_shutdown.connect(_worker_shutdown_wrapper(celery_app), weak=False) From d0c008599d88c5750a47a3ef97e2c21fec51ccfe Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 16 Oct 2025 10:17:28 +0200 Subject: [PATCH 45/58] mock celery app --- services/storage/tests/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index 965002e22439..de3d62fb9a3b 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -1017,6 +1017,7 @@ def app_server_factory_with_worker_mode( mocked_s3_server_envs: EnvVarsDict, datcore_adapter_service_mock: respx.MockRouter, mocked_redis_server: None, + mock_celery_app: Celery, monkeypatch: pytest.MonkeyPatch, ) -> Callable[[], FastAPIAppServer]: with monkeypatch.context() as patch: From c718176dd297835a9b79d45f9b6263716c7a2bd9 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 16 Oct 2025 11:41:19 +0200 Subject: [PATCH 46/58] fix --- services/storage/tests/conftest.py | 38 ++++++++++++------------------ 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index de3d62fb9a3b..3dff3010f586 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -219,9 +219,9 @@ def app_settings( enabled_rabbitmq: RabbitSettings, sqlalchemy_async_engine: AsyncEngine, postgres_host_config: dict[str, str], - mocked_s3_server_envs: EnvVarsDict, datcore_adapter_service_mock: respx.MockRouter, mocked_redis_server: None, + mocked_s3_server_envs: EnvVarsDict, # Moved to end to ensure S3_ENDPOINT override happens last ) -> ApplicationSettings: test_app_settings = ApplicationSettings.create_from_envs() print(f"{test_app_settings.model_dump_json(indent=2)=}") @@ -1009,34 +1009,26 @@ def _(celery_app: Celery) -> None: ... @pytest.fixture def app_server_factory_with_worker_mode( - enable_tracing, app_environment: EnvVarsDict, - enabled_rabbitmq: RabbitSettings, - sqlalchemy_async_engine: AsyncEngine, - postgres_host_config: dict[str, str], - mocked_s3_server_envs: EnvVarsDict, - datcore_adapter_service_mock: respx.MockRouter, - mocked_redis_server: None, - mock_celery_app: Celery, monkeypatch: pytest.MonkeyPatch, ) -> Callable[[], FastAPIAppServer]: - with monkeypatch.context() as patch: - patch.setenv("STORAGE_WORKER_MODE", "true") - patch.setenv("CELERY_POOL", "threads") + monkeypatch.setenv("STORAGE_WORKER_MODE", "true") - # Create settings with worker mode enabled - worker_mode_app_settings = ApplicationSettings.create_from_envs() + def _app_server_factory() -> FastAPIAppServer: + app_settings = ApplicationSettings.create_from_envs() - def _app_server_factory() -> FastAPIAppServer: - tracing_config = TracingConfig.create( - tracing_settings=None, # disable tracing in tests - service_name="storage-api", - ) - return FastAPIAppServer( - app=create_app(worker_mode_app_settings, tracing_config) - ) + # Verify that worker mode is actually enabled + assert app_settings.STORAGE_WORKER_MODE is True + + print(f"Worker: {app_settings.model_dump_json(indent=2)=}") + + tracing_config = TracingConfig.create( + tracing_settings=None, # disable tracing in tests + service_name="storage-api", + ) + return FastAPIAppServer(app=create_app(app_settings, tracing_config)) - return _app_server_factory + return _app_server_factory @pytest.fixture From 8f93ed4a9593634b4e58b53ba380b59b5d1f74ba Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 16 Oct 2025 11:49:22 +0200 Subject: [PATCH 47/58] fix mocks --- .../unit/api_functions/celery/conftest.py | 38 +++++++++++++------ services/storage/tests/conftest.py | 5 +-- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/services/api-server/tests/unit/api_functions/celery/conftest.py b/services/api-server/tests/unit/api_functions/celery/conftest.py index 4826279412c8..2b62b002b2b9 100644 --- a/services/api-server/tests/unit/api_functions/celery/conftest.py +++ b/services/api-server/tests/unit/api_functions/celery/conftest.py @@ -20,7 +20,7 @@ from pytest_simcore.helpers.monkeypatch_envs import delenvs_from_dict, setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.fastapi.celery.app_server import FastAPIAppServer -from settings_library.celery import CeleryPoolType +from servicelib.tracing import TracingConfig from settings_library.redis import RedisSettings from simcore_service_api_server.clients import celery_task_manager from simcore_service_api_server.core.application import create_app @@ -108,23 +108,37 @@ def add_worker_tasks() -> bool: @pytest.fixture -async def with_api_server_celery_worker( +def app_server_factory_with_worker_mode( app_environment: EnvVarsDict, - celery_app: Celery, monkeypatch: pytest.MonkeyPatch, - register_celery_tasks: Callable[[Celery], None], - add_worker_tasks: bool, -) -> AsyncIterator[TestWorkController]: - # Signals must be explicitily connected - monkeypatch.setenv("API_SERVER_WORKER_MODE", "true") - app_settings = ApplicationSettings.create_from_envs() +) -> Callable[[], FastAPIAppServer]: + monkeypatch.setenv("STORAGE_WORKER_MODE", "true") def _app_server_factory() -> FastAPIAppServer: - return FastAPIAppServer(app=create_app(app_settings)) + app_settings = ApplicationSettings.create_from_envs() + + assert app_settings.API_SERVER_WORKER_MODE is True + + tracing_config = TracingConfig.create( + tracing_settings=None, + service_name="api-worker", + ) + return FastAPIAppServer(app=create_app(app_settings, tracing_config)) + + return _app_server_factory + +@pytest.fixture +async def with_api_server_celery_worker( + celery_app: Celery, + app_server_factory_with_worker_mode: Callable[[], FastAPIAppServer], + register_celery_tasks: Callable[[Celery], None], + add_worker_tasks: bool, +) -> AsyncIterator[TestWorkController]: # NOTE: explicitly connect the signals in tests worker_init.connect( - _worker_init_wrapper(celery_app, _app_server_factory), weak=False + _worker_init_wrapper(celery_app, app_server_factory_with_worker_mode), + weak=False, ) worker_shutdown.connect(_worker_shutdown_wrapper(celery_app), weak=False) @@ -134,7 +148,7 @@ def _app_server_factory() -> FastAPIAppServer: with start_worker( celery_app, - pool=CeleryPoolType.THREADS, + pool="threads", concurrency=1, loglevel="info", perform_ping_check=False, diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index 3dff3010f586..c7436d55a45a 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -1017,14 +1017,11 @@ def app_server_factory_with_worker_mode( def _app_server_factory() -> FastAPIAppServer: app_settings = ApplicationSettings.create_from_envs() - # Verify that worker mode is actually enabled assert app_settings.STORAGE_WORKER_MODE is True - print(f"Worker: {app_settings.model_dump_json(indent=2)=}") - tracing_config = TracingConfig.create( tracing_settings=None, # disable tracing in tests - service_name="storage-api", + service_name="storage-worker", ) return FastAPIAppServer(app=create_app(app_settings, tracing_config)) From daf344775eee87053c7658d4723a4524ae3cc7e3 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 16 Oct 2025 12:57:35 +0200 Subject: [PATCH 48/58] add startup timeout --- .../src/servicelib/fastapi/celery/app_server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/service-library/src/servicelib/fastapi/celery/app_server.py b/packages/service-library/src/servicelib/fastapi/celery/app_server.py index 57e47a3441b3..be0106d25c5c 100644 --- a/packages/service-library/src/servicelib/fastapi/celery/app_server.py +++ b/packages/service-library/src/servicelib/fastapi/celery/app_server.py @@ -9,6 +9,7 @@ from ...celery.app_server import BaseAppServer from ...celery.task_manager import TaskManager +_STARTUP_TIMEOUT: Final[float] = datetime.timedelta(minutes=5).total_seconds() _SHUTDOWN_TIMEOUT: Final[float] = datetime.timedelta(seconds=10).total_seconds() _logger = logging.getLogger(__name__) @@ -27,7 +28,7 @@ async def run_until_shutdown( ) -> None: async with LifespanManager( self.app, - startup_timeout=None, # waits for full app initialization (DB migrations, etc.) + startup_timeout=_STARTUP_TIMEOUT, shutdown_timeout=_SHUTDOWN_TIMEOUT, ): _logger.info("FastAPI initialized: %s", self.app) From 20b56de1751d0ded3d9687b954a6ef88558ff6c6 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 16 Oct 2025 13:25:35 +0200 Subject: [PATCH 49/58] verbose storage --- scripts/common-service.Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/common-service.Makefile b/scripts/common-service.Makefile index 57fb6e3b5b46..c164bb01e1a3 100644 --- a/scripts/common-service.Makefile +++ b/scripts/common-service.Makefile @@ -174,7 +174,7 @@ _run-test-ci: _check_venv_active --keep-docker-up \ --log-date-format="%Y-%m-%d %H:%M:%S" \ --log-format="%(asctime)s %(levelname)s %(message)s" \ - --verbose \ + $(if $(filter simcore_service_storage,$(APP_PACKAGE_NAME)),-vv,--verbose) \ -m "not heavy_load" \ $(PYTEST_ADDITIONAL_PARAMETERS) \ $(TEST_TARGET) From bea45b425919e3ee31ce3cf7c3bce313de6997e9 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 16 Oct 2025 13:30:51 +0200 Subject: [PATCH 50/58] restore --- scripts/common-service.Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/common-service.Makefile b/scripts/common-service.Makefile index c164bb01e1a3..57fb6e3b5b46 100644 --- a/scripts/common-service.Makefile +++ b/scripts/common-service.Makefile @@ -174,7 +174,7 @@ _run-test-ci: _check_venv_active --keep-docker-up \ --log-date-format="%Y-%m-%d %H:%M:%S" \ --log-format="%(asctime)s %(levelname)s %(message)s" \ - $(if $(filter simcore_service_storage,$(APP_PACKAGE_NAME)),-vv,--verbose) \ + --verbose \ -m "not heavy_load" \ $(PYTEST_ADDITIONAL_PARAMETERS) \ $(TEST_TARGET) From 5970ecb322fbabfbee8159e514d11f4a4b0c7448 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 16 Oct 2025 15:26:25 +0200 Subject: [PATCH 51/58] force shutdown --- .../src/servicelib/fastapi/celery/app_server.py | 1 + services/storage/tests/conftest.py | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/service-library/src/servicelib/fastapi/celery/app_server.py b/packages/service-library/src/servicelib/fastapi/celery/app_server.py index be0106d25c5c..afc47d050f8c 100644 --- a/packages/service-library/src/servicelib/fastapi/celery/app_server.py +++ b/packages/service-library/src/servicelib/fastapi/celery/app_server.py @@ -34,3 +34,4 @@ async def run_until_shutdown( _logger.info("FastAPI initialized: %s", self.app) startup_completed_event.set() await self.shutdown_event.wait() # NOTE: wait here until shutdown is requested + _logger.info("FastAPI shutdown completed: %s", self.app) diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index c7436d55a45a..8e5ab854f1c3 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -23,7 +23,7 @@ from aws_library.s3 import SimcoreS3API from celery import Celery from celery.contrib.testing.worker import TestWorkController, start_worker -from celery.signals import worker_init, worker_shutdown +from celery.signals import worker_init from celery_library.worker.signals import _worker_init_wrapper, _worker_shutdown_wrapper from faker import Faker from fakeredis.aioredis import FakeRedis @@ -1040,7 +1040,7 @@ async def with_storage_celery_worker( _worker_init_wrapper(celery_app, app_server_factory_with_worker_mode), weak=False, ) - worker_shutdown.connect(_worker_shutdown_wrapper(celery_app), weak=False) + # worker_shutdown.connect(_worker_shutdown_wrapper(celery_app), weak=False) register_worker_tasks(celery_app) register_test_tasks(celery_app) @@ -1055,6 +1055,8 @@ async def with_storage_celery_worker( ) as worker: yield worker + _worker_shutdown_wrapper(celery_app)() + @pytest.fixture async def storage_rabbitmq_rpc_client( From cab7a6a365191ea1e351bbcf0d41ba2b62b0703b Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 16 Oct 2025 16:19:34 +0200 Subject: [PATCH 52/58] update --- services/storage/tests/conftest.py | 34 +++++++++++++++--------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index 8e5ab854f1c3..08250d8c842b 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -23,7 +23,7 @@ from aws_library.s3 import SimcoreS3API from celery import Celery from celery.contrib.testing.worker import TestWorkController, start_worker -from celery.signals import worker_init +from celery.signals import worker_init, worker_shutdown from celery_library.worker.signals import _worker_init_wrapper, _worker_shutdown_wrapper from faker import Faker from fakeredis.aioredis import FakeRedis @@ -1008,39 +1008,38 @@ def _(celery_app: Celery) -> None: ... @pytest.fixture -def app_server_factory_with_worker_mode( +def app_server_with_worker_mode( app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, -) -> Callable[[], FastAPIAppServer]: +) -> FastAPIAppServer: monkeypatch.setenv("STORAGE_WORKER_MODE", "true") - def _app_server_factory() -> FastAPIAppServer: - app_settings = ApplicationSettings.create_from_envs() - - assert app_settings.STORAGE_WORKER_MODE is True + app_settings = ApplicationSettings.create_from_envs() + assert app_settings.STORAGE_WORKER_MODE is True - tracing_config = TracingConfig.create( - tracing_settings=None, # disable tracing in tests - service_name="storage-worker", - ) - return FastAPIAppServer(app=create_app(app_settings, tracing_config)) - - return _app_server_factory + tracing_config = TracingConfig.create( + tracing_settings=None, # disable tracing in tests + service_name="storage-worker", + ) + return FastAPIAppServer(app=create_app(app_settings, tracing_config)) @pytest.fixture async def with_storage_celery_worker( celery_app: Celery, - app_server_factory_with_worker_mode: Callable[[], FastAPIAppServer], + app_server_with_worker_mode: FastAPIAppServer, register_test_tasks: Callable[[Celery], None], ) -> AsyncIterator[TestWorkController]: + def _app_server_factory() -> FastAPIAppServer: + return app_server_with_worker_mode + # NOTE: explicitly connect the signals in tests worker_init.connect( - _worker_init_wrapper(celery_app, app_server_factory_with_worker_mode), + _worker_init_wrapper(celery_app, _app_server_factory), weak=False, ) - # worker_shutdown.connect(_worker_shutdown_wrapper(celery_app), weak=False) + worker_shutdown.connect(_worker_shutdown_wrapper(celery_app), weak=False) register_worker_tasks(celery_app) register_test_tasks(celery_app) @@ -1055,6 +1054,7 @@ async def with_storage_celery_worker( ) as worker: yield worker + # Explicitly trigger shutdown signal for proper cleanup _worker_shutdown_wrapper(celery_app)() From 18ce2c7f10518a5cf87e6d76c8654c62bb1d11d0 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 16 Oct 2025 16:33:01 +0200 Subject: [PATCH 53/58] try --- services/storage/tests/conftest.py | 37 +++++++++++++++++------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index 08250d8c842b..c383d2e563ab 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -1008,35 +1008,41 @@ def _(celery_app: Celery) -> None: ... @pytest.fixture -def app_server_with_worker_mode( +def app_server_factory_with_worker_mode( app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, -) -> FastAPIAppServer: +) -> Callable[[], FastAPIAppServer]: monkeypatch.setenv("STORAGE_WORKER_MODE", "true") - app_settings = ApplicationSettings.create_from_envs() - assert app_settings.STORAGE_WORKER_MODE is True + # Cache to ensure we create the app server only once per test + _app_server_cache: list[FastAPIAppServer] = [] - tracing_config = TracingConfig.create( - tracing_settings=None, # disable tracing in tests - service_name="storage-worker", - ) - return FastAPIAppServer(app=create_app(app_settings, tracing_config)) + def _app_server_factory() -> FastAPIAppServer: + if not _app_server_cache: + app_settings = ApplicationSettings.create_from_envs() + assert app_settings.STORAGE_WORKER_MODE is True + + tracing_config = TracingConfig.create( + tracing_settings=None, # disable tracing in tests + service_name="storage-worker", + ) + _app_server_cache.append( + FastAPIAppServer(app=create_app(app_settings, tracing_config)) + ) + return _app_server_cache[0] + + return _app_server_factory @pytest.fixture async def with_storage_celery_worker( celery_app: Celery, - app_server_with_worker_mode: FastAPIAppServer, + app_server_factory_with_worker_mode: Callable[[], FastAPIAppServer], register_test_tasks: Callable[[Celery], None], ) -> AsyncIterator[TestWorkController]: - def _app_server_factory() -> FastAPIAppServer: - return app_server_with_worker_mode - - # NOTE: explicitly connect the signals in tests worker_init.connect( - _worker_init_wrapper(celery_app, _app_server_factory), + _worker_init_wrapper(celery_app, app_server_factory_with_worker_mode), weak=False, ) worker_shutdown.connect(_worker_shutdown_wrapper(celery_app), weak=False) @@ -1054,7 +1060,6 @@ def _app_server_factory() -> FastAPIAppServer: ) as worker: yield worker - # Explicitly trigger shutdown signal for proper cleanup _worker_shutdown_wrapper(celery_app)() From 2ceee86d4002030ab3b799c42451f79f5af2be0f Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 16 Oct 2025 20:19:53 +0200 Subject: [PATCH 54/58] revert --- services/storage/tests/conftest.py | 64 +++++++++++------------------- 1 file changed, 24 insertions(+), 40 deletions(-) diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index c383d2e563ab..1831fab085a9 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -24,7 +24,8 @@ from celery import Celery from celery.contrib.testing.worker import TestWorkController, start_worker from celery.signals import worker_init, worker_shutdown -from celery_library.worker.signals import _worker_init_wrapper, _worker_shutdown_wrapper +from celery.worker.worker import WorkController +from celery_library.signals import on_worker_init, on_worker_shutdown from faker import Faker from fakeredis.aioredis import FakeRedis from fastapi import FastAPI @@ -69,7 +70,7 @@ from settings_library.rabbit import RabbitSettings from simcore_postgres_database.models.tokens import tokens from simcore_postgres_database.storage_models import file_meta_data, projects, users -from simcore_service_storage.api._worker_tasks.tasks import register_worker_tasks +from simcore_service_storage.api._worker_tasks.tasks import setup_worker_tasks from simcore_service_storage.core.application import create_app from simcore_service_storage.core.settings import ApplicationSettings from simcore_service_storage.datcore_dsm import DatCoreDataManager @@ -219,9 +220,9 @@ def app_settings( enabled_rabbitmq: RabbitSettings, sqlalchemy_async_engine: AsyncEngine, postgres_host_config: dict[str, str], + mocked_s3_server_envs: EnvVarsDict, datcore_adapter_service_mock: respx.MockRouter, mocked_redis_server: None, - mocked_s3_server_envs: EnvVarsDict, # Moved to end to ensure S3_ENDPOINT override happens last ) -> ApplicationSettings: test_app_settings = ApplicationSettings.create_from_envs() print(f"{test_app_settings.model_dump_json(indent=2)=}") @@ -999,7 +1000,7 @@ def mock_celery_app(mocker: MockerFixture, celery_config: dict[str, Any]) -> Cel @pytest.fixture -def register_test_tasks() -> Callable[[Celery], None]: +def register_celery_tasks() -> Callable[[Celery], None]: """override if tasks are needed""" def _(celery_app: Celery) -> None: ... @@ -1008,47 +1009,32 @@ def _(celery_app: Celery) -> None: ... @pytest.fixture -def app_server_factory_with_worker_mode( +async def with_storage_celery_worker( app_environment: EnvVarsDict, + celery_app: Celery, monkeypatch: pytest.MonkeyPatch, -) -> Callable[[], FastAPIAppServer]: + register_celery_tasks: Callable[[Celery], None], +) -> AsyncIterator[TestWorkController]: + # Signals must be explicitily connected monkeypatch.setenv("STORAGE_WORKER_MODE", "true") + app_settings = ApplicationSettings.create_from_envs() + tracing_config = TracingConfig.create( + tracing_settings=None, # disable tracing in tests + service_name="storage-api", + ) - # Cache to ensure we create the app server only once per test - _app_server_cache: list[FastAPIAppServer] = [] - - def _app_server_factory() -> FastAPIAppServer: - if not _app_server_cache: - app_settings = ApplicationSettings.create_from_envs() - assert app_settings.STORAGE_WORKER_MODE is True - - tracing_config = TracingConfig.create( - tracing_settings=None, # disable tracing in tests - service_name="storage-worker", - ) - _app_server_cache.append( - FastAPIAppServer(app=create_app(app_settings, tracing_config)) - ) - return _app_server_cache[0] - - return _app_server_factory - + app_server = FastAPIAppServer( + app=create_app(app_settings, tracing_config=tracing_config) + ) -@pytest.fixture -async def with_storage_celery_worker( - celery_app: Celery, - app_server_factory_with_worker_mode: Callable[[], FastAPIAppServer], - register_test_tasks: Callable[[Celery], None], -) -> AsyncIterator[TestWorkController]: + def _on_worker_init_wrapper(sender: WorkController, **_kwargs): + return on_worker_init(sender, app_server, **_kwargs) - worker_init.connect( - _worker_init_wrapper(celery_app, app_server_factory_with_worker_mode), - weak=False, - ) - worker_shutdown.connect(_worker_shutdown_wrapper(celery_app), weak=False) + worker_init.connect(_on_worker_init_wrapper) + worker_shutdown.connect(on_worker_shutdown) - register_worker_tasks(celery_app) - register_test_tasks(celery_app) + setup_worker_tasks(celery_app) + register_celery_tasks(celery_app) with start_worker( celery_app, @@ -1060,8 +1046,6 @@ async def with_storage_celery_worker( ) as worker: yield worker - _worker_shutdown_wrapper(celery_app)() - @pytest.fixture async def storage_rabbitmq_rpc_client( From f8a575b7dc33d308b83c405c15f729ece881ecc4 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 16 Oct 2025 21:19:12 +0200 Subject: [PATCH 55/58] fix test --- services/storage/tests/conftest.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index 1831fab085a9..018ee74db4cf 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -24,8 +24,7 @@ from celery import Celery from celery.contrib.testing.worker import TestWorkController, start_worker from celery.signals import worker_init, worker_shutdown -from celery.worker.worker import WorkController -from celery_library.signals import on_worker_init, on_worker_shutdown +from celery_library.worker.signals import _worker_init_wrapper, _worker_shutdown_wrapper from faker import Faker from fakeredis.aioredis import FakeRedis from fastapi import FastAPI @@ -70,7 +69,7 @@ from settings_library.rabbit import RabbitSettings from simcore_postgres_database.models.tokens import tokens from simcore_postgres_database.storage_models import file_meta_data, projects, users -from simcore_service_storage.api._worker_tasks.tasks import setup_worker_tasks +from simcore_service_storage.api._worker_tasks.tasks import register_worker_tasks from simcore_service_storage.core.application import create_app from simcore_service_storage.core.settings import ApplicationSettings from simcore_service_storage.datcore_dsm import DatCoreDataManager @@ -1023,17 +1022,14 @@ async def with_storage_celery_worker( service_name="storage-api", ) - app_server = FastAPIAppServer( - app=create_app(app_settings, tracing_config=tracing_config) - ) - - def _on_worker_init_wrapper(sender: WorkController, **_kwargs): - return on_worker_init(sender, app_server, **_kwargs) + app_server = FastAPIAppServer(app=create_app(app_settings, tracing_config)) - worker_init.connect(_on_worker_init_wrapper) - worker_shutdown.connect(on_worker_shutdown) + worker_init.connect( + _worker_init_wrapper(celery_app, lambda: app_server), weak=False + ) + worker_shutdown.connect(_worker_shutdown_wrapper(celery_app), weak=False) - setup_worker_tasks(celery_app) + register_worker_tasks(celery_app) register_celery_tasks(celery_app) with start_worker( From ce82a27826d39fe785b1a06e270beee75744e955 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 17 Oct 2025 11:21:30 +0200 Subject: [PATCH 56/58] fix storage --- .../src/celery_library/worker/signals.py | 6 ++-- .../modules/celery/worker/main.py | 12 +++---- services/storage/tests/conftest.py | 31 ++++++++++++++----- 3 files changed, 31 insertions(+), 18 deletions(-) diff --git a/packages/celery-library/src/celery_library/worker/signals.py b/packages/celery-library/src/celery_library/worker/signals.py index 6bef1278de7e..fc39c28bff4a 100644 --- a/packages/celery-library/src/celery_library/worker/signals.py +++ b/packages/celery-library/src/celery_library/worker/signals.py @@ -67,7 +67,5 @@ def register_worker_signals( ) worker_process_shutdown.connect(_worker_shutdown_wrapper(app), weak=False) case _: - worker_init.connect( - _worker_init_wrapper(app, app_server_factory), weak=False - ) - worker_shutdown.connect(_worker_shutdown_wrapper(app), weak=False) + worker_init.connect(_worker_init_wrapper(app, app_server_factory)) + worker_shutdown.connect(_worker_shutdown_wrapper(app)) diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker/main.py b/services/storage/src/simcore_service_storage/modules/celery/worker/main.py index 20cd3c2b51f4..e6a455209706 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker/main.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker/main.py @@ -7,14 +7,14 @@ from ....core.application import create_app from ....core.settings import ApplicationSettings -_settings = ApplicationSettings.create_from_envs() -_tracing_config = TracingConfig.create( - tracing_settings=_settings.STORAGE_TRACING, - service_name="storage-celery-worker", -) - def get_app(): + _settings = ApplicationSettings.create_from_envs() + _tracing_config = TracingConfig.create( + tracing_settings=_settings.STORAGE_TRACING, + service_name="storage-celery-worker", + ) + setup_loggers( log_format_local_dev_enabled=_settings.STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=_settings.STORAGE_LOG_FILTER_MAPPING, diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index 018ee74db4cf..5c02fa3c4a42 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -1007,27 +1007,39 @@ def _(celery_app: Celery) -> None: ... return _ +@pytest.fixture +def worker_app_settings( + app_settings: ApplicationSettings, +) -> ApplicationSettings: + worker_test_app_settings = app_settings.model_copy( + update={"STORAGE_WORKER_MODE": True}, deep=True + ) + print(f"{worker_test_app_settings.model_dump_json(indent=2)=}") + return worker_test_app_settings + + +_logger = logging.getLogger(__name__) + + @pytest.fixture async def with_storage_celery_worker( - app_environment: EnvVarsDict, celery_app: Celery, + worker_app_settings: ApplicationSettings, monkeypatch: pytest.MonkeyPatch, register_celery_tasks: Callable[[Celery], None], ) -> AsyncIterator[TestWorkController]: # Signals must be explicitily connected - monkeypatch.setenv("STORAGE_WORKER_MODE", "true") - app_settings = ApplicationSettings.create_from_envs() tracing_config = TracingConfig.create( tracing_settings=None, # disable tracing in tests service_name="storage-api", ) - app_server = FastAPIAppServer(app=create_app(app_settings, tracing_config)) + app_server = FastAPIAppServer(app=create_app(worker_app_settings, tracing_config)) - worker_init.connect( - _worker_init_wrapper(celery_app, lambda: app_server), weak=False - ) - worker_shutdown.connect(_worker_shutdown_wrapper(celery_app), weak=False) + init_wrapper = _worker_init_wrapper(celery_app, lambda: app_server) + worker_init.connect(init_wrapper, weak=False) + shutdown_wrapper = _worker_shutdown_wrapper(celery_app) + worker_shutdown.connect(shutdown_wrapper, weak=False) register_worker_tasks(celery_app) register_celery_tasks(celery_app) @@ -1042,6 +1054,9 @@ async def with_storage_celery_worker( ) as worker: yield worker + worker_init.disconnect(init_wrapper) + worker_shutdown.disconnect(shutdown_wrapper) + @pytest.fixture async def storage_rabbitmq_rpc_client( From 945e0e5ca01aeadd4e8aba248c9adb2e61451ac1 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 17 Oct 2025 11:32:01 +0200 Subject: [PATCH 57/58] fix api server --- .../unit/api_functions/celery/conftest.py | 43 ++++++++----------- 1 file changed, 19 insertions(+), 24 deletions(-) diff --git a/services/api-server/tests/unit/api_functions/celery/conftest.py b/services/api-server/tests/unit/api_functions/celery/conftest.py index 2b62b002b2b9..9c5013a9e90f 100644 --- a/services/api-server/tests/unit/api_functions/celery/conftest.py +++ b/services/api-server/tests/unit/api_functions/celery/conftest.py @@ -108,39 +108,34 @@ def add_worker_tasks() -> bool: @pytest.fixture -def app_server_factory_with_worker_mode( - app_environment: EnvVarsDict, - monkeypatch: pytest.MonkeyPatch, -) -> Callable[[], FastAPIAppServer]: - monkeypatch.setenv("STORAGE_WORKER_MODE", "true") - - def _app_server_factory() -> FastAPIAppServer: - app_settings = ApplicationSettings.create_from_envs() - - assert app_settings.API_SERVER_WORKER_MODE is True - - tracing_config = TracingConfig.create( - tracing_settings=None, - service_name="api-worker", - ) - return FastAPIAppServer(app=create_app(app_settings, tracing_config)) - - return _app_server_factory +def worker_app_settings( + app_settings: ApplicationSettings, +) -> ApplicationSettings: + worker_test_app_settings = app_settings.model_copy( + update={"API_SERVER_WORKER_MODE": True}, deep=True + ) + print(f"{worker_test_app_settings.model_dump_json(indent=2)=}") + return worker_test_app_settings @pytest.fixture async def with_api_server_celery_worker( celery_app: Celery, - app_server_factory_with_worker_mode: Callable[[], FastAPIAppServer], + worker_app_settings: ApplicationSettings, register_celery_tasks: Callable[[Celery], None], add_worker_tasks: bool, ) -> AsyncIterator[TestWorkController]: - # NOTE: explicitly connect the signals in tests - worker_init.connect( - _worker_init_wrapper(celery_app, app_server_factory_with_worker_mode), - weak=False, + tracing_config = TracingConfig.create( + tracing_settings=None, # disable tracing in tests + service_name="api-server-worker-test", ) - worker_shutdown.connect(_worker_shutdown_wrapper(celery_app), weak=False) + + app_server = FastAPIAppServer(app=create_app(worker_app_settings, tracing_config)) + + _init_wrapper = _worker_init_wrapper(celery_app, lambda: app_server) + _shutdown_wrapper = _worker_shutdown_wrapper(celery_app) + worker_init.connect(_init_wrapper) + worker_shutdown.connect(_shutdown_wrapper) if add_worker_tasks: register_worker_tasks(celery_app) From c0cdaa231f2bbc9656dd8f094b18155a03f5bad1 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 17 Oct 2025 11:50:27 +0200 Subject: [PATCH 58/58] fix api server tests --- .../src/celery_library/worker/signals.py | 4 ++-- .../unit/api_functions/celery/conftest.py | 18 +++++------------- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/packages/celery-library/src/celery_library/worker/signals.py b/packages/celery-library/src/celery_library/worker/signals.py index fc39c28bff4a..042122d9586f 100644 --- a/packages/celery-library/src/celery_library/worker/signals.py +++ b/packages/celery-library/src/celery_library/worker/signals.py @@ -67,5 +67,5 @@ def register_worker_signals( ) worker_process_shutdown.connect(_worker_shutdown_wrapper(app), weak=False) case _: - worker_init.connect(_worker_init_wrapper(app, app_server_factory)) - worker_shutdown.connect(_worker_shutdown_wrapper(app)) + worker_init.connect(_worker_init_wrapper(app, app_server_factory), weak=False) + worker_shutdown.connect(_worker_shutdown_wrapper(app), weak=False) diff --git a/services/api-server/tests/unit/api_functions/celery/conftest.py b/services/api-server/tests/unit/api_functions/celery/conftest.py index 9c5013a9e90f..5774596c91d2 100644 --- a/services/api-server/tests/unit/api_functions/celery/conftest.py +++ b/services/api-server/tests/unit/api_functions/celery/conftest.py @@ -107,30 +107,22 @@ def add_worker_tasks() -> bool: return True -@pytest.fixture -def worker_app_settings( - app_settings: ApplicationSettings, -) -> ApplicationSettings: - worker_test_app_settings = app_settings.model_copy( - update={"API_SERVER_WORKER_MODE": True}, deep=True - ) - print(f"{worker_test_app_settings.model_dump_json(indent=2)=}") - return worker_test_app_settings - - @pytest.fixture async def with_api_server_celery_worker( celery_app: Celery, - worker_app_settings: ApplicationSettings, register_celery_tasks: Callable[[Celery], None], add_worker_tasks: bool, + monkeypatch: pytest.MonkeyPatch, ) -> AsyncIterator[TestWorkController]: tracing_config = TracingConfig.create( tracing_settings=None, # disable tracing in tests service_name="api-server-worker-test", ) + # Signals must be explicitily connected + monkeypatch.setenv("API_SERVER_WORKER_MODE", "true") + app_settings = ApplicationSettings.create_from_envs() - app_server = FastAPIAppServer(app=create_app(worker_app_settings, tracing_config)) + app_server = FastAPIAppServer(app=create_app(app_settings, tracing_config)) _init_wrapper = _worker_init_wrapper(celery_app, lambda: app_server) _shutdown_wrapper = _worker_shutdown_wrapper(celery_app)