From 119b9dac0a794e17274232c1c1bf196f71b896c5 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 12 Feb 2025 11:22:28 +0100 Subject: [PATCH 001/136] add distribute task queue --- services/storage/requirements/_base.in | 1 + 1 file changed, 1 insertion(+) diff --git a/services/storage/requirements/_base.in b/services/storage/requirements/_base.in index c457068e1223..4e7613749def 100644 --- a/services/storage/requirements/_base.in +++ b/services/storage/requirements/_base.in @@ -17,6 +17,7 @@ aioboto3 # s3 storage aiofiles # i/o asyncpg # database +celery httpx opentelemetry-instrumentation-botocore packaging From 235c8ed561278af8b79e88f9b1a56812e0d6b0ab Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 12 Feb 2025 11:30:55 +0100 Subject: [PATCH 002/136] add redis database --- packages/settings-library/src/settings_library/redis.py | 1 + services/docker-compose-ops.yml | 1 + services/docker-compose.yml | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/settings-library/src/settings_library/redis.py b/packages/settings-library/src/settings_library/redis.py index 7e3b0e7b6931..36516c9eadc8 100644 --- a/packages/settings-library/src/settings_library/redis.py +++ b/packages/settings-library/src/settings_library/redis.py @@ -18,6 +18,7 @@ class RedisDatabase(IntEnum): DISTRIBUTED_IDENTIFIERS = 6 DEFERRED_TASKS = 7 DYNAMIC_SERVICES = 8 + CELERY_TASKS = 9 class RedisSettings(BaseCustomSettings): diff --git a/services/docker-compose-ops.yml b/services/docker-compose-ops.yml index c80befe23164..b244d224059b 100644 --- a/services/docker-compose-ops.yml +++ b/services/docker-compose-ops.yml @@ -95,6 +95,7 @@ services: distributed_identifiers:${REDIS_HOST}:${REDIS_PORT}:6:${REDIS_PASSWORD}, deferred_tasks:${REDIS_HOST}:${REDIS_PORT}:7:${REDIS_PASSWORD}, dynamic_services:${REDIS_HOST}:${REDIS_PORT}:8:${REDIS_PASSWORD} + celery_tasks:${REDIS_HOST}:${REDIS_PORT}:9:${REDIS_PASSWORD} # If you add/remove a db, do not forget to update the --databases entry in the docker-compose.yml ports: - "18081:8081" diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 964b037c7082..b922de56c871 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -1238,7 +1238,7 @@ services: "--loglevel", "verbose", "--databases", - "9", + "10", "--appendonly", "yes", "--requirepass", From f3827c7fea99aba05dea5aa152d33d72bf765a31 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 12 Feb 2025 11:49:49 +0100 Subject: [PATCH 003/136] add settings --- .../settings-library/src/settings_library/celery.py | 11 +++++++++++ .../src/simcore_service_storage/core/settings.py | 5 +++++ 2 files changed, 16 insertions(+) create mode 100644 packages/settings-library/src/settings_library/celery.py diff --git a/packages/settings-library/src/settings_library/celery.py b/packages/settings-library/src/settings_library/celery.py new file mode 100644 index 000000000000..d61c41bf1e2d --- /dev/null +++ b/packages/settings-library/src/settings_library/celery.py @@ -0,0 +1,11 @@ +from pydantic_settings import SettingsConfigDict + +from .base import BaseCustomSettings + + +class CelerySettings(BaseCustomSettings): + model_config = SettingsConfigDict( + json_schema_extra={ + "examples": [], + } + ) diff --git a/services/storage/src/simcore_service_storage/core/settings.py b/services/storage/src/simcore_service_storage/core/settings.py index ca9af3c2ebed..a3551ac9f161 100644 --- a/services/storage/src/simcore_service_storage/core/settings.py +++ b/services/storage/src/simcore_service_storage/core/settings.py @@ -12,6 +12,7 @@ from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.application import BaseApplicationSettings from settings_library.basic_types import LogLevel, PortInt +from settings_library.celery import CelerySettings from settings_library.postgres import PostgresSettings from settings_library.redis import RedisSettings from settings_library.s3 import S3Settings @@ -54,6 +55,10 @@ class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): json_schema_extra={"auto_default_from_env": True} ) + STORAGE_CELERY: CelerySettings | None = Field( + json_schema_extra={"auto_default_from_env": True} + ) + STORAGE_TRACING: TracingSettings | None = Field( json_schema_extra={"auto_default_from_env": True} ) From 1a2cc8dc66efaf948dcebac18a9f45f1c5260da6 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 12 Feb 2025 12:17:42 +0100 Subject: [PATCH 004/136] update reqs --- services/storage/requirements/_base.in | 2 +- services/storage/requirements/_base.txt | 33 +++++++++++++++++++++++++ services/storage/requirements/_test.txt | 4 ++- 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/services/storage/requirements/_base.in b/services/storage/requirements/_base.in index 4e7613749def..551eede14c9a 100644 --- a/services/storage/requirements/_base.in +++ b/services/storage/requirements/_base.in @@ -17,7 +17,7 @@ aioboto3 # s3 storage aiofiles # i/o asyncpg # database -celery +celery[redis] httpx opentelemetry-instrumentation-botocore packaging diff --git a/services/storage/requirements/_base.txt b/services/storage/requirements/_base.txt index 26dccf1c4772..33dee0cfe698 100644 --- a/services/storage/requirements/_base.txt +++ b/services/storage/requirements/_base.txt @@ -67,6 +67,8 @@ aiosignal==1.3.2 # via aiohttp alembic==1.14.1 # via -r requirements/../../../packages/postgres-database/requirements/_base.in +amqp==5.3.1 + # via kombu annotated-types==0.7.0 # via pydantic anyio==4.8.0 @@ -96,6 +98,8 @@ attrs==25.1.0 # aiohttp # jsonschema # referencing +billiard==4.2.1 + # via celery boto3==1.35.81 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -132,6 +136,8 @@ botocore==1.35.81 # s3transfer botocore-stubs==1.36.16 # via types-aiobotocore +celery==5.4.0 + # via -r requirements/_base.in certifi==2025.1.31 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -167,9 +173,19 @@ charset-normalizer==3.4.1 # via requests click==8.1.8 # via + # celery + # click-didyoumean + # click-plugins + # click-repl # rich-toolkit # typer # uvicorn +click-didyoumean==0.3.1 + # via celery +click-plugins==1.1.1 + # via celery +click-repl==0.3.0 + # via celery deprecated==1.2.18 # via # opentelemetry-api @@ -302,6 +318,8 @@ jsonschema==4.23.0 # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in jsonschema-specifications==2024.10.1 # via jsonschema +kombu==5.4.2 + # via celery mako==1.3.9 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -496,6 +514,8 @@ prometheus-client==0.21.1 # prometheus-fastapi-instrumentator prometheus-fastapi-instrumentator==7.0.2 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in +prompt-toolkit==3.0.50 + # via click-repl propcache==0.2.1 # via # aiohttp @@ -611,6 +631,7 @@ python-dateutil==2.9.0.post0 # via # arrow # botocore + # celery python-dotenv==1.0.1 # via # pydantic-settings @@ -679,6 +700,7 @@ redis==5.2.1 # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in + # celery referencing==0.35.1 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -847,6 +869,10 @@ typing-extensions==4.12.2 # types-aiobotocore-ec2 # types-aiobotocore-s3 # types-aiobotocore-ssm +tzdata==2025.1 + # via + # celery + # kombu ujson==5.10.0 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -913,8 +939,15 @@ uvicorn==0.34.0 # fastapi-cli uvloop==0.21.0 # via uvicorn +vine==5.1.0 + # via + # amqp + # celery + # kombu watchfiles==1.0.4 # via uvicorn +wcwidth==0.2.13 + # via prompt-toolkit websockets==14.2 # via uvicorn wrapt==1.17.2 diff --git a/services/storage/requirements/_test.txt b/services/storage/requirements/_test.txt index 4e5563e7fd17..ff6163cf1f98 100644 --- a/services/storage/requirements/_test.txt +++ b/services/storage/requirements/_test.txt @@ -366,7 +366,9 @@ typing-extensions==4.12.2 # pydantic-core # sqlalchemy2-stubs tzdata==2025.1 - # via pandas + # via + # -c requirements/_base.txt + # pandas urllib3==2.3.0 # via # -c requirements/../../../requirements/constraints.txt From ce01b633c06b6e0b1216c1c373056891ada0ad5d Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 13 Feb 2025 00:25:38 +0100 Subject: [PATCH 005/136] add celery task --- .../core/application.py | 6 ++- .../modules/celery/__init__.py | 0 .../modules/celery/celery.py | 48 +++++++++++++++++++ .../modules/celery/tasks.py | 13 +++++ 4 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 services/storage/src/simcore_service_storage/modules/celery/__init__.py create mode 100644 services/storage/src/simcore_service_storage/modules/celery/celery.py create mode 100644 services/storage/src/simcore_service_storage/modules/celery/tasks.py diff --git a/services/storage/src/simcore_service_storage/core/application.py b/services/storage/src/simcore_service_storage/core/application.py index 9bfdf3660cdd..89b353d614a3 100644 --- a/services/storage/src/simcore_service_storage/core/application.py +++ b/services/storage/src/simcore_service_storage/core/application.py @@ -29,6 +29,7 @@ from ..dsm import setup_dsm from ..dsm_cleaner import setup_dsm_cleaner from ..exceptions.handlers import set_exception_handlers +from ..modules.celery.celery import setup_celery from ..modules.db import setup_db from ..modules.long_running_tasks import setup_rest_api_long_running_tasks_for_uploads from ..modules.redis import setup as setup_redis @@ -81,11 +82,14 @@ def create_app(settings: ApplicationSettings) -> FastAPI: setup_rest_api_routes(app, API_VTAG) set_exception_handlers(app) + setup_redis(app) + setup_dsm(app) if settings.STORAGE_CLEANER_INTERVAL_S: - setup_redis(app) setup_dsm_cleaner(app) + setup_celery(app) + if settings.STORAGE_PROFILING: app.add_middleware(ProfilerMiddleware) diff --git a/services/storage/src/simcore_service_storage/modules/celery/__init__.py b/services/storage/src/simcore_service_storage/modules/celery/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/services/storage/src/simcore_service_storage/modules/celery/celery.py b/services/storage/src/simcore_service_storage/modules/celery/celery.py new file mode 100644 index 000000000000..10397fdd5625 --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/celery/celery.py @@ -0,0 +1,48 @@ +import logging +from multiprocessing import Process +from typing import cast + +from celery import Celery +from celery.apps.worker import Worker +from fastapi import FastAPI +from settings_library.redis import RedisDatabase +from simcore_service_storage.modules.celery.tasks import setup_celery_tasks + +from ...core.settings import get_application_settings + +_log = logging.getLogger(__name__) + + +def setup_celery(app: FastAPI) -> None: + async def on_startup() -> None: + settings = get_application_settings(app) + assert settings.STORAGE_REDIS + + redis_dsn = settings.STORAGE_REDIS.build_redis_dsn( + RedisDatabase.CELERY_TASKS, + ) + + app.state.celery_app = Celery( + broker=redis_dsn, + backend=redis_dsn, + ) + + setup_celery_tasks(app.state.celery_app) + + # FIXME: Experiment: to start worker in a separate process + def worker_process(): + worker = Worker(app=app.state.celery_app) + worker.start() + + worker_proc = Process(target=worker_process) + worker_proc.start() + + async def on_shutdown() -> None: + _log.warning("Implementing shutdown of celery app") + + app.add_event_handler("startup", on_startup) + app.add_event_handler("shutdown", on_shutdown) + + +def get_celery_app(app: FastAPI) -> Celery: + return cast(Celery, app.state.celery_app) diff --git a/services/storage/src/simcore_service_storage/modules/celery/tasks.py b/services/storage/src/simcore_service_storage/modules/celery/tasks.py new file mode 100644 index 000000000000..eddcd6082a81 --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/celery/tasks.py @@ -0,0 +1,13 @@ +import logging + +from celery import Celery + +_log = logging.getLogger(__name__) + + +def archive(files: list[str]) -> None: + _log.info(f"Archiving {files=}") + + +def setup_celery_tasks(celery_app: Celery) -> None: + celery_app.task()(archive) From 866ca9b4a71d9adf27c2cae76504e0c481de4c76 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 13 Feb 2025 10:23:03 +0100 Subject: [PATCH 006/136] add celery task queue class --- .../modules/celery/celery.py | 46 ++++++++++++++----- .../modules/celery/tasks.py | 8 +--- 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/celery.py b/services/storage/src/simcore_service_storage/modules/celery/celery.py index 10397fdd5625..59b451661a0f 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/celery.py +++ b/services/storage/src/simcore_service_storage/modules/celery/celery.py @@ -4,30 +4,52 @@ from celery import Celery from celery.apps.worker import Worker +from celery.result import AsyncResult from fastapi import FastAPI from settings_library.redis import RedisDatabase -from simcore_service_storage.modules.celery.tasks import setup_celery_tasks +from simcore_service_storage.modules.celery.tasks import archive -from ...core.settings import get_application_settings +from ...core.settings import ApplicationSettings, get_application_settings _log = logging.getLogger(__name__) -def setup_celery(app: FastAPI) -> None: - async def on_startup() -> None: - settings = get_application_settings(app) - assert settings.STORAGE_REDIS - - redis_dsn = settings.STORAGE_REDIS.build_redis_dsn( +class CeleryTaskQueue: + def __init__(self, app_settings: ApplicationSettings): + assert app_settings.STORAGE_REDIS + redis_dsn = app_settings.STORAGE_REDIS.build_redis_dsn( RedisDatabase.CELERY_TASKS, ) - app.state.celery_app = Celery( + self._celery_app = Celery( broker=redis_dsn, backend=redis_dsn, ) - setup_celery_tasks(app.state.celery_app) + @property + def celery_app(self): + return self._celery_app + + def create_task(self, task): + self._celery_app.task()(task) + + def send_task(self, name: str, **kwargs) -> AsyncResult: + return self._celery_app.send_task(name, **kwargs) + + def cancel_task(self, task_id: str): + self._celery_app.control.revoke(task_id) + + +# TODO: use new FastAPI lifespan +def setup_celery(app: FastAPI) -> None: + async def on_startup() -> None: + settings = get_application_settings(app) + assert settings.STORAGE_REDIS + + task_queue = CeleryTaskQueue(settings) + task_queue.create_task(archive) + + app.state.task_queue = task_queue # FIXME: Experiment: to start worker in a separate process def worker_process(): @@ -44,5 +66,5 @@ async def on_shutdown() -> None: app.add_event_handler("shutdown", on_shutdown) -def get_celery_app(app: FastAPI) -> Celery: - return cast(Celery, app.state.celery_app) +def get_celery_task_queue(app: FastAPI) -> CeleryTaskQueue: + return cast(CeleryTaskQueue, app.state.task_queue) diff --git a/services/storage/src/simcore_service_storage/modules/celery/tasks.py b/services/storage/src/simcore_service_storage/modules/celery/tasks.py index eddcd6082a81..08b064fecb29 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/tasks.py +++ b/services/storage/src/simcore_service_storage/modules/celery/tasks.py @@ -1,13 +1,7 @@ import logging -from celery import Celery - _log = logging.getLogger(__name__) def archive(files: list[str]) -> None: - _log.info(f"Archiving {files=}") - - -def setup_celery_tasks(celery_app: Celery) -> None: - celery_app.task()(archive) + _log.info("Archiving: %s", ", ".join(files)) From 9e90c737dfc60e8b91173a201e29716e4c971883 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 13 Feb 2025 10:25:55 +0100 Subject: [PATCH 007/136] rename --- .../src/simcore_service_storage/modules/celery/celery.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/celery.py b/services/storage/src/simcore_service_storage/modules/celery/celery.py index 59b451661a0f..1732e5fe3f75 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/celery.py +++ b/services/storage/src/simcore_service_storage/modules/celery/celery.py @@ -1,6 +1,6 @@ import logging from multiprocessing import Process -from typing import cast +from typing import Callable, cast from celery import Celery from celery.apps.worker import Worker @@ -30,8 +30,8 @@ def __init__(self, app_settings: ApplicationSettings): def celery_app(self): return self._celery_app - def create_task(self, task): - self._celery_app.task()(task) + def create_task(self, task_fn: Callable): + self._celery_app.task()(task_fn) def send_task(self, name: str, **kwargs) -> AsyncResult: return self._celery_app.send_task(name, **kwargs) From b33cdae04f3fbf7dd25fbaa4e177bdfedc6e1f15 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 13 Feb 2025 10:57:51 +0100 Subject: [PATCH 008/136] make testable --- .../core/application.py | 2 +- .../modules/celery/{celery.py => core.py} | 32 +++++++++---------- 2 files changed, 16 insertions(+), 18 deletions(-) rename services/storage/src/simcore_service_storage/modules/celery/{celery.py => core.py} (79%) diff --git a/services/storage/src/simcore_service_storage/core/application.py b/services/storage/src/simcore_service_storage/core/application.py index 89b353d614a3..777ca11444d8 100644 --- a/services/storage/src/simcore_service_storage/core/application.py +++ b/services/storage/src/simcore_service_storage/core/application.py @@ -29,7 +29,7 @@ from ..dsm import setup_dsm from ..dsm_cleaner import setup_dsm_cleaner from ..exceptions.handlers import set_exception_handlers -from ..modules.celery.celery import setup_celery +from ..modules.celery.core import setup_celery from ..modules.db import setup_db from ..modules.long_running_tasks import setup_rest_api_long_running_tasks_for_uploads from ..modules.redis import setup as setup_redis diff --git a/services/storage/src/simcore_service_storage/modules/celery/celery.py b/services/storage/src/simcore_service_storage/modules/celery/core.py similarity index 79% rename from services/storage/src/simcore_service_storage/modules/celery/celery.py rename to services/storage/src/simcore_service_storage/modules/celery/core.py index 1732e5fe3f75..0f70ec4d6f48 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/celery.py +++ b/services/storage/src/simcore_service_storage/modules/celery/core.py @@ -9,26 +9,14 @@ from settings_library.redis import RedisDatabase from simcore_service_storage.modules.celery.tasks import archive -from ...core.settings import ApplicationSettings, get_application_settings +from ...core.settings import get_application_settings _log = logging.getLogger(__name__) class CeleryTaskQueue: - def __init__(self, app_settings: ApplicationSettings): - assert app_settings.STORAGE_REDIS - redis_dsn = app_settings.STORAGE_REDIS.build_redis_dsn( - RedisDatabase.CELERY_TASKS, - ) - - self._celery_app = Celery( - broker=redis_dsn, - backend=redis_dsn, - ) - - @property - def celery_app(self): - return self._celery_app + def __init__(self, celery_app: Celery): + self._celery_app = celery_app def create_task(self, task_fn: Callable): self._celery_app.task()(task_fn) @@ -40,13 +28,23 @@ def cancel_task(self, task_id: str): self._celery_app.control.revoke(task_id) -# TODO: use new FastAPI lifespan +# TODO: move and use new FastAPI lifespan def setup_celery(app: FastAPI) -> None: async def on_startup() -> None: settings = get_application_settings(app) assert settings.STORAGE_REDIS - task_queue = CeleryTaskQueue(settings) + assert settings.STORAGE_REDIS + redis_dsn = settings.STORAGE_REDIS.build_redis_dsn( + RedisDatabase.CELERY_TASKS, + ) + + celery_app = Celery( + broker=redis_dsn, + backend=redis_dsn, + ) + + task_queue = CeleryTaskQueue(celery_app) task_queue.create_task(archive) app.state.task_queue = task_queue From af41640b3f619268ae22398909038c9df7615593 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 13 Feb 2025 16:38:58 +0100 Subject: [PATCH 009/136] add storage worker --- services/docker-compose.yml | 39 +++++++++++++++++++ services/storage/docker/boot.sh | 15 +++++-- .../modules/celery/configurator.py | 17 ++++++++ .../modules/celery/core.py | 22 ++--------- .../modules/celery/tasks.py | 3 +- .../modules/celery/worker/__init__.py | 0 .../modules/celery/worker/main.py | 12 ++++++ 7 files changed, 84 insertions(+), 24 deletions(-) create mode 100644 services/storage/src/simcore_service_storage/modules/celery/configurator.py create mode 100644 services/storage/src/simcore_service_storage/modules/celery/worker/__init__.py create mode 100644 services/storage/src/simcore_service_storage/modules/celery/worker/main.py diff --git a/services/docker-compose.yml b/services/docker-compose.yml index b922de56c871..f78de6249bde 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -1143,6 +1143,45 @@ services: S3_ENDPOINT: ${S3_ENDPOINT} S3_REGION: ${S3_REGION} S3_SECRET_KEY: ${S3_SECRET_KEY} + STORAGE_MODE: NORMAL + STORAGE_LOGLEVEL: ${STORAGE_LOGLEVEL} + STORAGE_MONITORING_ENABLED: 1 + STORAGE_PROFILING: ${STORAGE_PROFILING} + STORAGE_PORT: ${STORAGE_PORT} + TRACING_OPENTELEMETRY_COLLECTOR_PORT: ${TRACING_OPENTELEMETRY_COLLECTOR_PORT} + TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT: ${TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT} + networks: + - default + - interactive_services_subnet + - storage_subnet + + storage-worker: + image: ${DOCKER_REGISTRY:-itisfoundation}/storage:${DOCKER_IMAGE_TAG:-latest} + init: true + hostname: "sto-{{.Node.Hostname}}-{{.Task.Slot}}" + environment: + BF_API_KEY: ${BF_API_KEY} + BF_API_SECRET: ${BF_API_SECRET} + DATCORE_ADAPTER_HOST: ${DATCORE_ADAPTER_HOST:-datcore-adapter} + LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + LOG_FILTER_MAPPING : ${LOG_FILTER_MAPPING} + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_ENDPOINT: ${POSTGRES_ENDPOINT} + POSTGRES_HOST: ${POSTGRES_HOST} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_PORT: ${POSTGRES_PORT} + POSTGRES_USER: ${POSTGRES_USER} + REDIS_HOST: ${REDIS_HOST} + REDIS_PORT: ${REDIS_PORT} + REDIS_SECURE: ${REDIS_SECURE} + REDIS_USER: ${REDIS_USER} + REDIS_PASSWORD: ${REDIS_PASSWORD} + S3_ACCESS_KEY: ${S3_ACCESS_KEY} + S3_BUCKET_NAME: ${S3_BUCKET_NAME} + S3_ENDPOINT: ${S3_ENDPOINT} + S3_REGION: ${S3_REGION} + S3_SECRET_KEY: ${S3_SECRET_KEY} + STORAGE_MODE: WORKER STORAGE_LOGLEVEL: ${STORAGE_LOGLEVEL} STORAGE_MONITORING_ENABLED: 1 STORAGE_PROFILING: ${STORAGE_PROFILING} diff --git a/services/storage/docker/boot.sh b/services/storage/docker/boot.sh index efe437aa521a..88fb21b5b34d 100755 --- a/services/storage/docker/boot.sh +++ b/services/storage/docker/boot.sh @@ -56,8 +56,15 @@ if [ "${SC_BOOT_MODE}" = "debug" ]; then --log-level \"${SERVER_LOG_LEVEL}\" " else - exec uvicorn simcore_service_storage.main:the_app \ - --host 0.0.0.0 \ - --port ${STORAGE_PORT} \ - --log-level "${SERVER_LOG_LEVEL}" + if [ "${STORAGE_MODE}" = "NORMAL" ]; then + exec uvicorn simcore_service_storage.main:the_app \ + --host 0.0.0.0 \ + --port ${STORAGE_PORT} \ + --log-level "${SERVER_LOG_LEVEL}" + else + exec celery \ + -A simcore_service_storage.modules.celery.worker.main:app \ + worker + --loglevel="${SERVER_LOG_LEVEL}" + fi fi diff --git a/services/storage/src/simcore_service_storage/modules/celery/configurator.py b/services/storage/src/simcore_service_storage/modules/celery/configurator.py new file mode 100644 index 000000000000..6477b25903ab --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/celery/configurator.py @@ -0,0 +1,17 @@ +import logging + +from celery import Celery +from settings_library.redis import RedisDatabase + +from ...core.settings import ApplicationSettings + +_log = logging.getLogger(__name__) + + +def create_celery_app(settings: ApplicationSettings) -> Celery: + assert settings.STORAGE_REDIS + app = Celery( + broker=settings.STORAGE_REDIS.build_redis_dsn(RedisDatabase.CELERY_TASKS), + backend=settings.STORAGE_REDIS.build_redis_dsn(RedisDatabase.CELERY_TASKS), + ) + return app diff --git a/services/storage/src/simcore_service_storage/modules/celery/core.py b/services/storage/src/simcore_service_storage/modules/celery/core.py index 0f70ec4d6f48..af9080fdce5b 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/core.py +++ b/services/storage/src/simcore_service_storage/modules/celery/core.py @@ -1,13 +1,10 @@ import logging -from multiprocessing import Process -from typing import Callable, cast +from typing import cast from celery import Celery -from celery.apps.worker import Worker from celery.result import AsyncResult from fastapi import FastAPI from settings_library.redis import RedisDatabase -from simcore_service_storage.modules.celery.tasks import archive from ...core.settings import get_application_settings @@ -18,11 +15,8 @@ class CeleryTaskQueue: def __init__(self, celery_app: Celery): self._celery_app = celery_app - def create_task(self, task_fn: Callable): - self._celery_app.task()(task_fn) - - def send_task(self, name: str, **kwargs) -> AsyncResult: - return self._celery_app.send_task(name, **kwargs) + def send_task(self, name: str, *args, **kwargs) -> AsyncResult: + return self._celery_app.send_task(name, args=args, kwargs=kwargs) def cancel_task(self, task_id: str): self._celery_app.control.revoke(task_id) @@ -34,7 +28,6 @@ async def on_startup() -> None: settings = get_application_settings(app) assert settings.STORAGE_REDIS - assert settings.STORAGE_REDIS redis_dsn = settings.STORAGE_REDIS.build_redis_dsn( RedisDatabase.CELERY_TASKS, ) @@ -45,18 +38,9 @@ async def on_startup() -> None: ) task_queue = CeleryTaskQueue(celery_app) - task_queue.create_task(archive) app.state.task_queue = task_queue - # FIXME: Experiment: to start worker in a separate process - def worker_process(): - worker = Worker(app=app.state.celery_app) - worker.start() - - worker_proc = Process(target=worker_process) - worker_proc.start() - async def on_shutdown() -> None: _log.warning("Implementing shutdown of celery app") diff --git a/services/storage/src/simcore_service_storage/modules/celery/tasks.py b/services/storage/src/simcore_service_storage/modules/celery/tasks.py index 08b064fecb29..1dac1792a83d 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/tasks.py +++ b/services/storage/src/simcore_service_storage/modules/celery/tasks.py @@ -3,5 +3,6 @@ _log = logging.getLogger(__name__) -def archive(files: list[str]) -> None: +def archive(files: list[str]) -> str: _log.info("Archiving: %s", ", ".join(files)) + return "".join(files) diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker/__init__.py b/services/storage/src/simcore_service_storage/modules/celery/worker/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker/main.py b/services/storage/src/simcore_service_storage/modules/celery/worker/main.py new file mode 100644 index 000000000000..780f3483d4c9 --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/celery/worker/main.py @@ -0,0 +1,12 @@ +from ....core.settings import ApplicationSettings +from ...celery.tasks import archive +from ..configurator import create_celery_app + +settings = ApplicationSettings.create_from_envs() + +app = create_celery_app(settings) + +app.task(name="archive")(archive) + + +__all__ = ["app"] From 6890444cf125abc64b2c8445b5387e541cdecd3f Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 14 Feb 2025 16:40:02 +0100 Subject: [PATCH 010/136] continue working --- services/storage/docker/boot.sh | 4 +- services/storage/requirements/_base.in | 1 + services/storage/requirements/_base.txt | 6 ++- services/storage/requirements/_test.in | 1 - services/storage/requirements/_test.txt | 3 -- .../core/application.py | 3 -- .../simcore_service_storage/core/settings.py | 2 + .../src/simcore_service_storage/main.py | 54 +++++++++++++++---- .../celery/{core.py => application.py} | 32 ++++------- .../modules/celery/configurator.py | 17 ------ .../modules/celery/tasks.py | 8 ++- .../modules/celery/worker/main.py | 12 ----- 12 files changed, 72 insertions(+), 71 deletions(-) rename services/storage/src/simcore_service_storage/modules/celery/{core.py => application.py} (50%) delete mode 100644 services/storage/src/simcore_service_storage/modules/celery/configurator.py delete mode 100644 services/storage/src/simcore_service_storage/modules/celery/worker/main.py diff --git a/services/storage/docker/boot.sh b/services/storage/docker/boot.sh index 88fb21b5b34d..6b20175840b0 100755 --- a/services/storage/docker/boot.sh +++ b/services/storage/docker/boot.sh @@ -57,13 +57,13 @@ if [ "${SC_BOOT_MODE}" = "debug" ]; then " else if [ "${STORAGE_MODE}" = "NORMAL" ]; then - exec uvicorn simcore_service_storage.main:the_app \ + exec uvicorn simcore_service_storage.main:app \ --host 0.0.0.0 \ --port ${STORAGE_PORT} \ --log-level "${SERVER_LOG_LEVEL}" else exec celery \ - -A simcore_service_storage.modules.celery.worker.main:app \ + -A simcore_service_storage.main:app \ worker --loglevel="${SERVER_LOG_LEVEL}" fi diff --git a/services/storage/requirements/_base.in b/services/storage/requirements/_base.in index 551eede14c9a..49e188dc605b 100644 --- a/services/storage/requirements/_base.in +++ b/services/storage/requirements/_base.in @@ -16,6 +16,7 @@ aioboto3 # s3 storage aiofiles # i/o +asgi_lifespan asyncpg # database celery[redis] httpx diff --git a/services/storage/requirements/_base.txt b/services/storage/requirements/_base.txt index 33dee0cfe698..1c79f2d807c6 100644 --- a/services/storage/requirements/_base.txt +++ b/services/storage/requirements/_base.txt @@ -87,6 +87,8 @@ arrow==1.3.0 # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in +asgi-lifespan==2.1.0 + # via -r requirements/_base.in asgiref==3.8.1 # via opentelemetry-instrumentation-asgi asyncpg==0.30.0 @@ -756,7 +758,9 @@ shellingham==1.5.4 six==1.17.0 # via python-dateutil sniffio==1.3.1 - # via anyio + # via + # anyio + # asgi-lifespan sqlalchemy==1.4.54 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt diff --git a/services/storage/requirements/_test.in b/services/storage/requirements/_test.in index 09a057e4a082..b17851ead06d 100644 --- a/services/storage/requirements/_test.in +++ b/services/storage/requirements/_test.in @@ -6,7 +6,6 @@ --constraint _base.txt -asgi_lifespan asyncpg-stubs coverage docker diff --git a/services/storage/requirements/_test.txt b/services/storage/requirements/_test.txt index ff6163cf1f98..7b460313e67b 100644 --- a/services/storage/requirements/_test.txt +++ b/services/storage/requirements/_test.txt @@ -21,8 +21,6 @@ anyio==4.8.0 # via # -c requirements/_base.txt # httpx -asgi-lifespan==2.1.0 - # via -r requirements/_test.in asyncpg==0.30.0 # via # -c requirements/_base.txt @@ -337,7 +335,6 @@ sniffio==1.3.1 # via # -c requirements/_base.txt # anyio - # asgi-lifespan sortedcontainers==2.4.0 # via fakeredis sqlalchemy==1.4.54 diff --git a/services/storage/src/simcore_service_storage/core/application.py b/services/storage/src/simcore_service_storage/core/application.py index 777ca11444d8..0d16c2ad3337 100644 --- a/services/storage/src/simcore_service_storage/core/application.py +++ b/services/storage/src/simcore_service_storage/core/application.py @@ -29,7 +29,6 @@ from ..dsm import setup_dsm from ..dsm_cleaner import setup_dsm_cleaner from ..exceptions.handlers import set_exception_handlers -from ..modules.celery.core import setup_celery from ..modules.db import setup_db from ..modules.long_running_tasks import setup_rest_api_long_running_tasks_for_uploads from ..modules.redis import setup as setup_redis @@ -88,8 +87,6 @@ def create_app(settings: ApplicationSettings) -> FastAPI: if settings.STORAGE_CLEANER_INTERVAL_S: setup_dsm_cleaner(app) - setup_celery(app) - if settings.STORAGE_PROFILING: app.add_middleware(ProfilerMiddleware) diff --git a/services/storage/src/simcore_service_storage/core/settings.py b/services/storage/src/simcore_service_storage/core/settings.py index a3551ac9f161..4f3f516ffa8a 100644 --- a/services/storage/src/simcore_service_storage/core/settings.py +++ b/services/storage/src/simcore_service_storage/core/settings.py @@ -101,6 +101,8 @@ class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of _logger message patterns that should be filtered out.", ) + STORAGE_MODE: str + @field_validator("LOG_LEVEL", mode="before") @classmethod def _validate_loglevel(cls, value: str) -> str: diff --git a/services/storage/src/simcore_service_storage/main.py b/services/storage/src/simcore_service_storage/main.py index 7af3c3605d2f..9921b0594f5f 100644 --- a/services/storage/src/simcore_service_storage/main.py +++ b/services/storage/src/simcore_service_storage/main.py @@ -1,23 +1,59 @@ """Main application to be deployed in for example uvicorn.""" +import asyncio import logging -from fastapi import FastAPI +from asgi_lifespan import LifespanManager +from celery.signals import worker_init, worker_shutdown from servicelib.logging_utils import config_all_loggers from simcore_service_storage.core.application import create_app from simcore_service_storage.core.settings import ApplicationSettings +from simcore_service_storage.modules.celery.application import create_celery_app +from simcore_service_storage.modules.celery.tasks import archive -_the_settings = ApplicationSettings.create_from_envs() +_settings = ApplicationSettings.create_from_envs() # SEE https://github.com/ITISFoundation/osparc-simcore/issues/3148 -logging.basicConfig(level=_the_settings.log_level) # NOSONAR -logging.root.setLevel(_the_settings.log_level) +logging.basicConfig(level=_settings.log_level) # NOSONAR +logging.root.setLevel(_settings.log_level) config_all_loggers( - log_format_local_dev_enabled=_the_settings.STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED, - logger_filter_mapping=_the_settings.STORAGE_LOG_FILTER_MAPPING, - tracing_settings=_the_settings.STORAGE_TRACING, + log_format_local_dev_enabled=_settings.STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED, + logger_filter_mapping=_settings.STORAGE_LOG_FILTER_MAPPING, + tracing_settings=_settings.STORAGE_TRACING, ) +_logger = logging.getLogger(__name__) -# SINGLETON FastAPI app -the_app: FastAPI = create_app(_the_settings) +fastapi_app = create_app(_settings) + +celery_app = create_celery_app(_settings) +celery_app.task(name="archive")(archive) + + +@worker_init.connect +def on_worker_init(**_kwargs): + loop = asyncio.new_event_loop() + # asyncio.set_event_loop(loop) + + async def lifespan(): + async with LifespanManager(fastapi_app): + _logger.error("FastAPI lifespan started") + await asyncio.Event().wait() + + fastapi_app.state.lifespan_task = loop.create_task(lifespan()) + _logger.error("Worker init: FastAPI lifespan task started") + + +@worker_shutdown.connect +def on_worker_shutdown(**_kwargs): + if fastapi_app.state.lifespan_task: + fastapi_app.state.lifespan_task.cancel() + _logger.info("FastAPI lifespan stopped.") + + +if _settings.STORAGE_MODE == "WORKER": + celery_app.conf.fastapi_app = fastapi_app + app = celery_app +else: + fastapi_app.state.celery = celery_app + app = fastapi_app diff --git a/services/storage/src/simcore_service_storage/modules/celery/core.py b/services/storage/src/simcore_service_storage/modules/celery/application.py similarity index 50% rename from services/storage/src/simcore_service_storage/modules/celery/core.py rename to services/storage/src/simcore_service_storage/modules/celery/application.py index af9080fdce5b..67eec70d8831 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/core.py +++ b/services/storage/src/simcore_service_storage/modules/celery/application.py @@ -6,7 +6,7 @@ from fastapi import FastAPI from settings_library.redis import RedisDatabase -from ...core.settings import get_application_settings +from ...core.settings import ApplicationSettings _log = logging.getLogger(__name__) @@ -23,29 +23,19 @@ def cancel_task(self, task_id: str): # TODO: move and use new FastAPI lifespan -def setup_celery(app: FastAPI) -> None: - async def on_startup() -> None: - settings = get_application_settings(app) - assert settings.STORAGE_REDIS +def create_celery_app(settings: ApplicationSettings) -> Celery: + assert settings.STORAGE_REDIS - redis_dsn = settings.STORAGE_REDIS.build_redis_dsn( - RedisDatabase.CELERY_TASKS, - ) + redis_dsn = settings.STORAGE_REDIS.build_redis_dsn( + RedisDatabase.CELERY_TASKS, + ) - celery_app = Celery( - broker=redis_dsn, - backend=redis_dsn, - ) + celery_app = Celery( + broker=redis_dsn, + backend=redis_dsn, + ) - task_queue = CeleryTaskQueue(celery_app) - - app.state.task_queue = task_queue - - async def on_shutdown() -> None: - _log.warning("Implementing shutdown of celery app") - - app.add_event_handler("startup", on_startup) - app.add_event_handler("shutdown", on_shutdown) + return celery_app def get_celery_task_queue(app: FastAPI) -> CeleryTaskQueue: diff --git a/services/storage/src/simcore_service_storage/modules/celery/configurator.py b/services/storage/src/simcore_service_storage/modules/celery/configurator.py deleted file mode 100644 index 6477b25903ab..000000000000 --- a/services/storage/src/simcore_service_storage/modules/celery/configurator.py +++ /dev/null @@ -1,17 +0,0 @@ -import logging - -from celery import Celery -from settings_library.redis import RedisDatabase - -from ...core.settings import ApplicationSettings - -_log = logging.getLogger(__name__) - - -def create_celery_app(settings: ApplicationSettings) -> Celery: - assert settings.STORAGE_REDIS - app = Celery( - broker=settings.STORAGE_REDIS.build_redis_dsn(RedisDatabase.CELERY_TASKS), - backend=settings.STORAGE_REDIS.build_redis_dsn(RedisDatabase.CELERY_TASKS), - ) - return app diff --git a/services/storage/src/simcore_service_storage/modules/celery/tasks.py b/services/storage/src/simcore_service_storage/modules/celery/tasks.py index 1dac1792a83d..e6f04b3e76a4 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/tasks.py +++ b/services/storage/src/simcore_service_storage/modules/celery/tasks.py @@ -1,8 +1,12 @@ import logging -_log = logging.getLogger(__name__) +from celery import current_app + +_logger = logging.getLogger(__name__) def archive(files: list[str]) -> str: - _log.info("Archiving: %s", ", ".join(files)) + _logger.error( + "Archiving: %s (conf=%s)", ", ".join(files), f"{current_app.conf.fastapi_app}" + ) return "".join(files) diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker/main.py b/services/storage/src/simcore_service_storage/modules/celery/worker/main.py deleted file mode 100644 index 780f3483d4c9..000000000000 --- a/services/storage/src/simcore_service_storage/modules/celery/worker/main.py +++ /dev/null @@ -1,12 +0,0 @@ -from ....core.settings import ApplicationSettings -from ...celery.tasks import archive -from ..configurator import create_celery_app - -settings = ApplicationSettings.create_from_envs() - -app = create_celery_app(settings) - -app.task(name="archive")(archive) - - -__all__ = ["app"] From d6b85be2ed230fb2f12e5c7a03c414a1077340c3 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 17 Feb 2025 10:25:48 +0100 Subject: [PATCH 011/136] continue --- services/storage/docker/boot.sh | 2 +- .../src/simcore_service_storage/main.py | 33 +------- .../modules/celery/application.py | 1 - .../modules/celery/configurator.py | 17 ++++ .../modules/celery/worker/main.py | 79 +++++++++++++++++++ 5 files changed, 99 insertions(+), 33 deletions(-) create mode 100644 services/storage/src/simcore_service_storage/modules/celery/configurator.py create mode 100644 services/storage/src/simcore_service_storage/modules/celery/worker/main.py diff --git a/services/storage/docker/boot.sh b/services/storage/docker/boot.sh index 6b20175840b0..fa354d02d265 100755 --- a/services/storage/docker/boot.sh +++ b/services/storage/docker/boot.sh @@ -63,7 +63,7 @@ else --log-level "${SERVER_LOG_LEVEL}" else exec celery \ - -A simcore_service_storage.main:app \ + -A simcore_service_storage.modules.celery.worker.main:app \ worker --loglevel="${SERVER_LOG_LEVEL}" fi diff --git a/services/storage/src/simcore_service_storage/main.py b/services/storage/src/simcore_service_storage/main.py index 9921b0594f5f..75dac40bddad 100644 --- a/services/storage/src/simcore_service_storage/main.py +++ b/services/storage/src/simcore_service_storage/main.py @@ -1,10 +1,7 @@ """Main application to be deployed in for example uvicorn.""" -import asyncio import logging -from asgi_lifespan import LifespanManager -from celery.signals import worker_init, worker_shutdown from servicelib.logging_utils import config_all_loggers from simcore_service_storage.core.application import create_app from simcore_service_storage.core.settings import ApplicationSettings @@ -29,31 +26,5 @@ celery_app = create_celery_app(_settings) celery_app.task(name="archive")(archive) - -@worker_init.connect -def on_worker_init(**_kwargs): - loop = asyncio.new_event_loop() - # asyncio.set_event_loop(loop) - - async def lifespan(): - async with LifespanManager(fastapi_app): - _logger.error("FastAPI lifespan started") - await asyncio.Event().wait() - - fastapi_app.state.lifespan_task = loop.create_task(lifespan()) - _logger.error("Worker init: FastAPI lifespan task started") - - -@worker_shutdown.connect -def on_worker_shutdown(**_kwargs): - if fastapi_app.state.lifespan_task: - fastapi_app.state.lifespan_task.cancel() - _logger.info("FastAPI lifespan stopped.") - - -if _settings.STORAGE_MODE == "WORKER": - celery_app.conf.fastapi_app = fastapi_app - app = celery_app -else: - fastapi_app.state.celery = celery_app - app = fastapi_app +fastapi_app.state.celery = celery_app +app = fastapi_app diff --git a/services/storage/src/simcore_service_storage/modules/celery/application.py b/services/storage/src/simcore_service_storage/modules/celery/application.py index 67eec70d8831..0a0d51c60d5b 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/application.py +++ b/services/storage/src/simcore_service_storage/modules/celery/application.py @@ -22,7 +22,6 @@ def cancel_task(self, task_id: str): self._celery_app.control.revoke(task_id) -# TODO: move and use new FastAPI lifespan def create_celery_app(settings: ApplicationSettings) -> Celery: assert settings.STORAGE_REDIS diff --git a/services/storage/src/simcore_service_storage/modules/celery/configurator.py b/services/storage/src/simcore_service_storage/modules/celery/configurator.py new file mode 100644 index 000000000000..6477b25903ab --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/celery/configurator.py @@ -0,0 +1,17 @@ +import logging + +from celery import Celery +from settings_library.redis import RedisDatabase + +from ...core.settings import ApplicationSettings + +_log = logging.getLogger(__name__) + + +def create_celery_app(settings: ApplicationSettings) -> Celery: + assert settings.STORAGE_REDIS + app = Celery( + broker=settings.STORAGE_REDIS.build_redis_dsn(RedisDatabase.CELERY_TASKS), + backend=settings.STORAGE_REDIS.build_redis_dsn(RedisDatabase.CELERY_TASKS), + ) + return app diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker/main.py b/services/storage/src/simcore_service_storage/modules/celery/worker/main.py new file mode 100644 index 000000000000..89ae2e0af7f1 --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/celery/worker/main.py @@ -0,0 +1,79 @@ +"""Main application to be deployed in for example uvicorn.""" + +import asyncio +import logging +import threading + +from asgi_lifespan import LifespanManager +from celery.signals import worker_init, worker_shutdown +from servicelib.logging_utils import config_all_loggers +from simcore_service_storage.core.application import create_app +from simcore_service_storage.core.settings import ApplicationSettings +from simcore_service_storage.modules.celery.application import create_celery_app +from simcore_service_storage.modules.celery.tasks import archive + +_settings = ApplicationSettings.create_from_envs() + +# SEE https://github.com/ITISFoundation/osparc-simcore/issues/3148 +logging.basicConfig(level=_settings.log_level) # NOSONAR +logging.root.setLevel(_settings.log_level) +config_all_loggers( + log_format_local_dev_enabled=_settings.STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED, + logger_filter_mapping=_settings.STORAGE_LOG_FILTER_MAPPING, + tracing_settings=_settings.STORAGE_TRACING, +) + +_logger = logging.getLogger(__name__) + +fastapi_app = create_app(_settings) + +celery_app = create_celery_app(_settings) +celery_app.task(name="archive")(archive) + + +@worker_init.connect +def on_worker_init(**_kwargs): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + shutdown_event = asyncio.Event() + + async def lifespan(): + async with LifespanManager(fastapi_app): + _logger.error("FastAPI lifespan started") + try: + await shutdown_event.wait() + except asyncio.CancelledError: + _logger.error("Lifespan task cancelled") + _logger.error("FastAPI lifespan ended") + + lifespan_task = loop.create_task(lifespan()) + fastapi_app.state.lifespan_task = lifespan_task + fastapi_app.state.shutdown_event = shutdown_event + fastapi_app.state.loop = loop # Store the loop for shutdown + + def run_loop(): + loop.run_forever() + + thread = threading.Thread(target=run_loop, daemon=True) + thread.start() + + +@worker_shutdown.connect +def on_worker_shutdown(**_kwargs): + loop = fastapi_app.state.loop + + async def shutdown(): + fastapi_app.state.shutdown_event.set() + fastapi_app.state.lifespan_task.cancel() + try: + await fastapi_app.state.lifespan_task + except asyncio.CancelledError: + pass + + asyncio.run_coroutine_threadsafe(shutdown(), loop) + + _logger.error("FastAPI lifespan stopped.") + + +celery_app.conf.fastapi_app = fastapi_app +app = celery_app From 00751db3bed4439e69c02ce2d3506a3745f539a9 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 17 Feb 2025 11:27:20 +0100 Subject: [PATCH 012/136] use rabbit --- .../src/simcore_service_storage/modules/celery/configurator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/configurator.py b/services/storage/src/simcore_service_storage/modules/celery/configurator.py index 6477b25903ab..90d22023cc7e 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/configurator.py +++ b/services/storage/src/simcore_service_storage/modules/celery/configurator.py @@ -9,9 +9,10 @@ def create_celery_app(settings: ApplicationSettings) -> Celery: + assert settings.STORAGE_RABBITMQ assert settings.STORAGE_REDIS app = Celery( - broker=settings.STORAGE_REDIS.build_redis_dsn(RedisDatabase.CELERY_TASKS), + broker=settings.STORAGE_RABBITMQ.dsn, backend=settings.STORAGE_REDIS.build_redis_dsn(RedisDatabase.CELERY_TASKS), ) return app From bd27fa5e6a2600cbdbf19bf69f83938aaf76630c Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 17 Feb 2025 13:07:11 +0100 Subject: [PATCH 013/136] continue --- .../modules/celery/tasks.py | 26 +++++++++++++++---- .../modules/celery/worker/main.py | 19 +++++++------- 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/tasks.py b/services/storage/src/simcore_service_storage/modules/celery/tasks.py index e6f04b3e76a4..6ba3c6344303 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/tasks.py +++ b/services/storage/src/simcore_service_storage/modules/celery/tasks.py @@ -1,12 +1,28 @@ +import asyncio import logging +from asyncio import AbstractEventLoop from celery import current_app +from fastapi import FastAPI _logger = logging.getLogger(__name__) -def archive(files: list[str]) -> str: - _logger.error( - "Archiving: %s (conf=%s)", ", ".join(files), f"{current_app.conf.fastapi_app}" - ) - return "".join(files) +def get_fastapi_app() -> FastAPI: + fast_api_app: FastAPI = current_app.conf.fastapi_app + return fast_api_app + + +def get_loop() -> AbstractEventLoop: # nosec + loop: AbstractEventLoop = current_app.conf.loop + return loop + + +async def _async_archive(files: list[str]) -> None: + fast_api_app: FastAPI = get_fastapi_app() + + _logger.error("Archiving: %s (conf=%s)", ", ".join(files), f"{fast_api_app}") + + +def archive(files: list[str]) -> None: + asyncio.run_coroutine_threadsafe(_async_archive(files), get_loop()) diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker/main.py b/services/storage/src/simcore_service_storage/modules/celery/worker/main.py index 89ae2e0af7f1..61f2e3202cca 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker/main.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker/main.py @@ -6,6 +6,7 @@ from asgi_lifespan import LifespanManager from celery.signals import worker_init, worker_shutdown +from servicelib.background_task import cancel_wait_task from servicelib.logging_utils import config_all_loggers from simcore_service_storage.core.application import create_app from simcore_service_storage.core.settings import ApplicationSettings @@ -25,7 +26,6 @@ _logger = logging.getLogger(__name__) -fastapi_app = create_app(_settings) celery_app = create_celery_app(_settings) celery_app.task(name="archive")(archive) @@ -37,6 +37,8 @@ def on_worker_init(**_kwargs): asyncio.set_event_loop(loop) shutdown_event = asyncio.Event() + fastapi_app = create_app(_settings) + async def lifespan(): async with LifespanManager(fastapi_app): _logger.error("FastAPI lifespan started") @@ -49,7 +51,9 @@ async def lifespan(): lifespan_task = loop.create_task(lifespan()) fastapi_app.state.lifespan_task = lifespan_task fastapi_app.state.shutdown_event = shutdown_event - fastapi_app.state.loop = loop # Store the loop for shutdown + + celery_app.conf.fastapi_app = fastapi_app + celery_app.conf.loop = loop def run_loop(): loop.run_forever() @@ -60,20 +64,17 @@ def run_loop(): @worker_shutdown.connect def on_worker_shutdown(**_kwargs): - loop = fastapi_app.state.loop + loop = celery_app.conf.loop + fastapi_app = celery_app.conf.fastapi_app async def shutdown(): fastapi_app.state.shutdown_event.set() - fastapi_app.state.lifespan_task.cancel() - try: - await fastapi_app.state.lifespan_task - except asyncio.CancelledError: - pass + + await cancel_wait_task(fastapi_app.state.lifespan_task, max_delay=5) asyncio.run_coroutine_threadsafe(shutdown(), loop) _logger.error("FastAPI lifespan stopped.") -celery_app.conf.fastapi_app = fastapi_app app = celery_app From 2794e5bfc5bd36383a4ed75236062e38b6ad1899 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 17 Feb 2025 15:20:46 +0100 Subject: [PATCH 014/136] continue --- services/storage/requirements/_test.in | 1 + services/storage/requirements/_test.txt | 70 ++++++++++++++++++- .../simcore_service_storage/core/settings.py | 2 +- .../modules/celery/tasks.py | 14 ++-- 4 files changed, 80 insertions(+), 7 deletions(-) diff --git a/services/storage/requirements/_test.in b/services/storage/requirements/_test.in index b17851ead06d..0e2518897773 100644 --- a/services/storage/requirements/_test.in +++ b/services/storage/requirements/_test.in @@ -16,6 +16,7 @@ moto[server] pandas pytest pytest-asyncio +pytest-celery pytest-cov pytest-icdiff pytest-instafail diff --git a/services/storage/requirements/_test.txt b/services/storage/requirements/_test.txt index 711a85db687d..5afedb4aa714 100644 --- a/services/storage/requirements/_test.txt +++ b/services/storage/requirements/_test.txt @@ -11,6 +11,10 @@ aiosignal==1.3.2 # via # -c requirements/_base.txt # aiohttp +amqp==5.3.1 + # via + # -c requirements/_base.txt + # kombu annotated-types==0.7.0 # via # -c requirements/_base.txt @@ -37,6 +41,10 @@ aws-sam-translator==1.94.0 # via cfn-lint aws-xray-sdk==2.14.0 # via moto +billiard==4.2.1 + # via + # -c requirements/_base.txt + # celery blinker==1.9.0 # via flask boto3==1.35.81 @@ -52,6 +60,10 @@ botocore==1.35.81 # boto3 # moto # s3transfer +celery==5.4.0 + # via + # -c requirements/_base.txt + # pytest-celery certifi==2025.1.31 # via # -c requirements/../../../requirements/constraints.txt @@ -71,7 +83,23 @@ charset-normalizer==3.4.1 click==8.1.8 # via # -c requirements/_base.txt + # celery + # click-didyoumean + # click-plugins + # click-repl # flask +click-didyoumean==0.3.1 + # via + # -c requirements/_base.txt + # celery +click-plugins==1.1.1 + # via + # -c requirements/_base.txt + # celery +click-repl==0.3.0 + # via + # -c requirements/_base.txt + # celery coverage==7.6.11 # via # -r requirements/_test.in @@ -81,10 +109,14 @@ cryptography==44.0.0 # -c requirements/../../../requirements/constraints.txt # joserfc # moto +debugpy==1.8.12 + # via pytest-celery docker==7.1.0 # via # -r requirements/_test.in # moto + # pytest-celery + # pytest-docker-tools faker==36.1.0 # via -r requirements/_test.in fakeredis==2.26.2 @@ -170,6 +202,10 @@ jsonschema-specifications==2024.10.1 # -c requirements/_base.txt # jsonschema # openapi-schema-validator +kombu==5.4.2 + # via + # -c requirements/_base.txt + # celery lazy-object-proxy==1.10.0 # via openapi-spec-validator lupa==2.4 @@ -217,11 +253,19 @@ ply==3.11 # via jsonpath-ng pprintpp==0.4.0 # via pytest-icdiff +prompt-toolkit==3.0.50 + # via + # -c requirements/_base.txt + # click-repl propcache==0.2.1 # via # -c requirements/_base.txt # aiohttp # yarl +psutil==6.1.1 + # via + # -c requirements/_base.txt + # pytest-celery py-partiql-parser==0.5.6 # via moto pycparser==2.22 @@ -242,6 +286,7 @@ pytest==8.3.4 # -r requirements/_test.in # pytest-asyncio # pytest-cov + # pytest-docker-tools # pytest-icdiff # pytest-instafail # pytest-mock @@ -250,8 +295,12 @@ pytest-asyncio==0.23.8 # via # -c requirements/../../../requirements/constraints.txt # -r requirements/_test.in +pytest-celery==1.1.3 + # via -r requirements/_test.in pytest-cov==6.0.0 # via -r requirements/_test.in +pytest-docker-tools==3.1.3 + # via pytest-celery pytest-icdiff==0.9 # via -r requirements/_test.in pytest-instafail==0.5.0 @@ -266,6 +315,7 @@ python-dateutil==2.9.0.post0 # via # -c requirements/_base.txt # botocore + # celery # moto # pandas # simcore-service-storage-sdk @@ -321,7 +371,9 @@ s3transfer==0.10.4 # -c requirements/_base.txt # boto3 setuptools==75.8.0 - # via moto + # via + # moto + # pytest-celery simcore-service-storage-sdk @ git+https://github.com/ITISFoundation/osparc-simcore.git@cfdf4f86d844ebb362f4f39e9c6571d561b72897#subdirectory=services/storage/client-sdk/python # via -r requirements/_test.in six==1.17.0 @@ -345,6 +397,10 @@ sqlalchemy2-stubs==0.0.2a38 # via sqlalchemy sympy==1.13.3 # via cfn-lint +tenacity==9.0.0 + # via + # -c requirements/_base.txt + # pytest-celery termcolor==2.5.0 # via pytest-sugar types-aiofiles==24.1.0.20241221 @@ -363,7 +419,9 @@ typing-extensions==4.12.2 tzdata==2025.1 # via # -c requirements/_base.txt + # celery # faker + # kombu # pandas urllib3==2.3.0 # via @@ -374,6 +432,16 @@ urllib3==2.3.0 # requests # responses # simcore-service-storage-sdk +vine==5.1.0 + # via + # -c requirements/_base.txt + # amqp + # celery + # kombu +wcwidth==0.2.13 + # via + # -c requirements/_base.txt + # prompt-toolkit werkzeug==3.1.3 # via # flask diff --git a/services/storage/src/simcore_service_storage/core/settings.py b/services/storage/src/simcore_service_storage/core/settings.py index b692db7a71db..8bb149e0b8d8 100644 --- a/services/storage/src/simcore_service_storage/core/settings.py +++ b/services/storage/src/simcore_service_storage/core/settings.py @@ -106,7 +106,7 @@ class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of _logger message patterns that should be filtered out.", ) - STORAGE_MODE: str + STORAGE_WORKER_MODE: bool | None = False @field_validator("LOG_LEVEL", mode="before") @classmethod diff --git a/services/storage/src/simcore_service_storage/modules/celery/tasks.py b/services/storage/src/simcore_service_storage/modules/celery/tasks.py index 6ba3c6344303..d08364e13b1c 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/tasks.py +++ b/services/storage/src/simcore_service_storage/modules/celery/tasks.py @@ -4,11 +4,13 @@ from celery import current_app from fastapi import FastAPI +from models_library.projects_nodes_io import StorageFileID +from models_library.users import UserID _logger = logging.getLogger(__name__) -def get_fastapi_app() -> FastAPI: +def get_fastapi_app(): fast_api_app: FastAPI = current_app.conf.fastapi_app return fast_api_app @@ -18,11 +20,13 @@ def get_loop() -> AbstractEventLoop: # nosec return loop -async def _async_archive(files: list[str]) -> None: +async def _async_archive(user_id: UserID, files: list[StorageFileID]) -> None: fast_api_app: FastAPI = get_fastapi_app() - _logger.error("Archiving: %s (conf=%s)", ", ".join(files), f"{fast_api_app}") + _logger.error( + "Archiving: %s (%s, %s)", ", ".join(files), f"{user_id=}", f"{fast_api_app=}" + ) -def archive(files: list[str]) -> None: - asyncio.run_coroutine_threadsafe(_async_archive(files), get_loop()) +def archive(user_id: UserID, files: list[StorageFileID]) -> None: + asyncio.run_coroutine_threadsafe(_async_archive(user_id, files), get_loop()) From f2e43b4164572c7c81ba480d0fb08cdcfdeae885 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 18 Feb 2025 12:16:14 +0100 Subject: [PATCH 015/136] add unit tests --- .../modules/celery/tasks.py | 29 ++++++++++++------- .../tests/unit/modules/celery/conftest.py | 26 +++++++++++++++++ .../tests/unit/modules/celery/test_tasks.py | 7 +++++ 3 files changed, 52 insertions(+), 10 deletions(-) create mode 100644 services/storage/tests/unit/modules/celery/conftest.py create mode 100644 services/storage/tests/unit/modules/celery/test_tasks.py diff --git a/services/storage/src/simcore_service_storage/modules/celery/tasks.py b/services/storage/src/simcore_service_storage/modules/celery/tasks.py index d08364e13b1c..88b34234000b 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/tasks.py +++ b/services/storage/src/simcore_service_storage/modules/celery/tasks.py @@ -2,31 +2,40 @@ import logging from asyncio import AbstractEventLoop -from celery import current_app +from celery import Task from fastapi import FastAPI from models_library.projects_nodes_io import StorageFileID from models_library.users import UserID +from .worker.main import celery_app + _logger = logging.getLogger(__name__) -def get_fastapi_app(): - fast_api_app: FastAPI = current_app.conf.fastapi_app +def get_fastapi_app(celery_app): + fast_api_app: FastAPI = celery_app.conf.get("fastapi_app") return fast_api_app -def get_loop() -> AbstractEventLoop: # nosec - loop: AbstractEventLoop = current_app.conf.loop +def get_loop(celery_app) -> AbstractEventLoop: # nosec + loop: AbstractEventLoop = celery_app.conf.get("loop") return loop -async def _async_archive(user_id: UserID, files: list[StorageFileID]) -> None: - fast_api_app: FastAPI = get_fastapi_app() +async def _async_archive( + celery_app, user_id: UserID, files: list[StorageFileID] +) -> StorageFileID: + fast_api_app: FastAPI = get_fastapi_app(celery_app) - _logger.error( + _logger.debug( "Archiving: %s (%s, %s)", ", ".join(files), f"{user_id=}", f"{fast_api_app=}" ) + return "_".join(files) + ".zip" + -def archive(user_id: UserID, files: list[StorageFileID]) -> None: - asyncio.run_coroutine_threadsafe(_async_archive(user_id, files), get_loop()) +@celery_app.task(name="archive", bind=True) +def archive(task: Task, user_id: UserID, files: list[StorageFileID]) -> StorageFileID: + return asyncio.run_coroutine_threadsafe( + _async_archive(task.app, user_id, files), get_loop(task.app) + ).result() diff --git a/services/storage/tests/unit/modules/celery/conftest.py b/services/storage/tests/unit/modules/celery/conftest.py new file mode 100644 index 000000000000..8df3780a3271 --- /dev/null +++ b/services/storage/tests/unit/modules/celery/conftest.py @@ -0,0 +1,26 @@ +import pytest +from celery.contrib.testing.worker import start_worker +from celery.signals import worker_init, worker_shutdown +from simcore_service_storage.modules.celery.worker.main import ( + app, + on_worker_init, + on_worker_shutdown, +) + +# Signals must be explicitily connected +worker_init.connect(on_worker_init) +worker_shutdown.connect(on_worker_shutdown) + + +@pytest.fixture +def celery_app(): + app.conf.update({"broker_url": "memory://"}) + return app + + +@pytest.fixture +def celery_worker(celery_app): + with start_worker(celery_app, perform_ping_check=False) as worker: + worker_init.send(sender=worker) + yield worker + worker_shutdown.send(sender=worker) diff --git a/services/storage/tests/unit/modules/celery/test_tasks.py b/services/storage/tests/unit/modules/celery/test_tasks.py new file mode 100644 index 000000000000..c6390b4dd196 --- /dev/null +++ b/services/storage/tests/unit/modules/celery/test_tasks.py @@ -0,0 +1,7 @@ +from faker import Faker +from simcore_service_storage.modules.celery.tasks import archive + + +def test_archive(celery_app, celery_worker, faker: Faker): + result = archive.apply(args=(faker.uuid4(), ["f1", "f2"])) + assert result.get() == "f1_f2.zip" From dbf0e9c2535a9f7410655523905cfae415d9fd8b Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 18 Feb 2025 14:37:09 +0100 Subject: [PATCH 016/136] continue --- .../modules/celery/worker/main.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker/main.py b/services/storage/src/simcore_service_storage/modules/celery/worker/main.py index 61f2e3202cca..210cc711c9b7 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker/main.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker/main.py @@ -11,7 +11,6 @@ from simcore_service_storage.core.application import create_app from simcore_service_storage.core.settings import ApplicationSettings from simcore_service_storage.modules.celery.application import create_celery_app -from simcore_service_storage.modules.celery.tasks import archive _settings = ApplicationSettings.create_from_envs() @@ -28,11 +27,10 @@ celery_app = create_celery_app(_settings) -celery_app.task(name="archive")(archive) @worker_init.connect -def on_worker_init(**_kwargs): +def on_worker_init(sender, **_kwargs): loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) shutdown_event = asyncio.Event() @@ -40,20 +38,22 @@ def on_worker_init(**_kwargs): fastapi_app = create_app(_settings) async def lifespan(): - async with LifespanManager(fastapi_app): + async with LifespanManager( + fastapi_app, startup_timeout=30, shutdown_timeout=30 + ): _logger.error("FastAPI lifespan started") try: await shutdown_event.wait() - except asyncio.CancelledError: - _logger.error("Lifespan task cancelled") + except asyncio.exceptions.CancelledError: + _logger.info("Lifespan task cancelled") _logger.error("FastAPI lifespan ended") lifespan_task = loop.create_task(lifespan()) fastapi_app.state.lifespan_task = lifespan_task fastapi_app.state.shutdown_event = shutdown_event - celery_app.conf.fastapi_app = fastapi_app - celery_app.conf.loop = loop + sender.app.conf["fastapi_app"] = fastapi_app + sender.app.conf["loop"] = loop def run_loop(): loop.run_forever() @@ -63,9 +63,9 @@ def run_loop(): @worker_shutdown.connect -def on_worker_shutdown(**_kwargs): - loop = celery_app.conf.loop - fastapi_app = celery_app.conf.fastapi_app +def on_worker_shutdown(sender, **_kwargs): + loop = sender.app.conf["loop"] + fastapi_app = sender.app.conf["fastapi_app"] async def shutdown(): fastapi_app.state.shutdown_event.set() From 2bc8ed4c8cc75d635f5be5e390967c0fcff908be Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 19 Feb 2025 11:49:02 +0100 Subject: [PATCH 017/136] base working tests --- .../modules/celery/application.py | 41 ----------- .../modules/celery/client/__init__.py | 3 + .../modules/celery/client/_interface.py | 33 +++++++++ .../modules/celery/client/client_utils.py | 8 +++ .../modules/celery/client/setup.py | 34 +++++++++ .../modules/celery/configurator.py | 18 ----- .../modules/celery/models.py | 3 + .../modules/celery/tasks.py | 41 ----------- .../modules/celery/worker/_interface.py | 12 ++++ .../celery/worker/{main.py => setup.py} | 13 ++-- .../modules/celery/worker/utils.py | 21 ++++++ .../tests/unit/modules/celery/conftest.py | 71 ++++++++++++++++--- .../tests/unit/modules/celery/test_core.py | 61 ++++++++++++++++ 13 files changed, 241 insertions(+), 118 deletions(-) delete mode 100644 services/storage/src/simcore_service_storage/modules/celery/application.py create mode 100644 services/storage/src/simcore_service_storage/modules/celery/client/__init__.py create mode 100644 services/storage/src/simcore_service_storage/modules/celery/client/_interface.py create mode 100644 services/storage/src/simcore_service_storage/modules/celery/client/client_utils.py create mode 100644 services/storage/src/simcore_service_storage/modules/celery/client/setup.py delete mode 100644 services/storage/src/simcore_service_storage/modules/celery/configurator.py create mode 100644 services/storage/src/simcore_service_storage/modules/celery/models.py delete mode 100644 services/storage/src/simcore_service_storage/modules/celery/tasks.py create mode 100644 services/storage/src/simcore_service_storage/modules/celery/worker/_interface.py rename services/storage/src/simcore_service_storage/modules/celery/worker/{main.py => setup.py} (86%) create mode 100644 services/storage/src/simcore_service_storage/modules/celery/worker/utils.py create mode 100644 services/storage/tests/unit/modules/celery/test_core.py diff --git a/services/storage/src/simcore_service_storage/modules/celery/application.py b/services/storage/src/simcore_service_storage/modules/celery/application.py deleted file mode 100644 index 0a0d51c60d5b..000000000000 --- a/services/storage/src/simcore_service_storage/modules/celery/application.py +++ /dev/null @@ -1,41 +0,0 @@ -import logging -from typing import cast - -from celery import Celery -from celery.result import AsyncResult -from fastapi import FastAPI -from settings_library.redis import RedisDatabase - -from ...core.settings import ApplicationSettings - -_log = logging.getLogger(__name__) - - -class CeleryTaskQueue: - def __init__(self, celery_app: Celery): - self._celery_app = celery_app - - def send_task(self, name: str, *args, **kwargs) -> AsyncResult: - return self._celery_app.send_task(name, args=args, kwargs=kwargs) - - def cancel_task(self, task_id: str): - self._celery_app.control.revoke(task_id) - - -def create_celery_app(settings: ApplicationSettings) -> Celery: - assert settings.STORAGE_REDIS - - redis_dsn = settings.STORAGE_REDIS.build_redis_dsn( - RedisDatabase.CELERY_TASKS, - ) - - celery_app = Celery( - broker=redis_dsn, - backend=redis_dsn, - ) - - return celery_app - - -def get_celery_task_queue(app: FastAPI) -> CeleryTaskQueue: - return cast(CeleryTaskQueue, app.state.task_queue) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client/__init__.py b/services/storage/src/simcore_service_storage/modules/celery/client/__init__.py new file mode 100644 index 000000000000..3d3e1162977e --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/celery/client/__init__.py @@ -0,0 +1,3 @@ +from ._interface import CeleryClientInterface + +__all__: tuple[str, ...] = ("CeleryClientInterface",) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client/_interface.py b/services/storage/src/simcore_service_storage/modules/celery/client/_interface.py new file mode 100644 index 000000000000..6580fad704b0 --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/celery/client/_interface.py @@ -0,0 +1,33 @@ +from typing import Any +from uuid import uuid4 + +from celery import Celery +from celery.result import AsyncResult +from models_library.users import UserID + +from ..models import TaskID + + +class CeleryClientInterface: + def __init__(self, celery_app: Celery): + self._celery_app = celery_app + + def submit(self, name: str, *, user_id: UserID, **kwargs) -> TaskID: + task_id = f"{user_id}_{name}_{uuid4()}" + return self._celery_app.send_task(name, task_id=task_id, kwargs=kwargs).id + + def _get_result(self, task_id: TaskID) -> AsyncResult: + return self._celery_app.AsyncResult(task_id) + + def get_state(self, task_id: TaskID) -> str: + # task_id , state, progress + return self._get_result(task_id).state + + def get_result(self, task_id: TaskID) -> Any: + return self._get_result(task_id).result + + def cancel(self, task_id: TaskID) -> None: + self._celery_app.control.revoke(task_id, terminate=True) + + def list(self, user_id: UserID) -> list[TaskID]: + return [] diff --git a/services/storage/src/simcore_service_storage/modules/celery/client/client_utils.py b/services/storage/src/simcore_service_storage/modules/celery/client/client_utils.py new file mode 100644 index 000000000000..28fda45d3982 --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/celery/client/client_utils.py @@ -0,0 +1,8 @@ +from typing import cast + +from fastapi import FastAPI +from simcore_service_storage.modules.celery.client import CeleryClientInterface + + +def get_celery_client_interface(app: FastAPI) -> CeleryClientInterface: + return cast(CeleryClientInterface, app.state.celery.conf["client_interface"]) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client/setup.py b/services/storage/src/simcore_service_storage/modules/celery/client/setup.py new file mode 100644 index 000000000000..245932e53776 --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/celery/client/setup.py @@ -0,0 +1,34 @@ +import logging + +from celery import Celery +from fastapi import FastAPI +from settings_library.redis import RedisDatabase + +from ....core.settings import ApplicationSettings +from ._interface import CeleryClientInterface + +_log = logging.getLogger(__name__) + + +def create_celery_app(settings: ApplicationSettings) -> Celery: + assert settings.STORAGE_RABBITMQ + assert settings.STORAGE_REDIS + + celery_app = Celery( + broker=settings.STORAGE_RABBITMQ.dsn, + backend=settings.STORAGE_REDIS.build_redis_dsn( + RedisDatabase.CELERY_TASKS, + ), + ) + celery_app.conf["client_interface"] = CeleryClientInterface(celery_app) + + return celery_app + + +def attach_to_fastapi(fastapi: FastAPI, celery: Celery) -> None: + fastapi.state.celery = celery + + +def get_celery_client(fastapi: FastAPI) -> CeleryClientInterface: + celery: Celery = fastapi.state.celery + return celery.conf.get("client_interface") diff --git a/services/storage/src/simcore_service_storage/modules/celery/configurator.py b/services/storage/src/simcore_service_storage/modules/celery/configurator.py deleted file mode 100644 index 90d22023cc7e..000000000000 --- a/services/storage/src/simcore_service_storage/modules/celery/configurator.py +++ /dev/null @@ -1,18 +0,0 @@ -import logging - -from celery import Celery -from settings_library.redis import RedisDatabase - -from ...core.settings import ApplicationSettings - -_log = logging.getLogger(__name__) - - -def create_celery_app(settings: ApplicationSettings) -> Celery: - assert settings.STORAGE_RABBITMQ - assert settings.STORAGE_REDIS - app = Celery( - broker=settings.STORAGE_RABBITMQ.dsn, - backend=settings.STORAGE_REDIS.build_redis_dsn(RedisDatabase.CELERY_TASKS), - ) - return app diff --git a/services/storage/src/simcore_service_storage/modules/celery/models.py b/services/storage/src/simcore_service_storage/modules/celery/models.py new file mode 100644 index 000000000000..83c4db48d22f --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/celery/models.py @@ -0,0 +1,3 @@ +from typing import TypeAlias + +TaskID: TypeAlias = str diff --git a/services/storage/src/simcore_service_storage/modules/celery/tasks.py b/services/storage/src/simcore_service_storage/modules/celery/tasks.py deleted file mode 100644 index 88b34234000b..000000000000 --- a/services/storage/src/simcore_service_storage/modules/celery/tasks.py +++ /dev/null @@ -1,41 +0,0 @@ -import asyncio -import logging -from asyncio import AbstractEventLoop - -from celery import Task -from fastapi import FastAPI -from models_library.projects_nodes_io import StorageFileID -from models_library.users import UserID - -from .worker.main import celery_app - -_logger = logging.getLogger(__name__) - - -def get_fastapi_app(celery_app): - fast_api_app: FastAPI = celery_app.conf.get("fastapi_app") - return fast_api_app - - -def get_loop(celery_app) -> AbstractEventLoop: # nosec - loop: AbstractEventLoop = celery_app.conf.get("loop") - return loop - - -async def _async_archive( - celery_app, user_id: UserID, files: list[StorageFileID] -) -> StorageFileID: - fast_api_app: FastAPI = get_fastapi_app(celery_app) - - _logger.debug( - "Archiving: %s (%s, %s)", ", ".join(files), f"{user_id=}", f"{fast_api_app=}" - ) - - return "_".join(files) + ".zip" - - -@celery_app.task(name="archive", bind=True) -def archive(task: Task, user_id: UserID, files: list[StorageFileID]) -> StorageFileID: - return asyncio.run_coroutine_threadsafe( - _async_archive(task.app, user_id, files), get_loop(task.app) - ).result() diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker/_interface.py b/services/storage/src/simcore_service_storage/modules/celery/worker/_interface.py new file mode 100644 index 000000000000..b63490ea67bd --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/celery/worker/_interface.py @@ -0,0 +1,12 @@ +from celery import Celery +from models_library.progress_bar import ProgressReport + +from ..models import TaskID + + +class CeleryWorkerInterface: + def __init__(self, celery_app: Celery) -> None: + self.celery_app = celery_app + + def set_progress(self, task_id: TaskID, report: ProgressReport) -> None: + pass diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker/main.py b/services/storage/src/simcore_service_storage/modules/celery/worker/setup.py similarity index 86% rename from services/storage/src/simcore_service_storage/modules/celery/worker/main.py rename to services/storage/src/simcore_service_storage/modules/celery/worker/setup.py index 210cc711c9b7..63455deb8941 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker/main.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker/setup.py @@ -10,7 +10,9 @@ from servicelib.logging_utils import config_all_loggers from simcore_service_storage.core.application import create_app from simcore_service_storage.core.settings import ApplicationSettings -from simcore_service_storage.modules.celery.application import create_celery_app +from simcore_service_storage.modules.celery.client.setup import create_celery_app + +from ._interface import CeleryWorkerInterface _settings = ApplicationSettings.create_from_envs() @@ -41,12 +43,10 @@ async def lifespan(): async with LifespanManager( fastapi_app, startup_timeout=30, shutdown_timeout=30 ): - _logger.error("FastAPI lifespan started") try: await shutdown_event.wait() - except asyncio.exceptions.CancelledError: - _logger.info("Lifespan task cancelled") - _logger.error("FastAPI lifespan ended") + except asyncio.CancelledError: + _logger.warning("Lifespan task cancelled") lifespan_task = loop.create_task(lifespan()) fastapi_app.state.lifespan_task = lifespan_task @@ -54,6 +54,7 @@ async def lifespan(): sender.app.conf["fastapi_app"] = fastapi_app sender.app.conf["loop"] = loop + sender.app.conf["worker_interface"] = CeleryWorkerInterface(sender.app) def run_loop(): loop.run_forever() @@ -74,7 +75,5 @@ async def shutdown(): asyncio.run_coroutine_threadsafe(shutdown(), loop) - _logger.error("FastAPI lifespan stopped.") - app = celery_app diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker/utils.py b/services/storage/src/simcore_service_storage/modules/celery/worker/utils.py new file mode 100644 index 000000000000..447093665e5b --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/celery/worker/utils.py @@ -0,0 +1,21 @@ +from asyncio import AbstractEventLoop + +from celery import Celery +from fastapi import FastAPI + +from ._interface import CeleryWorkerInterface + + +def get_fastapi_app(celery_app: Celery) -> FastAPI: + fast_api_app: FastAPI = celery_app.conf.get("fastapi_app") + return fast_api_app + + +def get_loop(celery_app: Celery) -> AbstractEventLoop: # nosec + loop: AbstractEventLoop = celery_app.conf.get("loop") + return loop + + +def get_worker_interface(celery_app: Celery) -> CeleryWorkerInterface: + worker_interface: CeleryWorkerInterface = celery_app.conf.get("worker_interface") + return worker_interface diff --git a/services/storage/tests/unit/modules/celery/conftest.py b/services/storage/tests/unit/modules/celery/conftest.py index 8df3780a3271..f1c1eb7ca920 100644 --- a/services/storage/tests/unit/modules/celery/conftest.py +++ b/services/storage/tests/unit/modules/celery/conftest.py @@ -1,26 +1,75 @@ +from asyncio import AbstractEventLoop +from typing import Callable, Iterable + import pytest -from celery.contrib.testing.worker import start_worker +from celery import Celery +from celery.contrib.testing.worker import TestWorkController, start_worker from celery.signals import worker_init, worker_shutdown -from simcore_service_storage.modules.celery.worker.main import ( - app, +from fastapi import FastAPI +from simcore_service_storage.main import celery_app as celery_app_client +from simcore_service_storage.modules.celery.client import CeleryClientInterface +from simcore_service_storage.modules.celery.worker._interface import ( + CeleryWorkerInterface, +) +from simcore_service_storage.modules.celery.worker.setup import ( + celery_app as celery_app_worker, +) +from simcore_service_storage.modules.celery.worker.setup import ( on_worker_init, on_worker_shutdown, ) -# Signals must be explicitily connected -worker_init.connect(on_worker_init) -worker_shutdown.connect(on_worker_shutdown) + +@pytest.fixture +def client_celery_app() -> Celery: + celery_app_client.conf.update( + {"broker_url": "memory://", "result_backend": "cache+memory://"} + ) + + assert isinstance(celery_app_client.conf["client_interface"], CeleryClientInterface) + assert "worker_interface" not in celery_app_client.conf + assert "loop" not in celery_app_client.conf + assert "fastapi_app" not in celery_app_client.conf + + return celery_app_client @pytest.fixture -def celery_app(): - app.conf.update({"broker_url": "memory://"}) - return app +def register_celery_tasks() -> Callable[[Celery], None]: + msg = "please define a callback that registers the tasks" + raise NotImplementedError(msg) @pytest.fixture -def celery_worker(celery_app): - with start_worker(celery_app, perform_ping_check=False) as worker: +def celery_worker( + register_celery_tasks: Callable[[Celery], None] +) -> Iterable[TestWorkController]: + celery_app_worker.conf.update( + {"broker_url": "memory://", "result_backend": "cache+memory://"} + ) + + register_celery_tasks(celery_app_worker) + + # Signals must be explicitily connected + worker_init.connect(on_worker_init) + worker_shutdown.connect(on_worker_shutdown) + + with start_worker( + celery_app_worker, loglevel="info", perform_ping_check=False + ) as worker: worker_init.send(sender=worker) + + assert isinstance( + celery_app_worker.conf["worker_interface"], CeleryWorkerInterface + ) + assert isinstance(celery_app_worker.conf["loop"], AbstractEventLoop) + assert isinstance(celery_app_worker.conf["fastapi_app"], FastAPI) + yield worker worker_shutdown.send(sender=worker) + + +@pytest.fixture +def worker_celery_app(celery_worker: TestWorkController) -> Celery: + assert isinstance(celery_worker.app, Celery) + return celery_worker.app diff --git a/services/storage/tests/unit/modules/celery/test_core.py b/services/storage/tests/unit/modules/celery/test_core.py new file mode 100644 index 000000000000..99556d13a65f --- /dev/null +++ b/services/storage/tests/unit/modules/celery/test_core.py @@ -0,0 +1,61 @@ +import asyncio +import time +from typing import Callable + +import pytest +from celery import Celery, Task +from models_library.progress_bar import ProgressReport +from simcore_service_storage.modules.celery.client.client_utils import ( + get_celery_client_interface, +) +from simcore_service_storage.modules.celery.worker.utils import ( + get_fastapi_app, + get_loop, + get_worker_interface, +) + + +async def _async_archive( + celery_app: Celery, task_id: str, param1: int, values: list[str] +) -> str: + fastapi_app = get_fastapi_app(celery_app) + worker_interface = get_worker_interface(celery_app) + + worker_interface.set_progress(task_id, ProgressReport(actual_value=0)) + print(fastapi_app, task_id, param1, values) + + return "result" + + +def sync_archive(task: Task, param1: int, values: list[str]) -> str: + return asyncio.run_coroutine_threadsafe( + _async_archive(task.app, task.request.id, param1, values), get_loop(task.app) + ).result() + + +@pytest.fixture +def register_celery_tasks() -> Callable[[Celery], None]: + def _(celery_app: Celery) -> None: + celery_app.task(name="sync_archive", bind=True)(sync_archive) + + return _ + + +def test_slow_task_ends_successfully( + client_celery_app: Celery, worker_celery_app: Celery +): + from simcore_service_storage.main import fastapi_app + + client_interface = get_celery_client_interface(fastapi_app) + + task_id = client_interface.submit( + "sync_archive", user_id=1, param1=1, values=["a", "b"] + ) + assert client_interface.get_state(task_id) == "PENDING" + assert client_interface.get_result(task_id) is None + + # use tnaticyt to wait for resutl + time.sleep(2) + + assert client_interface.get_state(task_id) == "SUCCESS" + assert client_interface.get_result(task_id) == "result" From 6f84e1da10fb1c75e97d84287861ad6722625445 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 20 Feb 2025 14:51:20 +0100 Subject: [PATCH 018/136] add progress --- .../src/simcore_service_storage/main.py | 9 ++- .../modules/celery/client/_interface.py | 76 +++++++++++++++---- .../modules/celery/models.py | 9 +++ .../modules/celery/worker/_interface.py | 10 ++- .../tests/unit/modules/celery/test_core.py | 75 +++++++++++++----- .../tests/unit/modules/celery/test_tasks.py | 7 -- 6 files changed, 139 insertions(+), 47 deletions(-) delete mode 100644 services/storage/tests/unit/modules/celery/test_tasks.py diff --git a/services/storage/src/simcore_service_storage/main.py b/services/storage/src/simcore_service_storage/main.py index 75dac40bddad..1965cd8b8f18 100644 --- a/services/storage/src/simcore_service_storage/main.py +++ b/services/storage/src/simcore_service_storage/main.py @@ -5,8 +5,10 @@ from servicelib.logging_utils import config_all_loggers from simcore_service_storage.core.application import create_app from simcore_service_storage.core.settings import ApplicationSettings -from simcore_service_storage.modules.celery.application import create_celery_app -from simcore_service_storage.modules.celery.tasks import archive +from simcore_service_storage.modules.celery.client.setup import ( + attach_to_fastapi, + create_celery_app, +) _settings = ApplicationSettings.create_from_envs() @@ -24,7 +26,6 @@ fastapi_app = create_app(_settings) celery_app = create_celery_app(_settings) -celery_app.task(name="archive")(archive) +attach_to_fastapi(fastapi_app, celery_app) -fastapi_app.state.celery = celery_app app = fastapi_app diff --git a/services/storage/src/simcore_service_storage/modules/celery/client/_interface.py b/services/storage/src/simcore_service_storage/modules/celery/client/_interface.py index 6580fad704b0..0ba96d2f639c 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client/_interface.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client/_interface.py @@ -1,33 +1,79 @@ -from typing import Any +from typing import Any, Final, TypeAlias from uuid import uuid4 from celery import Celery from celery.result import AsyncResult -from models_library.users import UserID +from models_library.progress_bar import ProgressReport +from pydantic import ValidationError -from ..models import TaskID +from ..models import TaskID, TaskProgress + +_PREFIX: Final = "AJ" + +TaskIdComponents: TypeAlias = dict[str, Any] + + +def _get_task_id_components(task_id_components: TaskIdComponents) -> list[str]: + return [f"{v}" for _, v in sorted(task_id_components.items())] + + +def _get_components_prefix( + name: str, task_id_components: TaskIdComponents +) -> list[str]: + return [_PREFIX, name, *_get_task_id_components(task_id_components)] + + +def _get_task_id_prefix(name: str, task_id_components: TaskIdComponents) -> TaskID: + return "::".join(_get_components_prefix(name, task_id_components)) + + +def _get_task_id(name: str, task_id_components: TaskIdComponents) -> TaskID: + return "::".join([*_get_components_prefix(name, task_id_components), f"{uuid4()}"]) class CeleryClientInterface: def __init__(self, celery_app: Celery): self._celery_app = celery_app - def submit(self, name: str, *, user_id: UserID, **kwargs) -> TaskID: - task_id = f"{user_id}_{name}_{uuid4()}" - return self._celery_app.send_task(name, task_id=task_id, kwargs=kwargs).id + def submit( + self, task_name: str, *, task_id_components: TaskIdComponents, **task_params + ) -> TaskID: + task_id = _get_task_id(task_name, task_id_components) + task = self._celery_app.send_task( + task_name, task_id=task_id, kwargs=task_params + ) + return task.id - def _get_result(self, task_id: TaskID) -> AsyncResult: - return self._celery_app.AsyncResult(task_id) + def get(self, task_id: TaskID) -> Any: + return self._celery_app.tasks(task_id) - def get_state(self, task_id: TaskID) -> str: - # task_id , state, progress - return self._get_result(task_id).state + def cancel(self, task_id: TaskID) -> None: + self._celery_app.control.revoke(task_id, terminate=True) + + def _get_async_result(self, task_id: TaskID) -> AsyncResult: + return self._celery_app.AsyncResult(task_id) def get_result(self, task_id: TaskID) -> Any: - return self._get_result(task_id).result + # se manca il risultato o se va in FAILURE, ritorna error + return self._get_async_result(task_id).result - def cancel(self, task_id: TaskID) -> None: - self._celery_app.control.revoke(task_id, terminate=True) + def _get_progress_report(self, task_id: TaskID) -> ProgressReport | None: + result = self._get_async_result(task_id).result + if result: + try: + return ProgressReport.model_validate(result) + except ValidationError: + return None + + def get_progress(self, task_id: TaskID) -> TaskProgress: + return TaskProgress( + task_id=task_id, + task_state=self._get_async_result(task_id).state, + progress_report=self._get_progress_report(task_id), + ) - def list(self, user_id: UserID) -> list[TaskID]: + def list( + self, task_name: str, *, task_id_components: TaskIdComponents + ) -> list[TaskID]: + prefix_to_search_in_redis = _get_task_id_prefix(task_name, task_id_components) return [] diff --git a/services/storage/src/simcore_service_storage/modules/celery/models.py b/services/storage/src/simcore_service_storage/modules/celery/models.py index 83c4db48d22f..9a137fefe649 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/models.py +++ b/services/storage/src/simcore_service_storage/modules/celery/models.py @@ -1,3 +1,12 @@ from typing import TypeAlias +from models_library.progress_bar import ProgressReport +from pydantic import BaseModel + TaskID: TypeAlias = str + + +class TaskProgress(BaseModel): + task_id: str + task_state: str # add enum + progress_report: ProgressReport | None = None diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker/_interface.py b/services/storage/src/simcore_service_storage/modules/celery/worker/_interface.py index b63490ea67bd..2c5afdf71e57 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker/_interface.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker/_interface.py @@ -8,5 +8,11 @@ class CeleryWorkerInterface: def __init__(self, celery_app: Celery) -> None: self.celery_app = celery_app - def set_progress(self, task_id: TaskID, report: ProgressReport) -> None: - pass + def set_progress( + self, task_name: str, task_id: TaskID, report: ProgressReport + ) -> None: + self.celery_app.tasks[task_name].update_state( + task_id=task_id, + state="PROGRESS", + meta=report.model_dump(mode="json"), + ) diff --git a/services/storage/tests/unit/modules/celery/test_core.py b/services/storage/tests/unit/modules/celery/test_core.py index 99556d13a65f..cef22829369c 100644 --- a/services/storage/tests/unit/modules/celery/test_core.py +++ b/services/storage/tests/unit/modules/celery/test_core.py @@ -1,61 +1,98 @@ import asyncio -import time from typing import Callable import pytest from celery import Celery, Task from models_library.progress_bar import ProgressReport +from simcore_service_storage.main import fastapi_app +from simcore_service_storage.modules.celery.client._interface import TaskIdComponents from simcore_service_storage.modules.celery.client.client_utils import ( get_celery_client_interface, ) from simcore_service_storage.modules.celery.worker.utils import ( - get_fastapi_app, get_loop, get_worker_interface, ) +from tenacity import Retrying, retry_if_exception_type, stop_after_delay, wait_fixed async def _async_archive( - celery_app: Celery, task_id: str, param1: int, values: list[str] + celery_app: Celery, task_name: str, task_id: str, files: list[str] ) -> str: - fastapi_app = get_fastapi_app(celery_app) worker_interface = get_worker_interface(celery_app) - worker_interface.set_progress(task_id, ProgressReport(actual_value=0)) - print(fastapi_app, task_id, param1, values) + for n in range(len(files)): + worker_interface.set_progress( + task_name=task_name, + task_id=task_id, + report=ProgressReport(actual_value=n / len(files) * 100), + ) + await asyncio.sleep(0.1) - return "result" + return "archive.zip" -def sync_archive(task: Task, param1: int, values: list[str]) -> str: +def sync_archive(task: Task, files: list[str]) -> str: + assert task.name return asyncio.run_coroutine_threadsafe( - _async_archive(task.app, task.request.id, param1, values), get_loop(task.app) + _async_archive(task.app, task.name, task.request.id, files), get_loop(task.app) ).result() +def sync_error(task: Task) -> str: + raise ValueError("my error here") + + @pytest.fixture def register_celery_tasks() -> Callable[[Celery], None]: def _(celery_app: Celery) -> None: celery_app.task(name="sync_archive", bind=True)(sync_archive) + celery_app.task(name="sync_error", bind=True)(sync_error) return _ -def test_slow_task_ends_successfully( - client_celery_app: Celery, worker_celery_app: Celery +def test_archive( + client_celery_app: Celery, + worker_celery_app: Celery, ): - from simcore_service_storage.main import fastapi_app + client_interface = get_celery_client_interface(fastapi_app) + task_id_components = TaskIdComponents(user_id=1) + + task_id = client_interface.submit( + "sync_archive", + task_id_components=task_id_components, + files=[f"file{n}" for n in range(100)], + ) + + for attempt in Retrying( + retry=retry_if_exception_type(AssertionError), + wait=wait_fixed(1), + stop=stop_after_delay(30), + ): + with attempt: + progress = client_interface.get_progress(task_id) + assert progress.task_state == "SUCCESS" + + assert client_interface.get_progress(task_id).task_state == "SUCCESS" + + +def test_sync_error( + client_celery_app: Celery, + worker_celery_app: Celery, +): client_interface = get_celery_client_interface(fastapi_app) task_id = client_interface.submit( - "sync_archive", user_id=1, param1=1, values=["a", "b"] + "sync_error", task_id_components=TaskIdComponents(user_id=1) ) - assert client_interface.get_state(task_id) == "PENDING" - assert client_interface.get_result(task_id) is None - # use tnaticyt to wait for resutl - time.sleep(2) + for attempt in Retrying( + retry=retry_if_exception_type(AssertionError), wait=wait_fixed(1) + ): + with attempt: + result = client_interface.get_result(task_id) + assert isinstance(result, ValueError) - assert client_interface.get_state(task_id) == "SUCCESS" - assert client_interface.get_result(task_id) == "result" + assert f"{client_interface.get_result(task_id)}" == "my error here" diff --git a/services/storage/tests/unit/modules/celery/test_tasks.py b/services/storage/tests/unit/modules/celery/test_tasks.py deleted file mode 100644 index c6390b4dd196..000000000000 --- a/services/storage/tests/unit/modules/celery/test_tasks.py +++ /dev/null @@ -1,7 +0,0 @@ -from faker import Faker -from simcore_service_storage.modules.celery.tasks import archive - - -def test_archive(celery_app, celery_worker, faker: Faker): - result = archive.apply(args=(faker.uuid4(), ["f1", "f2"])) - assert result.get() == "f1_f2.zip" From b8a010c417908532013802e28b20e395bd514963 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 21 Feb 2025 14:28:51 +0100 Subject: [PATCH 019/136] continue fixing --- .../modules/celery/client/_interface.py | 42 ++++++++--- .../client/{client_utils.py => utils.py} | 0 .../modules/celery/models.py | 2 +- services/storage/tests/conftest.py | 1 + .../tests/unit/modules/celery/conftest.py | 47 +++++++++++- .../celery/{test_core.py => test_celery.py} | 74 ++++++++++++++++--- 6 files changed, 142 insertions(+), 24 deletions(-) rename services/storage/src/simcore_service_storage/modules/celery/client/{client_utils.py => utils.py} (100%) rename services/storage/tests/unit/modules/celery/{test_core.py => test_celery.py} (54%) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client/_interface.py b/services/storage/src/simcore_service_storage/modules/celery/client/_interface.py index 0ba96d2f639c..f88750a9fb2e 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client/_interface.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client/_interface.py @@ -2,13 +2,14 @@ from uuid import uuid4 from celery import Celery +from celery.contrib.abortable import AbortableAsyncResult from celery.result import AsyncResult from models_library.progress_bar import ProgressReport from pydantic import ValidationError -from ..models import TaskID, TaskProgress +from ..models import TaskID, TaskStatus -_PREFIX: Final = "AJ" +_PREFIX: Final = "ct" TaskIdComponents: TypeAlias = dict[str, Any] @@ -31,6 +32,9 @@ def _get_task_id(name: str, task_id_components: TaskIdComponents) -> TaskID: return "::".join([*_get_components_prefix(name, task_id_components), f"{uuid4()}"]) +_CELERY_TASK_META_PREFIX = "celery-task-meta-" + + class CeleryClientInterface: def __init__(self, celery_app: Celery): self._celery_app = celery_app @@ -48,13 +52,13 @@ def get(self, task_id: TaskID) -> Any: return self._celery_app.tasks(task_id) def cancel(self, task_id: TaskID) -> None: - self._celery_app.control.revoke(task_id, terminate=True) + AbortableAsyncResult(task_id, app=self._celery_app).abort() def _get_async_result(self, task_id: TaskID) -> AsyncResult: return self._celery_app.AsyncResult(task_id) def get_result(self, task_id: TaskID) -> Any: - # se manca il risultato o se va in FAILURE, ritorna error + # if the result is missing or if it goes into FAILURE, return error return self._get_async_result(task_id).result def _get_progress_report(self, task_id: TaskID) -> ProgressReport | None: @@ -63,17 +67,37 @@ def _get_progress_report(self, task_id: TaskID) -> ProgressReport | None: try: return ProgressReport.model_validate(result) except ValidationError: - return None + pass + return None - def get_progress(self, task_id: TaskID) -> TaskProgress: - return TaskProgress( + def get_status(self, task_id: TaskID) -> TaskStatus: + return TaskStatus( task_id=task_id, task_state=self._get_async_result(task_id).state, progress_report=self._get_progress_report(task_id), ) + def _get_completed_task_ids( + self, task_name: str, task_id_components: TaskIdComponents + ) -> list[TaskID]: + search_key = ( + _CELERY_TASK_META_PREFIX + + _get_task_id_prefix(task_name, task_id_components) + + "*" + ) + redis = self._celery_app.backend.client + keys = redis.keys(search_key) + if keys: + return [f"{key}".lstrip(_CELERY_TASK_META_PREFIX) for key in keys] + return [] + def list( self, task_name: str, *, task_id_components: TaskIdComponents ) -> list[TaskID]: - prefix_to_search_in_redis = _get_task_id_prefix(task_name, task_id_components) - return [] + all_task_ids = self._get_completed_task_ids(task_name, task_id_components) + + for task_type in ["active", "registered", "scheduled", "revoked"]: + if task_ids := getattr(self._celery_app.control.inspect(), task_type)(): + all_task_ids.extend(task_ids) + + return all_task_ids diff --git a/services/storage/src/simcore_service_storage/modules/celery/client/client_utils.py b/services/storage/src/simcore_service_storage/modules/celery/client/utils.py similarity index 100% rename from services/storage/src/simcore_service_storage/modules/celery/client/client_utils.py rename to services/storage/src/simcore_service_storage/modules/celery/client/utils.py diff --git a/services/storage/src/simcore_service_storage/modules/celery/models.py b/services/storage/src/simcore_service_storage/modules/celery/models.py index 9a137fefe649..07faa7927301 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/models.py +++ b/services/storage/src/simcore_service_storage/modules/celery/models.py @@ -6,7 +6,7 @@ TaskID: TypeAlias = str -class TaskProgress(BaseModel): +class TaskStatus(BaseModel): task_id: str task_state: str # add enum progress_report: ProgressReport | None = None diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index fefbaa157eea..e2d4614e46c1 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -83,6 +83,7 @@ "pytest_simcore.openapi_specs", "pytest_simcore.postgres_service", "pytest_simcore.pytest_global_environs", + "pytest_simcore.rabbit_service", "pytest_simcore.repository_paths", "pytest_simcore.simcore_storage_data_models", "pytest_simcore.simcore_storage_datcore_adapter", diff --git a/services/storage/tests/unit/modules/celery/conftest.py b/services/storage/tests/unit/modules/celery/conftest.py index f1c1eb7ca920..0a5431f79581 100644 --- a/services/storage/tests/unit/modules/celery/conftest.py +++ b/services/storage/tests/unit/modules/celery/conftest.py @@ -1,4 +1,5 @@ from asyncio import AbstractEventLoop +from datetime import timedelta from typing import Callable, Iterable import pytest @@ -6,6 +7,7 @@ from celery.contrib.testing.worker import TestWorkController, start_worker from celery.signals import worker_init, worker_shutdown from fastapi import FastAPI +from settings_library.rabbit import RabbitSettings from simcore_service_storage.main import celery_app as celery_app_client from simcore_service_storage.modules.celery.client import CeleryClientInterface from simcore_service_storage.modules.celery.worker._interface import ( @@ -21,9 +23,43 @@ @pytest.fixture -def client_celery_app() -> Celery: +async def rabbit_service( + rabbit_settings: RabbitSettings, monkeypatch: pytest.MonkeyPatch +) -> RabbitSettings: + monkeypatch.setenv("RABBIT_HOST", rabbit_settings.RABBIT_HOST) + monkeypatch.setenv("RABBIT_PORT", f"{rabbit_settings.RABBIT_PORT}") + monkeypatch.setenv("RABBIT_USER", rabbit_settings.RABBIT_USER) + monkeypatch.setenv("RABBIT_SECURE", f"{rabbit_settings.RABBIT_SECURE}") + monkeypatch.setenv( + "RABBIT_PASSWORD", rabbit_settings.RABBIT_PASSWORD.get_secret_value() + ) + + return rabbit_settings + + +_CELERY_CONF = { + "result_backend": "redis://localhost", + "result_expires": timedelta(days=7), + "result_extended": True, + "task_always_eager": False, + "task_acks_late": True, + "result_persistent": True, + "broker_transport_options": {"visibility_timeout": 3600}, + "task_track_started": True, + "worker_concurrency": 1, + "worker_prefetch_multiplier": 1, + "worker_send_task_events": True, # Required for task monitoring + "task_send_sent_event": True, # Required for task monitoring +} + + +@pytest.fixture +def client_celery_app(rabbit_service: RabbitSettings) -> Celery: celery_app_client.conf.update( - {"broker_url": "memory://", "result_backend": "cache+memory://"} + **_CELERY_CONF + | { + "broker_url": rabbit_service.dsn, + } ) assert isinstance(celery_app_client.conf["client_interface"], CeleryClientInterface) @@ -42,10 +78,13 @@ def register_celery_tasks() -> Callable[[Celery], None]: @pytest.fixture def celery_worker( - register_celery_tasks: Callable[[Celery], None] + register_celery_tasks: Callable[[Celery], None], rabbit_service: RabbitSettings ) -> Iterable[TestWorkController]: celery_app_worker.conf.update( - {"broker_url": "memory://", "result_backend": "cache+memory://"} + **_CELERY_CONF + | { + "broker_url": rabbit_service.dsn, + } ) register_celery_tasks(celery_app_worker) diff --git a/services/storage/tests/unit/modules/celery/test_core.py b/services/storage/tests/unit/modules/celery/test_celery.py similarity index 54% rename from services/storage/tests/unit/modules/celery/test_core.py rename to services/storage/tests/unit/modules/celery/test_celery.py index cef22829369c..79af1efdffab 100644 --- a/services/storage/tests/unit/modules/celery/test_core.py +++ b/services/storage/tests/unit/modules/celery/test_celery.py @@ -1,12 +1,14 @@ import asyncio +import time from typing import Callable import pytest from celery import Celery, Task +from celery.contrib.abortable import AbortableTask from models_library.progress_bar import ProgressReport from simcore_service_storage.main import fastapi_app from simcore_service_storage.modules.celery.client._interface import TaskIdComponents -from simcore_service_storage.modules.celery.client.client_utils import ( +from simcore_service_storage.modules.celery.client.utils import ( get_celery_client_interface, ) from simcore_service_storage.modules.celery.worker.utils import ( @@ -15,6 +17,10 @@ ) from tenacity import Retrying, retry_if_exception_type, stop_after_delay, wait_fixed +pytest_simcore_core_services_selection = [ + "rabbit", +] + async def _async_archive( celery_app: Celery, task_name: str, task_id: str, files: list[str] @@ -25,7 +31,7 @@ async def _async_archive( worker_interface.set_progress( task_name=task_name, task_id=task_id, - report=ProgressReport(actual_value=n / len(files) * 100), + report=ProgressReport(actual_value=n / len(files) * 10), ) await asyncio.sleep(0.1) @@ -39,15 +45,23 @@ def sync_archive(task: Task, files: list[str]) -> str: ).result() -def sync_error(task: Task) -> str: - raise ValueError("my error here") +def failure_task(task: Task) -> str: + msg = "my error here" + raise ValueError(msg) + + +def sleeper_task(task: Task, seconds: int) -> None: + time.sleep(seconds) @pytest.fixture def register_celery_tasks() -> Callable[[Celery], None]: def _(celery_app: Celery) -> None: celery_app.task(name="sync_archive", bind=True)(sync_archive) - celery_app.task(name="sync_error", bind=True)(sync_error) + celery_app.task(name="failure_task", bind=True)(failure_task) + celery_app.task( + name="sleeper_task", acks_late=True, base=AbortableTask, bind=True + )(sleeper_task) return _ @@ -72,27 +86,67 @@ def test_archive( stop=stop_after_delay(30), ): with attempt: - progress = client_interface.get_progress(task_id) + progress = client_interface.get_status(task_id) assert progress.task_state == "SUCCESS" - assert client_interface.get_progress(task_id).task_state == "SUCCESS" + assert client_interface.get_status(task_id).task_state == "SUCCESS" -def test_sync_error( +def test_failure_task( client_celery_app: Celery, worker_celery_app: Celery, ): client_interface = get_celery_client_interface(fastapi_app) task_id = client_interface.submit( - "sync_error", task_id_components=TaskIdComponents(user_id=1) + "failure_task", task_id_components=TaskIdComponents(user_id=1) ) for attempt in Retrying( - retry=retry_if_exception_type(AssertionError), wait=wait_fixed(1) + retry=retry_if_exception_type(AssertionError), + wait=wait_fixed(1), ): with attempt: result = client_interface.get_result(task_id) assert isinstance(result, ValueError) + assert client_interface.get_status(task_id).task_state == "FAILURE" assert f"{client_interface.get_result(task_id)}" == "my error here" + + +def test_revoke( + client_celery_app: Celery, + worker_celery_app: Celery, +): + client_interface = get_celery_client_interface(fastapi_app) + + task_name = "sleeper_task" + components = TaskIdComponents(user_id=1) + + task_id = client_interface.submit( + task_name, + task_id_components=components, + seconds=10000, + ) + print(f"Submitted task_id: {task_id}") + + # Wait for task to be registered + time.sleep(5) # Give worker time to pick up task + + task_ids = client_interface.list(task_name, task_id_components=components) + + time.sleep(5) + assert task_id in task_ids + + # client_interface.cancel(task_id) + + # for attempt in Retrying( + # retry=retry_if_exception_type(AssertionError), + # wait=wait_fixed(1), + # stop=stop_after_delay(10), + # ): + # with attempt: + # task_state = client_interface.get_status(task_id).task_state + # assert task_state == "REVOKED" + + # assert client_interface.get_result(task_id) is None From fa3e919f967bd5c98ba95333bb11c7aa1cebee52 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 21 Feb 2025 14:45:08 +0100 Subject: [PATCH 020/136] continue fixing --- .../modules/celery/client/_interface.py | 6 +-- .../tests/unit/modules/celery/conftest.py | 38 +++-------------- .../tests/unit/modules/celery/test_celery.py | 42 ------------------- 3 files changed, 9 insertions(+), 77 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client/_interface.py b/services/storage/src/simcore_service_storage/modules/celery/client/_interface.py index f88750a9fb2e..ca317bf0e5a3 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client/_interface.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client/_interface.py @@ -86,9 +86,9 @@ def _get_completed_task_ids( + "*" ) redis = self._celery_app.backend.client - keys = redis.keys(search_key) - if keys: - return [f"{key}".lstrip(_CELERY_TASK_META_PREFIX) for key in keys] + if hasattr(redis, "keys"): + if keys := redis.keys(search_key): + return [f"{key}".lstrip(_CELERY_TASK_META_PREFIX) for key in keys] return [] def list( diff --git a/services/storage/tests/unit/modules/celery/conftest.py b/services/storage/tests/unit/modules/celery/conftest.py index 0a5431f79581..34785b607dc4 100644 --- a/services/storage/tests/unit/modules/celery/conftest.py +++ b/services/storage/tests/unit/modules/celery/conftest.py @@ -7,7 +7,6 @@ from celery.contrib.testing.worker import TestWorkController, start_worker from celery.signals import worker_init, worker_shutdown from fastapi import FastAPI -from settings_library.rabbit import RabbitSettings from simcore_service_storage.main import celery_app as celery_app_client from simcore_service_storage.modules.celery.client import CeleryClientInterface from simcore_service_storage.modules.celery.worker._interface import ( @@ -21,24 +20,9 @@ on_worker_shutdown, ) - -@pytest.fixture -async def rabbit_service( - rabbit_settings: RabbitSettings, monkeypatch: pytest.MonkeyPatch -) -> RabbitSettings: - monkeypatch.setenv("RABBIT_HOST", rabbit_settings.RABBIT_HOST) - monkeypatch.setenv("RABBIT_PORT", f"{rabbit_settings.RABBIT_PORT}") - monkeypatch.setenv("RABBIT_USER", rabbit_settings.RABBIT_USER) - monkeypatch.setenv("RABBIT_SECURE", f"{rabbit_settings.RABBIT_SECURE}") - monkeypatch.setenv( - "RABBIT_PASSWORD", rabbit_settings.RABBIT_PASSWORD.get_secret_value() - ) - - return rabbit_settings - - _CELERY_CONF = { - "result_backend": "redis://localhost", + "broker_url": "memory://", + "result_backend": "cache+memory://", "result_expires": timedelta(days=7), "result_extended": True, "task_always_eager": False, @@ -54,13 +38,8 @@ async def rabbit_service( @pytest.fixture -def client_celery_app(rabbit_service: RabbitSettings) -> Celery: - celery_app_client.conf.update( - **_CELERY_CONF - | { - "broker_url": rabbit_service.dsn, - } - ) +def client_celery_app() -> Celery: + celery_app_client.conf.update(_CELERY_CONF) assert isinstance(celery_app_client.conf["client_interface"], CeleryClientInterface) assert "worker_interface" not in celery_app_client.conf @@ -78,14 +57,9 @@ def register_celery_tasks() -> Callable[[Celery], None]: @pytest.fixture def celery_worker( - register_celery_tasks: Callable[[Celery], None], rabbit_service: RabbitSettings + register_celery_tasks: Callable[[Celery], None], ) -> Iterable[TestWorkController]: - celery_app_worker.conf.update( - **_CELERY_CONF - | { - "broker_url": rabbit_service.dsn, - } - ) + celery_app_worker.conf.update(_CELERY_CONF) register_celery_tasks(celery_app_worker) diff --git a/services/storage/tests/unit/modules/celery/test_celery.py b/services/storage/tests/unit/modules/celery/test_celery.py index 79af1efdffab..880ea341a077 100644 --- a/services/storage/tests/unit/modules/celery/test_celery.py +++ b/services/storage/tests/unit/modules/celery/test_celery.py @@ -17,10 +17,6 @@ ) from tenacity import Retrying, retry_if_exception_type, stop_after_delay, wait_fixed -pytest_simcore_core_services_selection = [ - "rabbit", -] - async def _async_archive( celery_app: Celery, task_name: str, task_id: str, files: list[str] @@ -112,41 +108,3 @@ def test_failure_task( assert client_interface.get_status(task_id).task_state == "FAILURE" assert f"{client_interface.get_result(task_id)}" == "my error here" - - -def test_revoke( - client_celery_app: Celery, - worker_celery_app: Celery, -): - client_interface = get_celery_client_interface(fastapi_app) - - task_name = "sleeper_task" - components = TaskIdComponents(user_id=1) - - task_id = client_interface.submit( - task_name, - task_id_components=components, - seconds=10000, - ) - print(f"Submitted task_id: {task_id}") - - # Wait for task to be registered - time.sleep(5) # Give worker time to pick up task - - task_ids = client_interface.list(task_name, task_id_components=components) - - time.sleep(5) - assert task_id in task_ids - - # client_interface.cancel(task_id) - - # for attempt in Retrying( - # retry=retry_if_exception_type(AssertionError), - # wait=wait_fixed(1), - # stop=stop_after_delay(10), - # ): - # with attempt: - # task_state = client_interface.get_status(task_id).task_state - # assert task_state == "REVOKED" - - # assert client_interface.get_result(task_id) is None From 6450c2eab796db993b25a80136c66554b04ae796 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 21 Feb 2025 18:49:56 +0100 Subject: [PATCH 021/136] fix docker --- services/docker-compose.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 375381e2b3bc..bfabac46caea 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -1170,7 +1170,7 @@ services: S3_ENDPOINT: ${S3_ENDPOINT} S3_REGION: ${S3_REGION} S3_SECRET_KEY: ${S3_SECRET_KEY} - STORAGE_MODE: NORMAL + STORAGE_WORKER_MODE: "False" STORAGE_LOGLEVEL: ${STORAGE_LOGLEVEL} STORAGE_MONITORING_ENABLED: 1 STORAGE_PROFILING: ${STORAGE_PROFILING} @@ -1198,6 +1198,11 @@ services: POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} POSTGRES_PORT: ${POSTGRES_PORT} POSTGRES_USER: ${POSTGRES_USER} + RABBIT_HOST: ${RABBIT_HOST} + RABBIT_PASSWORD: ${RABBIT_PASSWORD} + RABBIT_PORT: ${RABBIT_PORT} + RABBIT_SECURE: ${RABBIT_SECURE} + RABBIT_USER: ${RABBIT_USER} REDIS_HOST: ${REDIS_HOST} REDIS_PORT: ${REDIS_PORT} REDIS_SECURE: ${REDIS_SECURE} @@ -1208,7 +1213,7 @@ services: S3_ENDPOINT: ${S3_ENDPOINT} S3_REGION: ${S3_REGION} S3_SECRET_KEY: ${S3_SECRET_KEY} - STORAGE_MODE: WORKER + STORAGE_WORKER_MODE: "True" STORAGE_LOGLEVEL: ${STORAGE_LOGLEVEL} STORAGE_MONITORING_ENABLED: 1 STORAGE_PROFILING: ${STORAGE_PROFILING} From 4220788add0a95cacad79f2c63bb24082af8d3f5 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 24 Feb 2025 13:20:36 +0100 Subject: [PATCH 022/136] working --- services/storage/docker/boot.sh | 12 ++-- .../src/simcore_service_storage/main.py | 15 +++-- .../modules/celery/client/_interface.py | 3 + .../modules/celery/client/setup.py | 34 ---------- .../modules/celery/client/utils.py | 17 +++-- .../modules/celery/tasks.py | 37 +++++++++++ .../modules/celery/utils.py | 17 +++++ .../modules/celery/worker/_interface.py | 9 +++ .../modules/celery/worker/setup.py | 65 ++++++++++--------- 9 files changed, 127 insertions(+), 82 deletions(-) delete mode 100644 services/storage/src/simcore_service_storage/modules/celery/client/setup.py create mode 100644 services/storage/src/simcore_service_storage/modules/celery/tasks.py create mode 100644 services/storage/src/simcore_service_storage/modules/celery/utils.py diff --git a/services/storage/docker/boot.sh b/services/storage/docker/boot.sh index fa354d02d265..bbbb82bed201 100755 --- a/services/storage/docker/boot.sh +++ b/services/storage/docker/boot.sh @@ -56,15 +56,15 @@ if [ "${SC_BOOT_MODE}" = "debug" ]; then --log-level \"${SERVER_LOG_LEVEL}\" " else - if [ "${STORAGE_MODE}" = "NORMAL" ]; then + if [ "${STORAGE_WORKER_MODE}" = "True" ]; then + exec celery \ + --app=simcore_service_storage.modules.celery.worker.setup \ + worker --task-events --pool=threads \ + --loglevel=DEBUG + else exec uvicorn simcore_service_storage.main:app \ --host 0.0.0.0 \ --port ${STORAGE_PORT} \ --log-level "${SERVER_LOG_LEVEL}" - else - exec celery \ - -A simcore_service_storage.modules.celery.worker.main:app \ - worker - --loglevel="${SERVER_LOG_LEVEL}" fi fi diff --git a/services/storage/src/simcore_service_storage/main.py b/services/storage/src/simcore_service_storage/main.py index 1965cd8b8f18..744942ff7f6f 100644 --- a/services/storage/src/simcore_service_storage/main.py +++ b/services/storage/src/simcore_service_storage/main.py @@ -3,12 +3,12 @@ import logging from servicelib.logging_utils import config_all_loggers -from simcore_service_storage.core.application import create_app -from simcore_service_storage.core.settings import ApplicationSettings -from simcore_service_storage.modules.celery.client.setup import ( - attach_to_fastapi, - create_celery_app, -) + +from .core.application import create_app +from .core.settings import ApplicationSettings +from .modules.celery.client import CeleryClientInterface +from .modules.celery.client.utils import attach_to_fastapi +from .modules.celery.utils import create_celery_app _settings = ApplicationSettings.create_from_envs() @@ -24,8 +24,9 @@ _logger = logging.getLogger(__name__) fastapi_app = create_app(_settings) +celery_app = create_celery_app(ApplicationSettings.create_from_envs()) -celery_app = create_celery_app(_settings) +celery_app.conf["client_interface"] = CeleryClientInterface(celery_app) attach_to_fastapi(fastapi_app, celery_app) app = fastapi_app diff --git a/services/storage/src/simcore_service_storage/modules/celery/client/_interface.py b/services/storage/src/simcore_service_storage/modules/celery/client/_interface.py index ca317bf0e5a3..632dfe587340 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client/_interface.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client/_interface.py @@ -1,3 +1,4 @@ +import logging from typing import Any, Final, TypeAlias from uuid import uuid4 @@ -13,6 +14,8 @@ TaskIdComponents: TypeAlias = dict[str, Any] +_logger = logging.getLogger(__name__) + def _get_task_id_components(task_id_components: TaskIdComponents) -> list[str]: return [f"{v}" for _, v in sorted(task_id_components.items())] diff --git a/services/storage/src/simcore_service_storage/modules/celery/client/setup.py b/services/storage/src/simcore_service_storage/modules/celery/client/setup.py deleted file mode 100644 index 245932e53776..000000000000 --- a/services/storage/src/simcore_service_storage/modules/celery/client/setup.py +++ /dev/null @@ -1,34 +0,0 @@ -import logging - -from celery import Celery -from fastapi import FastAPI -from settings_library.redis import RedisDatabase - -from ....core.settings import ApplicationSettings -from ._interface import CeleryClientInterface - -_log = logging.getLogger(__name__) - - -def create_celery_app(settings: ApplicationSettings) -> Celery: - assert settings.STORAGE_RABBITMQ - assert settings.STORAGE_REDIS - - celery_app = Celery( - broker=settings.STORAGE_RABBITMQ.dsn, - backend=settings.STORAGE_REDIS.build_redis_dsn( - RedisDatabase.CELERY_TASKS, - ), - ) - celery_app.conf["client_interface"] = CeleryClientInterface(celery_app) - - return celery_app - - -def attach_to_fastapi(fastapi: FastAPI, celery: Celery) -> None: - fastapi.state.celery = celery - - -def get_celery_client(fastapi: FastAPI) -> CeleryClientInterface: - celery: Celery = fastapi.state.celery - return celery.conf.get("client_interface") diff --git a/services/storage/src/simcore_service_storage/modules/celery/client/utils.py b/services/storage/src/simcore_service_storage/modules/celery/client/utils.py index 28fda45d3982..94125cba93c6 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client/utils.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client/utils.py @@ -1,8 +1,17 @@ -from typing import cast +import logging +from celery import Celery from fastapi import FastAPI -from simcore_service_storage.modules.celery.client import CeleryClientInterface +from ._interface import CeleryClientInterface -def get_celery_client_interface(app: FastAPI) -> CeleryClientInterface: - return cast(CeleryClientInterface, app.state.celery.conf["client_interface"]) +_log = logging.getLogger(__name__) + + +def attach_to_fastapi(fastapi: FastAPI, celery: Celery) -> None: + fastapi.state.celery = celery + + +def get_celery_client(fastapi: FastAPI) -> CeleryClientInterface: + celery: Celery = fastapi.state.celery + return celery.conf.get("client_interface") diff --git a/services/storage/src/simcore_service_storage/modules/celery/tasks.py b/services/storage/src/simcore_service_storage/modules/celery/tasks.py new file mode 100644 index 000000000000..2175b596ad42 --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/celery/tasks.py @@ -0,0 +1,37 @@ +import asyncio +import logging + +from celery import Celery, Task +from models_library.progress_bar import ProgressReport + +from .worker.utils import get_loop, get_worker_interface + +_logger = logging.getLogger(__name__) + + +async def _async_archive( + celery_app: Celery, task_name: str, task_id: str, files: list[str] +) -> str: + print("am I being executed?") + worker_interface = get_worker_interface(celery_app) + + for n in range(len(files)): + _logger.error("Progressing %d", n) + worker_interface.set_progress( + task_name=task_name, + task_id=task_id, + report=ProgressReport(actual_value=n / len(files) * 10), + ) + await asyncio.sleep(0.1) + + _logger.error("execution completed") + return "archive.zip" + + +def sync_archive(task: Task, files: list[str]) -> str: + print("getting new task") + assert task.name + _logger.info("Calling async_archive") + return asyncio.run_coroutine_threadsafe( + _async_archive(task.app, task.name, task.request.id, files), get_loop(task.app) + ).result() diff --git a/services/storage/src/simcore_service_storage/modules/celery/utils.py b/services/storage/src/simcore_service_storage/modules/celery/utils.py new file mode 100644 index 000000000000..addd1679bd79 --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/celery/utils.py @@ -0,0 +1,17 @@ +from celery import Celery +from settings_library.redis import RedisDatabase + +from ...core.settings import ApplicationSettings + + +def create_celery_app(settings: ApplicationSettings) -> Celery: + assert settings.STORAGE_RABBITMQ + assert settings.STORAGE_REDIS + + app = Celery( + broker=settings.STORAGE_RABBITMQ.dsn, + backend=settings.STORAGE_REDIS.build_redis_dsn( + RedisDatabase.CELERY_TASKS, + ), + ) + return app diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker/_interface.py b/services/storage/src/simcore_service_storage/modules/celery/worker/_interface.py index 2c5afdf71e57..ec5e36d3a5d8 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker/_interface.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker/_interface.py @@ -1,13 +1,22 @@ +import logging +from typing import Callable + from celery import Celery from models_library.progress_bar import ProgressReport from ..models import TaskID +_logger = logging.getLogger(__name__) + class CeleryWorkerInterface: def __init__(self, celery_app: Celery) -> None: self.celery_app = celery_app + def register_task(self, fn: Callable): + _logger.info("Registering %s task", fn.__name__) + self.celery_app.task(name=fn.__name__, bind=True)(fn) + def set_progress( self, task_name: str, task_id: TaskID, report: ProgressReport ) -> None: diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker/setup.py b/services/storage/src/simcore_service_storage/modules/celery/worker/setup.py index 63455deb8941..c87fcee9034f 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker/setup.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker/setup.py @@ -8,10 +8,11 @@ from celery.signals import worker_init, worker_shutdown from servicelib.background_task import cancel_wait_task from servicelib.logging_utils import config_all_loggers -from simcore_service_storage.core.application import create_app -from simcore_service_storage.core.settings import ApplicationSettings -from simcore_service_storage.modules.celery.client.setup import create_celery_app +from ....core.application import create_app +from ....core.settings import ApplicationSettings +from ....modules.celery.tasks import sync_archive +from ....modules.celery.utils import create_celery_app from ._interface import CeleryWorkerInterface _settings = ApplicationSettings.create_from_envs() @@ -28,38 +29,37 @@ _logger = logging.getLogger(__name__) -celery_app = create_celery_app(_settings) - - @worker_init.connect def on_worker_init(sender, **_kwargs): - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - shutdown_event = asyncio.Event() - - fastapi_app = create_app(_settings) - - async def lifespan(): - async with LifespanManager( - fastapi_app, startup_timeout=30, shutdown_timeout=30 - ): - try: - await shutdown_event.wait() - except asyncio.CancelledError: - _logger.warning("Lifespan task cancelled") - - lifespan_task = loop.create_task(lifespan()) - fastapi_app.state.lifespan_task = lifespan_task - fastapi_app.state.shutdown_event = shutdown_event - - sender.app.conf["fastapi_app"] = fastapi_app - sender.app.conf["loop"] = loop - sender.app.conf["worker_interface"] = CeleryWorkerInterface(sender.app) - - def run_loop(): + def shhsshhshs(): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + shutdown_event = asyncio.Event() + + fastapi_app = create_app(_settings) + + async def lifespan(): + async with LifespanManager( + fastapi_app, startup_timeout=30, shutdown_timeout=30 + ): + try: + await shutdown_event.wait() + except asyncio.CancelledError: + _logger.warning("Lifespan task cancelled") + + lifespan_task = loop.create_task(lifespan()) + fastapi_app.state.lifespan_task = lifespan_task + fastapi_app.state.shutdown_event = shutdown_event + + celery_worker_interface = CeleryWorkerInterface(sender.app) + + sender.app.conf["fastapi_app"] = fastapi_app + sender.app.conf["loop"] = loop + sender.app.conf["worker_interface"] = celery_worker_interface + loop.run_forever() - thread = threading.Thread(target=run_loop, daemon=True) + thread = threading.Thread(target=shhsshhshs, daemon=True) thread.start() @@ -76,4 +76,7 @@ async def shutdown(): asyncio.run_coroutine_threadsafe(shutdown(), loop) +celery_app = create_celery_app(ApplicationSettings.create_from_envs()) +celery_app.task(name=sync_archive.__name__, bind=True)(sync_archive) + app = celery_app From b40a24ce72114b8c3efd076db69cd7bec538a787 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 24 Feb 2025 15:07:06 +0100 Subject: [PATCH 023/136] continue fix --- .../api/rest/_files.py | 10 ++++++ .../src/simcore_service_storage/main.py | 11 +++---- .../{client/_interface.py => client.py} | 12 +++++-- .../modules/celery/client/__init__.py | 3 -- .../modules/celery/client/utils.py | 17 ---------- .../modules/celery/{utils.py => common.py} | 6 +++- .../modules/celery/tasks.py | 16 +++++----- .../{worker/_interface.py => worker.py} | 15 +++++++-- .../modules/celery/worker/__init__.py | 0 .../modules/celery/worker/utils.py | 21 ------------- .../{worker/setup.py => worker_main.py} | 31 +++++++++++-------- .../tests/unit/modules/celery/conftest.py | 18 +++++------ .../tests/unit/modules/celery/test_celery.py | 11 +++---- 13 files changed, 80 insertions(+), 91 deletions(-) rename services/storage/src/simcore_service_storage/modules/celery/{client/_interface.py => client.py} (92%) delete mode 100644 services/storage/src/simcore_service_storage/modules/celery/client/__init__.py delete mode 100644 services/storage/src/simcore_service_storage/modules/celery/client/utils.py rename services/storage/src/simcore_service_storage/modules/celery/{utils.py => common.py} (77%) rename services/storage/src/simcore_service_storage/modules/celery/{worker/_interface.py => worker.py} (64%) delete mode 100644 services/storage/src/simcore_service_storage/modules/celery/worker/__init__.py delete mode 100644 services/storage/src/simcore_service_storage/modules/celery/worker/utils.py rename services/storage/src/simcore_service_storage/modules/celery/{worker/setup.py => worker_main.py} (74%) diff --git a/services/storage/src/simcore_service_storage/api/rest/_files.py b/services/storage/src/simcore_service_storage/api/rest/_files.py index 3b1bb4c8a465..2e88f956ec4f 100644 --- a/services/storage/src/simcore_service_storage/api/rest/_files.py +++ b/services/storage/src/simcore_service_storage/api/rest/_files.py @@ -33,6 +33,7 @@ StorageQueryParamsBase, UploadLinks, ) +from ...modules.celery.client import get_client from ...modules.long_running_tasks import get_completed_upload_tasks from ...simcore_s3_dsm import SimcoreS3DataManager from ...utils.utils import create_upload_completion_task_name @@ -55,6 +56,15 @@ async def list_files_metadata( location_id: LocationID, request: Request, ): + c = get_client(request.app) + components = {"user_id": 1} + + task_id = c.submit("sync_archive", task_id_components=components, files=["aaa.xyz"]) + _logger.info("Submitted task: %s", task_id) + + task_ids = c.list("sync_archive", task_id_components=components) + _logger.info("%s", task_ids) + dsm = get_dsm_provider(request.app).get(location_id) data: list[FileMetaData] = await dsm.list_files( user_id=query_params.user_id, diff --git a/services/storage/src/simcore_service_storage/main.py b/services/storage/src/simcore_service_storage/main.py index 744942ff7f6f..7a3aaf0d2fd9 100644 --- a/services/storage/src/simcore_service_storage/main.py +++ b/services/storage/src/simcore_service_storage/main.py @@ -6,9 +6,8 @@ from .core.application import create_app from .core.settings import ApplicationSettings -from .modules.celery.client import CeleryClientInterface -from .modules.celery.client.utils import attach_to_fastapi -from .modules.celery.utils import create_celery_app +from .modules.celery.client import CeleryTaskQueueClient +from .modules.celery.common import create_app _settings = ApplicationSettings.create_from_envs() @@ -24,9 +23,9 @@ _logger = logging.getLogger(__name__) fastapi_app = create_app(_settings) -celery_app = create_celery_app(ApplicationSettings.create_from_envs()) +celery_app = create_app(_settings) -celery_app.conf["client_interface"] = CeleryClientInterface(celery_app) -attach_to_fastapi(fastapi_app, celery_app) +celery_app.conf["client"] = CeleryTaskQueueClient(celery_app) +fastapi_app.state.celery_app = celery_app app = fastapi_app diff --git a/services/storage/src/simcore_service_storage/modules/celery/client/_interface.py b/services/storage/src/simcore_service_storage/modules/celery/client.py similarity index 92% rename from services/storage/src/simcore_service_storage/modules/celery/client/_interface.py rename to services/storage/src/simcore_service_storage/modules/celery/client.py index 632dfe587340..5dcf8c899cc3 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client/_interface.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -5,10 +5,11 @@ from celery import Celery from celery.contrib.abortable import AbortableAsyncResult from celery.result import AsyncResult +from fastapi import FastAPI from models_library.progress_bar import ProgressReport from pydantic import ValidationError -from ..models import TaskID, TaskStatus +from .models import TaskID, TaskStatus _PREFIX: Final = "ct" @@ -38,7 +39,7 @@ def _get_task_id(name: str, task_id_components: TaskIdComponents) -> TaskID: _CELERY_TASK_META_PREFIX = "celery-task-meta-" -class CeleryClientInterface: +class CeleryTaskQueueClient: def __init__(self, celery_app: Celery): self._celery_app = celery_app @@ -104,3 +105,10 @@ def list( all_task_ids.extend(task_ids) return all_task_ids + + +def get_client(fastapi: FastAPI) -> CeleryTaskQueueClient: + celery = fastapi.state.celery_app + assert isinstance(celery, Celery) + + return celery.conf["client"] diff --git a/services/storage/src/simcore_service_storage/modules/celery/client/__init__.py b/services/storage/src/simcore_service_storage/modules/celery/client/__init__.py deleted file mode 100644 index 3d3e1162977e..000000000000 --- a/services/storage/src/simcore_service_storage/modules/celery/client/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from ._interface import CeleryClientInterface - -__all__: tuple[str, ...] = ("CeleryClientInterface",) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client/utils.py b/services/storage/src/simcore_service_storage/modules/celery/client/utils.py deleted file mode 100644 index 94125cba93c6..000000000000 --- a/services/storage/src/simcore_service_storage/modules/celery/client/utils.py +++ /dev/null @@ -1,17 +0,0 @@ -import logging - -from celery import Celery -from fastapi import FastAPI - -from ._interface import CeleryClientInterface - -_log = logging.getLogger(__name__) - - -def attach_to_fastapi(fastapi: FastAPI, celery: Celery) -> None: - fastapi.state.celery = celery - - -def get_celery_client(fastapi: FastAPI) -> CeleryClientInterface: - celery: Celery = fastapi.state.celery - return celery.conf.get("client_interface") diff --git a/services/storage/src/simcore_service_storage/modules/celery/utils.py b/services/storage/src/simcore_service_storage/modules/celery/common.py similarity index 77% rename from services/storage/src/simcore_service_storage/modules/celery/utils.py rename to services/storage/src/simcore_service_storage/modules/celery/common.py index addd1679bd79..715bbe14753a 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/utils.py +++ b/services/storage/src/simcore_service_storage/modules/celery/common.py @@ -1,10 +1,14 @@ +import logging + from celery import Celery from settings_library.redis import RedisDatabase from ...core.settings import ApplicationSettings +_logger = logging.getLogger(__name__) + -def create_celery_app(settings: ApplicationSettings) -> Celery: +def create_app(settings: ApplicationSettings) -> Celery: assert settings.STORAGE_RABBITMQ assert settings.STORAGE_REDIS diff --git a/services/storage/src/simcore_service_storage/modules/celery/tasks.py b/services/storage/src/simcore_service_storage/modules/celery/tasks.py index 2175b596ad42..712a1c422400 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/tasks.py +++ b/services/storage/src/simcore_service_storage/modules/celery/tasks.py @@ -4,7 +4,7 @@ from celery import Celery, Task from models_library.progress_bar import ProgressReport -from .worker.utils import get_loop, get_worker_interface +from .worker import get_event_loop, get_worker _logger = logging.getLogger(__name__) @@ -12,26 +12,24 @@ async def _async_archive( celery_app: Celery, task_name: str, task_id: str, files: list[str] ) -> str: - print("am I being executed?") - worker_interface = get_worker_interface(celery_app) + worker = get_worker(celery_app) - for n in range(len(files)): - _logger.error("Progressing %d", n) - worker_interface.set_progress( + for n, file in enumerate(files, start=1): + _logger.info("Processing file %s", file) + worker.set_progress( task_name=task_name, task_id=task_id, report=ProgressReport(actual_value=n / len(files) * 10), ) await asyncio.sleep(0.1) - _logger.error("execution completed") return "archive.zip" def sync_archive(task: Task, files: list[str]) -> str: - print("getting new task") assert task.name _logger.info("Calling async_archive") return asyncio.run_coroutine_threadsafe( - _async_archive(task.app, task.name, task.request.id, files), get_loop(task.app) + _async_archive(task.app, task.name, task.request.id, files), + get_event_loop(task.app), ).result() diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker/_interface.py b/services/storage/src/simcore_service_storage/modules/celery/worker.py similarity index 64% rename from services/storage/src/simcore_service_storage/modules/celery/worker/_interface.py rename to services/storage/src/simcore_service_storage/modules/celery/worker.py index ec5e36d3a5d8..b7359af6b03a 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker/_interface.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker.py @@ -1,15 +1,21 @@ import logging +from asyncio import AbstractEventLoop from typing import Callable from celery import Celery from models_library.progress_bar import ProgressReport -from ..models import TaskID +from .models import TaskID _logger = logging.getLogger(__name__) -class CeleryWorkerInterface: +def get_event_loop(celery_app: Celery) -> AbstractEventLoop: # nosec + loop: AbstractEventLoop = celery_app.conf["loop"] + return loop + + +class CeleryTaskQueueWorker: def __init__(self, celery_app: Celery) -> None: self.celery_app = celery_app @@ -25,3 +31,8 @@ def set_progress( state="PROGRESS", meta=report.model_dump(mode="json"), ) + + +def get_worker(celery_app: Celery) -> CeleryTaskQueueWorker: + worker: CeleryTaskQueueWorker = celery_app.conf["worker"] + return worker diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker/__init__.py b/services/storage/src/simcore_service_storage/modules/celery/worker/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker/utils.py b/services/storage/src/simcore_service_storage/modules/celery/worker/utils.py deleted file mode 100644 index 447093665e5b..000000000000 --- a/services/storage/src/simcore_service_storage/modules/celery/worker/utils.py +++ /dev/null @@ -1,21 +0,0 @@ -from asyncio import AbstractEventLoop - -from celery import Celery -from fastapi import FastAPI - -from ._interface import CeleryWorkerInterface - - -def get_fastapi_app(celery_app: Celery) -> FastAPI: - fast_api_app: FastAPI = celery_app.conf.get("fastapi_app") - return fast_api_app - - -def get_loop(celery_app: Celery) -> AbstractEventLoop: # nosec - loop: AbstractEventLoop = celery_app.conf.get("loop") - return loop - - -def get_worker_interface(celery_app: Celery) -> CeleryWorkerInterface: - worker_interface: CeleryWorkerInterface = celery_app.conf.get("worker_interface") - return worker_interface diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker/setup.py b/services/storage/src/simcore_service_storage/modules/celery/worker_main.py similarity index 74% rename from services/storage/src/simcore_service_storage/modules/celery/worker/setup.py rename to services/storage/src/simcore_service_storage/modules/celery/worker_main.py index c87fcee9034f..4011c62d067e 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker/setup.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker_main.py @@ -5,15 +5,16 @@ import threading from asgi_lifespan import LifespanManager +from celery import Celery from celery.signals import worker_init, worker_shutdown from servicelib.background_task import cancel_wait_task from servicelib.logging_utils import config_all_loggers -from ....core.application import create_app -from ....core.settings import ApplicationSettings -from ....modules.celery.tasks import sync_archive -from ....modules.celery.utils import create_celery_app -from ._interface import CeleryWorkerInterface +from ...core.application import create_app +from ...core.settings import ApplicationSettings +from .common import create_app +from .tasks import sync_archive +from .worker import CeleryTaskQueueWorker _settings = ApplicationSettings.create_from_envs() @@ -31,7 +32,7 @@ @worker_init.connect def on_worker_init(sender, **_kwargs): - def shhsshhshs(): + def _init_fastapi(): loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) shutdown_event = asyncio.Event() @@ -40,7 +41,9 @@ def shhsshhshs(): async def lifespan(): async with LifespanManager( - fastapi_app, startup_timeout=30, shutdown_timeout=30 + fastapi_app, + startup_timeout=10, + shutdown_timeout=10, ): try: await shutdown_event.wait() @@ -51,20 +54,19 @@ async def lifespan(): fastapi_app.state.lifespan_task = lifespan_task fastapi_app.state.shutdown_event = shutdown_event - celery_worker_interface = CeleryWorkerInterface(sender.app) - sender.app.conf["fastapi_app"] = fastapi_app sender.app.conf["loop"] = loop - sender.app.conf["worker_interface"] = celery_worker_interface loop.run_forever() - thread = threading.Thread(target=shhsshhshs, daemon=True) + thread = threading.Thread(target=_init_fastapi, daemon=True) thread.start() @worker_shutdown.connect def on_worker_shutdown(sender, **_kwargs): + assert isinstance(sender.app, Celery) + loop = sender.app.conf["loop"] fastapi_app = sender.app.conf["fastapi_app"] @@ -76,7 +78,10 @@ async def shutdown(): asyncio.run_coroutine_threadsafe(shutdown(), loop) -celery_app = create_celery_app(ApplicationSettings.create_from_envs()) -celery_app.task(name=sync_archive.__name__, bind=True)(sync_archive) +celery_app = create_app(ApplicationSettings.create_from_envs()) +celery_worker = CeleryTaskQueueWorker(celery_app) +celery_app.conf["worker"] = celery_worker + +celery_worker.register_task(sync_archive) app = celery_app diff --git a/services/storage/tests/unit/modules/celery/conftest.py b/services/storage/tests/unit/modules/celery/conftest.py index 34785b607dc4..c4dd20103308 100644 --- a/services/storage/tests/unit/modules/celery/conftest.py +++ b/services/storage/tests/unit/modules/celery/conftest.py @@ -7,15 +7,13 @@ from celery.contrib.testing.worker import TestWorkController, start_worker from celery.signals import worker_init, worker_shutdown from fastapi import FastAPI +from simcore_service_storage.main import CeleryTaskQueueClient from simcore_service_storage.main import celery_app as celery_app_client -from simcore_service_storage.modules.celery.client import CeleryClientInterface -from simcore_service_storage.modules.celery.worker._interface import ( - CeleryWorkerInterface, -) -from simcore_service_storage.modules.celery.worker.setup import ( +from simcore_service_storage.modules.celery.worker import CeleryTaskQueueWorker +from simcore_service_storage.modules.celery.worker_main import ( celery_app as celery_app_worker, ) -from simcore_service_storage.modules.celery.worker.setup import ( +from simcore_service_storage.modules.celery.worker_main import ( on_worker_init, on_worker_shutdown, ) @@ -41,8 +39,8 @@ def client_celery_app() -> Celery: celery_app_client.conf.update(_CELERY_CONF) - assert isinstance(celery_app_client.conf["client_interface"], CeleryClientInterface) - assert "worker_interface" not in celery_app_client.conf + assert isinstance(celery_app_client.conf["client"], CeleryTaskQueueClient) + assert "worker" not in celery_app_client.conf assert "loop" not in celery_app_client.conf assert "fastapi_app" not in celery_app_client.conf @@ -72,9 +70,7 @@ def celery_worker( ) as worker: worker_init.send(sender=worker) - assert isinstance( - celery_app_worker.conf["worker_interface"], CeleryWorkerInterface - ) + assert isinstance(celery_app_worker.conf["worker"], CeleryTaskQueueWorker) assert isinstance(celery_app_worker.conf["loop"], AbstractEventLoop) assert isinstance(celery_app_worker.conf["fastapi_app"], FastAPI) diff --git a/services/storage/tests/unit/modules/celery/test_celery.py b/services/storage/tests/unit/modules/celery/test_celery.py index 880ea341a077..91e2f030421c 100644 --- a/services/storage/tests/unit/modules/celery/test_celery.py +++ b/services/storage/tests/unit/modules/celery/test_celery.py @@ -7,21 +7,20 @@ from celery.contrib.abortable import AbortableTask from models_library.progress_bar import ProgressReport from simcore_service_storage.main import fastapi_app -from simcore_service_storage.modules.celery.client._interface import TaskIdComponents +from simcore_service_storage.modules.celery.celery_task_queue_client import ( + TaskIdComponents, +) from simcore_service_storage.modules.celery.client.utils import ( get_celery_client_interface, ) -from simcore_service_storage.modules.celery.worker.utils import ( - get_loop, - get_worker_interface, -) +from simcore_service_storage.modules.celery.worker.utils import get_loop, get_worker from tenacity import Retrying, retry_if_exception_type, stop_after_delay, wait_fixed async def _async_archive( celery_app: Celery, task_name: str, task_id: str, files: list[str] ) -> str: - worker_interface = get_worker_interface(celery_app) + worker_interface = get_worker(celery_app) for n in range(len(files)): worker_interface.set_progress( From 341f7fa4c1c312625a0d4601d5c8ca8d376496af Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 24 Feb 2025 15:35:16 +0100 Subject: [PATCH 024/136] fix tests --- .../src/simcore_service_storage/main.py | 4 +- .../modules/celery/client.py | 34 +++++------ .../modules/celery/models.py | 3 +- .../modules/celery/worker_main.py | 4 +- .../tests/unit/modules/celery/test_celery.py | 59 +++++-------------- 5 files changed, 34 insertions(+), 70 deletions(-) diff --git a/services/storage/src/simcore_service_storage/main.py b/services/storage/src/simcore_service_storage/main.py index 7a3aaf0d2fd9..588037c2159c 100644 --- a/services/storage/src/simcore_service_storage/main.py +++ b/services/storage/src/simcore_service_storage/main.py @@ -7,7 +7,7 @@ from .core.application import create_app from .core.settings import ApplicationSettings from .modules.celery.client import CeleryTaskQueueClient -from .modules.celery.common import create_app +from .modules.celery.common import create_app as create_celery_app _settings = ApplicationSettings.create_from_envs() @@ -23,7 +23,7 @@ _logger = logging.getLogger(__name__) fastapi_app = create_app(_settings) -celery_app = create_app(_settings) +celery_app = create_celery_app(_settings) celery_app.conf["client"] = CeleryTaskQueueClient(celery_app) fastapi_app.state.celery_app = celery_app diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index 5dcf8c899cc3..2688784397e0 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -1,5 +1,5 @@ import logging -from typing import Any, Final, TypeAlias +from typing import Any, Final from uuid import uuid4 from celery import Celery @@ -9,30 +9,26 @@ from models_library.progress_bar import ProgressReport from pydantic import ValidationError -from .models import TaskID, TaskStatus +from .models import TaskID, TaskIDParts, TaskStatus _PREFIX: Final = "ct" -TaskIdComponents: TypeAlias = dict[str, Any] - _logger = logging.getLogger(__name__) -def _get_task_id_components(task_id_components: TaskIdComponents) -> list[str]: +def _get_task_id_components(task_id_components: TaskIDParts) -> list[str]: return [f"{v}" for _, v in sorted(task_id_components.items())] -def _get_components_prefix( - name: str, task_id_components: TaskIdComponents -) -> list[str]: - return [_PREFIX, name, *_get_task_id_components(task_id_components)] +def _get_components_prefix(name: str, task_id_parts: TaskIDParts) -> list[str]: + return [_PREFIX, name, *_get_task_id_components(task_id_parts)] -def _get_task_id_prefix(name: str, task_id_components: TaskIdComponents) -> TaskID: - return "::".join(_get_components_prefix(name, task_id_components)) +def _get_task_id_prefix(name: str, task_id_parts: TaskIDParts) -> TaskID: + return "::".join(_get_components_prefix(name, task_id_parts)) -def _get_task_id(name: str, task_id_components: TaskIdComponents) -> TaskID: +def _get_task_id(name: str, task_id_components: TaskIDParts) -> TaskID: return "::".join([*_get_components_prefix(name, task_id_components), f"{uuid4()}"]) @@ -44,9 +40,9 @@ def __init__(self, celery_app: Celery): self._celery_app = celery_app def submit( - self, task_name: str, *, task_id_components: TaskIdComponents, **task_params + self, task_name: str, *, task_id_parts: TaskIDParts, **task_params ) -> TaskID: - task_id = _get_task_id(task_name, task_id_components) + task_id = _get_task_id(task_name, task_id_parts) task = self._celery_app.send_task( task_name, task_id=task_id, kwargs=task_params ) @@ -82,11 +78,11 @@ def get_status(self, task_id: TaskID) -> TaskStatus: ) def _get_completed_task_ids( - self, task_name: str, task_id_components: TaskIdComponents + self, task_name: str, task_id_parts: TaskIDParts ) -> list[TaskID]: search_key = ( _CELERY_TASK_META_PREFIX - + _get_task_id_prefix(task_name, task_id_components) + + _get_task_id_prefix(task_name, task_id_parts) + "*" ) redis = self._celery_app.backend.client @@ -95,10 +91,8 @@ def _get_completed_task_ids( return [f"{key}".lstrip(_CELERY_TASK_META_PREFIX) for key in keys] return [] - def list( - self, task_name: str, *, task_id_components: TaskIdComponents - ) -> list[TaskID]: - all_task_ids = self._get_completed_task_ids(task_name, task_id_components) + def list(self, task_name: str, *, task_id_parts: TaskIDParts) -> list[TaskID]: + all_task_ids = self._get_completed_task_ids(task_name, task_id_parts) for task_type in ["active", "registered", "scheduled", "revoked"]: if task_ids := getattr(self._celery_app.control.inspect(), task_type)(): diff --git a/services/storage/src/simcore_service_storage/modules/celery/models.py b/services/storage/src/simcore_service_storage/modules/celery/models.py index 07faa7927301..35e9957c9990 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/models.py +++ b/services/storage/src/simcore_service_storage/modules/celery/models.py @@ -1,9 +1,10 @@ -from typing import TypeAlias +from typing import Any, TypeAlias from models_library.progress_bar import ProgressReport from pydantic import BaseModel TaskID: TypeAlias = str +TaskIDParts: TypeAlias = dict[str, Any] class TaskStatus(BaseModel): diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker_main.py b/services/storage/src/simcore_service_storage/modules/celery/worker_main.py index 4011c62d067e..ae187ef110c0 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker_main.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker_main.py @@ -12,7 +12,7 @@ from ...core.application import create_app from ...core.settings import ApplicationSettings -from .common import create_app +from .common import create_app as create_celery_app from .tasks import sync_archive from .worker import CeleryTaskQueueWorker @@ -78,7 +78,7 @@ async def shutdown(): asyncio.run_coroutine_threadsafe(shutdown(), loop) -celery_app = create_app(ApplicationSettings.create_from_envs()) +celery_app = create_celery_app(ApplicationSettings.create_from_envs()) celery_worker = CeleryTaskQueueWorker(celery_app) celery_app.conf["worker"] = celery_worker diff --git a/services/storage/tests/unit/modules/celery/test_celery.py b/services/storage/tests/unit/modules/celery/test_celery.py index 91e2f030421c..62afc1ec54c3 100644 --- a/services/storage/tests/unit/modules/celery/test_celery.py +++ b/services/storage/tests/unit/modules/celery/test_celery.py @@ -1,45 +1,16 @@ -import asyncio import time from typing import Callable import pytest from celery import Celery, Task from celery.contrib.abortable import AbortableTask -from models_library.progress_bar import ProgressReport from simcore_service_storage.main import fastapi_app -from simcore_service_storage.modules.celery.celery_task_queue_client import ( - TaskIdComponents, -) -from simcore_service_storage.modules.celery.client.utils import ( - get_celery_client_interface, -) -from simcore_service_storage.modules.celery.worker.utils import get_loop, get_worker +from simcore_service_storage.modules.celery.client import get_client +from simcore_service_storage.modules.celery.models import TaskIDParts +from simcore_service_storage.modules.celery.tasks import sync_archive from tenacity import Retrying, retry_if_exception_type, stop_after_delay, wait_fixed -async def _async_archive( - celery_app: Celery, task_name: str, task_id: str, files: list[str] -) -> str: - worker_interface = get_worker(celery_app) - - for n in range(len(files)): - worker_interface.set_progress( - task_name=task_name, - task_id=task_id, - report=ProgressReport(actual_value=n / len(files) * 10), - ) - await asyncio.sleep(0.1) - - return "archive.zip" - - -def sync_archive(task: Task, files: list[str]) -> str: - assert task.name - return asyncio.run_coroutine_threadsafe( - _async_archive(task.app, task.name, task.request.id, files), get_loop(task.app) - ).result() - - def failure_task(task: Task) -> str: msg = "my error here" raise ValueError(msg) @@ -65,13 +36,13 @@ def test_archive( client_celery_app: Celery, worker_celery_app: Celery, ): - client_interface = get_celery_client_interface(fastapi_app) + client = get_client(fastapi_app) - task_id_components = TaskIdComponents(user_id=1) + task_id_parts = TaskIDParts(user_id=1) - task_id = client_interface.submit( + task_id = client.submit( "sync_archive", - task_id_components=task_id_components, + task_id_parts=task_id_parts, files=[f"file{n}" for n in range(100)], ) @@ -81,29 +52,27 @@ def test_archive( stop=stop_after_delay(30), ): with attempt: - progress = client_interface.get_status(task_id) + progress = client.get_status(task_id) assert progress.task_state == "SUCCESS" - assert client_interface.get_status(task_id).task_state == "SUCCESS" + assert client.get_status(task_id).task_state == "SUCCESS" def test_failure_task( client_celery_app: Celery, worker_celery_app: Celery, ): - client_interface = get_celery_client_interface(fastapi_app) + client = get_client(fastapi_app) - task_id = client_interface.submit( - "failure_task", task_id_components=TaskIdComponents(user_id=1) - ) + task_id = client.submit("failure_task", task_id_parts=TaskIDParts(user_id=1)) for attempt in Retrying( retry=retry_if_exception_type(AssertionError), wait=wait_fixed(1), ): with attempt: - result = client_interface.get_result(task_id) + result = client.get_result(task_id) assert isinstance(result, ValueError) - assert client_interface.get_status(task_id).task_state == "FAILURE" - assert f"{client_interface.get_result(task_id)}" == "my error here" + assert client.get_status(task_id).task_state == "FAILURE" + assert f"{client.get_result(task_id)}" == "my error here" From 80753e7dff8759172ba2e0065758c16038d532bd Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 24 Feb 2025 15:36:32 +0100 Subject: [PATCH 025/136] update boot.sh --- services/storage/docker/boot.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/storage/docker/boot.sh b/services/storage/docker/boot.sh index bbbb82bed201..fc2b729110bc 100755 --- a/services/storage/docker/boot.sh +++ b/services/storage/docker/boot.sh @@ -58,9 +58,9 @@ if [ "${SC_BOOT_MODE}" = "debug" ]; then else if [ "${STORAGE_WORKER_MODE}" = "True" ]; then exec celery \ - --app=simcore_service_storage.modules.celery.worker.setup \ - worker --task-events --pool=threads \ - --loglevel=DEBUG + --app=simcore_service_storage.modules.celery.worker_main \ + worker --pool=threads \ + --loglevel="${SERVER_LOG_LEVEL}" else exec uvicorn simcore_service_storage.main:app \ --host 0.0.0.0 \ From 578573cd5a87c6dc86dc58472284a8f4a68eff56 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 24 Feb 2025 15:39:40 +0100 Subject: [PATCH 026/136] fix files endpoint --- .../src/simcore_service_storage/api/rest/_files.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/services/storage/src/simcore_service_storage/api/rest/_files.py b/services/storage/src/simcore_service_storage/api/rest/_files.py index 2e88f956ec4f..3b1bb4c8a465 100644 --- a/services/storage/src/simcore_service_storage/api/rest/_files.py +++ b/services/storage/src/simcore_service_storage/api/rest/_files.py @@ -33,7 +33,6 @@ StorageQueryParamsBase, UploadLinks, ) -from ...modules.celery.client import get_client from ...modules.long_running_tasks import get_completed_upload_tasks from ...simcore_s3_dsm import SimcoreS3DataManager from ...utils.utils import create_upload_completion_task_name @@ -56,15 +55,6 @@ async def list_files_metadata( location_id: LocationID, request: Request, ): - c = get_client(request.app) - components = {"user_id": 1} - - task_id = c.submit("sync_archive", task_id_components=components, files=["aaa.xyz"]) - _logger.info("Submitted task: %s", task_id) - - task_ids = c.list("sync_archive", task_id_components=components) - _logger.info("%s", task_ids) - dsm = get_dsm_provider(request.app).get(location_id) data: list[FileMetaData] = await dsm.list_files( user_id=query_params.user_id, From 499d290244ca0c124a5f8b4e097e93cd600724b8 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 24 Feb 2025 15:54:59 +0100 Subject: [PATCH 027/136] rename --- .../modules/celery/{tasks.py => example_tasks.py} | 0 .../src/simcore_service_storage/modules/celery/worker_main.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename services/storage/src/simcore_service_storage/modules/celery/{tasks.py => example_tasks.py} (100%) diff --git a/services/storage/src/simcore_service_storage/modules/celery/tasks.py b/services/storage/src/simcore_service_storage/modules/celery/example_tasks.py similarity index 100% rename from services/storage/src/simcore_service_storage/modules/celery/tasks.py rename to services/storage/src/simcore_service_storage/modules/celery/example_tasks.py diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker_main.py b/services/storage/src/simcore_service_storage/modules/celery/worker_main.py index ae187ef110c0..190909d51a3a 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker_main.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker_main.py @@ -13,7 +13,7 @@ from ...core.application import create_app from ...core.settings import ApplicationSettings from .common import create_app as create_celery_app -from .tasks import sync_archive +from .example_tasks import sync_archive from .worker import CeleryTaskQueueWorker _settings = ApplicationSettings.create_from_envs() From b305a5377d51cd3089003c04ca42027a9875abbc Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 24 Feb 2025 16:36:41 +0100 Subject: [PATCH 028/136] add abortable task --- .../modules/celery/client.py | 2 +- .../tests/unit/modules/celery/test_celery.py | 47 ++++++++++++++++--- 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index 2688784397e0..2d9f7f148884 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -52,7 +52,7 @@ def get(self, task_id: TaskID) -> Any: return self._celery_app.tasks(task_id) def cancel(self, task_id: TaskID) -> None: - AbortableAsyncResult(task_id, app=self._celery_app).abort() + AbortableAsyncResult(task_id).abort() def _get_async_result(self, task_id: TaskID) -> AsyncResult: return self._celery_app.AsyncResult(task_id) diff --git a/services/storage/tests/unit/modules/celery/test_celery.py b/services/storage/tests/unit/modules/celery/test_celery.py index 62afc1ec54c3..446b22622a4d 100644 --- a/services/storage/tests/unit/modules/celery/test_celery.py +++ b/services/storage/tests/unit/modules/celery/test_celery.py @@ -1,4 +1,6 @@ +import logging import time +from random import randint from typing import Callable import pytest @@ -6,18 +8,27 @@ from celery.contrib.abortable import AbortableTask from simcore_service_storage.main import fastapi_app from simcore_service_storage.modules.celery.client import get_client +from simcore_service_storage.modules.celery.example_tasks import sync_archive from simcore_service_storage.modules.celery.models import TaskIDParts -from simcore_service_storage.modules.celery.tasks import sync_archive from tenacity import Retrying, retry_if_exception_type, stop_after_delay, wait_fixed +_logger = logging.getLogger(__name__) + def failure_task(task: Task) -> str: msg = "my error here" raise ValueError(msg) -def sleeper_task(task: Task, seconds: int) -> None: - time.sleep(seconds) +def dreamer_task(task: AbortableTask) -> list[int]: + numbers = [] + for _ in range(30): + if task.is_aborted(): + _logger.warning("Alarm clock") + return numbers + numbers.append(randint(1, 90)) + time.sleep(1) + return numbers @pytest.fixture @@ -25,9 +36,9 @@ def register_celery_tasks() -> Callable[[Celery], None]: def _(celery_app: Celery) -> None: celery_app.task(name="sync_archive", bind=True)(sync_archive) celery_app.task(name="failure_task", bind=True)(failure_task) - celery_app.task( - name="sleeper_task", acks_late=True, base=AbortableTask, bind=True - )(sleeper_task) + celery_app.task(name="dreamer_task", base=AbortableTask, bind=True)( + dreamer_task + ) return _ @@ -76,3 +87,27 @@ def test_failure_task( assert client.get_status(task_id).task_state == "FAILURE" assert f"{client.get_result(task_id)}" == "my error here" + + +def test_dreamer_task( + client_celery_app: Celery, + worker_celery_app: Celery, +): + client = get_client(fastapi_app) + + task_id = client.submit("dreamer_task", task_id_parts=TaskIDParts(user_id=1)) + + time.sleep(1) + + client.cancel(task_id) + + for attempt in Retrying( + retry=retry_if_exception_type(AssertionError), + wait=wait_fixed(1), + stop=stop_after_delay(30), + ): + with attempt: + progress = client.get_status(task_id) + assert progress.task_state == "ABORTED" + + assert client.get_status(task_id).task_state == "ABORTED" From a5e1128be696f72dfb665d361665a76c9b2177cc Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 24 Feb 2025 16:55:06 +0100 Subject: [PATCH 029/136] typechecks --- .../src/simcore_service_storage/modules/celery/client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index 2d9f7f148884..0b33dda20164 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -46,6 +46,7 @@ def submit( task = self._celery_app.send_task( task_name, task_id=task_id, kwargs=task_params ) + assert isinstance(str, task.id) return task.id def get(self, task_id: TaskID) -> Any: @@ -105,4 +106,6 @@ def get_client(fastapi: FastAPI) -> CeleryTaskQueueClient: celery = fastapi.state.celery_app assert isinstance(celery, Celery) - return celery.conf["client"] + client = celery.conf["client"] + assert isinstance(client, CeleryTaskQueueClient) + return client From 6ab33bdedc64c5aba2766573f911b970fcbece6c Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 25 Feb 2025 10:51:58 +0100 Subject: [PATCH 030/136] add healthcheck --- services/docker-compose.yml | 6 ++-- services/storage/docker/boot.sh | 7 ++-- services/storage/docker/healthcheck.py | 34 ++++++++++++++++++- .../simcore_service_storage/core/settings.py | 7 +++- 4 files changed, 46 insertions(+), 8 deletions(-) mode change 100644 => 100755 services/storage/docker/healthcheck.py diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 4632ec7b2308..a3c62ef57269 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -1178,7 +1178,7 @@ services: S3_ENDPOINT: ${S3_ENDPOINT} S3_REGION: ${S3_REGION} S3_SECRET_KEY: ${S3_SECRET_KEY} - STORAGE_WORKER_MODE: "False" + STORAGE_WORKER_MODE: 0 STORAGE_LOGLEVEL: ${STORAGE_LOGLEVEL} STORAGE_MONITORING_ENABLED: 1 STORAGE_PROFILING: ${STORAGE_PROFILING} @@ -1193,7 +1193,7 @@ services: storage-worker: image: ${DOCKER_REGISTRY:-itisfoundation}/storage:${DOCKER_IMAGE_TAG:-latest} init: true - hostname: "sto-{{.Node.Hostname}}-{{.Task.Slot}}" + hostname: "stow-{{.Node.Hostname}}-{{.Task.Slot}}" environment: BF_API_KEY: ${BF_API_KEY} BF_API_SECRET: ${BF_API_SECRET} @@ -1221,7 +1221,7 @@ services: S3_ENDPOINT: ${S3_ENDPOINT} S3_REGION: ${S3_REGION} S3_SECRET_KEY: ${S3_SECRET_KEY} - STORAGE_WORKER_MODE: "True" + STORAGE_WORKER_MODE: 1 STORAGE_LOGLEVEL: ${STORAGE_LOGLEVEL} STORAGE_MONITORING_ENABLED: 1 STORAGE_PROFILING: ${STORAGE_PROFILING} diff --git a/services/storage/docker/boot.sh b/services/storage/docker/boot.sh index fc2b729110bc..a8ee42344bf4 100755 --- a/services/storage/docker/boot.sh +++ b/services/storage/docker/boot.sh @@ -56,11 +56,12 @@ if [ "${SC_BOOT_MODE}" = "debug" ]; then --log-level \"${SERVER_LOG_LEVEL}\" " else - if [ "${STORAGE_WORKER_MODE}" = "True" ]; then + if [ "${STORAGE_WORKER_MODE}" = "1" ]; then exec celery \ - --app=simcore_service_storage.modules.celery.worker_main \ + --app=simcore_service_storage.modules.celery.worker_main:app \ worker --pool=threads \ - --loglevel="${SERVER_LOG_LEVEL}" + --loglevel="${SERVER_LOG_LEVEL}" \ + --hostname="${HOSTNAME}" else exec uvicorn simcore_service_storage.main:app \ --host 0.0.0.0 \ diff --git a/services/storage/docker/healthcheck.py b/services/storage/docker/healthcheck.py old mode 100644 new mode 100755 index b6711cd55eb6..81b2c07885e3 --- a/services/storage/docker/healthcheck.py +++ b/services/storage/docker/healthcheck.py @@ -20,18 +20,50 @@ import os +import subprocess import sys from urllib.request import urlopen +from simcore_service_storage.main import ApplicationSettings + SUCCESS, UNHEALTHY = 0, 1 # Disabled if boots with debugger -ok = os.environ.get("SC_BOOT_MODE", "").lower() == "debug" +ok = os.getenv("SC_BOOT_MODE", "").lower() == "debug" # Queries host # pylint: disable=consider-using-with +app_settings = ApplicationSettings.create_from_envs() + +broker_url = app_settings.STORAGE_CELERY_BROKER.dsn + + +def _is_celery_worker_healthy(): + try: + result = subprocess.run( + [ + "celery", + "-b", + broker_url, + "inspect", + "ping", + "-d", + "celery@" + os.getenv("HOSTNAME", ""), + ], + capture_output=True, + text=True, + check=True, + ) + return "pong" in result.stdout + except subprocess.CalledProcessError: + return False + + +print(_is_celery_worker_healthy()) + ok = ( ok + or (bool(os.environ.get("STORAGE_WORKER_MODE", "") and _is_celery_worker_healthy())) or urlopen( "{host}{baseurl}".format( host=sys.argv[1], baseurl=os.environ.get("SIMCORE_NODE_BASEPATH", "") diff --git a/services/storage/src/simcore_service_storage/core/settings.py b/services/storage/src/simcore_service_storage/core/settings.py index 8bb149e0b8d8..1e7e45f2e9ab 100644 --- a/services/storage/src/simcore_service_storage/core/settings.py +++ b/services/storage/src/simcore_service_storage/core/settings.py @@ -106,7 +106,12 @@ class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of _logger message patterns that should be filtered out.", ) - STORAGE_WORKER_MODE: bool | None = False + STORAGE_WORKER_MODE: Annotated[ + bool | None, Field(description="If True, run as a worker") + ] = False + STORAGE_CELERY_BROKER: RabbitSettings | None = Field( + json_schema_extra={"auto_default_from_env": True} + ) @field_validator("LOG_LEVEL", mode="before") @classmethod From 5b86b64ae05828ab76204da25f4c377a33467dfc Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 25 Feb 2025 11:00:24 +0100 Subject: [PATCH 031/136] register abortable tasks by default --- .../src/simcore_service_storage/modules/celery/client.py | 3 ++- .../src/simcore_service_storage/modules/celery/worker.py | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index 0b33dda20164..2fe09f25bce3 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -43,6 +43,7 @@ def submit( self, task_name: str, *, task_id_parts: TaskIDParts, **task_params ) -> TaskID: task_id = _get_task_id(task_name, task_id_parts) + _logger.debug("Submitting task %s: %s", task_name, task_id) task = self._celery_app.send_task( task_name, task_id=task_id, kwargs=task_params ) @@ -53,13 +54,13 @@ def get(self, task_id: TaskID) -> Any: return self._celery_app.tasks(task_id) def cancel(self, task_id: TaskID) -> None: + _logger.info("Aborting task %s", task_id) AbortableAsyncResult(task_id).abort() def _get_async_result(self, task_id: TaskID) -> AsyncResult: return self._celery_app.AsyncResult(task_id) def get_result(self, task_id: TaskID) -> Any: - # if the result is missing or if it goes into FAILURE, return error return self._get_async_result(task_id).result def _get_progress_report(self, task_id: TaskID) -> ProgressReport | None: diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker.py b/services/storage/src/simcore_service_storage/modules/celery/worker.py index b7359af6b03a..b2e2637be7e6 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker.py @@ -3,6 +3,7 @@ from typing import Callable from celery import Celery +from celery.contrib.abortable import AbortableTask from models_library.progress_bar import ProgressReport from .models import TaskID @@ -20,12 +21,15 @@ def __init__(self, celery_app: Celery) -> None: self.celery_app = celery_app def register_task(self, fn: Callable): - _logger.info("Registering %s task", fn.__name__) - self.celery_app.task(name=fn.__name__, bind=True)(fn) + _logger.debug("Registering %s task", fn.__name__) + self.celery_app.task(name=fn.__name__, base=AbortableTask, bind=True)(fn) def set_progress( self, task_name: str, task_id: TaskID, report: ProgressReport ) -> None: + _logger.debug( + "Setting progress for %s: %s", task_name, report.model_dump_json() + ) self.celery_app.tasks[task_name].update_state( task_id=task_id, state="PROGRESS", From c27f5093afa42cce33d7befd486db3a325d76c5c Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 25 Feb 2025 11:03:16 +0100 Subject: [PATCH 032/136] continue --- services/storage/docker/healthcheck.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/storage/docker/healthcheck.py b/services/storage/docker/healthcheck.py index 81b2c07885e3..fee00bc0fde8 100755 --- a/services/storage/docker/healthcheck.py +++ b/services/storage/docker/healthcheck.py @@ -63,7 +63,7 @@ def _is_celery_worker_healthy(): ok = ( ok - or (bool(os.environ.get("STORAGE_WORKER_MODE", "") and _is_celery_worker_healthy())) + or (bool(os.getenv("STORAGE_WORKER_MODE", "") and _is_celery_worker_healthy())) or urlopen( "{host}{baseurl}".format( host=sys.argv[1], baseurl=os.environ.get("SIMCORE_NODE_BASEPATH", "") From e386ff95ac21b43b9b1980c44763dba7ece07aa7 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 25 Feb 2025 11:04:51 +0100 Subject: [PATCH 033/136] rename --- .../src/simcore_service_storage/modules/celery/client.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index 2fe09f25bce3..b6df3fc4d5bc 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -16,8 +16,8 @@ _logger = logging.getLogger(__name__) -def _get_task_id_components(task_id_components: TaskIDParts) -> list[str]: - return [f"{v}" for _, v in sorted(task_id_components.items())] +def _get_task_id_components(task_id_parts: TaskIDParts) -> list[str]: + return [f"{v}" for _, v in sorted(task_id_parts.items())] def _get_components_prefix(name: str, task_id_parts: TaskIDParts) -> list[str]: @@ -28,8 +28,8 @@ def _get_task_id_prefix(name: str, task_id_parts: TaskIDParts) -> TaskID: return "::".join(_get_components_prefix(name, task_id_parts)) -def _get_task_id(name: str, task_id_components: TaskIDParts) -> TaskID: - return "::".join([*_get_components_prefix(name, task_id_components), f"{uuid4()}"]) +def _get_task_id(name: str, task_id_parts: TaskIDParts) -> TaskID: + return "::".join([*_get_components_prefix(name, task_id_parts), f"{uuid4()}"]) _CELERY_TASK_META_PREFIX = "celery-task-meta-" From 2d4951aaf35b2f82c3893b110bc3dd06ad31b335 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 25 Feb 2025 11:46:39 +0100 Subject: [PATCH 034/136] add settings --- .../src/settings_library/celery.py | 12 ++ services/storage/docker/healthcheck.py | 2 +- .../simcore_service_storage/core/settings.py | 167 ++++++++++-------- .../modules/celery/common.py | 7 +- 4 files changed, 105 insertions(+), 83 deletions(-) diff --git a/packages/settings-library/src/settings_library/celery.py b/packages/settings-library/src/settings_library/celery.py index d61c41bf1e2d..c25ab3b036fb 100644 --- a/packages/settings-library/src/settings_library/celery.py +++ b/packages/settings-library/src/settings_library/celery.py @@ -1,9 +1,21 @@ +from typing import Annotated + +from pydantic import Field from pydantic_settings import SettingsConfigDict +from settings_library.rabbit import RabbitSettings +from settings_library.redis import RedisSettings from .base import BaseCustomSettings class CelerySettings(BaseCustomSettings): + CELERY_BROKER: Annotated[ + RabbitSettings, Field(json_schema_extra={"auto_default_from_env": True}) + ] + CELERY_RESULTS_BACKEND: Annotated[ + RedisSettings, Field(json_schema_extra={"auto_default_from_env": True}) + ] + model_config = SettingsConfigDict( json_schema_extra={ "examples": [], diff --git a/services/storage/docker/healthcheck.py b/services/storage/docker/healthcheck.py index fee00bc0fde8..d1cf1e267e1a 100755 --- a/services/storage/docker/healthcheck.py +++ b/services/storage/docker/healthcheck.py @@ -35,7 +35,7 @@ # pylint: disable=consider-using-with app_settings = ApplicationSettings.create_from_envs() -broker_url = app_settings.STORAGE_CELERY_BROKER.dsn +broker_url = app_settings.STORAGE_CELERY.CELERY_BROKER.dsn def _is_celery_worker_healthy(): diff --git a/services/storage/src/simcore_service_storage/core/settings.py b/services/storage/src/simcore_service_storage/core/settings.py index 1e7e45f2e9ab..43279642c8c3 100644 --- a/services/storage/src/simcore_service_storage/core/settings.py +++ b/services/storage/src/simcore_service_storage/core/settings.py @@ -1,14 +1,7 @@ from typing import Annotated, Self from fastapi import FastAPI -from pydantic import ( - AliasChoices, - Field, - PositiveInt, - TypeAdapter, - field_validator, - model_validator, -) +from pydantic import AliasChoices, Field, PositiveInt, field_validator, model_validator from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.application import BaseApplicationSettings from settings_library.basic_types import LogLevel, PortInt @@ -25,7 +18,7 @@ class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): STORAGE_HOST: str = "0.0.0.0" # nosec - STORAGE_PORT: PortInt = TypeAdapter(PortInt).validate_python(8080) + STORAGE_PORT: PortInt = 8080 LOG_LEVEL: Annotated[ LogLevel, @@ -37,81 +30,99 @@ class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): STORAGE_MONITORING_ENABLED: bool = False STORAGE_PROFILING: bool = False - BF_API_KEY: str | None = Field( - None, description="Pennsieve API key ONLY for testing purposes" - ) - BF_API_SECRET: str | None = Field( - None, description="Pennsieve API secret ONLY for testing purposes" - ) - - STORAGE_POSTGRES: PostgresSettings | None = Field( - json_schema_extra={"auto_default_from_env": True} - ) - - STORAGE_REDIS: RedisSettings | None = Field( - json_schema_extra={"auto_default_from_env": True} - ) - - STORAGE_S3: S3Settings | None = Field( - json_schema_extra={"auto_default_from_env": True} - ) - - STORAGE_CELERY: CelerySettings | None = Field( - json_schema_extra={"auto_default_from_env": True} - ) - - STORAGE_TRACING: TracingSettings | None = Field( - json_schema_extra={"auto_default_from_env": True} - ) - - DATCORE_ADAPTER: DatcoreAdapterSettings = Field( - json_schema_extra={"auto_default_from_env": True} - ) - - STORAGE_SYNC_METADATA_TIMEOUT: PositiveInt = Field( - 180, description="Timeout (seconds) for metadata sync task" - ) - - STORAGE_DEFAULT_PRESIGNED_LINK_EXPIRATION_SECONDS: int = Field( - 3600, description="Default expiration time in seconds for presigned links" - ) - - STORAGE_CLEANER_INTERVAL_S: int | None = Field( - 30, - description="Interval in seconds when task cleaning pending uploads runs. setting to NULL disables the cleaner.", - ) - - STORAGE_RABBITMQ: RabbitSettings | None = Field( - json_schema_extra={"auto_default_from_env": True}, - ) - - STORAGE_S3_CLIENT_MAX_TRANSFER_CONCURRENCY: int = Field( - 4, - description="Maximal amount of threads used by underlying S3 client to transfer data to S3 backend", - ) - - STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field( - default=False, - validation_alias=AliasChoices( - "STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED", - "LOG_FORMAT_LOCAL_DEV_ENABLED", + BF_API_KEY: Annotated[ + str | None, + Field(None, description="Pennsieve API key ONLY for testing purposes"), + ] + BF_API_SECRET: Annotated[ + str | None, + Field(None, description="Pennsieve API secret ONLY for testing purposes"), + ] + + STORAGE_POSTGRES: Annotated[ + PostgresSettings | None, + Field(json_schema_extra={"auto_default_from_env": True}), + ] + + STORAGE_REDIS: Annotated[ + RedisSettings | None, Field(json_schema_extra={"auto_default_from_env": True}) + ] + + STORAGE_S3: Annotated[ + S3Settings | None, Field(json_schema_extra={"auto_default_from_env": True}) + ] + + STORAGE_CELERY: Annotated[ + CelerySettings | None, Field(json_schema_extra={"auto_default_from_env": True}) + ] + + STORAGE_TRACING: Annotated[ + TracingSettings | None, Field(json_schema_extra={"auto_default_from_env": True}) + ] + + DATCORE_ADAPTER: Annotated[ + DatcoreAdapterSettings, Field(json_schema_extra={"auto_default_from_env": True}) + ] + + STORAGE_SYNC_METADATA_TIMEOUT: Annotated[ + PositiveInt, Field(180, description="Timeout (seconds) for metadata sync task") + ] + + STORAGE_DEFAULT_PRESIGNED_LINK_EXPIRATION_SECONDS: Annotated[ + int, + Field( + 3600, description="Default expiration time in seconds for presigned links" + ), + ] + + STORAGE_CLEANER_INTERVAL_S: Annotated[ + int | None, + Field( + 30, + description="Interval in seconds when task cleaning pending uploads runs. setting to NULL disables the cleaner.", + ), + ] + + STORAGE_RABBITMQ: Annotated[ + RabbitSettings | None, + Field( + json_schema_extra={"auto_default_from_env": True}, ), - description="Enables local development _logger format. WARNING: make sure it is disabled if you want to have structured logs!", - ) - STORAGE_LOG_FILTER_MAPPING: dict[LoggerName, list[MessageSubstring]] = Field( - default_factory=dict, - validation_alias=AliasChoices( - "STORAGE_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING" + ] + + STORAGE_S3_CLIENT_MAX_TRANSFER_CONCURRENCY: Annotated[ + int, + Field( + 4, + description="Maximal amount of threads used by underlying S3 client to transfer data to S3 backend", + ), + ] + + STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED: Annotated[ + bool, + Field( + default=False, + validation_alias=AliasChoices( + "STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED", + "LOG_FORMAT_LOCAL_DEV_ENABLED", + ), + description="Enables local development _logger format. WARNING: make sure it is disabled if you want to have structured logs!", + ), + ] + STORAGE_LOG_FILTER_MAPPING: Annotated[ + dict[LoggerName, list[MessageSubstring]], + Field( + default_factory=dict, + validation_alias=AliasChoices( + "STORAGE_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING" + ), + description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of _logger message patterns that should be filtered out.", ), - description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of _logger message patterns that should be filtered out.", - ) + ] STORAGE_WORKER_MODE: Annotated[ bool | None, Field(description="If True, run as a worker") ] = False - STORAGE_CELERY_BROKER: RabbitSettings | None = Field( - json_schema_extra={"auto_default_from_env": True} - ) @field_validator("LOG_LEVEL", mode="before") @classmethod diff --git a/services/storage/src/simcore_service_storage/modules/celery/common.py b/services/storage/src/simcore_service_storage/modules/celery/common.py index 715bbe14753a..57ed5857011a 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/common.py +++ b/services/storage/src/simcore_service_storage/modules/celery/common.py @@ -9,12 +9,11 @@ def create_app(settings: ApplicationSettings) -> Celery: - assert settings.STORAGE_RABBITMQ - assert settings.STORAGE_REDIS + assert settings.STORAGE_CELERY app = Celery( - broker=settings.STORAGE_RABBITMQ.dsn, - backend=settings.STORAGE_REDIS.build_redis_dsn( + broker=settings.STORAGE_CELERY.CELERY_BROKER.dsn, + backend=settings.STORAGE_CELERY.CELERY_RESULTS_BACKEND.build_redis_dsn( RedisDatabase.CELERY_TASKS, ), ) From 04ffc07fbc000b19873be651f3aca27532eab167 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 25 Feb 2025 11:52:04 +0100 Subject: [PATCH 035/136] remove debug --- services/storage/docker/healthcheck.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/services/storage/docker/healthcheck.py b/services/storage/docker/healthcheck.py index d1cf1e267e1a..47d30157f048 100755 --- a/services/storage/docker/healthcheck.py +++ b/services/storage/docker/healthcheck.py @@ -59,8 +59,6 @@ def _is_celery_worker_healthy(): return False -print(_is_celery_worker_healthy()) - ok = ( ok or (bool(os.getenv("STORAGE_WORKER_MODE", "") and _is_celery_worker_healthy())) From 54b9a118ed28f165d861fcf987deb033446756e2 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 25 Feb 2025 11:53:35 +0100 Subject: [PATCH 036/136] remuve unused pytest plugin --- services/storage/tests/conftest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index 1df60f190238..15a95dd919a0 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -83,7 +83,6 @@ "pytest_simcore.openapi_specs", "pytest_simcore.postgres_service", "pytest_simcore.pytest_global_environs", - "pytest_simcore.rabbit_service", "pytest_simcore.repository_paths", "pytest_simcore.simcore_storage_data_models", "pytest_simcore.simcore_storage_datcore_adapter", From 1caeca5cbee5f04e4a3c6c0f5f95387fd546fdd8 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 25 Feb 2025 12:28:04 +0100 Subject: [PATCH 037/136] fix healthcheck --- services/storage/docker/healthcheck.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/services/storage/docker/healthcheck.py b/services/storage/docker/healthcheck.py index 47d30157f048..8321aa75d2df 100755 --- a/services/storage/docker/healthcheck.py +++ b/services/storage/docker/healthcheck.py @@ -33,12 +33,14 @@ # Queries host # pylint: disable=consider-using-with -app_settings = ApplicationSettings.create_from_envs() - -broker_url = app_settings.STORAGE_CELERY.CELERY_BROKER.dsn def _is_celery_worker_healthy(): + app_settings = ApplicationSettings.create_from_envs() + + assert app_settings.STORAGE_CELERY + broker_url = app_settings.STORAGE_CELERY.CELERY_BROKER.dsn + try: result = subprocess.run( [ From 0d84c14018dfe5c1d7c328f64d6d16a102f48e6c Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 25 Feb 2025 13:00:10 +0100 Subject: [PATCH 038/136] typecheck --- .../src/simcore_service_storage/modules/celery/client.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index b6df3fc4d5bc..98ef9d916e3c 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -53,7 +53,7 @@ def submit( def get(self, task_id: TaskID) -> Any: return self._celery_app.tasks(task_id) - def cancel(self, task_id: TaskID) -> None: + def cancel(self, task_id: TaskID) -> None: # pylint: disable=R6301 _logger.info("Aborting task %s", task_id) AbortableAsyncResult(task_id).abort() @@ -88,9 +88,8 @@ def _get_completed_task_ids( + "*" ) redis = self._celery_app.backend.client - if hasattr(redis, "keys"): - if keys := redis.keys(search_key): - return [f"{key}".lstrip(_CELERY_TASK_META_PREFIX) for key in keys] + if hasattr(redis, "keys") and (keys := redis.keys(search_key)): + return [f"{key}".lstrip(_CELERY_TASK_META_PREFIX) for key in keys] return [] def list(self, task_name: str, *, task_id_parts: TaskIDParts) -> list[TaskID]: From f297a3bfc7dbb904e875764e06fcdec34e60472b Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 25 Feb 2025 14:17:38 +0100 Subject: [PATCH 039/136] fix tests --- .../src/settings_library/celery.py | 32 ++++++++++- .../modules/celery/client.py | 11 ---- .../modules/celery/common.py | 9 ++-- .../modules/celery/utils.py | 12 +++++ .../tests/unit/modules/celery/conftest.py | 39 +++++++------- .../tests/unit/modules/celery/test_celery.py | 53 +++++++++---------- 6 files changed, 93 insertions(+), 63 deletions(-) create mode 100644 services/storage/src/simcore_service_storage/modules/celery/utils.py diff --git a/packages/settings-library/src/settings_library/celery.py b/packages/settings-library/src/settings_library/celery.py index c25ab3b036fb..157afe59242a 100644 --- a/packages/settings-library/src/settings_library/celery.py +++ b/packages/settings-library/src/settings_library/celery.py @@ -1,3 +1,4 @@ +from datetime import timedelta from typing import Annotated from pydantic import Field @@ -12,12 +13,39 @@ class CelerySettings(BaseCustomSettings): CELERY_BROKER: Annotated[ RabbitSettings, Field(json_schema_extra={"auto_default_from_env": True}) ] - CELERY_RESULTS_BACKEND: Annotated[ + CELERY_RESULT_BACKEND: Annotated[ RedisSettings, Field(json_schema_extra={"auto_default_from_env": True}) ] + CELERY_RESULT_EXPIRES: Annotated[ + timedelta, + Field( + description="Time (in seconds, or a timedelta object) for when after stored task tombstones will be deleted." + ), + ] = timedelta(days=7) + CELERY_RESULT_PERSISTENT: Annotated[ + bool, + Field( + description="If set to True, result messages will be persistent (after a broker restart)." + ), + ] = False model_config = SettingsConfigDict( json_schema_extra={ - "examples": [], + "examples": [ + { + "CELERY_BROKER": { + "RABBITMQ_USER": "guest", + "RABBITMQ_PASSWORD": "guest", + "RABBITMQ_HOST": "localhost", + "RABBITMQ_PORT": 5672, + }, + "CELERY_RESULT_BACKEND": { + "REDIS_HOST": "localhost", + "REDIS_PORT": 6379, + }, + "CELERY_RESULT_EXPIRES": "3600", + "CELERY_RESULT_PERSISTENT": True, + } + ], } ) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index 98ef9d916e3c..ca79f8ec23b4 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -5,7 +5,6 @@ from celery import Celery from celery.contrib.abortable import AbortableAsyncResult from celery.result import AsyncResult -from fastapi import FastAPI from models_library.progress_bar import ProgressReport from pydantic import ValidationError @@ -47,7 +46,6 @@ def submit( task = self._celery_app.send_task( task_name, task_id=task_id, kwargs=task_params ) - assert isinstance(str, task.id) return task.id def get(self, task_id: TaskID) -> Any: @@ -100,12 +98,3 @@ def list(self, task_name: str, *, task_id_parts: TaskIDParts) -> list[TaskID]: all_task_ids.extend(task_ids) return all_task_ids - - -def get_client(fastapi: FastAPI) -> CeleryTaskQueueClient: - celery = fastapi.state.celery_app - assert isinstance(celery, Celery) - - client = celery.conf["client"] - assert isinstance(client, CeleryTaskQueueClient) - return client diff --git a/services/storage/src/simcore_service_storage/modules/celery/common.py b/services/storage/src/simcore_service_storage/modules/celery/common.py index 57ed5857011a..c9860123d626 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/common.py +++ b/services/storage/src/simcore_service_storage/modules/celery/common.py @@ -9,12 +9,15 @@ def create_app(settings: ApplicationSettings) -> Celery: - assert settings.STORAGE_CELERY + celery_settings = settings.STORAGE_CELERY + assert celery_settings app = Celery( - broker=settings.STORAGE_CELERY.CELERY_BROKER.dsn, - backend=settings.STORAGE_CELERY.CELERY_RESULTS_BACKEND.build_redis_dsn( + broker=celery_settings.CELERY_BROKER.dsn, + backend=celery_settings.CELERY_RESULT_BACKEND.build_redis_dsn( RedisDatabase.CELERY_TASKS, ), ) + app.conf.result_expires = celery_settings.CELERY_RESULT_EXPIRES + app.conf.result_extended = True # original args are included in the results return app diff --git a/services/storage/src/simcore_service_storage/modules/celery/utils.py b/services/storage/src/simcore_service_storage/modules/celery/utils.py new file mode 100644 index 000000000000..6c5016f30937 --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/celery/utils.py @@ -0,0 +1,12 @@ +from celery import Celery +from fastapi import FastAPI +from simcore_service_storage.main import CeleryTaskQueueClient + + +def get_celery_client(fastapi: FastAPI) -> CeleryTaskQueueClient: + celery = fastapi.state.celery_app + assert isinstance(celery, Celery) + + client = celery.conf["client"] + assert isinstance(client, CeleryTaskQueueClient) + return client diff --git a/services/storage/tests/unit/modules/celery/conftest.py b/services/storage/tests/unit/modules/celery/conftest.py index c4dd20103308..209ecb15b141 100644 --- a/services/storage/tests/unit/modules/celery/conftest.py +++ b/services/storage/tests/unit/modules/celery/conftest.py @@ -1,6 +1,6 @@ from asyncio import AbstractEventLoop +from collections.abc import Callable, Iterable from datetime import timedelta -from typing import Callable, Iterable import pytest from celery import Celery @@ -23,20 +23,18 @@ "result_backend": "cache+memory://", "result_expires": timedelta(days=7), "result_extended": True, - "task_always_eager": False, - "task_acks_late": True, - "result_persistent": True, - "broker_transport_options": {"visibility_timeout": 3600}, - "task_track_started": True, - "worker_concurrency": 1, - "worker_prefetch_multiplier": 1, - "worker_send_task_events": True, # Required for task monitoring - "task_send_sent_event": True, # Required for task monitoring + "pool": "threads", } @pytest.fixture -def client_celery_app() -> Celery: +def register_celery_tasks() -> Callable[[Celery], None]: + msg = "please define a callback that registers the tasks" + raise NotImplementedError(msg) + + +@pytest.fixture +def celery_client_app() -> Celery: celery_app_client.conf.update(_CELERY_CONF) assert isinstance(celery_app_client.conf["client"], CeleryTaskQueueClient) @@ -47,12 +45,6 @@ def client_celery_app() -> Celery: return celery_app_client -@pytest.fixture -def register_celery_tasks() -> Callable[[Celery], None]: - msg = "please define a callback that registers the tasks" - raise NotImplementedError(msg) - - @pytest.fixture def celery_worker( register_celery_tasks: Callable[[Celery], None], @@ -75,10 +67,21 @@ def celery_worker( assert isinstance(celery_app_worker.conf["fastapi_app"], FastAPI) yield worker + worker_shutdown.send(sender=worker) @pytest.fixture -def worker_celery_app(celery_worker: TestWorkController) -> Celery: +def celery_worker_app(celery_worker: TestWorkController) -> Celery: assert isinstance(celery_worker.app, Celery) return celery_worker.app + + +@pytest.fixture +def celery_task_queue_client(celery_worker_app: Celery) -> CeleryTaskQueueClient: + return CeleryTaskQueueClient(celery_worker_app) + + +@pytest.fixture +def celery_task_queue_worker(celery_worker_app: Celery) -> CeleryTaskQueueWorker: + return CeleryTaskQueueWorker(celery_worker_app) diff --git a/services/storage/tests/unit/modules/celery/test_celery.py b/services/storage/tests/unit/modules/celery/test_celery.py index 446b22622a4d..8563908fa340 100644 --- a/services/storage/tests/unit/modules/celery/test_celery.py +++ b/services/storage/tests/unit/modules/celery/test_celery.py @@ -1,13 +1,12 @@ import logging import time +from collections.abc import Callable from random import randint -from typing import Callable import pytest from celery import Celery, Task from celery.contrib.abortable import AbortableTask -from simcore_service_storage.main import fastapi_app -from simcore_service_storage.modules.celery.client import get_client +from simcore_service_storage.main import CeleryTaskQueueClient from simcore_service_storage.modules.celery.example_tasks import sync_archive from simcore_service_storage.modules.celery.models import TaskIDParts from tenacity import Retrying, retry_if_exception_type, stop_after_delay, wait_fixed @@ -43,18 +42,16 @@ def _(celery_app: Celery) -> None: return _ +@pytest.mark.usefixtures("celery_client_app", "celery_worker_app") def test_archive( - client_celery_app: Celery, - worker_celery_app: Celery, + celery_task_queue_client: CeleryTaskQueueClient, ): - client = get_client(fastapi_app) - task_id_parts = TaskIDParts(user_id=1) - task_id = client.submit( + task_id = celery_task_queue_client.submit( "sync_archive", task_id_parts=task_id_parts, - files=[f"file{n}" for n in range(100)], + files=[f"file{n}" for n in range(30)], ) for attempt in Retrying( @@ -63,43 +60,41 @@ def test_archive( stop=stop_after_delay(30), ): with attempt: - progress = client.get_status(task_id) + progress = celery_task_queue_client.get_status(task_id) assert progress.task_state == "SUCCESS" - assert client.get_status(task_id).task_state == "SUCCESS" + assert celery_task_queue_client.get_status(task_id).task_state == "SUCCESS" +@pytest.mark.usefixtures("celery_client_app", "celery_worker_app") def test_failure_task( - client_celery_app: Celery, - worker_celery_app: Celery, + celery_task_queue_client: CeleryTaskQueueClient, ): - client = get_client(fastapi_app) - - task_id = client.submit("failure_task", task_id_parts=TaskIDParts(user_id=1)) + task_id = celery_task_queue_client.submit( + "failure_task", task_id_parts=TaskIDParts(user_id=1) + ) for attempt in Retrying( retry=retry_if_exception_type(AssertionError), wait=wait_fixed(1), ): with attempt: - result = client.get_result(task_id) + result = celery_task_queue_client.get_result(task_id) assert isinstance(result, ValueError) - assert client.get_status(task_id).task_state == "FAILURE" - assert f"{client.get_result(task_id)}" == "my error here" + assert celery_task_queue_client.get_status(task_id).task_state == "FAILURE" + assert f"{celery_task_queue_client.get_result(task_id)}" == "my error here" +@pytest.mark.usefixtures("celery_client_app", "celery_worker_app") def test_dreamer_task( - client_celery_app: Celery, - worker_celery_app: Celery, + celery_task_queue_client: CeleryTaskQueueClient, ): - client = get_client(fastapi_app) - - task_id = client.submit("dreamer_task", task_id_parts=TaskIDParts(user_id=1)) - - time.sleep(1) + task_id = celery_task_queue_client.submit( + "dreamer_task", task_id_parts=TaskIDParts(user_id=1) + ) - client.cancel(task_id) + celery_task_queue_client.cancel(task_id) for attempt in Retrying( retry=retry_if_exception_type(AssertionError), @@ -107,7 +102,7 @@ def test_dreamer_task( stop=stop_after_delay(30), ): with attempt: - progress = client.get_status(task_id) + progress = celery_task_queue_client.get_status(task_id) assert progress.task_state == "ABORTED" - assert client.get_status(task_id).task_state == "ABORTED" + assert celery_task_queue_client.get_status(task_id).task_state == "ABORTED" From d7602773d707f124f5cc069abae8aedddfa489d9 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 25 Feb 2025 14:22:28 +0100 Subject: [PATCH 040/136] fix tests --- services/storage/tests/unit/modules/celery/test_celery.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/storage/tests/unit/modules/celery/test_celery.py b/services/storage/tests/unit/modules/celery/test_celery.py index 8563908fa340..7a8ae0fc62a9 100644 --- a/services/storage/tests/unit/modules/celery/test_celery.py +++ b/services/storage/tests/unit/modules/celery/test_celery.py @@ -43,7 +43,7 @@ def _(celery_app: Celery) -> None: @pytest.mark.usefixtures("celery_client_app", "celery_worker_app") -def test_archive( +def test_sumitting_task_calling_async_function_results_with_success_state( celery_task_queue_client: CeleryTaskQueueClient, ): task_id_parts = TaskIDParts(user_id=1) @@ -67,7 +67,7 @@ def test_archive( @pytest.mark.usefixtures("celery_client_app", "celery_worker_app") -def test_failure_task( +def test_submitting_task_with_failure_results_with_error( celery_task_queue_client: CeleryTaskQueueClient, ): task_id = celery_task_queue_client.submit( @@ -87,7 +87,7 @@ def test_failure_task( @pytest.mark.usefixtures("celery_client_app", "celery_worker_app") -def test_dreamer_task( +def test_aborting_task_results_with_aborted_state( celery_task_queue_client: CeleryTaskQueueClient, ): task_id = celery_task_queue_client.submit( From c768f0e9fc0fd3bd90ca5a10e6d28be87450d0f4 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 25 Feb 2025 14:45:56 +0100 Subject: [PATCH 041/136] fix import --- services/storage/tests/unit/modules/celery/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/storage/tests/unit/modules/celery/conftest.py b/services/storage/tests/unit/modules/celery/conftest.py index 209ecb15b141..686faae48e64 100644 --- a/services/storage/tests/unit/modules/celery/conftest.py +++ b/services/storage/tests/unit/modules/celery/conftest.py @@ -7,8 +7,8 @@ from celery.contrib.testing.worker import TestWorkController, start_worker from celery.signals import worker_init, worker_shutdown from fastapi import FastAPI -from simcore_service_storage.main import CeleryTaskQueueClient from simcore_service_storage.main import celery_app as celery_app_client +from simcore_service_storage.modules.celery.client import CeleryTaskQueueClient from simcore_service_storage.modules.celery.worker import CeleryTaskQueueWorker from simcore_service_storage.modules.celery.worker_main import ( celery_app as celery_app_worker, From 3c8eb4c2a040352e8cbf57c351132cc1c268e2fb Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 25 Feb 2025 15:34:08 +0100 Subject: [PATCH 042/136] add utils --- .../src/simcore_service_storage/main.py | 8 +++-- .../modules/celery/utils.py | 32 +++++++++++++++++-- .../modules/celery/worker_main.py | 7 ++-- .../tests/unit/modules/celery/conftest.py | 19 ++++++++++- 4 files changed, 57 insertions(+), 9 deletions(-) diff --git a/services/storage/src/simcore_service_storage/main.py b/services/storage/src/simcore_service_storage/main.py index 588037c2159c..f0785da026b3 100644 --- a/services/storage/src/simcore_service_storage/main.py +++ b/services/storage/src/simcore_service_storage/main.py @@ -3,6 +3,10 @@ import logging from servicelib.logging_utils import config_all_loggers +from simcore_service_storage.modules.celery.utils import ( + set_celery_app, + set_celery_client, +) from .core.application import create_app from .core.settings import ApplicationSettings @@ -25,7 +29,7 @@ fastapi_app = create_app(_settings) celery_app = create_celery_app(_settings) -celery_app.conf["client"] = CeleryTaskQueueClient(celery_app) -fastapi_app.state.celery_app = celery_app +set_celery_client(celery_app, CeleryTaskQueueClient(celery_app)) +set_celery_app(fastapi_app, celery_app) app = fastapi_app diff --git a/services/storage/src/simcore_service_storage/modules/celery/utils.py b/services/storage/src/simcore_service_storage/modules/celery/utils.py index 6c5016f30937..9c050d86bb38 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/utils.py +++ b/services/storage/src/simcore_service_storage/modules/celery/utils.py @@ -1,12 +1,38 @@ from celery import Celery from fastapi import FastAPI -from simcore_service_storage.main import CeleryTaskQueueClient +from .client import CeleryTaskQueueClient +from .worker import CeleryTaskQueueWorker -def get_celery_client(fastapi: FastAPI) -> CeleryTaskQueueClient: +_CLIENT_KEY = "client" +_WORKER_KEY = "worker" + + +def get_celery_app(fastapi: FastAPI) -> Celery: celery = fastapi.state.celery_app assert isinstance(celery, Celery) + return celery + - client = celery.conf["client"] +def set_celery_app(fastapi: FastAPI, celery: Celery) -> None: + fastapi.state.celery_app = celery + + +def get_celery_client(celery_app: Celery) -> CeleryTaskQueueClient: + client = celery_app.conf[_CLIENT_KEY] assert isinstance(client, CeleryTaskQueueClient) return client + + +def set_celery_client(celery_app: Celery, celery_client: CeleryTaskQueueClient) -> None: + celery_app.conf[_CLIENT_KEY] = celery_client + + +def get_celery_worker(celery_app: Celery) -> CeleryTaskQueueWorker: + worker = celery_app.conf[_WORKER_KEY] + assert isinstance(worker, CeleryTaskQueueWorker) + return worker + + +def set_celery_worker(celery_app: Celery, celery_worker: CeleryTaskQueueWorker) -> None: + celery_app.conf[_WORKER_KEY] = celery_worker diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker_main.py b/services/storage/src/simcore_service_storage/modules/celery/worker_main.py index 190909d51a3a..7f02f5f46e2e 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker_main.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker_main.py @@ -9,6 +9,7 @@ from celery.signals import worker_init, worker_shutdown from servicelib.background_task import cancel_wait_task from servicelib.logging_utils import config_all_loggers +from simcore_service_storage.modules.celery.utils import set_celery_worker from ...core.application import create_app from ...core.settings import ApplicationSettings @@ -78,10 +79,10 @@ async def shutdown(): asyncio.run_coroutine_threadsafe(shutdown(), loop) -celery_app = create_celery_app(ApplicationSettings.create_from_envs()) -celery_worker = CeleryTaskQueueWorker(celery_app) -celery_app.conf["worker"] = celery_worker +celery_app = create_celery_app(_settings) +celery_worker = CeleryTaskQueueWorker(celery_app) celery_worker.register_task(sync_archive) +set_celery_worker(celery_app, CeleryTaskQueueWorker(celery_app)) app = celery_app diff --git a/services/storage/tests/unit/modules/celery/conftest.py b/services/storage/tests/unit/modules/celery/conftest.py index 686faae48e64..dd3b22fd98fb 100644 --- a/services/storage/tests/unit/modules/celery/conftest.py +++ b/services/storage/tests/unit/modules/celery/conftest.py @@ -7,6 +7,8 @@ from celery.contrib.testing.worker import TestWorkController, start_worker from celery.signals import worker_init, worker_shutdown from fastapi import FastAPI +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +from pytest_simcore.helpers.typing_env import EnvVarsDict from simcore_service_storage.main import celery_app as celery_app_client from simcore_service_storage.modules.celery.client import CeleryTaskQueueClient from simcore_service_storage.modules.celery.worker import CeleryTaskQueueWorker @@ -18,6 +20,21 @@ on_worker_shutdown, ) + +@pytest.fixture +def app_environment( + monkeypatch: pytest.MonkeyPatch, + app_environment: EnvVarsDict, +) -> EnvVarsDict: + return setenvs_from_dict( + monkeypatch, + { + **app_environment, + "SC_BOOT_MODE": "local-development", + }, + ) + + _CELERY_CONF = { "broker_url": "memory://", "result_backend": "cache+memory://", @@ -34,7 +51,7 @@ def register_celery_tasks() -> Callable[[Celery], None]: @pytest.fixture -def celery_client_app() -> Celery: +def celery_client_app(app_environment: EnvVarsDict) -> Celery: celery_app_client.conf.update(_CELERY_CONF) assert isinstance(celery_app_client.conf["client"], CeleryTaskQueueClient) From 780a9479f488398a7f8fa83c4df01b381992f807 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 25 Feb 2025 15:47:30 +0100 Subject: [PATCH 043/136] update utils --- .../src/simcore_service_storage/modules/celery/client.py | 2 +- .../src/simcore_service_storage/modules/celery/utils.py | 8 ++++++-- services/storage/tests/unit/modules/celery/test_celery.py | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index ca79f8ec23b4..4722470c6db4 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -51,7 +51,7 @@ def submit( def get(self, task_id: TaskID) -> Any: return self._celery_app.tasks(task_id) - def cancel(self, task_id: TaskID) -> None: # pylint: disable=R6301 + def abort(self, task_id: TaskID) -> None: # pylint: disable=R6301 _logger.info("Aborting task %s", task_id) AbortableAsyncResult(task_id).abort() diff --git a/services/storage/src/simcore_service_storage/modules/celery/utils.py b/services/storage/src/simcore_service_storage/modules/celery/utils.py index 9c050d86bb38..2d8ddb3f7593 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/utils.py +++ b/services/storage/src/simcore_service_storage/modules/celery/utils.py @@ -18,13 +18,17 @@ def set_celery_app(fastapi: FastAPI, celery: Celery) -> None: fastapi.state.celery_app = celery -def get_celery_client(celery_app: Celery) -> CeleryTaskQueueClient: +def get_celery_client(fastapi_app: FastAPI) -> CeleryTaskQueueClient: + celery_app = get_celery_app(fastapi_app) client = celery_app.conf[_CLIENT_KEY] assert isinstance(client, CeleryTaskQueueClient) return client -def set_celery_client(celery_app: Celery, celery_client: CeleryTaskQueueClient) -> None: +def set_celery_client( + fastapi_app: FastAPI, celery_client: CeleryTaskQueueClient +) -> None: + celery_app = get_celery_app(fastapi_app) celery_app.conf[_CLIENT_KEY] = celery_client diff --git a/services/storage/tests/unit/modules/celery/test_celery.py b/services/storage/tests/unit/modules/celery/test_celery.py index 7a8ae0fc62a9..12456f71c754 100644 --- a/services/storage/tests/unit/modules/celery/test_celery.py +++ b/services/storage/tests/unit/modules/celery/test_celery.py @@ -94,7 +94,7 @@ def test_aborting_task_results_with_aborted_state( "dreamer_task", task_id_parts=TaskIDParts(user_id=1) ) - celery_task_queue_client.cancel(task_id) + celery_task_queue_client.abort(task_id) for attempt in Retrying( retry=retry_if_exception_type(AssertionError), From 830d56263ebfa640c1be017710383be2787fca68 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 25 Feb 2025 16:02:44 +0100 Subject: [PATCH 044/136] fix tests --- services/storage/src/simcore_service_storage/main.py | 2 +- services/storage/tests/unit/modules/celery/test_celery.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/services/storage/src/simcore_service_storage/main.py b/services/storage/src/simcore_service_storage/main.py index f0785da026b3..80f522708edd 100644 --- a/services/storage/src/simcore_service_storage/main.py +++ b/services/storage/src/simcore_service_storage/main.py @@ -29,7 +29,7 @@ fastapi_app = create_app(_settings) celery_app = create_celery_app(_settings) -set_celery_client(celery_app, CeleryTaskQueueClient(celery_app)) set_celery_app(fastapi_app, celery_app) +set_celery_client(fastapi_app, CeleryTaskQueueClient(celery_app)) app = fastapi_app diff --git a/services/storage/tests/unit/modules/celery/test_celery.py b/services/storage/tests/unit/modules/celery/test_celery.py index 12456f71c754..c37f72d0bccc 100644 --- a/services/storage/tests/unit/modules/celery/test_celery.py +++ b/services/storage/tests/unit/modules/celery/test_celery.py @@ -83,7 +83,9 @@ def test_submitting_task_with_failure_results_with_error( assert isinstance(result, ValueError) assert celery_task_queue_client.get_status(task_id).task_state == "FAILURE" - assert f"{celery_task_queue_client.get_result(task_id)}" == "my error here" + result = celery_task_queue_client.get_result(task_id) + assert isinstance(result, ValueError) + assert f"{result}" == "my error here" @pytest.mark.usefixtures("celery_client_app", "celery_worker_app") From b22abeaace74b69a191a717ee88aa04d1cbc49f8 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 26 Feb 2025 09:26:20 +0100 Subject: [PATCH 045/136] update example --- .../settings-library/src/settings_library/celery.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/settings-library/src/settings_library/celery.py b/packages/settings-library/src/settings_library/celery.py index 157afe59242a..ce6506126f44 100644 --- a/packages/settings-library/src/settings_library/celery.py +++ b/packages/settings-library/src/settings_library/celery.py @@ -34,16 +34,17 @@ class CelerySettings(BaseCustomSettings): "examples": [ { "CELERY_BROKER": { - "RABBITMQ_USER": "guest", - "RABBITMQ_PASSWORD": "guest", - "RABBITMQ_HOST": "localhost", - "RABBITMQ_PORT": 5672, + "RABBIT_USER": "guest", + "RABBIT_SECURE": False, + "RABBIT_PASSWORD": "guest", + "RABBIT_HOST": "localhost", + "RABBIT_PORT": 5672, }, "CELERY_RESULT_BACKEND": { "REDIS_HOST": "localhost", "REDIS_PORT": 6379, }, - "CELERY_RESULT_EXPIRES": "3600", + "CELERY_RESULT_EXPIRES": timedelta(days=1), "CELERY_RESULT_PERSISTENT": True, } ], From 5072b3b22fa1cc499870f6fee3b43ce3d6c1be93 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 26 Feb 2025 10:56:47 +0100 Subject: [PATCH 046/136] add async interface --- .../src/common_library/async_utils.py | 22 ++++++++ .../modules/celery/client.py | 53 +++++++++++++------ .../tests/unit/modules/celery/test_celery.py | 34 +++++++----- 3 files changed, 79 insertions(+), 30 deletions(-) create mode 100644 packages/common-library/src/common_library/async_utils.py diff --git a/packages/common-library/src/common_library/async_utils.py b/packages/common-library/src/common_library/async_utils.py new file mode 100644 index 000000000000..6ad0666a9260 --- /dev/null +++ b/packages/common-library/src/common_library/async_utils.py @@ -0,0 +1,22 @@ +import asyncio +import functools +from collections.abc import Callable, Coroutine +from typing import Any, TypeVar + +R = TypeVar("R") + + +def make_async( + executor=None, +) -> Callable[[Callable[..., R]], Callable[..., Coroutine[Any, Any, R]]]: + def decorator(func) -> Callable[..., Coroutine[Any, Any, R]]: + @functools.wraps(func) + async def wrapper(*args, **kwargs) -> R: + loop = asyncio.get_running_loop() + return await loop.run_in_executor( + executor, functools.partial(func, *args, **kwargs) + ) + + return wrapper + + return decorator diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index 4722470c6db4..48b5fcf9e5f3 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -1,10 +1,10 @@ import logging -from typing import Any, Final +from typing import Any, Coroutine, Final from uuid import uuid4 from celery import Celery from celery.contrib.abortable import AbortableAsyncResult -from celery.result import AsyncResult +from common_library.async_utils import make_async from models_library.progress_bar import ProgressReport from pydantic import ValidationError @@ -38,31 +38,42 @@ class CeleryTaskQueueClient: def __init__(self, celery_app: Celery): self._celery_app = celery_app - def submit( + @make_async() + def __send_task(self, task_name: str, task_id: TaskID, **task_params): + return self._celery_app.send_task(task_name, task_i=task_id, **task_params) + + async def submit( self, task_name: str, *, task_id_parts: TaskIDParts, **task_params ) -> TaskID: task_id = _get_task_id(task_name, task_id_parts) _logger.debug("Submitting task %s: %s", task_name, task_id) - task = self._celery_app.send_task( - task_name, task_id=task_id, kwargs=task_params - ) + task = await self.__send_task(task_name, task_id=task_id, kwargs=task_params) return task.id - def get(self, task_id: TaskID) -> Any: + @make_async() + def __get_task(self, task_id: TaskID) -> Coroutine[Any, Any, Any]: return self._celery_app.tasks(task_id) - def abort(self, task_id: TaskID) -> None: # pylint: disable=R6301 + async def get_task(self, task_id: TaskID) -> Coroutine[Any, Any, Any]: + return await self.__get_task(task_id) + + @make_async() + def __abort_task(self, task_id: TaskID) -> Any: # pylint: disable=R6301 _logger.info("Aborting task %s", task_id) AbortableAsyncResult(task_id).abort() - def _get_async_result(self, task_id: TaskID) -> AsyncResult: - return self._celery_app.AsyncResult(task_id) + async def abort_task(self, task_id: TaskID) -> None: + return await self.__abort_task(task_id) + + @make_async() + def __get_result(self, task_id: TaskID) -> Any: + return self._celery_app.AsyncResult(task_id).result - def get_result(self, task_id: TaskID) -> Any: - return self._get_async_result(task_id).result + async def get_result(self, task_id: TaskID) -> Any: + return await self.__get_result(task_id) def _get_progress_report(self, task_id: TaskID) -> ProgressReport | None: - result = self._get_async_result(task_id).result + result = self.__get_result(task_id) if result: try: return ProgressReport.model_validate(result) @@ -70,13 +81,17 @@ def _get_progress_report(self, task_id: TaskID) -> ProgressReport | None: pass return None - def get_status(self, task_id: TaskID) -> TaskStatus: + @make_async() + def __get_status(self, task_id: TaskID) -> TaskStatus: return TaskStatus( task_id=task_id, - task_state=self._get_async_result(task_id).state, + task_state=self._celery_app.AsyncResult(task_id).state, progress_report=self._get_progress_report(task_id), ) + async def get_task_status(self, task_id: TaskID) -> TaskStatus: + return await self.__get_status(task_id) + def _get_completed_task_ids( self, task_name: str, task_id_parts: TaskIDParts ) -> list[TaskID]: @@ -90,7 +105,8 @@ def _get_completed_task_ids( return [f"{key}".lstrip(_CELERY_TASK_META_PREFIX) for key in keys] return [] - def list(self, task_name: str, *, task_id_parts: TaskIDParts) -> list[TaskID]: + @make_async() + def __list_tasks(self, task_name: str, *, task_id_parts: TaskIDParts) -> Any: all_task_ids = self._get_completed_task_ids(task_name, task_id_parts) for task_type in ["active", "registered", "scheduled", "revoked"]: @@ -98,3 +114,8 @@ def list(self, task_name: str, *, task_id_parts: TaskIDParts) -> list[TaskID]: all_task_ids.extend(task_ids) return all_task_ids + + async def list_tasks( + self, task_name: str, *, task_id_parts: TaskIDParts + ) -> list[TaskID]: + return await self.__list_tasks(task_name, task_id_parts=task_id_parts) diff --git a/services/storage/tests/unit/modules/celery/test_celery.py b/services/storage/tests/unit/modules/celery/test_celery.py index c37f72d0bccc..1b95404ea8d2 100644 --- a/services/storage/tests/unit/modules/celery/test_celery.py +++ b/services/storage/tests/unit/modules/celery/test_celery.py @@ -43,12 +43,12 @@ def _(celery_app: Celery) -> None: @pytest.mark.usefixtures("celery_client_app", "celery_worker_app") -def test_sumitting_task_calling_async_function_results_with_success_state( +async def test_sumitting_task_calling_async_function_results_with_success_state( celery_task_queue_client: CeleryTaskQueueClient, ): task_id_parts = TaskIDParts(user_id=1) - task_id = celery_task_queue_client.submit( + task_id = await celery_task_queue_client.submit( "sync_archive", task_id_parts=task_id_parts, files=[f"file{n}" for n in range(30)], @@ -60,17 +60,19 @@ def test_sumitting_task_calling_async_function_results_with_success_state( stop=stop_after_delay(30), ): with attempt: - progress = celery_task_queue_client.get_status(task_id) + progress = await celery_task_queue_client.get_task_status(task_id) assert progress.task_state == "SUCCESS" - assert celery_task_queue_client.get_status(task_id).task_state == "SUCCESS" + assert ( + await celery_task_queue_client.get_task_status(task_id) + ).task_state == "SUCCESS" @pytest.mark.usefixtures("celery_client_app", "celery_worker_app") -def test_submitting_task_with_failure_results_with_error( +async def test_submitting_task_with_failure_results_with_error( celery_task_queue_client: CeleryTaskQueueClient, ): - task_id = celery_task_queue_client.submit( + task_id = await celery_task_queue_client.submit( "failure_task", task_id_parts=TaskIDParts(user_id=1) ) @@ -79,24 +81,26 @@ def test_submitting_task_with_failure_results_with_error( wait=wait_fixed(1), ): with attempt: - result = celery_task_queue_client.get_result(task_id) + result = await celery_task_queue_client.get_result(task_id) assert isinstance(result, ValueError) - assert celery_task_queue_client.get_status(task_id).task_state == "FAILURE" - result = celery_task_queue_client.get_result(task_id) + assert ( + await celery_task_queue_client.get_task_status(task_id) + ).task_state == "FAILURE" + result = await celery_task_queue_client.get_result(task_id) assert isinstance(result, ValueError) assert f"{result}" == "my error here" @pytest.mark.usefixtures("celery_client_app", "celery_worker_app") -def test_aborting_task_results_with_aborted_state( +async def test_aborting_task_results_with_aborted_state( celery_task_queue_client: CeleryTaskQueueClient, ): - task_id = celery_task_queue_client.submit( + task_id = await celery_task_queue_client.submit( "dreamer_task", task_id_parts=TaskIDParts(user_id=1) ) - celery_task_queue_client.abort(task_id) + await celery_task_queue_client.abort_task(task_id) for attempt in Retrying( retry=retry_if_exception_type(AssertionError), @@ -104,7 +108,9 @@ def test_aborting_task_results_with_aborted_state( stop=stop_after_delay(30), ): with attempt: - progress = celery_task_queue_client.get_status(task_id) + progress = await celery_task_queue_client.get_task_status(task_id) assert progress.task_state == "ABORTED" - assert celery_task_queue_client.get_status(task_id).task_state == "ABORTED" + assert ( + await celery_task_queue_client.get_task_status(task_id) + ).task_state == "ABORTED" From e09a66b063722720d06c40c1ccb8e585579fbec9 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 26 Feb 2025 11:01:12 +0100 Subject: [PATCH 047/136] add tests --- .../{async_utils.py => async_tools.py} | 0 .../common-library/tests/test_async_tools.py | 45 +++++++++++++++++++ .../modules/celery/client.py | 2 +- 3 files changed, 46 insertions(+), 1 deletion(-) rename packages/common-library/src/common_library/{async_utils.py => async_tools.py} (100%) create mode 100644 packages/common-library/tests/test_async_tools.py diff --git a/packages/common-library/src/common_library/async_utils.py b/packages/common-library/src/common_library/async_tools.py similarity index 100% rename from packages/common-library/src/common_library/async_utils.py rename to packages/common-library/src/common_library/async_tools.py diff --git a/packages/common-library/tests/test_async_tools.py b/packages/common-library/tests/test_async_tools.py new file mode 100644 index 000000000000..961bf6f9fde4 --- /dev/null +++ b/packages/common-library/tests/test_async_tools.py @@ -0,0 +1,45 @@ +import asyncio +from concurrent.futures import ThreadPoolExecutor + +import pytest +from common_library.async_tools import make_async + + +@make_async() +def sync_function(x: int, y: int) -> int: + return x + y + + +@make_async() +def sync_function_with_exception() -> None: + raise ValueError("This is an error!") + + +@pytest.mark.asyncio +async def test_make_async_returns_coroutine(): + result = sync_function(2, 3) + assert asyncio.iscoroutine(result), "Function should return a coroutine" + + +@pytest.mark.asyncio +async def test_make_async_execution(): + result = await sync_function(2, 3) + assert result == 5, "Function should return 5" + + +@pytest.mark.asyncio +async def test_make_async_exception(): + with pytest.raises(ValueError, match="This is an error!"): + await sync_function_with_exception() + + +@pytest.mark.asyncio +async def test_make_async_with_executor(): + executor = ThreadPoolExecutor() + + @make_async(executor) + def heavy_computation(x: int) -> int: + return x * x + + result = await heavy_computation(4) + assert result == 16, "Function should return 16" diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index 48b5fcf9e5f3..e67aa51311f0 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -4,7 +4,7 @@ from celery import Celery from celery.contrib.abortable import AbortableAsyncResult -from common_library.async_utils import make_async +from common_library.async_tools import make_async from models_library.progress_bar import ProgressReport from pydantic import ValidationError From e94f73655f3808c41401a1325c813efbd8aee106 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 26 Feb 2025 11:09:10 +0100 Subject: [PATCH 048/136] improve typehints --- packages/common-library/src/common_library/async_tools.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/common-library/src/common_library/async_tools.py b/packages/common-library/src/common_library/async_tools.py index 6ad0666a9260..4b7402ab1eb5 100644 --- a/packages/common-library/src/common_library/async_tools.py +++ b/packages/common-library/src/common_library/async_tools.py @@ -1,15 +1,16 @@ import asyncio import functools from collections.abc import Callable, Coroutine +from concurrent.futures import Executor from typing import Any, TypeVar R = TypeVar("R") def make_async( - executor=None, + executor: Executor | None = None, ) -> Callable[[Callable[..., R]], Callable[..., Coroutine[Any, Any, R]]]: - def decorator(func) -> Callable[..., Coroutine[Any, Any, R]]: + def decorator(func: Callable[..., R]) -> Callable[..., Coroutine[Any, Any, R]]: @functools.wraps(func) async def wrapper(*args, **kwargs) -> R: loop = asyncio.get_running_loop() From 079fd1b5f9ede4e7a607cab980ca1430ca58d372 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 26 Feb 2025 12:36:15 +0100 Subject: [PATCH 049/136] improve typehint --- .../src/common_library/async_tools.py | 11 ++--- .../modules/celery/client.py | 40 +++++-------------- .../tests/unit/modules/celery/test_celery.py | 6 +-- 3 files changed, 20 insertions(+), 37 deletions(-) diff --git a/packages/common-library/src/common_library/async_tools.py b/packages/common-library/src/common_library/async_tools.py index 4b7402ab1eb5..d92944299e79 100644 --- a/packages/common-library/src/common_library/async_tools.py +++ b/packages/common-library/src/common_library/async_tools.py @@ -1,18 +1,19 @@ import asyncio import functools -from collections.abc import Callable, Coroutine +from collections.abc import Awaitable, Callable from concurrent.futures import Executor -from typing import Any, TypeVar +from typing import ParamSpec, TypeVar R = TypeVar("R") +P = ParamSpec("P") def make_async( executor: Executor | None = None, -) -> Callable[[Callable[..., R]], Callable[..., Coroutine[Any, Any, R]]]: - def decorator(func: Callable[..., R]) -> Callable[..., Coroutine[Any, Any, R]]: +) -> Callable[[Callable[P, R]], Callable[P, Awaitable[R]]]: + def decorator(func: Callable[P, R]) -> Callable[P, Awaitable[R]]: @functools.wraps(func) - async def wrapper(*args, **kwargs) -> R: + async def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: loop = asyncio.get_running_loop() return await loop.run_in_executor( executor, functools.partial(func, *args, **kwargs) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index e67aa51311f0..96dcb6e7032e 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -1,5 +1,5 @@ import logging -from typing import Any, Coroutine, Final +from typing import Any, Final from uuid import uuid4 from celery import Celery @@ -39,41 +39,31 @@ def __init__(self, celery_app: Celery): self._celery_app = celery_app @make_async() - def __send_task(self, task_name: str, task_id: TaskID, **task_params): - return self._celery_app.send_task(task_name, task_i=task_id, **task_params) - - async def submit( + def send_task( self, task_name: str, *, task_id_parts: TaskIDParts, **task_params ) -> TaskID: task_id = _get_task_id(task_name, task_id_parts) _logger.debug("Submitting task %s: %s", task_name, task_id) - task = await self.__send_task(task_name, task_id=task_id, kwargs=task_params) + task = self._celery_app.send_task( + task_name, task_id=task_id, kwargs=task_params + ) return task.id @make_async() - def __get_task(self, task_id: TaskID) -> Coroutine[Any, Any, Any]: + def get_task(self, task_id: TaskID) -> Any: return self._celery_app.tasks(task_id) - async def get_task(self, task_id: TaskID) -> Coroutine[Any, Any, Any]: - return await self.__get_task(task_id) - @make_async() - def __abort_task(self, task_id: TaskID) -> Any: # pylint: disable=R6301 + def abort_task(self, task_id: TaskID) -> None: # pylint: disable=R6301 _logger.info("Aborting task %s", task_id) AbortableAsyncResult(task_id).abort() - async def abort_task(self, task_id: TaskID) -> None: - return await self.__abort_task(task_id) - @make_async() - def __get_result(self, task_id: TaskID) -> Any: + def get_result(self, task_id: TaskID) -> Any: return self._celery_app.AsyncResult(task_id).result - async def get_result(self, task_id: TaskID) -> Any: - return await self.__get_result(task_id) - def _get_progress_report(self, task_id: TaskID) -> ProgressReport | None: - result = self.__get_result(task_id) + result = self._celery_app.AsyncResult(task_id).result if result: try: return ProgressReport.model_validate(result) @@ -82,16 +72,13 @@ def _get_progress_report(self, task_id: TaskID) -> ProgressReport | None: return None @make_async() - def __get_status(self, task_id: TaskID) -> TaskStatus: + def get_task_status(self, task_id: TaskID) -> TaskStatus: return TaskStatus( task_id=task_id, task_state=self._celery_app.AsyncResult(task_id).state, progress_report=self._get_progress_report(task_id), ) - async def get_task_status(self, task_id: TaskID) -> TaskStatus: - return await self.__get_status(task_id) - def _get_completed_task_ids( self, task_name: str, task_id_parts: TaskIDParts ) -> list[TaskID]: @@ -106,7 +93,7 @@ def _get_completed_task_ids( return [] @make_async() - def __list_tasks(self, task_name: str, *, task_id_parts: TaskIDParts) -> Any: + def list_tasks(self, task_name: str, *, task_id_parts: TaskIDParts) -> Any: all_task_ids = self._get_completed_task_ids(task_name, task_id_parts) for task_type in ["active", "registered", "scheduled", "revoked"]: @@ -114,8 +101,3 @@ def __list_tasks(self, task_name: str, *, task_id_parts: TaskIDParts) -> Any: all_task_ids.extend(task_ids) return all_task_ids - - async def list_tasks( - self, task_name: str, *, task_id_parts: TaskIDParts - ) -> list[TaskID]: - return await self.__list_tasks(task_name, task_id_parts=task_id_parts) diff --git a/services/storage/tests/unit/modules/celery/test_celery.py b/services/storage/tests/unit/modules/celery/test_celery.py index 1b95404ea8d2..28f103e2b358 100644 --- a/services/storage/tests/unit/modules/celery/test_celery.py +++ b/services/storage/tests/unit/modules/celery/test_celery.py @@ -48,7 +48,7 @@ async def test_sumitting_task_calling_async_function_results_with_success_state( ): task_id_parts = TaskIDParts(user_id=1) - task_id = await celery_task_queue_client.submit( + task_id = await celery_task_queue_client.send_task( "sync_archive", task_id_parts=task_id_parts, files=[f"file{n}" for n in range(30)], @@ -72,7 +72,7 @@ async def test_sumitting_task_calling_async_function_results_with_success_state( async def test_submitting_task_with_failure_results_with_error( celery_task_queue_client: CeleryTaskQueueClient, ): - task_id = await celery_task_queue_client.submit( + task_id = await celery_task_queue_client.send_task( "failure_task", task_id_parts=TaskIDParts(user_id=1) ) @@ -96,7 +96,7 @@ async def test_submitting_task_with_failure_results_with_error( async def test_aborting_task_results_with_aborted_state( celery_task_queue_client: CeleryTaskQueueClient, ): - task_id = await celery_task_queue_client.submit( + task_id = await celery_task_queue_client.send_task( "dreamer_task", task_id_parts=TaskIDParts(user_id=1) ) From 975b4d347909764df7184f06e1b6de217a62f1b8 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 26 Feb 2025 12:50:53 +0100 Subject: [PATCH 050/136] remove unused --- .../modules/celery/example_tasks.py | 5 +++-- .../simcore_service_storage/modules/celery/utils.py | 9 +++++++++ .../modules/celery/worker.py | 13 +------------ 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/example_tasks.py b/services/storage/src/simcore_service_storage/modules/celery/example_tasks.py index 712a1c422400..d4e27a7eb035 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/example_tasks.py +++ b/services/storage/src/simcore_service_storage/modules/celery/example_tasks.py @@ -4,7 +4,8 @@ from celery import Celery, Task from models_library.progress_bar import ProgressReport -from .worker import get_event_loop, get_worker +from .utils import get_celery_worker, get_event_loop +from .worker import CeleryTaskQueueWorker _logger = logging.getLogger(__name__) @@ -12,7 +13,7 @@ async def _async_archive( celery_app: Celery, task_name: str, task_id: str, files: list[str] ) -> str: - worker = get_worker(celery_app) + worker: CeleryTaskQueueWorker = get_celery_worker(celery_app) for n, file in enumerate(files, start=1): _logger.info("Processing file %s", file) diff --git a/services/storage/src/simcore_service_storage/modules/celery/utils.py b/services/storage/src/simcore_service_storage/modules/celery/utils.py index 2d8ddb3f7593..134e663eabd4 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/utils.py +++ b/services/storage/src/simcore_service_storage/modules/celery/utils.py @@ -1,3 +1,5 @@ +from asyncio import AbstractEventLoop + from celery import Celery from fastapi import FastAPI @@ -6,6 +8,7 @@ _CLIENT_KEY = "client" _WORKER_KEY = "worker" +_EVENT_LOOP_KEY = "loop" def get_celery_app(fastapi: FastAPI) -> Celery: @@ -40,3 +43,9 @@ def get_celery_worker(celery_app: Celery) -> CeleryTaskQueueWorker: def set_celery_worker(celery_app: Celery, celery_worker: CeleryTaskQueueWorker) -> None: celery_app.conf[_WORKER_KEY] = celery_worker + + +def get_event_loop(celery_app: Celery) -> AbstractEventLoop: # nosec + loop = celery_app.conf[_EVENT_LOOP_KEY] + assert isinstance(loop, AbstractEventLoop) + return loop diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker.py b/services/storage/src/simcore_service_storage/modules/celery/worker.py index b2e2637be7e6..c593c27c4cdf 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker.py @@ -1,6 +1,5 @@ import logging -from asyncio import AbstractEventLoop -from typing import Callable +from collections.abc import Callable from celery import Celery from celery.contrib.abortable import AbortableTask @@ -11,11 +10,6 @@ _logger = logging.getLogger(__name__) -def get_event_loop(celery_app: Celery) -> AbstractEventLoop: # nosec - loop: AbstractEventLoop = celery_app.conf["loop"] - return loop - - class CeleryTaskQueueWorker: def __init__(self, celery_app: Celery) -> None: self.celery_app = celery_app @@ -35,8 +29,3 @@ def set_progress( state="PROGRESS", meta=report.model_dump(mode="json"), ) - - -def get_worker(celery_app: Celery) -> CeleryTaskQueueWorker: - worker: CeleryTaskQueueWorker = celery_app.conf["worker"] - return worker From 2efae6bf11df4c8437e172bc9f0d849975550d10 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 26 Feb 2025 13:02:48 +0100 Subject: [PATCH 051/136] update --- .../modules/celery/example_tasks.py | 2 +- .../src/simcore_service_storage/modules/celery/worker.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/example_tasks.py b/services/storage/src/simcore_service_storage/modules/celery/example_tasks.py index d4e27a7eb035..8b37c708963f 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/example_tasks.py +++ b/services/storage/src/simcore_service_storage/modules/celery/example_tasks.py @@ -17,7 +17,7 @@ async def _async_archive( for n, file in enumerate(files, start=1): _logger.info("Processing file %s", file) - worker.set_progress( + worker.set_task_progress( task_name=task_name, task_id=task_id, report=ProgressReport(actual_value=n / len(files) * 10), diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker.py b/services/storage/src/simcore_service_storage/modules/celery/worker.py index c593c27c4cdf..7c757fd1e6af 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker.py @@ -14,11 +14,12 @@ class CeleryTaskQueueWorker: def __init__(self, celery_app: Celery) -> None: self.celery_app = celery_app - def register_task(self, fn: Callable): - _logger.debug("Registering %s task", fn.__name__) - self.celery_app.task(name=fn.__name__, base=AbortableTask, bind=True)(fn) + def register_task(self, fn: Callable, task_name: str | None = None) -> None: + name = task_name or fn.__name__ + _logger.info("Registering %s task", name) + self.celery_app.task(name=name, base=AbortableTask, bind=True)(fn) - def set_progress( + def set_task_progress( self, task_name: str, task_id: TaskID, report: ProgressReport ) -> None: _logger.debug( From 754c065f553acf8b61d1a255171cf22c11945d59 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 26 Feb 2025 13:22:57 +0100 Subject: [PATCH 052/136] fix util --- .../src/simcore_service_storage/modules/celery/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index 96dcb6e7032e..daa9ab8d7e93 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -16,7 +16,7 @@ def _get_task_id_components(task_id_parts: TaskIDParts) -> list[str]: - return [f"{v}" for _, v in sorted(task_id_parts.items())] + return sorted(map(str, task_id_parts.values())) def _get_components_prefix(name: str, task_id_parts: TaskIDParts) -> list[str]: From 9c9adf02098cc9346690900ded3930f90b9710f9 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 26 Feb 2025 15:29:05 +0100 Subject: [PATCH 053/136] refactor --- .../api/rpc/_data_export.py | 18 ++++++++++--- .../modules/celery/client.py | 26 +++++++------------ 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/services/storage/src/simcore_service_storage/api/rpc/_data_export.py b/services/storage/src/simcore_service_storage/api/rpc/_data_export.py index 644daf245861..0b9392edee66 100644 --- a/services/storage/src/simcore_service_storage/api/rpc/_data_export.py +++ b/services/storage/src/simcore_service_storage/api/rpc/_data_export.py @@ -1,5 +1,3 @@ -from uuid import uuid4 - from fastapi import FastAPI from models_library.api_schemas_rpc_async_jobs.async_jobs import AsyncJobGet, AsyncJobId from models_library.api_schemas_storage.data_export_async_jobs import ( @@ -10,6 +8,9 @@ ) from servicelib.rabbitmq import RPCRouter +from ...modules.celery.client import CeleryTaskQueueClient, TaskIDParts +from ...modules.celery.utils import get_celery_client + router = RPCRouter() @@ -24,7 +25,18 @@ async def start_data_export( app: FastAPI, paths: DataExportTaskStartInput ) -> AsyncJobGet: assert app # nosec + + client: CeleryTaskQueueClient = get_celery_client(app) + + task_id = await client.send_task( + task_name="sync_archive", + task_id_parts=TaskIDParts( + user_id=paths.user_id, product_name=paths.product_name + ), + files=paths.paths, + ) + return AsyncJobGet( - job_id=AsyncJobId(f"{uuid4()}"), + job_id=AsyncJobId(task_id), job_name=", ".join(str(p) for p in paths.paths), ) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index daa9ab8d7e93..ca2da21fa5b3 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -10,28 +10,22 @@ from .models import TaskID, TaskIDParts, TaskStatus -_PREFIX: Final = "ct" - _logger = logging.getLogger(__name__) - -def _get_task_id_components(task_id_parts: TaskIDParts) -> list[str]: - return sorted(map(str, task_id_parts.values())) - - -def _get_components_prefix(name: str, task_id_parts: TaskIDParts) -> list[str]: - return [_PREFIX, name, *_get_task_id_components(task_id_parts)] +_CELERY_TASK_META_PREFIX = "celery-task-meta-" +_PREFIX: Final[str] = "ct" -def _get_task_id_prefix(name: str, task_id_parts: TaskIDParts) -> TaskID: - return "::".join(_get_components_prefix(name, task_id_parts)) +def _build_parts_prefix(name: str, task_id_parts: TaskIDParts) -> list[str]: + return [_PREFIX, name, *[f"{task_id_parts[key]}" for key in sorted(task_id_parts)]] -def _get_task_id(name: str, task_id_parts: TaskIDParts) -> TaskID: - return "::".join([*_get_components_prefix(name, task_id_parts), f"{uuid4()}"]) +def build_task_id_prefix(name: str, task_id_parts: TaskIDParts) -> TaskID: + return "::".join(_build_parts_prefix(name, task_id_parts)) -_CELERY_TASK_META_PREFIX = "celery-task-meta-" +def build_task_id(name: str, task_id_parts: TaskIDParts) -> TaskID: + return "::".join([*_build_parts_prefix(name, task_id_parts), f"{uuid4()}"]) class CeleryTaskQueueClient: @@ -42,7 +36,7 @@ def __init__(self, celery_app: Celery): def send_task( self, task_name: str, *, task_id_parts: TaskIDParts, **task_params ) -> TaskID: - task_id = _get_task_id(task_name, task_id_parts) + task_id = build_task_id(task_name, task_id_parts) _logger.debug("Submitting task %s: %s", task_name, task_id) task = self._celery_app.send_task( task_name, task_id=task_id, kwargs=task_params @@ -84,7 +78,7 @@ def _get_completed_task_ids( ) -> list[TaskID]: search_key = ( _CELERY_TASK_META_PREFIX - + _get_task_id_prefix(task_name, task_id_parts) + + build_task_id_prefix(task_name, task_id_parts) + "*" ) redis = self._celery_app.backend.client From 3810eda247c6c1146f5dc87a40c4c41bf974e9ad Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 26 Feb 2025 16:06:04 +0100 Subject: [PATCH 054/136] update docker --- services/docker-compose.yml | 46 +++++-------------------------------- 1 file changed, 6 insertions(+), 40 deletions(-) diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 0d848396c6a8..b01969864d51 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -1152,7 +1152,7 @@ services: image: ${DOCKER_REGISTRY:-itisfoundation}/storage:${DOCKER_IMAGE_TAG:-latest} init: true hostname: "sto-{{.Node.Hostname}}-{{.Task.Slot}}" - environment: + environment: &storage_environment BF_API_KEY: ${BF_API_KEY} BF_API_SECRET: ${BF_API_SECRET} DATCORE_ADAPTER_HOST: ${DATCORE_ADAPTER_HOST:-datcore-adapter} @@ -1186,53 +1186,19 @@ services: STORAGE_PORT: ${STORAGE_PORT} TRACING_OPENTELEMETRY_COLLECTOR_PORT: ${TRACING_OPENTELEMETRY_COLLECTOR_PORT} TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT: ${TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT} - networks: + networks: &storage_networks - default - interactive_services_subnet - storage_subnet - storage-worker: + sto-worker: image: ${DOCKER_REGISTRY:-itisfoundation}/storage:${DOCKER_IMAGE_TAG:-latest} init: true - hostname: "stow-{{.Node.Hostname}}-{{.Task.Slot}}" + hostname: "sto-worker-{{.Node.Hostname}}-{{.Task.Slot}}" environment: - BF_API_KEY: ${BF_API_KEY} - BF_API_SECRET: ${BF_API_SECRET} - DATCORE_ADAPTER_HOST: ${DATCORE_ADAPTER_HOST:-datcore-adapter} - LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} - LOG_FILTER_MAPPING : ${LOG_FILTER_MAPPING} - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_ENDPOINT: ${POSTGRES_ENDPOINT} - POSTGRES_HOST: ${POSTGRES_HOST} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_USER: ${POSTGRES_USER} - RABBIT_HOST: ${RABBIT_HOST} - RABBIT_PASSWORD: ${RABBIT_PASSWORD} - RABBIT_PORT: ${RABBIT_PORT} - RABBIT_SECURE: ${RABBIT_SECURE} - RABBIT_USER: ${RABBIT_USER} - REDIS_HOST: ${REDIS_HOST} - REDIS_PORT: ${REDIS_PORT} - REDIS_SECURE: ${REDIS_SECURE} - REDIS_USER: ${REDIS_USER} - REDIS_PASSWORD: ${REDIS_PASSWORD} - S3_ACCESS_KEY: ${S3_ACCESS_KEY} - S3_BUCKET_NAME: ${S3_BUCKET_NAME} - S3_ENDPOINT: ${S3_ENDPOINT} - S3_REGION: ${S3_REGION} - S3_SECRET_KEY: ${S3_SECRET_KEY} + <<: *storage_environment STORAGE_WORKER_MODE: 1 - STORAGE_LOGLEVEL: ${STORAGE_LOGLEVEL} - STORAGE_MONITORING_ENABLED: 1 - STORAGE_PROFILING: ${STORAGE_PROFILING} - STORAGE_PORT: ${STORAGE_PORT} - TRACING_OPENTELEMETRY_COLLECTOR_PORT: ${TRACING_OPENTELEMETRY_COLLECTOR_PORT} - TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT: ${TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT} - networks: - - default - - interactive_services_subnet - - storage_subnet + networks: *storage_networks rabbit: image: itisfoundation/rabbitmq:3.11.2-management From e7f3fa27b7916748ef11bc84b285ffc69ac5dd34 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 26 Feb 2025 16:34:16 +0100 Subject: [PATCH 055/136] typecheck --- packages/settings-library/src/settings_library/celery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/settings-library/src/settings_library/celery.py b/packages/settings-library/src/settings_library/celery.py index ce6506126f44..314cc74d6cd5 100644 --- a/packages/settings-library/src/settings_library/celery.py +++ b/packages/settings-library/src/settings_library/celery.py @@ -44,7 +44,7 @@ class CelerySettings(BaseCustomSettings): "REDIS_HOST": "localhost", "REDIS_PORT": 6379, }, - "CELERY_RESULT_EXPIRES": timedelta(days=1), + "CELERY_RESULT_EXPIRES": timedelta(days=1), # type: ignore[dict-item] "CELERY_RESULT_PERSISTENT": True, } ], From dcd9da35487139dd03e250b962002af7024d7271 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 26 Feb 2025 16:40:28 +0100 Subject: [PATCH 056/136] fix typecheck --- .../src/simcore_service_storage/modules/celery/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index ca2da21fa5b3..db35d7565809 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -13,7 +13,7 @@ _logger = logging.getLogger(__name__) _CELERY_TASK_META_PREFIX = "celery-task-meta-" -_PREFIX: Final[str] = "ct" +_PREFIX: Final[str] = "ct" # short for celery task, not Catania def _build_parts_prefix(name: str, task_id_parts: TaskIDParts) -> list[str]: @@ -83,7 +83,7 @@ def _get_completed_task_ids( ) redis = self._celery_app.backend.client if hasattr(redis, "keys") and (keys := redis.keys(search_key)): - return [f"{key}".lstrip(_CELERY_TASK_META_PREFIX) for key in keys] + return [f"{key}".removeprefix(_CELERY_TASK_META_PREFIX) for key in keys] return [] @make_async() From abec2234408598675c7c6b9cb521e9f0e1605d84 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 27 Feb 2025 07:48:58 +0100 Subject: [PATCH 057/136] change list to set --- .../modules/celery/client.py | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index db35d7565809..6290876a446b 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -14,6 +14,12 @@ _CELERY_TASK_META_PREFIX = "celery-task-meta-" _PREFIX: Final[str] = "ct" # short for celery task, not Catania +_CELERY_INSPECT_TASK_STATUSES = ( + "active", + "registered", + "scheduled", + "revoked", +) def _build_parts_prefix(name: str, task_id_parts: TaskIDParts) -> list[str]: @@ -75,7 +81,7 @@ def get_task_status(self, task_id: TaskID) -> TaskStatus: def _get_completed_task_ids( self, task_name: str, task_id_parts: TaskIDParts - ) -> list[TaskID]: + ) -> set[TaskID]: search_key = ( _CELERY_TASK_META_PREFIX + build_task_id_prefix(task_name, task_id_parts) @@ -83,15 +89,19 @@ def _get_completed_task_ids( ) redis = self._celery_app.backend.client if hasattr(redis, "keys") and (keys := redis.keys(search_key)): - return [f"{key}".removeprefix(_CELERY_TASK_META_PREFIX) for key in keys] - return [] + return {f"{key}".removeprefix(_CELERY_TASK_META_PREFIX) for key in keys} + return set() @make_async() - def list_tasks(self, task_name: str, *, task_id_parts: TaskIDParts) -> Any: + def get_task_ids( + self, task_name: str, *, task_id_parts: TaskIDParts + ) -> set[TaskID]: all_task_ids = self._get_completed_task_ids(task_name, task_id_parts) - for task_type in ["active", "registered", "scheduled", "revoked"]: - if task_ids := getattr(self._celery_app.control.inspect(), task_type)(): - all_task_ids.extend(task_ids) + for task_inspect_status in _CELERY_INSPECT_TASK_STATUSES: + if task_ids := getattr( + self._celery_app.control.inspect(), task_inspect_status + )(): + all_task_ids.add(task_ids) return all_task_ids From 0ce1aa480ba351ef735eed73ca6481b4476318dd Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 27 Feb 2025 07:51:44 +0100 Subject: [PATCH 058/136] rename --- .../simcore_service_storage/modules/celery/client.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index 6290876a446b..fbbb03ea94a9 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -12,18 +12,22 @@ _logger = logging.getLogger(__name__) -_CELERY_TASK_META_PREFIX = "celery-task-meta-" -_PREFIX: Final[str] = "ct" # short for celery task, not Catania _CELERY_INSPECT_TASK_STATUSES = ( "active", "registered", "scheduled", "revoked", ) +_CELERY_TASK_META_PREFIX = "celery-task-meta-" +_CELERY_TASK_ID_PREFIX: Final[str] = "ct" # short for celery task, not Catania def _build_parts_prefix(name: str, task_id_parts: TaskIDParts) -> list[str]: - return [_PREFIX, name, *[f"{task_id_parts[key]}" for key in sorted(task_id_parts)]] + return [ + _CELERY_TASK_ID_PREFIX, + name, + *[f"{task_id_parts[key]}" for key in sorted(task_id_parts)], + ] def build_task_id_prefix(name: str, task_id_parts: TaskIDParts) -> TaskID: From ce770461c88326b6e8dd36150dd40d03b769ffb8 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 27 Feb 2025 10:05:15 +0100 Subject: [PATCH 059/136] add task context --- .../api/rpc/_data_export.py | 4 +-- .../modules/celery/client.py | 35 +++++++++---------- .../modules/celery/models.py | 2 +- .../tests/unit/modules/celery/test_celery.py | 12 ++++--- 4 files changed, 26 insertions(+), 27 deletions(-) diff --git a/services/storage/src/simcore_service_storage/api/rpc/_data_export.py b/services/storage/src/simcore_service_storage/api/rpc/_data_export.py index 0b9392edee66..fb706f28fe0f 100644 --- a/services/storage/src/simcore_service_storage/api/rpc/_data_export.py +++ b/services/storage/src/simcore_service_storage/api/rpc/_data_export.py @@ -8,7 +8,7 @@ ) from servicelib.rabbitmq import RPCRouter -from ...modules.celery.client import CeleryTaskQueueClient, TaskIDParts +from ...modules.celery.client import CeleryTaskQueueClient, TaskContext from ...modules.celery.utils import get_celery_client router = RPCRouter() @@ -30,7 +30,7 @@ async def start_data_export( task_id = await client.send_task( task_name="sync_archive", - task_id_parts=TaskIDParts( + task_context=TaskContext( user_id=paths.user_id, product_name=paths.product_name ), files=paths.paths, diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index fbbb03ea94a9..73f166499ae8 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -8,7 +8,7 @@ from models_library.progress_bar import ProgressReport from pydantic import ValidationError -from .models import TaskID, TaskIDParts, TaskStatus +from .models import TaskContext, TaskID, TaskStatus _logger = logging.getLogger(__name__) @@ -22,20 +22,20 @@ _CELERY_TASK_ID_PREFIX: Final[str] = "ct" # short for celery task, not Catania -def _build_parts_prefix(name: str, task_id_parts: TaskIDParts) -> list[str]: +def _build_parts_prefix(name: str, task_context: TaskContext) -> list[str]: return [ _CELERY_TASK_ID_PREFIX, name, - *[f"{task_id_parts[key]}" for key in sorted(task_id_parts)], + *[f"{task_context[key]}" for key in sorted(task_context)], ] -def build_task_id_prefix(name: str, task_id_parts: TaskIDParts) -> TaskID: - return "::".join(_build_parts_prefix(name, task_id_parts)) +def build_task_id_prefix(name: str, task_context: TaskContext) -> TaskID: + return "::".join(_build_parts_prefix(name, task_context)) -def build_task_id(name: str, task_id_parts: TaskIDParts) -> TaskID: - return "::".join([*_build_parts_prefix(name, task_id_parts), f"{uuid4()}"]) +def build_task_id(name: str, task_context: TaskContext, task_uuid: str) -> TaskID: + return "::".join([*_build_parts_prefix(name, task_context), f"{task_uuid}"]) class CeleryTaskQueueClient: @@ -44,14 +44,13 @@ def __init__(self, celery_app: Celery): @make_async() def send_task( - self, task_name: str, *, task_id_parts: TaskIDParts, **task_params + self, task_name: str, *, task_context: TaskContext, **task_params ) -> TaskID: - task_id = build_task_id(task_name, task_id_parts) + task_uuid = f"{uuid4()}" + task_id = build_task_id(task_name, task_context, task_uuid) _logger.debug("Submitting task %s: %s", task_name, task_id) - task = self._celery_app.send_task( - task_name, task_id=task_id, kwargs=task_params - ) - return task.id + self._celery_app.send_task(task_name, task_id=task_id, kwargs=task_params) + return task_uuid @make_async() def get_task(self, task_id: TaskID) -> Any: @@ -84,11 +83,11 @@ def get_task_status(self, task_id: TaskID) -> TaskStatus: ) def _get_completed_task_ids( - self, task_name: str, task_id_parts: TaskIDParts + self, task_name: str, task_context: TaskContext ) -> set[TaskID]: search_key = ( _CELERY_TASK_META_PREFIX - + build_task_id_prefix(task_name, task_id_parts) + + build_task_id_prefix(task_name, task_context) + "*" ) redis = self._celery_app.backend.client @@ -97,10 +96,8 @@ def _get_completed_task_ids( return set() @make_async() - def get_task_ids( - self, task_name: str, *, task_id_parts: TaskIDParts - ) -> set[TaskID]: - all_task_ids = self._get_completed_task_ids(task_name, task_id_parts) + def get_task_ids(self, task_name: str, *, task_context: TaskContext) -> set[TaskID]: + all_task_ids = self._get_completed_task_ids(task_name, task_context) for task_inspect_status in _CELERY_INSPECT_TASK_STATUSES: if task_ids := getattr( diff --git a/services/storage/src/simcore_service_storage/modules/celery/models.py b/services/storage/src/simcore_service_storage/modules/celery/models.py index 35e9957c9990..682fe1e09a6d 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/models.py +++ b/services/storage/src/simcore_service_storage/modules/celery/models.py @@ -4,7 +4,7 @@ from pydantic import BaseModel TaskID: TypeAlias = str -TaskIDParts: TypeAlias = dict[str, Any] +TaskContext: TypeAlias = dict[str, Any] class TaskStatus(BaseModel): diff --git a/services/storage/tests/unit/modules/celery/test_celery.py b/services/storage/tests/unit/modules/celery/test_celery.py index 28f103e2b358..cf9fef02529e 100644 --- a/services/storage/tests/unit/modules/celery/test_celery.py +++ b/services/storage/tests/unit/modules/celery/test_celery.py @@ -8,7 +8,7 @@ from celery.contrib.abortable import AbortableTask from simcore_service_storage.main import CeleryTaskQueueClient from simcore_service_storage.modules.celery.example_tasks import sync_archive -from simcore_service_storage.modules.celery.models import TaskIDParts +from simcore_service_storage.modules.celery.models import TaskContext from tenacity import Retrying, retry_if_exception_type, stop_after_delay, wait_fixed _logger = logging.getLogger(__name__) @@ -46,11 +46,11 @@ def _(celery_app: Celery) -> None: async def test_sumitting_task_calling_async_function_results_with_success_state( celery_task_queue_client: CeleryTaskQueueClient, ): - task_id_parts = TaskIDParts(user_id=1) + task_context = TaskContext(user_id=1, product_name="test") task_id = await celery_task_queue_client.send_task( "sync_archive", - task_id_parts=task_id_parts, + task_context=task_context, files=[f"file{n}" for n in range(30)], ) @@ -73,7 +73,8 @@ async def test_submitting_task_with_failure_results_with_error( celery_task_queue_client: CeleryTaskQueueClient, ): task_id = await celery_task_queue_client.send_task( - "failure_task", task_id_parts=TaskIDParts(user_id=1) + "failure_task", + task_context=TaskContext(user_id=1, product_name="test"), ) for attempt in Retrying( @@ -97,7 +98,8 @@ async def test_aborting_task_results_with_aborted_state( celery_task_queue_client: CeleryTaskQueueClient, ): task_id = await celery_task_queue_client.send_task( - "dreamer_task", task_id_parts=TaskIDParts(user_id=1) + "dreamer_task", + task_context=TaskContext(user_id=1, product_name="test"), ) await celery_task_queue_client.abort_task(task_id) From f28cbc95f4fe3edb7949242bc6fe055f344e5830 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 27 Feb 2025 11:59:15 +0100 Subject: [PATCH 060/136] add rabbit --- services/storage/tests/unit/modules/celery/conftest.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/services/storage/tests/unit/modules/celery/conftest.py b/services/storage/tests/unit/modules/celery/conftest.py index dd3b22fd98fb..be0f325539a3 100644 --- a/services/storage/tests/unit/modules/celery/conftest.py +++ b/services/storage/tests/unit/modules/celery/conftest.py @@ -9,6 +9,7 @@ from fastapi import FastAPI from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict +from simcore_service_storage.core.settings import RabbitSettings from simcore_service_storage.main import celery_app as celery_app_client from simcore_service_storage.modules.celery.client import CeleryTaskQueueClient from simcore_service_storage.modules.celery.worker import CeleryTaskQueueWorker @@ -24,6 +25,7 @@ @pytest.fixture def app_environment( monkeypatch: pytest.MonkeyPatch, + rabbit_service: RabbitSettings, app_environment: EnvVarsDict, ) -> EnvVarsDict: return setenvs_from_dict( @@ -31,6 +33,11 @@ def app_environment( { **app_environment, "SC_BOOT_MODE": "local-development", + "RABBIT_HOST": rabbit_service.RABBIT_HOST, + "RABBIT_PORT": f"{rabbit_service.RABBIT_PORT}", + "RABBIT_USER": rabbit_service.RABBIT_USER, + "RABBIT_SECURE": f"{rabbit_service.RABBIT_SECURE}", + "RABBIT_PASSWORD": rabbit_service.RABBIT_PASSWORD.get_secret_value(), }, ) From ad2c55ca5950b9b3e20b0fb7dae6c9b1feefd4a1 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 27 Feb 2025 12:01:55 +0100 Subject: [PATCH 061/136] add fixture --- .../tests/unit/modules/celery/conftest.py | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/services/storage/tests/unit/modules/celery/conftest.py b/services/storage/tests/unit/modules/celery/conftest.py index be0f325539a3..9cabbfd35b38 100644 --- a/services/storage/tests/unit/modules/celery/conftest.py +++ b/services/storage/tests/unit/modules/celery/conftest.py @@ -1,6 +1,7 @@ from asyncio import AbstractEventLoop from collections.abc import Callable, Iterable from datetime import timedelta +from typing import Any import pytest from celery import Celery @@ -42,13 +43,15 @@ def app_environment( ) -_CELERY_CONF = { - "broker_url": "memory://", - "result_backend": "cache+memory://", - "result_expires": timedelta(days=7), - "result_extended": True, - "pool": "threads", -} +@pytest.mark.fixture +def celery_conf() -> dict[str, Any]: + return { + "broker_url": "memory://", + "result_backend": "cache+memory://", + "result_expires": timedelta(days=7), + "result_extended": True, + "pool": "threads", + } @pytest.fixture @@ -58,8 +61,10 @@ def register_celery_tasks() -> Callable[[Celery], None]: @pytest.fixture -def celery_client_app(app_environment: EnvVarsDict) -> Celery: - celery_app_client.conf.update(_CELERY_CONF) +def celery_client_app( + app_environment: EnvVarsDict, celery_conf: dict[str, Any] +) -> Celery: + celery_app_client.conf.update(celery_conf) assert isinstance(celery_app_client.conf["client"], CeleryTaskQueueClient) assert "worker" not in celery_app_client.conf @@ -72,8 +77,9 @@ def celery_client_app(app_environment: EnvVarsDict) -> Celery: @pytest.fixture def celery_worker( register_celery_tasks: Callable[[Celery], None], + celery_conf: dict[str, Any], ) -> Iterable[TestWorkController]: - celery_app_worker.conf.update(_CELERY_CONF) + celery_app_worker.conf.update(celery_conf) register_celery_tasks(celery_app_worker) From 3da45155bb7e7cf6ab9cac58b3d531feafc1f4be Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 27 Feb 2025 12:56:18 +0100 Subject: [PATCH 062/136] update interface --- .../modules/celery/client.py | 61 +++++++++++-------- .../modules/celery/models.py | 6 +- .../tests/unit/modules/celery/conftest.py | 14 ++--- .../tests/unit/modules/celery/test_celery.py | 43 +++++++------ 4 files changed, 70 insertions(+), 54 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index 73f166499ae8..99d5a99903bd 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -8,7 +8,7 @@ from models_library.progress_bar import ProgressReport from pydantic import ValidationError -from .models import TaskContext, TaskID, TaskStatus +from .models import TaskContext, TaskID, TaskStatus, TaskUUID _logger = logging.getLogger(__name__) @@ -22,20 +22,19 @@ _CELERY_TASK_ID_PREFIX: Final[str] = "ct" # short for celery task, not Catania -def _build_parts_prefix(name: str, task_context: TaskContext) -> list[str]: +def _build_context_prefix(task_context: TaskContext) -> list[str]: return [ _CELERY_TASK_ID_PREFIX, - name, *[f"{task_context[key]}" for key in sorted(task_context)], ] -def build_task_id_prefix(name: str, task_context: TaskContext) -> TaskID: - return "::".join(_build_parts_prefix(name, task_context)) +def _build_task_id_prefix(task_context: TaskContext) -> str: + return "::".join(_build_context_prefix(task_context)) -def build_task_id(name: str, task_context: TaskContext, task_uuid: str) -> TaskID: - return "::".join([*_build_parts_prefix(name, task_context), f"{task_uuid}"]) +def _build_task_id(task_context: TaskContext, task_uuid: TaskUUID) -> TaskID: + return "::".join([_build_task_id_prefix(task_context), f"{task_uuid}"]) class CeleryTaskQueueClient: @@ -45,27 +44,35 @@ def __init__(self, celery_app: Celery): @make_async() def send_task( self, task_name: str, *, task_context: TaskContext, **task_params - ) -> TaskID: - task_uuid = f"{uuid4()}" - task_id = build_task_id(task_name, task_context, task_uuid) + ) -> TaskUUID: + task_uuid = uuid4() + task_id = _build_task_id(task_context, task_uuid) _logger.debug("Submitting task %s: %s", task_name, task_id) self._celery_app.send_task(task_name, task_id=task_id, kwargs=task_params) return task_uuid @make_async() - def get_task(self, task_id: TaskID) -> Any: + def get_task(self, task_context: TaskContext, task_uuid: TaskUUID) -> Any: + task_id = _build_task_id(task_context, task_uuid) return self._celery_app.tasks(task_id) @make_async() - def abort_task(self, task_id: TaskID) -> None: # pylint: disable=R6301 + def abort_task( + self, task_context: TaskContext, task_uuid: TaskUUID + ) -> None: # pylint: disable=R6301 + task_id = _build_task_id(task_context, task_uuid) _logger.info("Aborting task %s", task_id) AbortableAsyncResult(task_id).abort() @make_async() - def get_result(self, task_id: TaskID) -> Any: + def get_result(self, task_context: TaskContext, task_uuid: TaskUUID) -> Any: + task_id = _build_task_id(task_context, task_uuid) return self._celery_app.AsyncResult(task_id).result - def _get_progress_report(self, task_id: TaskID) -> ProgressReport | None: + def _get_progress_report( + self, task_context: TaskContext, task_uuid: TaskUUID + ) -> ProgressReport | None: + task_id = _build_task_id(task_context, task_uuid) result = self._celery_app.AsyncResult(task_id).result if result: try: @@ -75,29 +82,31 @@ def _get_progress_report(self, task_id: TaskID) -> ProgressReport | None: return None @make_async() - def get_task_status(self, task_id: TaskID) -> TaskStatus: + def get_task_status( + self, task_context: TaskContext, task_uuid: TaskUUID + ) -> TaskStatus: + task_id = _build_task_id(task_context, task_uuid) return TaskStatus( - task_id=task_id, + task_uuid=task_uuid, task_state=self._celery_app.AsyncResult(task_id).state, - progress_report=self._get_progress_report(task_id), + progress_report=self._get_progress_report(task_context, task_uuid), ) - def _get_completed_task_ids( - self, task_name: str, task_context: TaskContext - ) -> set[TaskID]: + def _get_completed_task_ids(self, task_context: TaskContext) -> set[TaskUUID]: search_key = ( - _CELERY_TASK_META_PREFIX - + build_task_id_prefix(task_name, task_context) - + "*" + _CELERY_TASK_META_PREFIX + _build_task_id_prefix(task_context) + "*" ) redis = self._celery_app.backend.client if hasattr(redis, "keys") and (keys := redis.keys(search_key)): - return {f"{key}".removeprefix(_CELERY_TASK_META_PREFIX) for key in keys} + return { + TaskUUID(f"{key}".removeprefix(_CELERY_TASK_META_PREFIX)) + for key in keys + } return set() @make_async() - def get_task_ids(self, task_name: str, *, task_context: TaskContext) -> set[TaskID]: - all_task_ids = self._get_completed_task_ids(task_name, task_context) + def get_task_ids(self, task_context: TaskContext) -> set[TaskUUID]: + all_task_ids = self._get_completed_task_ids(task_context) for task_inspect_status in _CELERY_INSPECT_TASK_STATUSES: if task_ids := getattr( diff --git a/services/storage/src/simcore_service_storage/modules/celery/models.py b/services/storage/src/simcore_service_storage/modules/celery/models.py index 682fe1e09a6d..d85e72d22cb3 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/models.py +++ b/services/storage/src/simcore_service_storage/modules/celery/models.py @@ -1,13 +1,15 @@ from typing import Any, TypeAlias +from uuid import UUID from models_library.progress_bar import ProgressReport from pydantic import BaseModel -TaskID: TypeAlias = str TaskContext: TypeAlias = dict[str, Any] +TaskID: TypeAlias = str +TaskUUID: TypeAlias = UUID class TaskStatus(BaseModel): - task_id: str + task_uuid: TaskUUID task_state: str # add enum progress_report: ProgressReport | None = None diff --git a/services/storage/tests/unit/modules/celery/conftest.py b/services/storage/tests/unit/modules/celery/conftest.py index 9cabbfd35b38..f52ee0eef0d1 100644 --- a/services/storage/tests/unit/modules/celery/conftest.py +++ b/services/storage/tests/unit/modules/celery/conftest.py @@ -10,7 +10,6 @@ from fastapi import FastAPI from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict -from simcore_service_storage.core.settings import RabbitSettings from simcore_service_storage.main import celery_app as celery_app_client from simcore_service_storage.modules.celery.client import CeleryTaskQueueClient from simcore_service_storage.modules.celery.worker import CeleryTaskQueueWorker @@ -26,7 +25,6 @@ @pytest.fixture def app_environment( monkeypatch: pytest.MonkeyPatch, - rabbit_service: RabbitSettings, app_environment: EnvVarsDict, ) -> EnvVarsDict: return setenvs_from_dict( @@ -34,16 +32,16 @@ def app_environment( { **app_environment, "SC_BOOT_MODE": "local-development", - "RABBIT_HOST": rabbit_service.RABBIT_HOST, - "RABBIT_PORT": f"{rabbit_service.RABBIT_PORT}", - "RABBIT_USER": rabbit_service.RABBIT_USER, - "RABBIT_SECURE": f"{rabbit_service.RABBIT_SECURE}", - "RABBIT_PASSWORD": rabbit_service.RABBIT_PASSWORD.get_secret_value(), + "RABBIT_HOST": "localhost", + "RABBIT_PORT": "5672", + "RABBIT_USER": "mock", + "RABBIT_SECURE": True, + "RABBIT_PASSWORD": "", }, ) -@pytest.mark.fixture +@pytest.fixture def celery_conf() -> dict[str, Any]: return { "broker_url": "memory://", diff --git a/services/storage/tests/unit/modules/celery/test_celery.py b/services/storage/tests/unit/modules/celery/test_celery.py index cf9fef02529e..2707d5f64c55 100644 --- a/services/storage/tests/unit/modules/celery/test_celery.py +++ b/services/storage/tests/unit/modules/celery/test_celery.py @@ -42,13 +42,16 @@ def _(celery_app: Celery) -> None: return _ +@pytest.fixture +def task_context() -> TaskContext: + return TaskContext(user_id=1, product_name="test") + + @pytest.mark.usefixtures("celery_client_app", "celery_worker_app") async def test_sumitting_task_calling_async_function_results_with_success_state( - celery_task_queue_client: CeleryTaskQueueClient, + celery_task_queue_client: CeleryTaskQueueClient, task_context: TaskContext ): - task_context = TaskContext(user_id=1, product_name="test") - - task_id = await celery_task_queue_client.send_task( + task_uuid = await celery_task_queue_client.send_task( "sync_archive", task_context=task_context, files=[f"file{n}" for n in range(30)], @@ -60,21 +63,23 @@ async def test_sumitting_task_calling_async_function_results_with_success_state( stop=stop_after_delay(30), ): with attempt: - progress = await celery_task_queue_client.get_task_status(task_id) + progress = await celery_task_queue_client.get_task_status( + task_context, task_uuid + ) assert progress.task_state == "SUCCESS" assert ( - await celery_task_queue_client.get_task_status(task_id) + await celery_task_queue_client.get_task_status(task_context, task_uuid) ).task_state == "SUCCESS" @pytest.mark.usefixtures("celery_client_app", "celery_worker_app") async def test_submitting_task_with_failure_results_with_error( celery_task_queue_client: CeleryTaskQueueClient, + task_context: TaskContext, ): - task_id = await celery_task_queue_client.send_task( - "failure_task", - task_context=TaskContext(user_id=1, product_name="test"), + task_uuid = await celery_task_queue_client.send_task( + "failure_task", task_context=task_context ) for attempt in Retrying( @@ -82,27 +87,27 @@ async def test_submitting_task_with_failure_results_with_error( wait=wait_fixed(1), ): with attempt: - result = await celery_task_queue_client.get_result(task_id) + result = await celery_task_queue_client.get_result(task_context, task_uuid) assert isinstance(result, ValueError) assert ( - await celery_task_queue_client.get_task_status(task_id) + await celery_task_queue_client.get_task_status(task_context, task_uuid) ).task_state == "FAILURE" - result = await celery_task_queue_client.get_result(task_id) + result = await celery_task_queue_client.get_result(task_context, task_uuid) assert isinstance(result, ValueError) assert f"{result}" == "my error here" @pytest.mark.usefixtures("celery_client_app", "celery_worker_app") async def test_aborting_task_results_with_aborted_state( - celery_task_queue_client: CeleryTaskQueueClient, + celery_task_queue_client: CeleryTaskQueueClient, task_context: TaskContext ): - task_id = await celery_task_queue_client.send_task( + task_uuid = await celery_task_queue_client.send_task( "dreamer_task", - task_context=TaskContext(user_id=1, product_name="test"), + task_context=task_context, ) - await celery_task_queue_client.abort_task(task_id) + await celery_task_queue_client.abort_task(task_context, task_uuid) for attempt in Retrying( retry=retry_if_exception_type(AssertionError), @@ -110,9 +115,11 @@ async def test_aborting_task_results_with_aborted_state( stop=stop_after_delay(30), ): with attempt: - progress = await celery_task_queue_client.get_task_status(task_id) + progress = await celery_task_queue_client.get_task_status( + task_context, task_uuid + ) assert progress.task_state == "ABORTED" assert ( - await celery_task_queue_client.get_task_status(task_id) + await celery_task_queue_client.get_task_status(task_context, task_uuid) ).task_state == "ABORTED" From 87a457fd5d2cb32f2622c9762fba78e3e99c7443 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 27 Feb 2025 13:03:34 +0100 Subject: [PATCH 063/136] typecheck --- .../src/simcore_service_storage/modules/celery/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index 99d5a99903bd..17fd3bb8e5f9 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -57,9 +57,9 @@ def get_task(self, task_context: TaskContext, task_uuid: TaskUUID) -> Any: return self._celery_app.tasks(task_id) @make_async() - def abort_task( + def abort_task( # pylint: disable=R6301 self, task_context: TaskContext, task_uuid: TaskUUID - ) -> None: # pylint: disable=R6301 + ) -> None: task_id = _build_task_id(task_context, task_uuid) _logger.info("Aborting task %s", task_id) AbortableAsyncResult(task_id).abort() From d0a97fbcbf8a05a23c8a50d9ef3f8e0e762aa8a6 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 27 Feb 2025 13:15:10 +0100 Subject: [PATCH 064/136] adapt code --- .../api_schemas_rpc_async_jobs/async_jobs.py | 2 +- .../api/rpc/_async_jobs.py | 34 ++++++++++++++----- .../api/rpc/_data_export.py | 10 +++--- .../modules/celery/client.py | 2 +- 4 files changed, 33 insertions(+), 15 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_rpc_async_jobs/async_jobs.py b/packages/models-library/src/models_library/api_schemas_rpc_async_jobs/async_jobs.py index 8901e66a7ad5..c89363b1ed70 100644 --- a/packages/models-library/src/models_library/api_schemas_rpc_async_jobs/async_jobs.py +++ b/packages/models-library/src/models_library/api_schemas_rpc_async_jobs/async_jobs.py @@ -13,7 +13,7 @@ class AsyncJobStatus(BaseModel): job_id: AsyncJobId - progress: ProgressReport + progress: ProgressReport | None done: bool started: datetime stopped: datetime | None diff --git a/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py b/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py index c4aed1aa65d8..7e91420840f5 100644 --- a/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py +++ b/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py @@ -1,6 +1,5 @@ # pylint: disable=unused-argument from datetime import datetime -from uuid import uuid4 from fastapi import FastAPI from models_library.api_schemas_rpc_async_jobs.async_jobs import ( @@ -15,9 +14,11 @@ ResultError, StatusError, ) -from models_library.progress_bar import ProgressReport from servicelib.rabbitmq import RPCRouter +from ...modules.celery.models import TaskStatus, TaskUUID +from ...modules.celery.utils import get_celery_client + router = RPCRouter() @@ -36,13 +37,17 @@ async def get_status( ) -> AsyncJobStatus: assert app # nosec assert job_id_data # nosec - progress_report = ProgressReport(actual_value=0.5, total=1.0, attempt=1) + + task_status: TaskStatus = await get_celery_client(app).get_task_status( + task_context=job_id_data.model_dump(), + task_uuid=job_id, + ) return AsyncJobStatus( job_id=job_id, - progress=progress_report, + progress=task_status.progress_report, done=False, - started=datetime.now(), - stopped=None, + started=datetime.now(), # TODO: retrieve that + stopped=None, # TODO: retrieve that ) @@ -53,12 +58,23 @@ async def get_result( assert app # nosec assert job_id # nosec assert job_id_data # nosec - return AsyncJobResult(result="Here's your result.", error=None) + + result = await get_celery_client(app).get_result( + task_context=job_id_data.model_dump(), + task_uuid=job_id, + ) + + return AsyncJobResult(result=result, error=None) @router.expose() async def list_jobs( - app: FastAPI, filter_: str, job_id_data: AsyncJobNameData + app: FastAPI, filter_: str, job_id_data: AsyncJobNameData # TODO: implement filter ) -> list[AsyncJobGet]: assert app # nosec - return [AsyncJobGet(job_id=AsyncJobId(f"{uuid4()}"))] + + task_uuids: set[TaskUUID] = await get_celery_client(app).get_task_uuids( + task_context=job_id_data.model_dump(), + ) + + return [AsyncJobGet(job_id=task_uuid) for task_uuid in task_uuids] diff --git a/services/storage/src/simcore_service_storage/api/rpc/_data_export.py b/services/storage/src/simcore_service_storage/api/rpc/_data_export.py index 7e31bddce38d..26eb82977fec 100644 --- a/services/storage/src/simcore_service_storage/api/rpc/_data_export.py +++ b/services/storage/src/simcore_service_storage/api/rpc/_data_export.py @@ -1,9 +1,6 @@ -from uuid import uuid4 - from fastapi import FastAPI from models_library.api_schemas_rpc_async_jobs.async_jobs import ( AsyncJobGet, - AsyncJobId, AsyncJobNameData, ) from models_library.api_schemas_storage.data_export_async_jobs import ( @@ -17,6 +14,7 @@ from ...datcore_dsm import DatCoreDataManager from ...dsm import get_dsm_provider from ...exceptions.errors import FileAccessRightError +from ...modules.celery.utils import get_celery_client from ...modules.datcore_adapter.datcore_adapter_exceptions import DatcoreAdapterError from ...simcore_s3_dsm import SimcoreS3DataManager @@ -53,6 +51,10 @@ async def start_data_export( location_id=data_export_start.location_id, ) from err + task_uuid = await get_celery_client(app).send_task( + "export_data", task_context=job_id_data.model_dump() + ) + return AsyncJobGet( - job_id=AsyncJobId(f"{uuid4()}"), + job_id=task_uuid, ) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index 17fd3bb8e5f9..1aa86d6d5c8b 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -105,7 +105,7 @@ def _get_completed_task_ids(self, task_context: TaskContext) -> set[TaskUUID]: return set() @make_async() - def get_task_ids(self, task_context: TaskContext) -> set[TaskUUID]: + def get_task_uuids(self, task_context: TaskContext) -> set[TaskUUID]: all_task_ids = self._get_completed_task_ids(task_context) for task_inspect_status in _CELERY_INSPECT_TASK_STATUSES: From 6a0fc4aada4f01b67ef30786814d3e400bc3b559 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 27 Feb 2025 13:30:54 +0100 Subject: [PATCH 065/136] rename --- .../src/simcore_service_storage/modules/celery/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index 1aa86d6d5c8b..5af79866f4d3 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -92,7 +92,7 @@ def get_task_status( progress_report=self._get_progress_report(task_context, task_uuid), ) - def _get_completed_task_ids(self, task_context: TaskContext) -> set[TaskUUID]: + def _get_completed_task_uuids(self, task_context: TaskContext) -> set[TaskUUID]: search_key = ( _CELERY_TASK_META_PREFIX + _build_task_id_prefix(task_context) + "*" ) @@ -106,7 +106,7 @@ def _get_completed_task_ids(self, task_context: TaskContext) -> set[TaskUUID]: @make_async() def get_task_uuids(self, task_context: TaskContext) -> set[TaskUUID]: - all_task_ids = self._get_completed_task_ids(task_context) + all_task_ids = self._get_completed_task_uuids(task_context) for task_inspect_status in _CELERY_INSPECT_TASK_STATUSES: if task_ids := getattr( From 6f070ff1dc9d5856fc91b600f6355b512a8ce872 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 3 Mar 2025 09:38:45 +0100 Subject: [PATCH 066/136] add stop --- services/storage/tests/unit/modules/celery/test_celery.py | 1 + 1 file changed, 1 insertion(+) diff --git a/services/storage/tests/unit/modules/celery/test_celery.py b/services/storage/tests/unit/modules/celery/test_celery.py index 2707d5f64c55..952e81a1f637 100644 --- a/services/storage/tests/unit/modules/celery/test_celery.py +++ b/services/storage/tests/unit/modules/celery/test_celery.py @@ -85,6 +85,7 @@ async def test_submitting_task_with_failure_results_with_error( for attempt in Retrying( retry=retry_if_exception_type(AssertionError), wait=wait_fixed(1), + stop=stop_after_delay(30), ): with attempt: result = await celery_task_queue_client.get_result(task_context, task_uuid) From 5b44bb97e691700b7e37719740386303b2ce183c Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 3 Mar 2025 09:48:10 +0100 Subject: [PATCH 067/136] fix import --- .../src/simcore_service_storage/modules/celery/worker_main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker_main.py b/services/storage/src/simcore_service_storage/modules/celery/worker_main.py index 7f02f5f46e2e..f57065663260 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker_main.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker_main.py @@ -9,10 +9,10 @@ from celery.signals import worker_init, worker_shutdown from servicelib.background_task import cancel_wait_task from servicelib.logging_utils import config_all_loggers -from simcore_service_storage.modules.celery.utils import set_celery_worker from ...core.application import create_app from ...core.settings import ApplicationSettings +from ...modules.celery.utils import set_celery_worker from .common import create_app as create_celery_app from .example_tasks import sync_archive from .worker import CeleryTaskQueueWorker From 8bb905586815b8a8fa7e966873f0269fdbac4660 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 3 Mar 2025 09:51:16 +0100 Subject: [PATCH 068/136] move example --- .../modules/celery/example_tasks.py | 36 ------------------- .../tests/unit/modules/celery/test_celery.py | 34 +++++++++++++++++- 2 files changed, 33 insertions(+), 37 deletions(-) delete mode 100644 services/storage/src/simcore_service_storage/modules/celery/example_tasks.py diff --git a/services/storage/src/simcore_service_storage/modules/celery/example_tasks.py b/services/storage/src/simcore_service_storage/modules/celery/example_tasks.py deleted file mode 100644 index 8b37c708963f..000000000000 --- a/services/storage/src/simcore_service_storage/modules/celery/example_tasks.py +++ /dev/null @@ -1,36 +0,0 @@ -import asyncio -import logging - -from celery import Celery, Task -from models_library.progress_bar import ProgressReport - -from .utils import get_celery_worker, get_event_loop -from .worker import CeleryTaskQueueWorker - -_logger = logging.getLogger(__name__) - - -async def _async_archive( - celery_app: Celery, task_name: str, task_id: str, files: list[str] -) -> str: - worker: CeleryTaskQueueWorker = get_celery_worker(celery_app) - - for n, file in enumerate(files, start=1): - _logger.info("Processing file %s", file) - worker.set_task_progress( - task_name=task_name, - task_id=task_id, - report=ProgressReport(actual_value=n / len(files) * 10), - ) - await asyncio.sleep(0.1) - - return "archive.zip" - - -def sync_archive(task: Task, files: list[str]) -> str: - assert task.name - _logger.info("Calling async_archive") - return asyncio.run_coroutine_threadsafe( - _async_archive(task.app, task.name, task.request.id, files), - get_event_loop(task.app), - ).result() diff --git a/services/storage/tests/unit/modules/celery/test_celery.py b/services/storage/tests/unit/modules/celery/test_celery.py index 952e81a1f637..0022e79accb4 100644 --- a/services/storage/tests/unit/modules/celery/test_celery.py +++ b/services/storage/tests/unit/modules/celery/test_celery.py @@ -1,3 +1,4 @@ +import asyncio import logging import time from collections.abc import Callable @@ -6,14 +7,45 @@ import pytest from celery import Celery, Task from celery.contrib.abortable import AbortableTask +from models_library.progress_bar import ProgressReport from simcore_service_storage.main import CeleryTaskQueueClient -from simcore_service_storage.modules.celery.example_tasks import sync_archive from simcore_service_storage.modules.celery.models import TaskContext +from simcore_service_storage.modules.celery.utils import ( + get_celery_worker, + get_event_loop, +) +from simcore_service_storage.modules.celery.worker import CeleryTaskQueueWorker from tenacity import Retrying, retry_if_exception_type, stop_after_delay, wait_fixed _logger = logging.getLogger(__name__) +async def _async_archive( + celery_app: Celery, task_name: str, task_id: str, files: list[str] +) -> str: + worker: CeleryTaskQueueWorker = get_celery_worker(celery_app) + + for n, file in enumerate(files, start=1): + _logger.info("Processing file %s", file) + worker.set_task_progress( + task_name=task_name, + task_id=task_id, + report=ProgressReport(actual_value=n / len(files) * 10), + ) + await asyncio.sleep(0.1) + + return "archive.zip" + + +def sync_archive(task: Task, files: list[str]) -> str: + assert task.name + _logger.info("Calling async_archive") + return asyncio.run_coroutine_threadsafe( + _async_archive(task.app, task.name, task.request.id, files), + get_event_loop(task.app), + ).result() + + def failure_task(task: Task) -> str: msg = "my error here" raise ValueError(msg) From ea709b3bba974acbcab74909291d76b3f602b894 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 3 Mar 2025 09:54:25 +0100 Subject: [PATCH 069/136] change default --- packages/settings-library/src/settings_library/celery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/settings-library/src/settings_library/celery.py b/packages/settings-library/src/settings_library/celery.py index 314cc74d6cd5..6104e091a21e 100644 --- a/packages/settings-library/src/settings_library/celery.py +++ b/packages/settings-library/src/settings_library/celery.py @@ -27,7 +27,7 @@ class CelerySettings(BaseCustomSettings): Field( description="If set to True, result messages will be persistent (after a broker restart)." ), - ] = False + ] = True model_config = SettingsConfigDict( json_schema_extra={ From f9ecc62d2c769788a684cbe7c6873e288d2341c4 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 3 Mar 2025 09:55:16 +0100 Subject: [PATCH 070/136] update description --- packages/settings-library/src/settings_library/celery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/settings-library/src/settings_library/celery.py b/packages/settings-library/src/settings_library/celery.py index 6104e091a21e..4b98af96f4cf 100644 --- a/packages/settings-library/src/settings_library/celery.py +++ b/packages/settings-library/src/settings_library/celery.py @@ -19,7 +19,7 @@ class CelerySettings(BaseCustomSettings): CELERY_RESULT_EXPIRES: Annotated[ timedelta, Field( - description="Time (in seconds, or a timedelta object) for when after stored task tombstones will be deleted." + description="Time after which task results will be deleted (default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formating)." ), ] = timedelta(days=7) CELERY_RESULT_PERSISTENT: Annotated[ From 2b5f1436f9acac8abd5600d89da5a2ecccc180c6 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 3 Mar 2025 09:57:07 +0100 Subject: [PATCH 071/136] update prefix --- .../src/simcore_service_storage/modules/celery/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index 5af79866f4d3..10add61ab21c 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -19,7 +19,7 @@ "revoked", ) _CELERY_TASK_META_PREFIX = "celery-task-meta-" -_CELERY_TASK_ID_PREFIX: Final[str] = "ct" # short for celery task, not Catania +_CELERY_TASK_ID_PREFIX: Final[str] = "celery" def _build_context_prefix(task_context: TaskContext) -> list[str]: From 872baf7bd01bbfb2ae159564523d7452f1ba1565 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 3 Mar 2025 09:58:18 +0100 Subject: [PATCH 072/136] add types --- .../src/simcore_service_storage/modules/celery/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index 10add61ab21c..cb122bb6e344 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -12,13 +12,13 @@ _logger = logging.getLogger(__name__) -_CELERY_INSPECT_TASK_STATUSES = ( +_CELERY_INSPECT_TASK_STATUSES: Final[tuple[str, ...]] = ( "active", "registered", "scheduled", "revoked", ) -_CELERY_TASK_META_PREFIX = "celery-task-meta-" +_CELERY_TASK_META_PREFIX: Final[str] = "celery-task-meta-" _CELERY_TASK_ID_PREFIX: Final[str] = "celery" From 320095dde0b420c940b191c413163a829948c228 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 3 Mar 2025 10:00:20 +0100 Subject: [PATCH 073/136] refactor --- .../src/simcore_service_storage/api/rpc/_async_jobs.py | 2 +- .../src/simcore_service_storage/api/rpc/_data_export.py | 2 +- services/storage/src/simcore_service_storage/main.py | 4 ++-- .../modules/celery/{common.py => _common.py} | 0 .../modules/celery/{utils.py => _utils.py} | 0 .../src/simcore_service_storage/modules/celery/worker_main.py | 4 ++-- services/storage/tests/unit/modules/celery/test_celery.py | 4 ++-- 7 files changed, 8 insertions(+), 8 deletions(-) rename services/storage/src/simcore_service_storage/modules/celery/{common.py => _common.py} (100%) rename services/storage/src/simcore_service_storage/modules/celery/{utils.py => _utils.py} (100%) diff --git a/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py b/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py index 7e91420840f5..c28361fc36ae 100644 --- a/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py +++ b/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py @@ -16,8 +16,8 @@ ) from servicelib.rabbitmq import RPCRouter +from ...modules.celery._utils import get_celery_client from ...modules.celery.models import TaskStatus, TaskUUID -from ...modules.celery.utils import get_celery_client router = RPCRouter() diff --git a/services/storage/src/simcore_service_storage/api/rpc/_data_export.py b/services/storage/src/simcore_service_storage/api/rpc/_data_export.py index 26eb82977fec..8fe790624b68 100644 --- a/services/storage/src/simcore_service_storage/api/rpc/_data_export.py +++ b/services/storage/src/simcore_service_storage/api/rpc/_data_export.py @@ -14,7 +14,7 @@ from ...datcore_dsm import DatCoreDataManager from ...dsm import get_dsm_provider from ...exceptions.errors import FileAccessRightError -from ...modules.celery.utils import get_celery_client +from ...modules.celery._utils import get_celery_client from ...modules.datcore_adapter.datcore_adapter_exceptions import DatcoreAdapterError from ...simcore_s3_dsm import SimcoreS3DataManager diff --git a/services/storage/src/simcore_service_storage/main.py b/services/storage/src/simcore_service_storage/main.py index 80f522708edd..a971d3501397 100644 --- a/services/storage/src/simcore_service_storage/main.py +++ b/services/storage/src/simcore_service_storage/main.py @@ -3,15 +3,15 @@ import logging from servicelib.logging_utils import config_all_loggers -from simcore_service_storage.modules.celery.utils import ( +from simcore_service_storage.modules.celery._utils import ( set_celery_app, set_celery_client, ) from .core.application import create_app from .core.settings import ApplicationSettings +from .modules.celery._common import create_app as create_celery_app from .modules.celery.client import CeleryTaskQueueClient -from .modules.celery.common import create_app as create_celery_app _settings = ApplicationSettings.create_from_envs() diff --git a/services/storage/src/simcore_service_storage/modules/celery/common.py b/services/storage/src/simcore_service_storage/modules/celery/_common.py similarity index 100% rename from services/storage/src/simcore_service_storage/modules/celery/common.py rename to services/storage/src/simcore_service_storage/modules/celery/_common.py diff --git a/services/storage/src/simcore_service_storage/modules/celery/utils.py b/services/storage/src/simcore_service_storage/modules/celery/_utils.py similarity index 100% rename from services/storage/src/simcore_service_storage/modules/celery/utils.py rename to services/storage/src/simcore_service_storage/modules/celery/_utils.py diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker_main.py b/services/storage/src/simcore_service_storage/modules/celery/worker_main.py index f57065663260..82ebab928e42 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker_main.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker_main.py @@ -12,8 +12,8 @@ from ...core.application import create_app from ...core.settings import ApplicationSettings -from ...modules.celery.utils import set_celery_worker -from .common import create_app as create_celery_app +from ._common import create_app as create_celery_app +from ._utils import set_celery_worker from .example_tasks import sync_archive from .worker import CeleryTaskQueueWorker diff --git a/services/storage/tests/unit/modules/celery/test_celery.py b/services/storage/tests/unit/modules/celery/test_celery.py index 0022e79accb4..2528d1f4771b 100644 --- a/services/storage/tests/unit/modules/celery/test_celery.py +++ b/services/storage/tests/unit/modules/celery/test_celery.py @@ -9,11 +9,11 @@ from celery.contrib.abortable import AbortableTask from models_library.progress_bar import ProgressReport from simcore_service_storage.main import CeleryTaskQueueClient -from simcore_service_storage.modules.celery.models import TaskContext -from simcore_service_storage.modules.celery.utils import ( +from simcore_service_storage.modules.celery._utils import ( get_celery_worker, get_event_loop, ) +from simcore_service_storage.modules.celery.models import TaskContext from simcore_service_storage.modules.celery.worker import CeleryTaskQueueWorker from tenacity import Retrying, retry_if_exception_type, stop_after_delay, wait_fixed From 319570e788b9d466fd946d7c005e8dd77ad4932e Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 3 Mar 2025 10:06:35 +0100 Subject: [PATCH 074/136] use suppress --- .../src/simcore_service_storage/modules/celery/client.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index cb122bb6e344..ba8b43401adf 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -1,3 +1,4 @@ +import contextlib import logging from typing import Any, Final from uuid import uuid4 @@ -75,10 +76,8 @@ def _get_progress_report( task_id = _build_task_id(task_context, task_uuid) result = self._celery_app.AsyncResult(task_id).result if result: - try: + with contextlib.suppress(ValidationError): return ProgressReport.model_validate(result) - except ValidationError: - pass return None @make_async() From 432f940fa8cddc19feb86273c2c962c4a1a17d46 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 3 Mar 2025 10:15:34 +0100 Subject: [PATCH 075/136] use longnames --- services/storage/docker/healthcheck.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/storage/docker/healthcheck.py b/services/storage/docker/healthcheck.py index 8321aa75d2df..9cb4f2421d7a 100755 --- a/services/storage/docker/healthcheck.py +++ b/services/storage/docker/healthcheck.py @@ -39,17 +39,17 @@ def _is_celery_worker_healthy(): app_settings = ApplicationSettings.create_from_envs() assert app_settings.STORAGE_CELERY - broker_url = app_settings.STORAGE_CELERY.CELERY_BROKER.dsn + broker_url = app_settings.STORAGE_CELERY.CELERY_RABBIT_BROKER.dsn try: result = subprocess.run( [ "celery", - "-b", + "--broker", broker_url, "inspect", "ping", - "-d", + "--destination", "celery@" + os.getenv("HOSTNAME", ""), ], capture_output=True, From 966baf6e383dbe063b2b25c6a90cd540e80abece Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 3 Mar 2025 10:15:49 +0100 Subject: [PATCH 076/136] change var names --- packages/settings-library/src/settings_library/celery.py | 8 ++++---- .../src/simcore_service_storage/modules/celery/_common.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/settings-library/src/settings_library/celery.py b/packages/settings-library/src/settings_library/celery.py index 4b98af96f4cf..9259b574047b 100644 --- a/packages/settings-library/src/settings_library/celery.py +++ b/packages/settings-library/src/settings_library/celery.py @@ -10,10 +10,10 @@ class CelerySettings(BaseCustomSettings): - CELERY_BROKER: Annotated[ + CELERY_RABBIT_BROKER: Annotated[ RabbitSettings, Field(json_schema_extra={"auto_default_from_env": True}) ] - CELERY_RESULT_BACKEND: Annotated[ + CELERY_REDIS_RESULT_BACKEND: Annotated[ RedisSettings, Field(json_schema_extra={"auto_default_from_env": True}) ] CELERY_RESULT_EXPIRES: Annotated[ @@ -33,14 +33,14 @@ class CelerySettings(BaseCustomSettings): json_schema_extra={ "examples": [ { - "CELERY_BROKER": { + "CELERY_RABBIT_BROKER": { "RABBIT_USER": "guest", "RABBIT_SECURE": False, "RABBIT_PASSWORD": "guest", "RABBIT_HOST": "localhost", "RABBIT_PORT": 5672, }, - "CELERY_RESULT_BACKEND": { + "CELERY_REDIS_RESULT_BACKEND": { "REDIS_HOST": "localhost", "REDIS_PORT": 6379, }, diff --git a/services/storage/src/simcore_service_storage/modules/celery/_common.py b/services/storage/src/simcore_service_storage/modules/celery/_common.py index c9860123d626..38a2826895f1 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/_common.py +++ b/services/storage/src/simcore_service_storage/modules/celery/_common.py @@ -13,8 +13,8 @@ def create_app(settings: ApplicationSettings) -> Celery: assert celery_settings app = Celery( - broker=celery_settings.CELERY_BROKER.dsn, - backend=celery_settings.CELERY_RESULT_BACKEND.build_redis_dsn( + broker=celery_settings.CELERY_RABBIT_BROKER.dsn, + backend=celery_settings.CELERY_REDIS_RESULT_BACKEND.build_redis_dsn( RedisDatabase.CELERY_TASKS, ), ) From 4a2cada2bdf5fbf934433b0e660197e81f696587 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 3 Mar 2025 10:19:18 +0100 Subject: [PATCH 077/136] fix settings --- .../storage/src/simcore_service_storage/core/application.py | 3 ++- services/storage/src/simcore_service_storage/core/settings.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/services/storage/src/simcore_service_storage/core/application.py b/services/storage/src/simcore_service_storage/core/application.py index 4f68c350945f..297e7d0b1390 100644 --- a/services/storage/src/simcore_service_storage/core/application.py +++ b/services/storage/src/simcore_service_storage/core/application.py @@ -88,7 +88,8 @@ def create_app(settings: ApplicationSettings) -> FastAPI: setup_rest_api_routes(app, API_VTAG) set_exception_handlers(app) - setup_redis(app) + if settings.STORAGE_WORKER_MODE: + setup_redis(app) setup_dsm(app) if settings.STORAGE_CLEANER_INTERVAL_S: diff --git a/services/storage/src/simcore_service_storage/core/settings.py b/services/storage/src/simcore_service_storage/core/settings.py index 43279642c8c3..df720366aa7f 100644 --- a/services/storage/src/simcore_service_storage/core/settings.py +++ b/services/storage/src/simcore_service_storage/core/settings.py @@ -121,7 +121,7 @@ class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): ] STORAGE_WORKER_MODE: Annotated[ - bool | None, Field(description="If True, run as a worker") + bool, Field(description="If True, run as a worker") ] = False @field_validator("LOG_LEVEL", mode="before") From ce4c2fa8e38a8211a4ac3a9978d312278e7a3a99 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 3 Mar 2025 10:30:59 +0100 Subject: [PATCH 078/136] log_context --- .../modules/celery/worker.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker.py b/services/storage/src/simcore_service_storage/modules/celery/worker.py index 7c757fd1e6af..d46a2a94347c 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker.py @@ -4,6 +4,7 @@ from celery import Celery from celery.contrib.abortable import AbortableTask from models_library.progress_bar import ProgressReport +from servicelib.logging_utils import log_context from .models import TaskID @@ -22,11 +23,13 @@ def register_task(self, fn: Callable, task_name: str | None = None) -> None: def set_task_progress( self, task_name: str, task_id: TaskID, report: ProgressReport ) -> None: - _logger.debug( - "Setting progress for %s: %s", task_name, report.model_dump_json() - ) - self.celery_app.tasks[task_name].update_state( - task_id=task_id, - state="PROGRESS", - meta=report.model_dump(mode="json"), - ) + with log_context( + _logger, + logging.DEBUG, + msg=f"Setting progress for {task_name}: {report.model_dump_json()}", + ): + self.celery_app.tasks[task_name].update_state( + task_id=task_id, + state="PROGRESS", + meta=report.model_dump(mode="json"), + ) From 7ce7d6f904bd5fe1381cad5e5a53157ee52c4211 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 3 Mar 2025 10:34:45 +0100 Subject: [PATCH 079/136] remove example --- .../src/simcore_service_storage/modules/celery/worker_main.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker_main.py b/services/storage/src/simcore_service_storage/modules/celery/worker_main.py index 82ebab928e42..4c699c512752 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker_main.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker_main.py @@ -14,7 +14,6 @@ from ...core.settings import ApplicationSettings from ._common import create_app as create_celery_app from ._utils import set_celery_worker -from .example_tasks import sync_archive from .worker import CeleryTaskQueueWorker _settings = ApplicationSettings.create_from_envs() @@ -82,7 +81,6 @@ async def shutdown(): celery_app = create_celery_app(_settings) celery_worker = CeleryTaskQueueWorker(celery_app) -celery_worker.register_task(sync_archive) set_celery_worker(celery_app, CeleryTaskQueueWorker(celery_app)) app = celery_app From d1714fdfdcbce67b76e05b0fa3016a02772d81a7 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 3 Mar 2025 10:43:01 +0100 Subject: [PATCH 080/136] update docker compose --- services/docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/docker-compose.yml b/services/docker-compose.yml index b01969864d51..f0967e1996f4 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -1179,7 +1179,7 @@ services: S3_ENDPOINT: ${S3_ENDPOINT} S3_REGION: ${S3_REGION} S3_SECRET_KEY: ${S3_SECRET_KEY} - STORAGE_WORKER_MODE: 0 + STORAGE_WORKER_MODE: "false" STORAGE_LOGLEVEL: ${STORAGE_LOGLEVEL} STORAGE_MONITORING_ENABLED: 1 STORAGE_PROFILING: ${STORAGE_PROFILING} @@ -1192,12 +1192,12 @@ services: - storage_subnet sto-worker: - image: ${DOCKER_REGISTRY:-itisfoundation}/storage:${DOCKER_IMAGE_TAG:-latest} + image: ${DOCKER_REGISTRY:-itisfoundation}/storage:${DOCKER_IMAGE_TAG:-master-github-latest} init: true hostname: "sto-worker-{{.Node.Hostname}}-{{.Task.Slot}}" environment: <<: *storage_environment - STORAGE_WORKER_MODE: 1 + STORAGE_WORKER_MODE: "true" networks: *storage_networks rabbit: From 67ec039ba03cf42319c9a8dfd8c01e4f91c69a05 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 3 Mar 2025 10:51:38 +0100 Subject: [PATCH 081/136] remove typehint --- .../src/simcore_service_storage/api/rpc/_async_jobs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py b/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py index c28361fc36ae..fb0d82a31cf0 100644 --- a/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py +++ b/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py @@ -17,7 +17,7 @@ from servicelib.rabbitmq import RPCRouter from ...modules.celery._utils import get_celery_client -from ...modules.celery.models import TaskStatus, TaskUUID +from ...modules.celery.models import TaskStatus router = RPCRouter() @@ -73,7 +73,7 @@ async def list_jobs( ) -> list[AsyncJobGet]: assert app # nosec - task_uuids: set[TaskUUID] = await get_celery_client(app).get_task_uuids( + task_uuids = await get_celery_client(app).get_task_uuids( task_context=job_id_data.model_dump(), ) From 38967942064ff3c024d57e28a6e1552d8ad9a1e8 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 3 Mar 2025 10:57:29 +0100 Subject: [PATCH 082/136] add log --- .../src/simcore_service_storage/modules/celery/worker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker.py b/services/storage/src/simcore_service_storage/modules/celery/worker.py index d46a2a94347c..346b4574a21c 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker.py @@ -17,8 +17,8 @@ def __init__(self, celery_app: Celery) -> None: def register_task(self, fn: Callable, task_name: str | None = None) -> None: name = task_name or fn.__name__ - _logger.info("Registering %s task", name) - self.celery_app.task(name=name, base=AbortableTask, bind=True)(fn) + with log_context(_logger, logging.INFO, msg=f"Registering {name} task"): + self.celery_app.task(name=name, base=AbortableTask, bind=True)(fn) def set_task_progress( self, task_name: str, task_id: TaskID, report: ProgressReport From 4aeb1c9912953238561e950f8d37ec4ad13f2c0a Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 3 Mar 2025 11:00:42 +0100 Subject: [PATCH 083/136] add log --- .../simcore_service_storage/modules/celery/client.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index ba8b43401adf..d9f9301d2547 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -8,6 +8,7 @@ from common_library.async_tools import make_async from models_library.progress_bar import ProgressReport from pydantic import ValidationError +from servicelib.logging_utils import log_context from .models import TaskContext, TaskID, TaskStatus, TaskUUID @@ -48,9 +49,13 @@ def send_task( ) -> TaskUUID: task_uuid = uuid4() task_id = _build_task_id(task_context, task_uuid) - _logger.debug("Submitting task %s: %s", task_name, task_id) - self._celery_app.send_task(task_name, task_id=task_id, kwargs=task_params) - return task_uuid + with log_context( + _logger, + logging.DEBUG, + msg=f"Submitting task {task_name}: {task_id=} {task_params=}", + ): + self._celery_app.send_task(task_name, task_id=task_id, kwargs=task_params) + return task_uuid @make_async() def get_task(self, task_context: TaskContext, task_uuid: TaskUUID) -> Any: From 558cd5de8badf49a4d1e427102cee86b2c983dae Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 3 Mar 2025 11:17:36 +0100 Subject: [PATCH 084/136] remove comment --- .../src/simcore_service_storage/modules/celery/worker_main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker_main.py b/services/storage/src/simcore_service_storage/modules/celery/worker_main.py index 4c699c512752..2d0aa8b0d4b9 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker_main.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker_main.py @@ -18,7 +18,6 @@ _settings = ApplicationSettings.create_from_envs() -# SEE https://github.com/ITISFoundation/osparc-simcore/issues/3148 logging.basicConfig(level=_settings.log_level) # NOSONAR logging.root.setLevel(_settings.log_level) config_all_loggers( From 11074353eed7082db68e2a382c2fcdd96e5d8eb5 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 3 Mar 2025 11:43:29 +0100 Subject: [PATCH 085/136] add task params --- .../src/simcore_service_storage/api/rpc/_data_export.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/services/storage/src/simcore_service_storage/api/rpc/_data_export.py b/services/storage/src/simcore_service_storage/api/rpc/_data_export.py index 8fe790624b68..8f2f23043ca0 100644 --- a/services/storage/src/simcore_service_storage/api/rpc/_data_export.py +++ b/services/storage/src/simcore_service_storage/api/rpc/_data_export.py @@ -52,7 +52,9 @@ async def start_data_export( ) from err task_uuid = await get_celery_client(app).send_task( - "export_data", task_context=job_id_data.model_dump() + "export_data", + task_context=job_id_data.model_dump(), + files=data_export_start.file_and_folder_ids, # ANE: adapt here your signature ) return AsyncJobGet( From dd7f2fa64f8c0d90c345f4aa479334b6a921b1e0 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 3 Mar 2025 12:22:29 +0100 Subject: [PATCH 086/136] track started tasks --- .../src/simcore_service_storage/modules/celery/_common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/services/storage/src/simcore_service_storage/modules/celery/_common.py b/services/storage/src/simcore_service_storage/modules/celery/_common.py index 38a2826895f1..5d20716fff44 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/_common.py +++ b/services/storage/src/simcore_service_storage/modules/celery/_common.py @@ -20,4 +20,5 @@ def create_app(settings: ApplicationSettings) -> Celery: ) app.conf.result_expires = celery_settings.CELERY_RESULT_EXPIRES app.conf.result_extended = True # original args are included in the results + app.conf.task_track_started = True return app From 4b4d5a2fa427738b556ef9318dfdc2a6b3f274cd Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 3 Mar 2025 12:27:32 +0100 Subject: [PATCH 087/136] add enums --- .../simcore_service_storage/modules/celery/models.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/models.py b/services/storage/src/simcore_service_storage/modules/celery/models.py index d85e72d22cb3..75a63a9b7808 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/models.py +++ b/services/storage/src/simcore_service_storage/modules/celery/models.py @@ -1,3 +1,4 @@ +from enum import StrEnum, auto from typing import Any, TypeAlias from uuid import UUID @@ -9,7 +10,15 @@ TaskUUID: TypeAlias = UUID +class TaskState(StrEnum): + PENDING = auto() + STARTED = auto() + SUCCESS = auto() + FAILURE = auto() + ABORTED = auto() + + class TaskStatus(BaseModel): task_uuid: TaskUUID - task_state: str # add enum + task_state: TaskState progress_report: ProgressReport | None = None From 50550b4abbf220bcdac52d56d88a6556eb594ed9 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 3 Mar 2025 13:32:40 +0100 Subject: [PATCH 088/136] add redis on startup --- .../storage/src/simcore_service_storage/core/application.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/services/storage/src/simcore_service_storage/core/application.py b/services/storage/src/simcore_service_storage/core/application.py index 297e7d0b1390..4f68c350945f 100644 --- a/services/storage/src/simcore_service_storage/core/application.py +++ b/services/storage/src/simcore_service_storage/core/application.py @@ -88,8 +88,7 @@ def create_app(settings: ApplicationSettings) -> FastAPI: setup_rest_api_routes(app, API_VTAG) set_exception_handlers(app) - if settings.STORAGE_WORKER_MODE: - setup_redis(app) + setup_redis(app) setup_dsm(app) if settings.STORAGE_CLEANER_INTERVAL_S: From aa08b2ab042836347540a6508d33607a2c378295 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 3 Mar 2025 13:46:57 +0100 Subject: [PATCH 089/136] add enum --- .../src/simcore_service_storage/modules/celery/models.py | 1 + .../src/simcore_service_storage/modules/celery/worker.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/models.py b/services/storage/src/simcore_service_storage/modules/celery/models.py index 75a63a9b7808..112ce10b02f9 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/models.py +++ b/services/storage/src/simcore_service_storage/modules/celery/models.py @@ -13,6 +13,7 @@ class TaskState(StrEnum): PENDING = auto() STARTED = auto() + PROGRESS = auto() SUCCESS = auto() FAILURE = auto() ABORTED = auto() diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker.py b/services/storage/src/simcore_service_storage/modules/celery/worker.py index 346b4574a21c..6a44007a5264 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker.py @@ -6,7 +6,7 @@ from models_library.progress_bar import ProgressReport from servicelib.logging_utils import log_context -from .models import TaskID +from .models import TaskID, TaskState _logger = logging.getLogger(__name__) @@ -30,6 +30,6 @@ def set_task_progress( ): self.celery_app.tasks[task_name].update_state( task_id=task_id, - state="PROGRESS", + state=TaskState.PROGRESS.value, meta=report.model_dump(mode="json"), ) From 363b15d38b61918b7ce01e89cce3161364286360 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 3 Mar 2025 14:44:11 +0100 Subject: [PATCH 090/136] fix worker startup --- services/storage/docker/boot.sh | 4 ++-- .../simcore_service_storage/modules/celery/_utils.py | 12 ++++++++---- .../modules/celery/worker_main.py | 11 ++--------- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/services/storage/docker/boot.sh b/services/storage/docker/boot.sh index a8ee42344bf4..fda30e846c19 100755 --- a/services/storage/docker/boot.sh +++ b/services/storage/docker/boot.sh @@ -56,10 +56,10 @@ if [ "${SC_BOOT_MODE}" = "debug" ]; then --log-level \"${SERVER_LOG_LEVEL}\" " else - if [ "${STORAGE_WORKER_MODE}" = "1" ]; then + if [ "${STORAGE_WORKER_MODE}" = "true" ]; then exec celery \ --app=simcore_service_storage.modules.celery.worker_main:app \ - worker --pool=threads \ + worker \ --loglevel="${SERVER_LOG_LEVEL}" \ --hostname="${HOSTNAME}" else diff --git a/services/storage/src/simcore_service_storage/modules/celery/_utils.py b/services/storage/src/simcore_service_storage/modules/celery/_utils.py index 134e663eabd4..37466adaab1a 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/_utils.py +++ b/services/storage/src/simcore_service_storage/modules/celery/_utils.py @@ -3,6 +3,8 @@ from celery import Celery from fastapi import FastAPI +from ...core.settings import ApplicationSettings +from ._common import create_app from .client import CeleryTaskQueueClient from .worker import CeleryTaskQueueWorker @@ -11,6 +13,12 @@ _EVENT_LOOP_KEY = "loop" +def create_celery_app_worker(settings: ApplicationSettings) -> Celery: + celery_app = create_app(settings) + celery_app.conf[_WORKER_KEY] = CeleryTaskQueueWorker(celery_app) + return celery_app + + def get_celery_app(fastapi: FastAPI) -> Celery: celery = fastapi.state.celery_app assert isinstance(celery, Celery) @@ -41,10 +49,6 @@ def get_celery_worker(celery_app: Celery) -> CeleryTaskQueueWorker: return worker -def set_celery_worker(celery_app: Celery, celery_worker: CeleryTaskQueueWorker) -> None: - celery_app.conf[_WORKER_KEY] = celery_worker - - def get_event_loop(celery_app: Celery) -> AbstractEventLoop: # nosec loop = celery_app.conf[_EVENT_LOOP_KEY] assert isinstance(loop, AbstractEventLoop) diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker_main.py b/services/storage/src/simcore_service_storage/modules/celery/worker_main.py index 2d0aa8b0d4b9..ca66b66ab440 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker_main.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker_main.py @@ -12,9 +12,7 @@ from ...core.application import create_app from ...core.settings import ApplicationSettings -from ._common import create_app as create_celery_app -from ._utils import set_celery_worker -from .worker import CeleryTaskQueueWorker +from ._utils import create_celery_app_worker _settings = ApplicationSettings.create_from_envs() @@ -77,9 +75,4 @@ async def shutdown(): asyncio.run_coroutine_threadsafe(shutdown(), loop) -celery_app = create_celery_app(_settings) - -celery_worker = CeleryTaskQueueWorker(celery_app) -set_celery_worker(celery_app, CeleryTaskQueueWorker(celery_app)) - -app = celery_app +app = create_celery_app_worker(_settings) From 0fdd564d377d679b1c7fed38469dd1011728f90d Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 3 Mar 2025 16:50:55 +0100 Subject: [PATCH 091/136] fix state --- .../src/simcore_service_storage/modules/celery/worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker.py b/services/storage/src/simcore_service_storage/modules/celery/worker.py index 6a44007a5264..b963d187f5f1 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker.py @@ -30,6 +30,6 @@ def set_task_progress( ): self.celery_app.tasks[task_name].update_state( task_id=task_id, - state=TaskState.PROGRESS.value, + state=TaskState.RUNNING.value, meta=report.model_dump(mode="json"), ) From 46ac5df1516be8e7106fff49d03e7fa399b57fc2 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 4 Mar 2025 09:31:55 +0100 Subject: [PATCH 092/136] continue --- services/storage/docker/boot.sh | 5 +++-- .../modules/celery/client.py | 15 +++++++++++---- .../modules/celery/models.py | 5 ++--- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/services/storage/docker/boot.sh b/services/storage/docker/boot.sh index fda30e846c19..0fd1d2b4edcf 100755 --- a/services/storage/docker/boot.sh +++ b/services/storage/docker/boot.sh @@ -59,9 +59,10 @@ else if [ "${STORAGE_WORKER_MODE}" = "true" ]; then exec celery \ --app=simcore_service_storage.modules.celery.worker_main:app \ - worker \ + worker --pool=threads \ --loglevel="${SERVER_LOG_LEVEL}" \ - --hostname="${HOSTNAME}" + --hostname="${HOSTNAME}" \ + --concurrency="${CELERY_CONCURRENCY}" else exec uvicorn simcore_service_storage.main:app \ --host 0.0.0.0 \ diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index d9f9301d2547..3adb308cc942 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -10,7 +10,7 @@ from pydantic import ValidationError from servicelib.logging_utils import log_context -from .models import TaskContext, TaskID, TaskStatus, TaskUUID +from .models import TaskContext, TaskID, TaskState, TaskStatus, TaskUUID _logger = logging.getLogger(__name__) @@ -77,13 +77,20 @@ def get_result(self, task_context: TaskContext, task_uuid: TaskUUID) -> Any: def _get_progress_report( self, task_context: TaskContext, task_uuid: TaskUUID - ) -> ProgressReport | None: + ) -> ProgressReport: task_id = _build_task_id(task_context, task_uuid) result = self._celery_app.AsyncResult(task_id).result - if result: + state = self._celery_app.AsyncResult(task_id).state + if result and state == TaskState.RUNNING.value: with contextlib.suppress(ValidationError): return ProgressReport.model_validate(result) - return None + if state in ( + TaskState.ABORTED.value, + TaskState.FAILURE.value, + TaskState.SUCCESS.value, + ): + return ProgressReport(actual_value=100.0) + return ProgressReport(actual_value=0.0) @make_async() def get_task_status( diff --git a/services/storage/src/simcore_service_storage/modules/celery/models.py b/services/storage/src/simcore_service_storage/modules/celery/models.py index 112ce10b02f9..af82cad506be 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/models.py +++ b/services/storage/src/simcore_service_storage/modules/celery/models.py @@ -12,8 +12,7 @@ class TaskState(StrEnum): PENDING = auto() - STARTED = auto() - PROGRESS = auto() + RUNNING = auto() SUCCESS = auto() FAILURE = auto() ABORTED = auto() @@ -22,4 +21,4 @@ class TaskState(StrEnum): class TaskStatus(BaseModel): task_uuid: TaskUUID task_state: TaskState - progress_report: ProgressReport | None = None + progress_report: ProgressReport From 5d7115b61116ab054192c027cc00f35b43fdd7f4 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 4 Mar 2025 09:50:57 +0100 Subject: [PATCH 093/136] add validation --- .../modules/celery/models.py | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/models.py b/services/storage/src/simcore_service_storage/modules/celery/models.py index af82cad506be..feb67d2d25a1 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/models.py +++ b/services/storage/src/simcore_service_storage/modules/celery/models.py @@ -1,14 +1,17 @@ from enum import StrEnum, auto -from typing import Any, TypeAlias +from typing import Any, Final, Self, TypeAlias from uuid import UUID from models_library.progress_bar import ProgressReport -from pydantic import BaseModel +from pydantic import BaseModel, model_validator TaskContext: TypeAlias = dict[str, Any] TaskID: TypeAlias = str TaskUUID: TypeAlias = UUID +_MIN_PROGRESS: Final[float] = 0.0 +_MAX_PROGRESS: Final[float] = 100.0 + class TaskState(StrEnum): PENDING = auto() @@ -22,3 +25,21 @@ class TaskStatus(BaseModel): task_uuid: TaskUUID task_state: TaskState progress_report: ProgressReport + + @model_validator(mode="after") + def _check_consistency(self) -> Self: + value = self.progress_report.actual_value + + valid_states = { + TaskState.PENDING: value == _MIN_PROGRESS, + TaskState.RUNNING: _MIN_PROGRESS <= value <= _MAX_PROGRESS, + TaskState.SUCCESS: value == _MAX_PROGRESS, + TaskState.ABORTED: value == _MAX_PROGRESS, + TaskState.FAILURE: value == _MAX_PROGRESS, + } + + if not valid_states.get(self.task_state, True): + msg = f"Inconsistent progress actual value for state={self.task_state}: {value}" + raise ValueError(msg) + + return self From 52d9219a8c60ce1e226472b5338ae39cad761ee1 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 4 Mar 2025 09:51:39 +0100 Subject: [PATCH 094/136] remove started and stopped --- .../api_schemas_rpc_async_jobs/async_jobs.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_rpc_async_jobs/async_jobs.py b/packages/models-library/src/models_library/api_schemas_rpc_async_jobs/async_jobs.py index c89363b1ed70..bb410add1253 100644 --- a/packages/models-library/src/models_library/api_schemas_rpc_async_jobs/async_jobs.py +++ b/packages/models-library/src/models_library/api_schemas_rpc_async_jobs/async_jobs.py @@ -1,10 +1,8 @@ -from datetime import datetime from typing import Any, TypeAlias from uuid import UUID from models_library.users import UserID -from pydantic import BaseModel, model_validator -from typing_extensions import Self +from pydantic import BaseModel from ..progress_bar import ProgressReport @@ -15,18 +13,6 @@ class AsyncJobStatus(BaseModel): job_id: AsyncJobId progress: ProgressReport | None done: bool - started: datetime - stopped: datetime | None - - @model_validator(mode="after") - def _check_consistency(self) -> Self: - is_done = self.done - is_stopped = self.stopped is not None - - if is_done != is_stopped: - msg = f"Inconsistent data: {self.done=}, {self.stopped=}" - raise ValueError(msg) - return self class AsyncJobResult(BaseModel): From 38b4be79937649739aa34e16279173232a8d0ac9 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 4 Mar 2025 10:04:15 +0100 Subject: [PATCH 095/136] progress not nullable --- .../src/models_library/api_schemas_rpc_async_jobs/async_jobs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/models-library/src/models_library/api_schemas_rpc_async_jobs/async_jobs.py b/packages/models-library/src/models_library/api_schemas_rpc_async_jobs/async_jobs.py index bb410add1253..953cd1f819cc 100644 --- a/packages/models-library/src/models_library/api_schemas_rpc_async_jobs/async_jobs.py +++ b/packages/models-library/src/models_library/api_schemas_rpc_async_jobs/async_jobs.py @@ -11,7 +11,7 @@ class AsyncJobStatus(BaseModel): job_id: AsyncJobId - progress: ProgressReport | None + progress: ProgressReport done: bool From c3b2f4d6c7a00e1fa5c40b33adc5fd7b5d263835 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 4 Mar 2025 10:30:30 +0100 Subject: [PATCH 096/136] removed for now --- .../src/models_library/api_schemas_webserver/storage.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/storage.py b/packages/models-library/src/models_library/api_schemas_webserver/storage.py index 9c793ad2ee7d..8dacbdd25539 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/storage.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/storage.py @@ -1,4 +1,3 @@ -from datetime import datetime from pathlib import Path from typing import Any @@ -47,8 +46,6 @@ class StorageAsyncJobStatus(OutputSchema): job_id: AsyncJobId progress: ProgressReport done: bool - started: datetime - stopped: datetime | None @classmethod def from_rpc_schema( @@ -58,8 +55,6 @@ def from_rpc_schema( job_id=async_job_rpc_status.job_id, progress=async_job_rpc_status.progress, done=async_job_rpc_status.done, - started=async_job_rpc_status.started, - stopped=async_job_rpc_status.stopped, ) From 319f2691bafb28cbdb823eebf2844c223bb74b66 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 4 Mar 2025 10:31:47 +0100 Subject: [PATCH 097/136] typecheck --- .../src/simcore_service_storage/api/rpc/_async_jobs.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py b/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py index fb0d82a31cf0..2b90d5061ba6 100644 --- a/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py +++ b/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py @@ -1,5 +1,4 @@ # pylint: disable=unused-argument -from datetime import datetime from fastapi import FastAPI from models_library.api_schemas_rpc_async_jobs.async_jobs import ( @@ -46,8 +45,6 @@ async def get_status( job_id=job_id, progress=task_status.progress_report, done=False, - started=datetime.now(), # TODO: retrieve that - stopped=None, # TODO: retrieve that ) @@ -69,7 +66,7 @@ async def get_result( @router.expose() async def list_jobs( - app: FastAPI, filter_: str, job_id_data: AsyncJobNameData # TODO: implement filter + app: FastAPI, filter_: str, job_id_data: AsyncJobNameData ) -> list[AsyncJobGet]: assert app # nosec From 096c3b7ef49f53469a17f256210de92be2d6944c Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 4 Mar 2025 11:36:31 +0100 Subject: [PATCH 098/136] fix port --- packages/settings-library/src/settings_library/redis.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/settings-library/src/settings_library/redis.py b/packages/settings-library/src/settings_library/redis.py index 36516c9eadc8..322991e9c533 100644 --- a/packages/settings-library/src/settings_library/redis.py +++ b/packages/settings-library/src/settings_library/redis.py @@ -1,6 +1,5 @@ from enum import IntEnum -from pydantic import TypeAdapter from pydantic.networks import RedisDsn from pydantic.types import SecretStr @@ -25,7 +24,7 @@ class RedisSettings(BaseCustomSettings): # host REDIS_SECURE: bool = False REDIS_HOST: str = "redis" - REDIS_PORT: PortInt = TypeAdapter(PortInt).validate_python(6789) + REDIS_PORT: PortInt = 6789 # auth REDIS_USER: str | None = None From 02d401acf237cae8314475e56ec2586f8ed67265 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 4 Mar 2025 15:44:15 +0100 Subject: [PATCH 099/136] fix tests --- .../api/rpc/_async_jobs.py | 2 +- .../api/rpc/_data_export.py | 2 +- .../src/simcore_service_storage/main.py | 12 +-- .../modules/celery/__init__.py | 36 +++++++ .../modules/celery/_common.py | 6 +- .../modules/celery/_utils.py | 55 ----------- .../modules/celery/client.py | 25 +++-- .../modules/celery/utils.py | 27 ++++++ .../modules/celery/worker.py | 4 +- .../modules/celery/worker_main.py | 32 +++++-- .../tests/unit/modules/celery/conftest.py | 61 ++++-------- .../tests/unit/modules/celery/test_celery.py | 96 +++++++++---------- 12 files changed, 178 insertions(+), 180 deletions(-) delete mode 100644 services/storage/src/simcore_service_storage/modules/celery/_utils.py create mode 100644 services/storage/src/simcore_service_storage/modules/celery/utils.py diff --git a/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py b/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py index 2b90d5061ba6..d3408a98c31b 100644 --- a/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py +++ b/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py @@ -15,7 +15,7 @@ ) from servicelib.rabbitmq import RPCRouter -from ...modules.celery._utils import get_celery_client +from ...modules.celery import get_celery_client from ...modules.celery.models import TaskStatus router = RPCRouter() diff --git a/services/storage/src/simcore_service_storage/api/rpc/_data_export.py b/services/storage/src/simcore_service_storage/api/rpc/_data_export.py index 8f2f23043ca0..aab9d7339f62 100644 --- a/services/storage/src/simcore_service_storage/api/rpc/_data_export.py +++ b/services/storage/src/simcore_service_storage/api/rpc/_data_export.py @@ -14,7 +14,7 @@ from ...datcore_dsm import DatCoreDataManager from ...dsm import get_dsm_provider from ...exceptions.errors import FileAccessRightError -from ...modules.celery._utils import get_celery_client +from ...modules.celery import get_celery_client from ...modules.datcore_adapter.datcore_adapter_exceptions import DatcoreAdapterError from ...simcore_s3_dsm import SimcoreS3DataManager diff --git a/services/storage/src/simcore_service_storage/main.py b/services/storage/src/simcore_service_storage/main.py index a971d3501397..f0639c753685 100644 --- a/services/storage/src/simcore_service_storage/main.py +++ b/services/storage/src/simcore_service_storage/main.py @@ -3,15 +3,10 @@ import logging from servicelib.logging_utils import config_all_loggers -from simcore_service_storage.modules.celery._utils import ( - set_celery_app, - set_celery_client, -) +from simcore_service_storage.modules.celery import setup_celery from .core.application import create_app from .core.settings import ApplicationSettings -from .modules.celery._common import create_app as create_celery_app -from .modules.celery.client import CeleryTaskQueueClient _settings = ApplicationSettings.create_from_envs() @@ -27,9 +22,6 @@ _logger = logging.getLogger(__name__) fastapi_app = create_app(_settings) -celery_app = create_celery_app(_settings) - -set_celery_app(fastapi_app, celery_app) -set_celery_client(fastapi_app, CeleryTaskQueueClient(celery_app)) +setup_celery(fastapi_app) app = fastapi_app diff --git a/services/storage/src/simcore_service_storage/modules/celery/__init__.py b/services/storage/src/simcore_service_storage/modules/celery/__init__.py index e69de29bb2d1..2852b74eab40 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/__init__.py +++ b/services/storage/src/simcore_service_storage/modules/celery/__init__.py @@ -0,0 +1,36 @@ +import logging +from asyncio import AbstractEventLoop + +from fastapi import FastAPI +from simcore_service_storage.modules.celery._common import create_app +from simcore_service_storage.modules.celery.client import CeleryTaskQueueClient + +from ...core.settings import get_application_settings + +_logger = logging.getLogger(__name__) + + +def setup_celery(app: FastAPI) -> None: + async def on_startup() -> None: + celery_settings = get_application_settings(app).STORAGE_CELERY + assert celery_settings # nosec + celery_app = create_app(celery_settings) + app.state.celery_client = CeleryTaskQueueClient(celery_app) + + app.add_event_handler("startup", on_startup) + + +def get_celery_client(app: FastAPI) -> CeleryTaskQueueClient: + celery_client = app.state.celery_client + assert isinstance(celery_client, CeleryTaskQueueClient) + return celery_client + + +def get_event_loop(app: FastAPI) -> AbstractEventLoop: + event_loop = app.state.event_loop + assert isinstance(event_loop, AbstractEventLoop) + return event_loop + + +def set_event_loop(app: FastAPI, event_loop: AbstractEventLoop) -> None: + app.state.event_loop = event_loop diff --git a/services/storage/src/simcore_service_storage/modules/celery/_common.py b/services/storage/src/simcore_service_storage/modules/celery/_common.py index 5d20716fff44..dfdce92bc23e 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/_common.py +++ b/services/storage/src/simcore_service_storage/modules/celery/_common.py @@ -1,15 +1,13 @@ import logging from celery import Celery +from settings_library.celery import CelerySettings from settings_library.redis import RedisDatabase -from ...core.settings import ApplicationSettings - _logger = logging.getLogger(__name__) -def create_app(settings: ApplicationSettings) -> Celery: - celery_settings = settings.STORAGE_CELERY +def create_app(celery_settings: CelerySettings) -> Celery: assert celery_settings app = Celery( diff --git a/services/storage/src/simcore_service_storage/modules/celery/_utils.py b/services/storage/src/simcore_service_storage/modules/celery/_utils.py deleted file mode 100644 index 37466adaab1a..000000000000 --- a/services/storage/src/simcore_service_storage/modules/celery/_utils.py +++ /dev/null @@ -1,55 +0,0 @@ -from asyncio import AbstractEventLoop - -from celery import Celery -from fastapi import FastAPI - -from ...core.settings import ApplicationSettings -from ._common import create_app -from .client import CeleryTaskQueueClient -from .worker import CeleryTaskQueueWorker - -_CLIENT_KEY = "client" -_WORKER_KEY = "worker" -_EVENT_LOOP_KEY = "loop" - - -def create_celery_app_worker(settings: ApplicationSettings) -> Celery: - celery_app = create_app(settings) - celery_app.conf[_WORKER_KEY] = CeleryTaskQueueWorker(celery_app) - return celery_app - - -def get_celery_app(fastapi: FastAPI) -> Celery: - celery = fastapi.state.celery_app - assert isinstance(celery, Celery) - return celery - - -def set_celery_app(fastapi: FastAPI, celery: Celery) -> None: - fastapi.state.celery_app = celery - - -def get_celery_client(fastapi_app: FastAPI) -> CeleryTaskQueueClient: - celery_app = get_celery_app(fastapi_app) - client = celery_app.conf[_CLIENT_KEY] - assert isinstance(client, CeleryTaskQueueClient) - return client - - -def set_celery_client( - fastapi_app: FastAPI, celery_client: CeleryTaskQueueClient -) -> None: - celery_app = get_celery_app(fastapi_app) - celery_app.conf[_CLIENT_KEY] = celery_client - - -def get_celery_worker(celery_app: Celery) -> CeleryTaskQueueWorker: - worker = celery_app.conf[_WORKER_KEY] - assert isinstance(worker, CeleryTaskQueueWorker) - return worker - - -def get_event_loop(celery_app: Celery) -> AbstractEventLoop: # nosec - loop = celery_app.conf[_EVENT_LOOP_KEY] - assert isinstance(loop, AbstractEventLoop) - return loop diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index 3adb308cc942..6aa2b0c340ae 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -22,6 +22,14 @@ ) _CELERY_TASK_META_PREFIX: Final[str] = "celery-task-meta-" _CELERY_TASK_ID_PREFIX: Final[str] = "celery" +_CELERY_STATES_MAPPING: Final[dict[str, TaskState]] = { + "PENDING": TaskState.PENDING, + "STARTED": TaskState.PENDING, + "RUNNING": TaskState.RUNNING, + "SUCCESS": TaskState.SUCCESS, + "ABORTED": TaskState.ABORTED, + "FAILURE": TaskState.FAILURE, +} def _build_context_prefix(task_context: TaskContext) -> list[str]: @@ -80,26 +88,29 @@ def _get_progress_report( ) -> ProgressReport: task_id = _build_task_id(task_context, task_uuid) result = self._celery_app.AsyncResult(task_id).result - state = self._celery_app.AsyncResult(task_id).state - if result and state == TaskState.RUNNING.value: + state = self._get_state(task_context, task_uuid) + if result and state == TaskState.RUNNING: with contextlib.suppress(ValidationError): return ProgressReport.model_validate(result) if state in ( - TaskState.ABORTED.value, - TaskState.FAILURE.value, - TaskState.SUCCESS.value, + TaskState.ABORTED, + TaskState.FAILURE, + TaskState.SUCCESS, ): return ProgressReport(actual_value=100.0) return ProgressReport(actual_value=0.0) + def _get_state(self, task_context: TaskContext, task_uuid: TaskUUID) -> TaskState: + task_id = _build_task_id(task_context, task_uuid) + return _CELERY_STATES_MAPPING[self._celery_app.AsyncResult(task_id).state] + @make_async() def get_task_status( self, task_context: TaskContext, task_uuid: TaskUUID ) -> TaskStatus: - task_id = _build_task_id(task_context, task_uuid) return TaskStatus( task_uuid=task_uuid, - task_state=self._celery_app.AsyncResult(task_id).state, + task_state=self._get_state(task_context, task_uuid), progress_report=self._get_progress_report(task_context, task_uuid), ) diff --git a/services/storage/src/simcore_service_storage/modules/celery/utils.py b/services/storage/src/simcore_service_storage/modules/celery/utils.py new file mode 100644 index 000000000000..6eff8a9b081f --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/celery/utils.py @@ -0,0 +1,27 @@ +from celery import Celery +from fastapi import FastAPI + +from .worker import CeleryTaskQueueWorker + +_WORKER_KEY = "celery_worker" +_FASTAPI_APP_KEY = "fastapi_app" + + +def get_celery_worker(celery_app: Celery) -> CeleryTaskQueueWorker: + worker = celery_app.conf[_WORKER_KEY] + assert isinstance(worker, CeleryTaskQueueWorker) + return worker + + +def get_fastapi_app(celery_app: Celery) -> FastAPI: + fastapi_app = celery_app.conf[_FASTAPI_APP_KEY] + assert isinstance(fastapi_app, FastAPI) + return fastapi_app + + +def set_celery_worker(celery_app: Celery, worker: CeleryTaskQueueWorker) -> None: + celery_app.conf[_WORKER_KEY] = worker + + +def set_fastapi_app(celery_app: Celery, fastapi_app: FastAPI) -> None: + celery_app.conf[_FASTAPI_APP_KEY] = fastapi_app diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker.py b/services/storage/src/simcore_service_storage/modules/celery/worker.py index b963d187f5f1..a07ff8d13efe 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker.py @@ -6,7 +6,7 @@ from models_library.progress_bar import ProgressReport from servicelib.logging_utils import log_context -from .models import TaskID, TaskState +from .models import TaskID _logger = logging.getLogger(__name__) @@ -30,6 +30,6 @@ def set_task_progress( ): self.celery_app.tasks[task_name].update_state( task_id=task_id, - state=TaskState.RUNNING.value, + state="RUNNING", meta=report.model_dump(mode="json"), ) diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker_main.py b/services/storage/src/simcore_service_storage/modules/celery/worker_main.py index ca66b66ab440..99f36526b5bc 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker_main.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker_main.py @@ -3,16 +3,25 @@ import asyncio import logging import threading +from typing import Final from asgi_lifespan import LifespanManager from celery import Celery from celery.signals import worker_init, worker_shutdown +from fastapi import FastAPI from servicelib.background_task import cancel_wait_task from servicelib.logging_utils import config_all_loggers +from simcore_service_storage.modules.celery import get_event_loop, set_event_loop +from simcore_service_storage.modules.celery.utils import ( + CeleryTaskQueueWorker, + get_fastapi_app, + set_celery_worker, + set_fastapi_app, +) from ...core.application import create_app from ...core.settings import ApplicationSettings -from ._utils import create_celery_app_worker +from ._common import create_app as create_celery_app _settings = ApplicationSettings.create_from_envs() @@ -26,6 +35,8 @@ _logger = logging.getLogger(__name__) +_LIFESPAN_TIMEOUT: Final[int] = 10 + @worker_init.connect def on_worker_init(sender, **_kwargs): @@ -39,8 +50,8 @@ def _init_fastapi(): async def lifespan(): async with LifespanManager( fastapi_app, - startup_timeout=10, - shutdown_timeout=10, + startup_timeout=_LIFESPAN_TIMEOUT, + shutdown_timeout=_LIFESPAN_TIMEOUT, ): try: await shutdown_event.wait() @@ -50,9 +61,10 @@ async def lifespan(): lifespan_task = loop.create_task(lifespan()) fastapi_app.state.lifespan_task = lifespan_task fastapi_app.state.shutdown_event = shutdown_event + set_event_loop(fastapi_app, loop) - sender.app.conf["fastapi_app"] = fastapi_app - sender.app.conf["loop"] = loop + set_fastapi_app(sender.app, fastapi_app) + set_celery_worker(sender.app, CeleryTaskQueueWorker(sender.app)) loop.run_forever() @@ -64,15 +76,17 @@ async def lifespan(): def on_worker_shutdown(sender, **_kwargs): assert isinstance(sender.app, Celery) - loop = sender.app.conf["loop"] - fastapi_app = sender.app.conf["fastapi_app"] + fastapi_app = get_fastapi_app(sender.app) + assert isinstance(fastapi_app, FastAPI) + event_loop = get_event_loop(fastapi_app) async def shutdown(): fastapi_app.state.shutdown_event.set() await cancel_wait_task(fastapi_app.state.lifespan_task, max_delay=5) - asyncio.run_coroutine_threadsafe(shutdown(), loop) + asyncio.run_coroutine_threadsafe(shutdown(), event_loop) -app = create_celery_app_worker(_settings) +assert _settings.STORAGE_CELERY +app = create_celery_app(_settings.STORAGE_CELERY) diff --git a/services/storage/tests/unit/modules/celery/conftest.py b/services/storage/tests/unit/modules/celery/conftest.py index f52ee0eef0d1..74285c9c38b1 100644 --- a/services/storage/tests/unit/modules/celery/conftest.py +++ b/services/storage/tests/unit/modules/celery/conftest.py @@ -1,4 +1,3 @@ -from asyncio import AbstractEventLoop from collections.abc import Callable, Iterable from datetime import timedelta from typing import Any @@ -7,15 +6,10 @@ from celery import Celery from celery.contrib.testing.worker import TestWorkController, start_worker from celery.signals import worker_init, worker_shutdown -from fastapi import FastAPI from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict -from simcore_service_storage.main import celery_app as celery_app_client from simcore_service_storage.modules.celery.client import CeleryTaskQueueClient from simcore_service_storage.modules.celery.worker import CeleryTaskQueueWorker -from simcore_service_storage.modules.celery.worker_main import ( - celery_app as celery_app_worker, -) from simcore_service_storage.modules.celery.worker_main import ( on_worker_init, on_worker_shutdown, @@ -52,6 +46,11 @@ def celery_conf() -> dict[str, Any]: } +@pytest.fixture +def celery_app(celery_conf: dict[str, Any]): + return Celery(**celery_conf) + + @pytest.fixture def register_celery_tasks() -> Callable[[Celery], None]: msg = "please define a callback that registers the tasks" @@ -59,40 +58,26 @@ def register_celery_tasks() -> Callable[[Celery], None]: @pytest.fixture -def celery_client_app( - app_environment: EnvVarsDict, celery_conf: dict[str, Any] -) -> Celery: - celery_app_client.conf.update(celery_conf) - - assert isinstance(celery_app_client.conf["client"], CeleryTaskQueueClient) - assert "worker" not in celery_app_client.conf - assert "loop" not in celery_app_client.conf - assert "fastapi_app" not in celery_app_client.conf - - return celery_app_client +def celery_client( + app_environment: EnvVarsDict, celery_app: Celery +) -> CeleryTaskQueueClient: + return CeleryTaskQueueClient(celery_app) @pytest.fixture -def celery_worker( +def celery_worker_controller( register_celery_tasks: Callable[[Celery], None], - celery_conf: dict[str, Any], + celery_app: Celery, ) -> Iterable[TestWorkController]: - celery_app_worker.conf.update(celery_conf) - - register_celery_tasks(celery_app_worker) # Signals must be explicitily connected worker_init.connect(on_worker_init) worker_shutdown.connect(on_worker_shutdown) - with start_worker( - celery_app_worker, loglevel="info", perform_ping_check=False - ) as worker: - worker_init.send(sender=worker) + register_celery_tasks(celery_app) - assert isinstance(celery_app_worker.conf["worker"], CeleryTaskQueueWorker) - assert isinstance(celery_app_worker.conf["loop"], AbstractEventLoop) - assert isinstance(celery_app_worker.conf["fastapi_app"], FastAPI) + with start_worker(celery_app, loglevel="info", perform_ping_check=False) as worker: + worker_init.send(sender=worker) yield worker @@ -100,16 +85,8 @@ def celery_worker( @pytest.fixture -def celery_worker_app(celery_worker: TestWorkController) -> Celery: - assert isinstance(celery_worker.app, Celery) - return celery_worker.app - - -@pytest.fixture -def celery_task_queue_client(celery_worker_app: Celery) -> CeleryTaskQueueClient: - return CeleryTaskQueueClient(celery_worker_app) - - -@pytest.fixture -def celery_task_queue_worker(celery_worker_app: Celery) -> CeleryTaskQueueWorker: - return CeleryTaskQueueWorker(celery_worker_app) +def celery_worker( + celery_worker_controller: TestWorkController, +) -> CeleryTaskQueueWorker: + assert isinstance(celery_worker_controller.app, Celery) + return CeleryTaskQueueWorker(celery_worker_controller.app) diff --git a/services/storage/tests/unit/modules/celery/test_celery.py b/services/storage/tests/unit/modules/celery/test_celery.py index 2528d1f4771b..15ecfe90be3a 100644 --- a/services/storage/tests/unit/modules/celery/test_celery.py +++ b/services/storage/tests/unit/modules/celery/test_celery.py @@ -8,13 +8,14 @@ from celery import Celery, Task from celery.contrib.abortable import AbortableTask from models_library.progress_bar import ProgressReport -from simcore_service_storage.main import CeleryTaskQueueClient -from simcore_service_storage.modules.celery._utils import ( +from servicelib.logging_utils import log_context +from simcore_service_storage.modules.celery import get_event_loop +from simcore_service_storage.modules.celery.client import CeleryTaskQueueClient +from simcore_service_storage.modules.celery.models import TaskContext, TaskState +from simcore_service_storage.modules.celery.utils import ( get_celery_worker, - get_event_loop, + get_fastapi_app, ) -from simcore_service_storage.modules.celery.models import TaskContext -from simcore_service_storage.modules.celery.worker import CeleryTaskQueueWorker from tenacity import Retrying, retry_if_exception_type, stop_after_delay, wait_fixed _logger = logging.getLogger(__name__) @@ -23,16 +24,19 @@ async def _async_archive( celery_app: Celery, task_name: str, task_id: str, files: list[str] ) -> str: - worker: CeleryTaskQueueWorker = get_celery_worker(celery_app) + worker = get_celery_worker(celery_app) + + def sleep_for(seconds: float) -> None: + time.sleep(seconds) for n, file in enumerate(files, start=1): - _logger.info("Processing file %s", file) - worker.set_task_progress( - task_name=task_name, - task_id=task_id, - report=ProgressReport(actual_value=n / len(files) * 10), - ) - await asyncio.sleep(0.1) + with log_context(_logger, logging.INFO, msg=f"Processing file {file}"): + worker.set_task_progress( + task_name=task_name, + task_id=task_id, + report=ProgressReport(actual_value=n / len(files) * 10), + ) + await asyncio.get_event_loop().run_in_executor(None, sleep_for, 1) return "archive.zip" @@ -42,7 +46,7 @@ def sync_archive(task: Task, files: list[str]) -> str: _logger.info("Calling async_archive") return asyncio.run_coroutine_threadsafe( _async_archive(task.app, task.name, task.request.id, files), - get_event_loop(task.app), + get_event_loop(get_fastapi_app(task.app)), ).result() @@ -74,19 +78,16 @@ def _(celery_app: Celery) -> None: return _ -@pytest.fixture -def task_context() -> TaskContext: - return TaskContext(user_id=1, product_name="test") - - -@pytest.mark.usefixtures("celery_client_app", "celery_worker_app") +@pytest.mark.usefixtures("celery_worker") async def test_sumitting_task_calling_async_function_results_with_success_state( - celery_task_queue_client: CeleryTaskQueueClient, task_context: TaskContext + celery_client: CeleryTaskQueueClient, ): - task_uuid = await celery_task_queue_client.send_task( + task_context = TaskContext(user_id=42) + + task_uuid = await celery_client.send_task( "sync_archive", task_context=task_context, - files=[f"file{n}" for n in range(30)], + files=[f"file{n}" for n in range(5)], ) for attempt in Retrying( @@ -95,24 +96,21 @@ async def test_sumitting_task_calling_async_function_results_with_success_state( stop=stop_after_delay(30), ): with attempt: - progress = await celery_task_queue_client.get_task_status( - task_context, task_uuid - ) - assert progress.task_state == "SUCCESS" + progress = await celery_client.get_task_status(task_context, task_uuid) + assert progress.task_state == TaskState.SUCCESS assert ( - await celery_task_queue_client.get_task_status(task_context, task_uuid) - ).task_state == "SUCCESS" + await celery_client.get_task_status(task_context, task_uuid) + ).task_state == TaskState.SUCCESS -@pytest.mark.usefixtures("celery_client_app", "celery_worker_app") +@pytest.mark.usefixtures("celery_worker") async def test_submitting_task_with_failure_results_with_error( - celery_task_queue_client: CeleryTaskQueueClient, - task_context: TaskContext, + celery_client: CeleryTaskQueueClient, ): - task_uuid = await celery_task_queue_client.send_task( - "failure_task", task_context=task_context - ) + task_context = TaskContext(user_id=42) + + task_uuid = await celery_client.send_task("failure_task", task_context=task_context) for attempt in Retrying( retry=retry_if_exception_type(AssertionError), @@ -120,27 +118,29 @@ async def test_submitting_task_with_failure_results_with_error( stop=stop_after_delay(30), ): with attempt: - result = await celery_task_queue_client.get_result(task_context, task_uuid) + result = await celery_client.get_result(task_context, task_uuid) assert isinstance(result, ValueError) assert ( - await celery_task_queue_client.get_task_status(task_context, task_uuid) - ).task_state == "FAILURE" - result = await celery_task_queue_client.get_result(task_context, task_uuid) + await celery_client.get_task_status(task_context, task_uuid) + ).task_state == TaskState.FAILURE + result = await celery_client.get_result(task_context, task_uuid) assert isinstance(result, ValueError) assert f"{result}" == "my error here" -@pytest.mark.usefixtures("celery_client_app", "celery_worker_app") +@pytest.mark.usefixtures("celery_worker") async def test_aborting_task_results_with_aborted_state( - celery_task_queue_client: CeleryTaskQueueClient, task_context: TaskContext + celery_client: CeleryTaskQueueClient, ): - task_uuid = await celery_task_queue_client.send_task( + task_context = TaskContext(user_id=42) + + task_uuid = await celery_client.send_task( "dreamer_task", task_context=task_context, ) - await celery_task_queue_client.abort_task(task_context, task_uuid) + await celery_client.abort_task(task_context, task_uuid) for attempt in Retrying( retry=retry_if_exception_type(AssertionError), @@ -148,11 +148,9 @@ async def test_aborting_task_results_with_aborted_state( stop=stop_after_delay(30), ): with attempt: - progress = await celery_task_queue_client.get_task_status( - task_context, task_uuid - ) - assert progress.task_state == "ABORTED" + progress = await celery_client.get_task_status(task_context, task_uuid) + assert progress.task_state == TaskState.ABORTED assert ( - await celery_task_queue_client.get_task_status(task_context, task_uuid) - ).task_state == "ABORTED" + await celery_client.get_task_status(task_context, task_uuid) + ).task_state == TaskState.ABORTED From 0b624ec0e7b5a7172aace244ac26cc653d37a40b Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 4 Mar 2025 15:54:10 +0100 Subject: [PATCH 100/136] remove register --- .../src/simcore_service_storage/modules/celery/worker.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker.py b/services/storage/src/simcore_service_storage/modules/celery/worker.py index a07ff8d13efe..65aa341bc4a6 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker.py @@ -1,8 +1,6 @@ import logging -from collections.abc import Callable from celery import Celery -from celery.contrib.abortable import AbortableTask from models_library.progress_bar import ProgressReport from servicelib.logging_utils import log_context @@ -15,11 +13,6 @@ class CeleryTaskQueueWorker: def __init__(self, celery_app: Celery) -> None: self.celery_app = celery_app - def register_task(self, fn: Callable, task_name: str | None = None) -> None: - name = task_name or fn.__name__ - with log_context(_logger, logging.INFO, msg=f"Registering {name} task"): - self.celery_app.task(name=name, base=AbortableTask, bind=True)(fn) - def set_task_progress( self, task_name: str, task_id: TaskID, report: ProgressReport ) -> None: From e77d2dd0c31491d8ad06624e937aa5928e89bb0a Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 4 Mar 2025 16:29:39 +0100 Subject: [PATCH 101/136] add concurrency --- services/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/services/docker-compose.yml b/services/docker-compose.yml index f0967e1996f4..a8f6c3e5066a 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -1198,6 +1198,7 @@ services: environment: <<: *storage_environment STORAGE_WORKER_MODE: "true" + CELERY_CONCURRENCY: 1 networks: *storage_networks rabbit: From 26160a8aa15e55229ede69abb4e7ce9d264c5802 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 4 Mar 2025 16:32:37 +0100 Subject: [PATCH 102/136] add fixture --- services/storage/tests/unit/modules/celery/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/services/storage/tests/unit/modules/celery/conftest.py b/services/storage/tests/unit/modules/celery/conftest.py index 74285c9c38b1..406ce566d1bc 100644 --- a/services/storage/tests/unit/modules/celery/conftest.py +++ b/services/storage/tests/unit/modules/celery/conftest.py @@ -66,6 +66,7 @@ def celery_client( @pytest.fixture def celery_worker_controller( + app_environment: EnvVarsDict, register_celery_tasks: Callable[[Celery], None], celery_app: Celery, ) -> Iterable[TestWorkController]: From 3bb2c6bb8107651c76d2f7669f5b8b47b37ea473 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 4 Mar 2025 21:26:56 +0100 Subject: [PATCH 103/136] fix tests --- .../modules/celery/signals.py | 70 +++++++++++++++++ .../modules/celery/tasks.py | 27 +++++++ .../modules/celery/worker_main.py | 75 ++----------------- 3 files changed, 105 insertions(+), 67 deletions(-) create mode 100644 services/storage/src/simcore_service_storage/modules/celery/signals.py create mode 100644 services/storage/src/simcore_service_storage/modules/celery/tasks.py diff --git a/services/storage/src/simcore_service_storage/modules/celery/signals.py b/services/storage/src/simcore_service_storage/modules/celery/signals.py new file mode 100644 index 000000000000..c10bb71dab89 --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/celery/signals.py @@ -0,0 +1,70 @@ +import asyncio +import logging +import threading +from typing import Final + +from asgi_lifespan import LifespanManager +from celery import Celery +from fastapi import FastAPI +from servicelib.async_utils import cancel_wait_task +from simcore_service_storage.core.application import create_app +from simcore_service_storage.core.settings import ApplicationSettings +from simcore_service_storage.modules.celery import get_event_loop, set_event_loop +from simcore_service_storage.modules.celery.utils import ( + get_fastapi_app, + set_celery_worker, + set_fastapi_app, +) +from simcore_service_storage.modules.celery.worker import CeleryTaskQueueWorker + +_logger = logging.getLogger(__name__) + +_LIFESPAN_TIMEOUT: Final[int] = 10 + + +def on_worker_init(sender, **_kwargs): + def _init_fastapi(): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + shutdown_event = asyncio.Event() + + fastapi_app = create_app(ApplicationSettings.create_from_envs()) + + async def lifespan(): + async with LifespanManager( + fastapi_app, + startup_timeout=_LIFESPAN_TIMEOUT, + shutdown_timeout=_LIFESPAN_TIMEOUT, + ): + try: + await shutdown_event.wait() + except asyncio.CancelledError: + _logger.warning("Lifespan task cancelled") + + lifespan_task = loop.create_task(lifespan()) + fastapi_app.state.lifespan_task = lifespan_task + fastapi_app.state.shutdown_event = shutdown_event + set_event_loop(fastapi_app, loop) + + set_fastapi_app(sender.app, fastapi_app) + set_celery_worker(sender.app, CeleryTaskQueueWorker(sender.app)) + + loop.run_forever() + + thread = threading.Thread(target=_init_fastapi, daemon=True) + thread.start() + + +def on_worker_shutdown(sender, **_kwargs): + assert isinstance(sender.app, Celery) + + fastapi_app = get_fastapi_app(sender.app) + assert isinstance(fastapi_app, FastAPI) + event_loop = get_event_loop(fastapi_app) + + async def shutdown(): + fastapi_app.state.shutdown_event.set() + + await cancel_wait_task(fastapi_app.state.lifespan_task, max_delay=5) + + asyncio.run_coroutine_threadsafe(shutdown(), event_loop) diff --git a/services/storage/src/simcore_service_storage/modules/celery/tasks.py b/services/storage/src/simcore_service_storage/modules/celery/tasks.py new file mode 100644 index 000000000000..e5ca99aa771e --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/celery/tasks.py @@ -0,0 +1,27 @@ +import logging +import time + +from celery import Task +from models_library.progress_bar import ProgressReport +from models_library.projects_nodes_io import StorageFileID +from servicelib.logging_utils import log_context +from simcore_service_storage.modules.celery.utils import get_celery_worker + +_logger = logging.getLogger(__name__) + + +def export_data(task: Task, files: list[StorageFileID]): + for n, file in enumerate(files, start=1): + with log_context( + _logger, + logging.INFO, + msg=f"Exporting {file=} ({n}/{len(files)})", + ): + assert task.name + get_celery_worker(task.app).set_task_progress( + task_name=task.name, + task_id=task.request.id, + report=ProgressReport(actual_value=n / len(files) * 100), + ) + time.sleep(10) + return "done" diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker_main.py b/services/storage/src/simcore_service_storage/modules/celery/worker_main.py index 99f36526b5bc..1958d1fed6da 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker_main.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker_main.py @@ -1,27 +1,18 @@ """Main application to be deployed in for example uvicorn.""" -import asyncio import logging -import threading -from typing import Final -from asgi_lifespan import LifespanManager -from celery import Celery +from celery.contrib.abortable import AbortableTask from celery.signals import worker_init, worker_shutdown -from fastapi import FastAPI -from servicelib.background_task import cancel_wait_task from servicelib.logging_utils import config_all_loggers -from simcore_service_storage.modules.celery import get_event_loop, set_event_loop -from simcore_service_storage.modules.celery.utils import ( - CeleryTaskQueueWorker, - get_fastapi_app, - set_celery_worker, - set_fastapi_app, +from simcore_service_storage.modules.celery.signals import ( + on_worker_init, + on_worker_shutdown, ) -from ...core.application import create_app from ...core.settings import ApplicationSettings from ._common import create_app as create_celery_app +from .tasks import export_data _settings = ApplicationSettings.create_from_envs() @@ -35,58 +26,8 @@ _logger = logging.getLogger(__name__) -_LIFESPAN_TIMEOUT: Final[int] = 10 - - -@worker_init.connect -def on_worker_init(sender, **_kwargs): - def _init_fastapi(): - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - shutdown_event = asyncio.Event() - - fastapi_app = create_app(_settings) - - async def lifespan(): - async with LifespanManager( - fastapi_app, - startup_timeout=_LIFESPAN_TIMEOUT, - shutdown_timeout=_LIFESPAN_TIMEOUT, - ): - try: - await shutdown_event.wait() - except asyncio.CancelledError: - _logger.warning("Lifespan task cancelled") - - lifespan_task = loop.create_task(lifespan()) - fastapi_app.state.lifespan_task = lifespan_task - fastapi_app.state.shutdown_event = shutdown_event - set_event_loop(fastapi_app, loop) - - set_fastapi_app(sender.app, fastapi_app) - set_celery_worker(sender.app, CeleryTaskQueueWorker(sender.app)) - - loop.run_forever() - - thread = threading.Thread(target=_init_fastapi, daemon=True) - thread.start() - - -@worker_shutdown.connect -def on_worker_shutdown(sender, **_kwargs): - assert isinstance(sender.app, Celery) - - fastapi_app = get_fastapi_app(sender.app) - assert isinstance(fastapi_app, FastAPI) - event_loop = get_event_loop(fastapi_app) - - async def shutdown(): - fastapi_app.state.shutdown_event.set() - - await cancel_wait_task(fastapi_app.state.lifespan_task, max_delay=5) - - asyncio.run_coroutine_threadsafe(shutdown(), event_loop) - - assert _settings.STORAGE_CELERY app = create_celery_app(_settings.STORAGE_CELERY) +worker_init.connect(on_worker_init) +worker_shutdown.connect(on_worker_shutdown) +app.task(name="export_data", bind=True, base=AbortableTask)(export_data) From 964a22e047c56c7c7b1ffe635c0261cc6fe28f93 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 4 Mar 2025 21:46:52 +0100 Subject: [PATCH 104/136] fix import --- services/storage/tests/unit/modules/celery/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/storage/tests/unit/modules/celery/conftest.py b/services/storage/tests/unit/modules/celery/conftest.py index 406ce566d1bc..3cd06195b286 100644 --- a/services/storage/tests/unit/modules/celery/conftest.py +++ b/services/storage/tests/unit/modules/celery/conftest.py @@ -9,11 +9,11 @@ from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from simcore_service_storage.modules.celery.client import CeleryTaskQueueClient -from simcore_service_storage.modules.celery.worker import CeleryTaskQueueWorker -from simcore_service_storage.modules.celery.worker_main import ( +from simcore_service_storage.modules.celery.signals import ( on_worker_init, on_worker_shutdown, ) +from simcore_service_storage.modules.celery.worker import CeleryTaskQueueWorker @pytest.fixture From c620103499801a74255cec7ca6204427a127e473 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 5 Mar 2025 10:48:35 +0100 Subject: [PATCH 105/136] progress --- api/specs/web-server/_storage.py | 6 +- services/storage/Makefile | 2 +- .../storage/tests/unit/test_db_data_export.py | 79 ++++++++++++++++--- .../api/v0/openapi.yaml | 30 ------- .../storage/_rest.py | 25 ++++-- 5 files changed, 89 insertions(+), 53 deletions(-) diff --git a/api/specs/web-server/_storage.py b/api/specs/web-server/_storage.py index fac4c4fde874..848fac10a7b0 100644 --- a/api/specs/web-server/_storage.py +++ b/api/specs/web-server/_storage.py @@ -210,7 +210,7 @@ async def export_data(data_export: DataExportPost, location_id: LocationID): response_model=Envelope[StorageAsyncJobStatus], name="get_async_job_status", ) -async def get_async_job_status(storage_async_job_get: StorageAsyncJobGet, job_id: UUID): +async def get_async_job_status(job_id: UUID): """Get async job status""" @@ -218,7 +218,7 @@ async def get_async_job_status(storage_async_job_get: StorageAsyncJobGet, job_id "/storage/async-jobs/{job_id}:abort", name="abort_async_job", ) -async def abort_async_job(storage_async_job_get: StorageAsyncJobGet, job_id: UUID): +async def abort_async_job(job_id: UUID): """aborts execution of an async job""" @@ -227,7 +227,7 @@ async def abort_async_job(storage_async_job_get: StorageAsyncJobGet, job_id: UUI response_model=Envelope[StorageAsyncJobResult], name="get_async_job_result", ) -async def get_async_job_result(storage_async_job_get: StorageAsyncJobGet, job_id: UUID): +async def get_async_job_result(job_id: UUID): """Get the result of the async job""" diff --git a/services/storage/Makefile b/services/storage/Makefile index 4bbab60c8a52..194e144f4cfb 100644 --- a/services/storage/Makefile +++ b/services/storage/Makefile @@ -15,7 +15,7 @@ openapi.json: .env-ignore @set -o allexport; \ source $<; \ set +o allexport; \ - python3 -c "import json; from $(APP_PACKAGE_NAME).main import *; print( json.dumps(the_app.openapi(), indent=2) )" > $@ + python3 -c "import json; from $(APP_PACKAGE_NAME).main import *; print( json.dumps(app.openapi(), indent=2) )" > $@ # validates OAS file: $@ $(call validate_openapi_specs,$@) diff --git a/services/storage/tests/unit/test_db_data_export.py b/services/storage/tests/unit/test_db_data_export.py index 1c9d3b01edbc..acefa909615d 100644 --- a/services/storage/tests/unit/test_db_data_export.py +++ b/services/storage/tests/unit/test_db_data_export.py @@ -18,6 +18,7 @@ from models_library.api_schemas_storage.data_export_async_jobs import ( DataExportTaskStartInput, ) +from models_library.progress_bar import ProgressReport from models_library.projects_nodes_io import NodeID, SimcoreS3FileID from models_library.users import UserID from pydantic import ByteSize, TypeAdapter @@ -28,9 +29,11 @@ from servicelib.rabbitmq import RabbitMQRPCClient from servicelib.rabbitmq.rpc_interfaces.async_jobs import async_jobs from settings_library.rabbit import RabbitSettings -from simcore_service_storage.api.rpc._async_jobs import AsyncJobNameData +from simcore_service_storage.api.rpc._async_jobs import AsyncJobNameData, TaskStatus from simcore_service_storage.api.rpc._data_export import AccessRightError from simcore_service_storage.core.settings import ApplicationSettings +from simcore_service_storage.modules.celery.client import TaskUUID +from simcore_service_storage.modules.celery.models import TaskState pytest_plugins = [ "pytest_simcore.rabbit_service", @@ -42,6 +45,8 @@ "postgres", ] +_faker = Faker() + @pytest.fixture async def mock_rabbit_setup(mocker: MockerFixture): @@ -49,6 +54,38 @@ async def mock_rabbit_setup(mocker: MockerFixture): pass +class _MockCeleryClient: + async def send_task(self, *args, **kwargs) -> TaskUUID: + return _faker.uuid4() + + async def get_task_status(self, *args, **kwargs) -> TaskStatus: + return TaskStatus( + task_uuid=_faker.uuid4(), + task_state=TaskState.RUNNING, + progress_report=ProgressReport(actual_value=42.0), + ) + + async def get_result(self, *args, **kwargs) -> Any: + return {} + + async def get_task_uuids(self, *args, **kwargs) -> set[TaskUUID]: + return {_faker.uuid4()} + + +@pytest.fixture +async def mock_celery_client(mocker: MockerFixture) -> MockerFixture: + _celery_client = _MockCeleryClient() + mocker.patch( + "simcore_service_storage.api.rpc._async_jobs.get_celery_client", + return_value=_celery_client, + ) + mocker.patch( + "simcore_service_storage.api.rpc._data_export.get_celery_client", + return_value=_celery_client, + ) + return mocker + + @pytest.fixture async def app_environment( app_environment: EnvVarsDict, @@ -110,6 +147,7 @@ class UserWithFile(NamedTuple): ) async def test_start_data_export_success( rpc_client: RabbitMQRPCClient, + mock_celery_client: MockerFixture, with_random_project_with_files: tuple[ dict[str, Any], dict[NodeID, dict[SimcoreS3FileID, FileIDDict]], @@ -150,7 +188,10 @@ async def test_start_data_export_success( async def test_start_data_export_fail( - rpc_client: RabbitMQRPCClient, user_id: UserID, faker: Faker + rpc_client: RabbitMQRPCClient, + mock_celery_client: MockerFixture, + user_id: UserID, + faker: Faker, ): with pytest.raises(AccessRightError): @@ -168,13 +209,16 @@ async def test_start_data_export_fail( ) -async def test_abort_data_export(rpc_client: RabbitMQRPCClient, faker: Faker): - _job_id = AsyncJobId(faker.uuid4()) +async def test_abort_data_export( + rpc_client: RabbitMQRPCClient, + mock_celery_client: MockerFixture, +): + _job_id = AsyncJobId(_faker.uuid4()) result = await async_jobs.abort( rpc_client, rpc_namespace=STORAGE_RPC_NAMESPACE, job_id_data=AsyncJobNameData( - user_id=faker.pyint(min_value=1, max_value=100), product_name="osparc" + user_id=_faker.pyint(min_value=1, max_value=100), product_name="osparc" ), job_id=_job_id, ) @@ -182,39 +226,48 @@ async def test_abort_data_export(rpc_client: RabbitMQRPCClient, faker: Faker): assert result.job_id == _job_id -async def test_get_data_export_status(rpc_client: RabbitMQRPCClient, faker: Faker): - _job_id = AsyncJobId(faker.uuid4()) +async def test_get_data_export_status( + rpc_client: RabbitMQRPCClient, + mock_celery_client: MockerFixture, +): + _job_id = AsyncJobId(_faker.uuid4()) result = await async_jobs.get_status( rpc_client, rpc_namespace=STORAGE_RPC_NAMESPACE, job_id=_job_id, job_id_data=AsyncJobNameData( - user_id=faker.pyint(min_value=1, max_value=100), product_name="osparc" + user_id=_faker.pyint(min_value=1, max_value=100), product_name="osparc" ), ) assert isinstance(result, AsyncJobStatus) assert result.job_id == _job_id -async def test_get_data_export_result(rpc_client: RabbitMQRPCClient, faker: Faker): - _job_id = AsyncJobId(faker.uuid4()) +async def test_get_data_export_result( + rpc_client: RabbitMQRPCClient, + mock_celery_client: MockerFixture, +): + _job_id = AsyncJobId(_faker.uuid4()) result = await async_jobs.get_result( rpc_client, rpc_namespace=STORAGE_RPC_NAMESPACE, job_id=_job_id, job_id_data=AsyncJobNameData( - user_id=faker.pyint(min_value=1, max_value=100), product_name="osparc" + user_id=_faker.pyint(min_value=1, max_value=100), product_name="osparc" ), ) assert isinstance(result, AsyncJobResult) -async def test_list_jobs(rpc_client: RabbitMQRPCClient, faker: Faker): +async def test_list_jobs( + rpc_client: RabbitMQRPCClient, + mock_celery_client: MockerFixture, +): result = await async_jobs.list_jobs( rpc_client, rpc_namespace=STORAGE_RPC_NAMESPACE, job_id_data=AsyncJobNameData( - user_id=faker.pyint(min_value=1, max_value=100), product_name="osparc" + user_id=_faker.pyint(min_value=1, max_value=100), product_name="osparc" ), filter_="", ) diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 1cd96ab7b28b..9edc45bd2756 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -6405,12 +6405,6 @@ paths: type: string format: uuid title: Job Id - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/StorageAsyncJobGet' responses: '200': description: Successful Response @@ -6433,12 +6427,6 @@ paths: type: string format: uuid title: Job Id - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/StorageAsyncJobGet' responses: '200': description: Successful Response @@ -6460,12 +6448,6 @@ paths: type: string format: uuid title: Job Id - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/StorageAsyncJobGet' responses: '200': description: Successful Response @@ -14715,23 +14697,11 @@ components: done: type: boolean title: Done - started: - type: string - format: date-time - title: Started - stopped: - anyOf: - - type: string - format: date-time - - type: 'null' - title: Stopped type: object required: - jobId - progress - done - - started - - stopped title: StorageAsyncJobStatus Structure: properties: diff --git a/services/web/server/src/simcore_service_webserver/storage/_rest.py b/services/web/server/src/simcore_service_webserver/storage/_rest.py index a3839db9d550..669bad63f218 100644 --- a/services/web/server/src/simcore_service_webserver/storage/_rest.py +++ b/services/web/server/src/simcore_service_webserver/storage/_rest.py @@ -7,6 +7,7 @@ import urllib.parse from typing import Any, Final, NamedTuple from urllib.parse import quote, unquote +from uuid import UUID from aiohttp import ClientTimeout, web from models_library.api_schemas_rpc_async_jobs.async_jobs import AsyncJobNameData @@ -471,10 +472,14 @@ async def get_async_jobs(request: web.Request) -> web.Response: @permission_required("storage.files.*") @handle_data_export_exceptions async def get_async_job_status(request: web.Request) -> web.Response: + + class _PathParams(BaseModel): + job_id: UUID + _req_ctx = RequestContext.model_validate(request) rabbitmq_rpc_client = get_rabbitmq_rpc_client(request.app) - async_job_get = parse_request_path_parameters_as(StorageAsyncJobGet, request) + async_job_get = parse_request_path_parameters_as(_PathParams, request) async_job_rpc_status = await get_status( rabbitmq_rpc_client=rabbitmq_rpc_client, rpc_namespace=STORAGE_RPC_NAMESPACE, @@ -497,10 +502,13 @@ async def get_async_job_status(request: web.Request) -> web.Response: @permission_required("storage.files.*") @handle_data_export_exceptions async def abort_async_job(request: web.Request) -> web.Response: + class _PathParams(BaseModel): + job_id: UUID + _req_ctx = RequestContext.model_validate(request) rabbitmq_rpc_client = get_rabbitmq_rpc_client(request.app) - async_job_get = parse_request_path_parameters_as(StorageAsyncJobGet, request) + async_job_get = parse_request_path_parameters_as(_PathParams, request) async_job_rpc_abort = await abort( rabbitmq_rpc_client=rabbitmq_rpc_client, rpc_namespace=STORAGE_RPC_NAMESPACE, @@ -510,9 +518,11 @@ async def abort_async_job(request: web.Request) -> web.Response: ), ) return web.Response( - status=status.HTTP_200_OK - if async_job_rpc_abort.result - else status.HTTP_500_INTERNAL_SERVER_ERROR + status=( + status.HTTP_200_OK + if async_job_rpc_abort.result + else status.HTTP_500_INTERNAL_SERVER_ERROR + ) ) @@ -524,10 +534,13 @@ async def abort_async_job(request: web.Request) -> web.Response: @permission_required("storage.files.*") @handle_data_export_exceptions async def get_async_job_result(request: web.Request) -> web.Response: + class _PathParams(BaseModel): + job_id: UUID + _req_ctx = RequestContext.model_validate(request) rabbitmq_rpc_client = get_rabbitmq_rpc_client(request.app) - async_job_get = parse_request_path_parameters_as(StorageAsyncJobGet, request) + async_job_get = parse_request_path_parameters_as(_PathParams, request) async_job_rpc_result = await get_result( rabbitmq_rpc_client=rabbitmq_rpc_client, rpc_namespace=STORAGE_RPC_NAMESPACE, From c75ccae778250311eb809e19df73468969b9b774 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 5 Mar 2025 12:32:40 +0100 Subject: [PATCH 106/136] fix done property --- .../src/simcore_service_storage/api/rpc/_async_jobs.py | 2 +- .../src/simcore_service_storage/modules/celery/models.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py b/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py index d3408a98c31b..629a7366f09d 100644 --- a/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py +++ b/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py @@ -44,7 +44,7 @@ async def get_status( return AsyncJobStatus( job_id=job_id, progress=task_status.progress_report, - done=False, + done=task_status.is_done, ) diff --git a/services/storage/src/simcore_service_storage/modules/celery/models.py b/services/storage/src/simcore_service_storage/modules/celery/models.py index feb67d2d25a1..39e79e809a1e 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/models.py +++ b/services/storage/src/simcore_service_storage/modules/celery/models.py @@ -21,11 +21,18 @@ class TaskState(StrEnum): ABORTED = auto() +_TASK_DONE = {TaskState.SUCCESS, TaskState.FAILURE, TaskState.ABORTED} + + class TaskStatus(BaseModel): task_uuid: TaskUUID task_state: TaskState progress_report: ProgressReport + @property + def is_done(self) -> bool: + return self.task_state in _TASK_DONE + @model_validator(mode="after") def _check_consistency(self) -> Self: value = self.progress_report.actual_value From 317ee6607e2a3960cad8b67b2ab2d06a9032ddf0 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 5 Mar 2025 12:55:23 +0100 Subject: [PATCH 107/136] update keys --- .../simcore_service_storage/modules/celery/client.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index 6aa2b0c340ae..91d237b3843d 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -21,7 +21,6 @@ "revoked", ) _CELERY_TASK_META_PREFIX: Final[str] = "celery-task-meta-" -_CELERY_TASK_ID_PREFIX: Final[str] = "celery" _CELERY_STATES_MAPPING: Final[dict[str, TaskState]] = { "PENDING": TaskState.PENDING, "STARTED": TaskState.PENDING, @@ -33,18 +32,15 @@ def _build_context_prefix(task_context: TaskContext) -> list[str]: - return [ - _CELERY_TASK_ID_PREFIX, - *[f"{task_context[key]}" for key in sorted(task_context)], - ] + return [f"{task_context[key]}" for key in sorted(task_context)] def _build_task_id_prefix(task_context: TaskContext) -> str: - return "::".join(_build_context_prefix(task_context)) + return ":".join(_build_context_prefix(task_context)) def _build_task_id(task_context: TaskContext, task_uuid: TaskUUID) -> TaskID: - return "::".join([_build_task_id_prefix(task_context), f"{task_uuid}"]) + return ":".join([_build_task_id_prefix(task_context), f"{task_uuid}"]) class CeleryTaskQueueClient: From 1eab85b6d52b3f6b394ace232e17dac347e61ff3 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 5 Mar 2025 16:28:06 +0100 Subject: [PATCH 108/136] add retry --- .../storage/src/simcore_service_storage/modules/celery/client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index 91d237b3843d..a3a67d7e07a9 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -24,6 +24,7 @@ _CELERY_STATES_MAPPING: Final[dict[str, TaskState]] = { "PENDING": TaskState.PENDING, "STARTED": TaskState.PENDING, + "RETRY": TaskState.PENDING, "RUNNING": TaskState.RUNNING, "SUCCESS": TaskState.SUCCESS, "ABORTED": TaskState.ABORTED, From 3a7f7e9cf7dce3fb18c6277d289edbbd0f21baf6 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 6 Mar 2025 10:41:20 +0100 Subject: [PATCH 109/136] improve error handling --- .../api/rpc/_async_jobs.py | 2 +- .../core/application.py | 5 ++- .../modules/celery/_common.py | 40 ++++++++++++++++++- .../modules/celery/client.py | 29 ++++++++------ .../modules/celery/models.py | 20 +++++++--- .../modules/celery/worker_main.py | 9 +++-- .../tests/unit/modules/celery/test_celery.py | 28 +++++++------ 7 files changed, 95 insertions(+), 38 deletions(-) diff --git a/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py b/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py index 629a7366f09d..f901502dda8d 100644 --- a/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py +++ b/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py @@ -56,7 +56,7 @@ async def get_result( assert job_id # nosec assert job_id_data # nosec - result = await get_celery_client(app).get_result( + result = await get_celery_client(app).get_task_result( task_context=job_id_data.model_dump(), task_uuid=job_id, ) diff --git a/services/storage/src/simcore_service_storage/core/application.py b/services/storage/src/simcore_service_storage/core/application.py index 4f68c350945f..13082a00fb06 100644 --- a/services/storage/src/simcore_service_storage/core/application.py +++ b/services/storage/src/simcore_service_storage/core/application.py @@ -83,7 +83,8 @@ def create_app(settings: ApplicationSettings) -> FastAPI: setup_client_session(app) setup_rabbitmq(app) - setup_rpc_api_routes(app) + if not settings.STORAGE_WORKER_MODE: + setup_rpc_api_routes(app) setup_rest_api_long_running_tasks_for_uploads(app) setup_rest_api_routes(app, API_VTAG) set_exception_handlers(app) @@ -91,7 +92,7 @@ def create_app(settings: ApplicationSettings) -> FastAPI: setup_redis(app) setup_dsm(app) - if settings.STORAGE_CLEANER_INTERVAL_S: + if settings.STORAGE_CLEANER_INTERVAL_S and not settings.STORAGE_WORKER_MODE: setup_dsm_cleaner(app) if settings.STORAGE_PROFILING: diff --git a/services/storage/src/simcore_service_storage/modules/celery/_common.py b/services/storage/src/simcore_service_storage/modules/celery/_common.py index dfdce92bc23e..151ae0b4a6f4 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/_common.py +++ b/services/storage/src/simcore_service_storage/modules/celery/_common.py @@ -1,9 +1,16 @@ +from collections.abc import Callable +from functools import wraps import logging +import traceback -from celery import Celery +from celery import Celery, Task +from celery.exceptions import Ignore +from celery.contrib.abortable import AbortableTask from settings_library.celery import CelerySettings from settings_library.redis import RedisDatabase +from .models import TaskError + _logger = logging.getLogger(__name__) @@ -18,5 +25,36 @@ def create_app(celery_settings: CelerySettings) -> Celery: ) app.conf.result_expires = celery_settings.CELERY_RESULT_EXPIRES app.conf.result_extended = True # original args are included in the results + app.conf.result_serializer = "json" app.conf.task_track_started = True return app + + +def error_handling(func: Callable): + @wraps(func) + def wrapper(task: Task, *args, **kwargs): + try: + return func(task, *args, **kwargs) + except Exception as exc: + exc_type = type(exc).__name__ + exc_message = f"{exc}" + exc_traceback = traceback.format_exc().split('\n') + + task.update_state( + state="ERROR", + meta=TaskError( + exc_type=exc_type, + exc_msg=exc_message, + ).model_dump(mode="json"), + traceback=exc_traceback + ) + raise Ignore from exc + return wrapper + + +def define_task(app: Celery, fn: Callable, task_name: str | None = None): + app.task( + name=task_name or fn.__name__, + bind=True, + base=AbortableTask, + )(error_handling(fn)) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index a3a67d7e07a9..00719994965f 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -1,16 +1,16 @@ import contextlib import logging -from typing import Any, Final +from typing import Any, Final, Type from uuid import uuid4 from celery import Celery from celery.contrib.abortable import AbortableAsyncResult from common_library.async_tools import make_async from models_library.progress_bar import ProgressReport -from pydantic import ValidationError +from pydantic import TypeAdapter, ValidationError from servicelib.logging_utils import log_context -from .models import TaskContext, TaskID, TaskState, TaskStatus, TaskUUID +from .models import TaskContext, TaskError, TaskID, TaskResult, TaskState, TaskStatus, TaskUUID _logger = logging.getLogger(__name__) @@ -28,8 +28,11 @@ "RUNNING": TaskState.RUNNING, "SUCCESS": TaskState.SUCCESS, "ABORTED": TaskState.ABORTED, - "FAILURE": TaskState.FAILURE, + "FAILURE": TaskState.ERROR, + "ERROR": TaskState.ERROR, } +_CELERY_TASK_ID_KEY_SEPARATOR: Final[str] = ":" +_CELERY_TASK_ID_KEY_ENCODING = "utf-8" def _build_context_prefix(task_context: TaskContext) -> list[str]: @@ -37,11 +40,11 @@ def _build_context_prefix(task_context: TaskContext) -> list[str]: def _build_task_id_prefix(task_context: TaskContext) -> str: - return ":".join(_build_context_prefix(task_context)) + return _CELERY_TASK_ID_KEY_SEPARATOR.join(_build_context_prefix(task_context)) def _build_task_id(task_context: TaskContext, task_uuid: TaskUUID) -> TaskID: - return ":".join([_build_task_id_prefix(task_context), f"{task_uuid}"]) + return _CELERY_TASK_ID_KEY_SEPARATOR.join([_build_task_id_prefix(task_context), f"{task_uuid}"]) class CeleryTaskQueueClient: @@ -76,9 +79,11 @@ def abort_task( # pylint: disable=R6301 AbortableAsyncResult(task_id).abort() @make_async() - def get_result(self, task_context: TaskContext, task_uuid: TaskUUID) -> Any: + def get_task_result(self, task_context: TaskContext, task_uuid: TaskUUID) -> TaskResult: task_id = _build_task_id(task_context, task_uuid) - return self._celery_app.AsyncResult(task_id).result + return TypeAdapter(TaskResult).validate_python( + self._celery_app.AsyncResult(task_id).result + ) def _get_progress_report( self, task_context: TaskContext, task_uuid: TaskUUID @@ -91,7 +96,7 @@ def _get_progress_report( return ProgressReport.model_validate(result) if state in ( TaskState.ABORTED, - TaskState.FAILURE, + TaskState.ERROR, TaskState.SUCCESS, ): return ProgressReport(actual_value=100.0) @@ -113,12 +118,12 @@ def get_task_status( def _get_completed_task_uuids(self, task_context: TaskContext) -> set[TaskUUID]: search_key = ( - _CELERY_TASK_META_PREFIX + _build_task_id_prefix(task_context) + "*" + _CELERY_TASK_META_PREFIX + _build_task_id_prefix(task_context) ) redis = self._celery_app.backend.client - if hasattr(redis, "keys") and (keys := redis.keys(search_key)): + if hasattr(redis, "keys") and (keys := redis.keys(search_key + "*")): return { - TaskUUID(f"{key}".removeprefix(_CELERY_TASK_META_PREFIX)) + TaskUUID(f"{key.decode(_CELERY_TASK_ID_KEY_ENCODING).removeprefix(search_key + _CELERY_TASK_ID_KEY_SEPARATOR)}") for key in keys } return set() diff --git a/services/storage/src/simcore_service_storage/modules/celery/models.py b/services/storage/src/simcore_service_storage/modules/celery/models.py index 39e79e809a1e..b8fa7db6cd88 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/models.py +++ b/services/storage/src/simcore_service_storage/modules/celery/models.py @@ -1,9 +1,9 @@ from enum import StrEnum, auto -from typing import Any, Final, Self, TypeAlias +from typing import Annotated, Any, Final, Self, TypeAlias from uuid import UUID from models_library.progress_bar import ProgressReport -from pydantic import BaseModel, model_validator +from pydantic import BaseModel, Field, model_validator TaskContext: TypeAlias = dict[str, Any] TaskID: TypeAlias = str @@ -17,11 +17,11 @@ class TaskState(StrEnum): PENDING = auto() RUNNING = auto() SUCCESS = auto() - FAILURE = auto() + ERROR = auto() ABORTED = auto() -_TASK_DONE = {TaskState.SUCCESS, TaskState.FAILURE, TaskState.ABORTED} +_TASK_DONE = {TaskState.SUCCESS, TaskState.ERROR, TaskState.ABORTED} class TaskStatus(BaseModel): @@ -42,7 +42,7 @@ def _check_consistency(self) -> Self: TaskState.RUNNING: _MIN_PROGRESS <= value <= _MAX_PROGRESS, TaskState.SUCCESS: value == _MAX_PROGRESS, TaskState.ABORTED: value == _MAX_PROGRESS, - TaskState.FAILURE: value == _MAX_PROGRESS, + TaskState.ERROR: value == _MAX_PROGRESS, } if not valid_states.get(self.task_state, True): @@ -50,3 +50,13 @@ def _check_consistency(self) -> Self: raise ValueError(msg) return self + + +class TaskError(BaseModel): + exc_type: str + exc_msg: str + + +TaskResult: TypeAlias = Annotated[ + TaskError | Any, Field(union_mode="left_to_right") +] diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker_main.py b/services/storage/src/simcore_service_storage/modules/celery/worker_main.py index 1958d1fed6da..fc4bd68be381 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker_main.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker_main.py @@ -2,7 +2,6 @@ import logging -from celery.contrib.abortable import AbortableTask from celery.signals import worker_init, worker_shutdown from servicelib.logging_utils import config_all_loggers from simcore_service_storage.modules.celery.signals import ( @@ -11,8 +10,8 @@ ) from ...core.settings import ApplicationSettings -from ._common import create_app as create_celery_app -from .tasks import export_data +from ._common import create_app as create_celery_app, define_task +from .tasks import export_data, export_data_with_error _settings = ApplicationSettings.create_from_envs() @@ -30,4 +29,6 @@ app = create_celery_app(_settings.STORAGE_CELERY) worker_init.connect(on_worker_init) worker_shutdown.connect(on_worker_shutdown) -app.task(name="export_data", bind=True, base=AbortableTask)(export_data) + +define_task(app, export_data) +define_task(app, export_data_with_error) diff --git a/services/storage/tests/unit/modules/celery/test_celery.py b/services/storage/tests/unit/modules/celery/test_celery.py index 15ecfe90be3a..736d6ca7b278 100644 --- a/services/storage/tests/unit/modules/celery/test_celery.py +++ b/services/storage/tests/unit/modules/celery/test_celery.py @@ -4,14 +4,16 @@ from collections.abc import Callable from random import randint +from pydantic import ValidationError import pytest from celery import Celery, Task from celery.contrib.abortable import AbortableTask from models_library.progress_bar import ProgressReport from servicelib.logging_utils import log_context from simcore_service_storage.modules.celery import get_event_loop +from simcore_service_storage.modules.celery._common import define_task from simcore_service_storage.modules.celery.client import CeleryTaskQueueClient -from simcore_service_storage.modules.celery.models import TaskContext, TaskState +from simcore_service_storage.modules.celery.models import TaskContext, TaskError, TaskState from simcore_service_storage.modules.celery.utils import ( get_celery_worker, get_fastapi_app, @@ -69,11 +71,9 @@ def dreamer_task(task: AbortableTask) -> list[int]: @pytest.fixture def register_celery_tasks() -> Callable[[Celery], None]: def _(celery_app: Celery) -> None: - celery_app.task(name="sync_archive", bind=True)(sync_archive) - celery_app.task(name="failure_task", bind=True)(failure_task) - celery_app.task(name="dreamer_task", base=AbortableTask, bind=True)( - dreamer_task - ) + define_task(celery_app, sync_archive) + define_task(celery_app, failure_task) + define_task(celery_app, dreamer_task) return _ @@ -99,6 +99,9 @@ async def test_sumitting_task_calling_async_function_results_with_success_state( progress = await celery_client.get_task_status(task_context, task_uuid) assert progress.task_state == TaskState.SUCCESS + assert ( + await celery_client.get_task_result(task_context, task_uuid) + ) == "archive.zip" assert ( await celery_client.get_task_status(task_context, task_uuid) ).task_state == TaskState.SUCCESS @@ -113,20 +116,19 @@ async def test_submitting_task_with_failure_results_with_error( task_uuid = await celery_client.send_task("failure_task", task_context=task_context) for attempt in Retrying( - retry=retry_if_exception_type(AssertionError), + retry=retry_if_exception_type((AssertionError, ValidationError)), wait=wait_fixed(1), stop=stop_after_delay(30), ): with attempt: - result = await celery_client.get_result(task_context, task_uuid) - assert isinstance(result, ValueError) + result = await celery_client.get_task_result(task_context, task_uuid) + assert isinstance(result, TaskError) assert ( await celery_client.get_task_status(task_context, task_uuid) - ).task_state == TaskState.FAILURE - result = await celery_client.get_result(task_context, task_uuid) - assert isinstance(result, ValueError) - assert f"{result}" == "my error here" + ).task_state == TaskState.ERROR + result = await celery_client.get_task_result(task_context, task_uuid) + assert f"{result.exc_msg}" == "my error here" @pytest.mark.usefixtures("celery_worker") From 8e5b1f9fc923b197964558f55391f506bdea0130 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 6 Mar 2025 10:44:34 +0100 Subject: [PATCH 110/136] add task --- .../modules/celery/client.py | 4 ++-- .../modules/celery/tasks.py | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index 00719994965f..423c10e69de0 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -1,6 +1,6 @@ import contextlib import logging -from typing import Any, Final, Type +from typing import Any, Final from uuid import uuid4 from celery import Celery @@ -10,7 +10,7 @@ from pydantic import TypeAdapter, ValidationError from servicelib.logging_utils import log_context -from .models import TaskContext, TaskError, TaskID, TaskResult, TaskState, TaskStatus, TaskUUID +from .models import TaskContext, TaskID, TaskResult, TaskState, TaskStatus, TaskUUID _logger = logging.getLogger(__name__) diff --git a/services/storage/src/simcore_service_storage/modules/celery/tasks.py b/services/storage/src/simcore_service_storage/modules/celery/tasks.py index e5ca99aa771e..3237dd5a587f 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/tasks.py +++ b/services/storage/src/simcore_service_storage/modules/celery/tasks.py @@ -1,16 +1,20 @@ import logging import time + from celery import Task +from common_library.errors_classes import OsparcErrorMixin from models_library.progress_bar import ProgressReport from models_library.projects_nodes_io import StorageFileID from servicelib.logging_utils import log_context -from simcore_service_storage.modules.celery.utils import get_celery_worker + +from .utils import get_celery_worker _logger = logging.getLogger(__name__) def export_data(task: Task, files: list[StorageFileID]): + _logger.info("Exporting files: %s", files) for n, file in enumerate(files, start=1): with log_context( _logger, @@ -25,3 +29,12 @@ def export_data(task: Task, files: list[StorageFileID]): ) time.sleep(10) return "done" + + +class MyError(OsparcErrorMixin, Exception): + msg_template = "Something strange happened: {msg}" + + +def export_data_with_error(task: Task, files: list[StorageFileID]): + msg = "BOOM!" + raise MyError(msg=msg) From c2be8f0de120fdc0e5fd10e903f914e4b8e15c7b Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 6 Mar 2025 11:10:55 +0100 Subject: [PATCH 111/136] update --- .../src/simcore_service_storage/modules/celery/client.py | 6 ++---- services/storage/tests/unit/modules/celery/test_celery.py | 8 +++++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index 423c10e69de0..3b6ec2b56f3a 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -79,11 +79,9 @@ def abort_task( # pylint: disable=R6301 AbortableAsyncResult(task_id).abort() @make_async() - def get_task_result(self, task_context: TaskContext, task_uuid: TaskUUID) -> TaskResult: + def get_task_result(self, task_context: TaskContext, task_uuid: TaskUUID) -> Any: task_id = _build_task_id(task_context, task_uuid) - return TypeAdapter(TaskResult).validate_python( - self._celery_app.AsyncResult(task_id).result - ) + return self._celery_app.AsyncResult(task_id).result def _get_progress_report( self, task_context: TaskContext, task_uuid: TaskUUID diff --git a/services/storage/tests/unit/modules/celery/test_celery.py b/services/storage/tests/unit/modules/celery/test_celery.py index 736d6ca7b278..bcce3580cc12 100644 --- a/services/storage/tests/unit/modules/celery/test_celery.py +++ b/services/storage/tests/unit/modules/celery/test_celery.py @@ -4,7 +4,7 @@ from collections.abc import Callable from random import randint -from pydantic import ValidationError +from pydantic import TypeAdapter, ValidationError import pytest from celery import Celery, Task from celery.contrib.abortable import AbortableTask @@ -121,13 +121,15 @@ async def test_submitting_task_with_failure_results_with_error( stop=stop_after_delay(30), ): with attempt: - result = await celery_client.get_task_result(task_context, task_uuid) + raw_result = await celery_client.get_task_result(task_context, task_uuid) + result = TypeAdapter(TaskError).validate_python(raw_result) assert isinstance(result, TaskError) assert ( await celery_client.get_task_status(task_context, task_uuid) ).task_state == TaskState.ERROR - result = await celery_client.get_task_result(task_context, task_uuid) + raw_result = await celery_client.get_task_result(task_context, task_uuid) + result = TypeAdapter(TaskError).validate_python(raw_result) assert f"{result.exc_msg}" == "my error here" From 87fb925f43f8db878d369f8788d3a0756f048cdc Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 6 Mar 2025 11:22:28 +0100 Subject: [PATCH 112/136] fix get --- .../src/simcore_service_storage/modules/celery/client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index 3b6ec2b56f3a..a2b0fefdb95c 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -134,6 +134,7 @@ def get_task_uuids(self, task_context: TaskContext) -> set[TaskUUID]: if task_ids := getattr( self._celery_app.control.inspect(), task_inspect_status )(): - all_task_ids.add(task_ids) + for values in task_ids.values(): + all_task_ids.add(values) return all_task_ids From 9a96dd8fd4cfc42717019b814a70f9a84fabc0fd Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 6 Mar 2025 11:29:52 +0100 Subject: [PATCH 113/136] fix get_uuids --- .../src/simcore_service_storage/modules/celery/client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index a2b0fefdb95c..57add0031a78 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -135,6 +135,7 @@ def get_task_uuids(self, task_context: TaskContext) -> set[TaskUUID]: self._celery_app.control.inspect(), task_inspect_status )(): for values in task_ids.values(): - all_task_ids.add(values) + for value in values: + all_task_ids.add(TaskUUID(value.decode(_CELERY_TASK_ID_KEY_ENCODING))) return all_task_ids From 8df0a5cadbc6a259b75e838c8cf51b84411ac816 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 6 Mar 2025 11:43:36 +0100 Subject: [PATCH 114/136] fix --- .../src/simcore_service_storage/modules/celery/client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index 57add0031a78..a3feca55b77e 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -130,12 +130,15 @@ def _get_completed_task_uuids(self, task_context: TaskContext) -> set[TaskUUID]: def get_task_uuids(self, task_context: TaskContext) -> set[TaskUUID]: all_task_ids = self._get_completed_task_uuids(task_context) + search_key = ( + _CELERY_TASK_META_PREFIX + _build_task_id_prefix(task_context) + ) for task_inspect_status in _CELERY_INSPECT_TASK_STATUSES: if task_ids := getattr( self._celery_app.control.inspect(), task_inspect_status )(): for values in task_ids.values(): for value in values: - all_task_ids.add(TaskUUID(value.decode(_CELERY_TASK_ID_KEY_ENCODING))) + all_task_ids.add(TaskUUID(value.removeprefix(search_key + _CELERY_TASK_ID_KEY_SEPARATOR))) return all_task_ids From c56b8b7fd434d7e8228046cf593ea24a5b54cfd7 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 6 Mar 2025 11:46:05 +0100 Subject: [PATCH 115/136] fix import --- .../src/simcore_service_storage/modules/celery/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index a3feca55b77e..72dcc7e3853c 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -7,10 +7,10 @@ from celery.contrib.abortable import AbortableAsyncResult from common_library.async_tools import make_async from models_library.progress_bar import ProgressReport -from pydantic import TypeAdapter, ValidationError +from pydantic import ValidationError from servicelib.logging_utils import log_context -from .models import TaskContext, TaskID, TaskResult, TaskState, TaskStatus, TaskUUID +from .models import TaskContext, TaskID, TaskState, TaskStatus, TaskUUID _logger = logging.getLogger(__name__) From 0ec94b6663821dd05d9319d219d9187ca6eb2800 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 6 Mar 2025 14:05:25 +0100 Subject: [PATCH 116/136] update --- .../simcore_service_storage/modules/celery/_common.py | 6 ++++++ .../simcore_service_storage/modules/celery/client.py | 7 +++++-- .../storage/tests/unit/modules/celery/test_celery.py | 10 +++++----- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/_common.py b/services/storage/src/simcore_service_storage/modules/celery/_common.py index 151ae0b4a6f4..4ca47d64afae 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/_common.py +++ b/services/storage/src/simcore_service_storage/modules/celery/_common.py @@ -40,6 +40,12 @@ def wrapper(task: Task, *args, **kwargs): exc_message = f"{exc}" exc_traceback = traceback.format_exc().split('\n') + _logger.exception( + "Task %s failed with exception: %s", + task.request.id, + exc_message, + ) + task.update_state( state="ERROR", meta=TaskError( diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index 72dcc7e3853c..60576fcd2a8e 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -34,6 +34,9 @@ _CELERY_TASK_ID_KEY_SEPARATOR: Final[str] = ":" _CELERY_TASK_ID_KEY_ENCODING = "utf-8" +_MIN_PROGRESS_VALUE = 0.0 +_MAX_PROGRESS_VALUE = 100.0 + def _build_context_prefix(task_context: TaskContext) -> list[str]: return [f"{task_context[key]}" for key in sorted(task_context)] @@ -97,8 +100,8 @@ def _get_progress_report( TaskState.ERROR, TaskState.SUCCESS, ): - return ProgressReport(actual_value=100.0) - return ProgressReport(actual_value=0.0) + return ProgressReport(actual_value=_MAX_PROGRESS_VALUE) + return ProgressReport(actual_value=_MIN_PROGRESS_VALUE) def _get_state(self, task_context: TaskContext, task_uuid: TaskUUID) -> TaskState: task_id = _build_task_id(task_context, task_uuid) diff --git a/services/storage/tests/unit/modules/celery/test_celery.py b/services/storage/tests/unit/modules/celery/test_celery.py index bcce3580cc12..0d5edf7789c2 100644 --- a/services/storage/tests/unit/modules/celery/test_celery.py +++ b/services/storage/tests/unit/modules/celery/test_celery.py @@ -96,15 +96,15 @@ async def test_sumitting_task_calling_async_function_results_with_success_state( stop=stop_after_delay(30), ): with attempt: - progress = await celery_client.get_task_status(task_context, task_uuid) - assert progress.task_state == TaskState.SUCCESS + status = await celery_client.get_task_status(task_context, task_uuid) + assert status.task_state == TaskState.SUCCESS - assert ( - await celery_client.get_task_result(task_context, task_uuid) - ) == "archive.zip" assert ( await celery_client.get_task_status(task_context, task_uuid) ).task_state == TaskState.SUCCESS + assert ( + await celery_client.get_task_result(task_context, task_uuid) + ) == "archive.zip" @pytest.mark.usefixtures("celery_worker") From 403aaa168f0cb969e35373a47b3c5dc11b5050e4 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 6 Mar 2025 15:28:34 +0100 Subject: [PATCH 117/136] use taskstate --- .../src/simcore_service_storage/modules/celery/_common.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/_common.py b/services/storage/src/simcore_service_storage/modules/celery/_common.py index 4ca47d64afae..fea06be05948 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/_common.py +++ b/services/storage/src/simcore_service_storage/modules/celery/_common.py @@ -9,7 +9,7 @@ from settings_library.celery import CelerySettings from settings_library.redis import RedisDatabase -from .models import TaskError +from .models import TaskError, TaskState _logger = logging.getLogger(__name__) @@ -47,14 +47,14 @@ def wrapper(task: Task, *args, **kwargs): ) task.update_state( - state="ERROR", + state=TaskState.ERROR.upper(), meta=TaskError( exc_type=exc_type, exc_msg=exc_message, ).model_dump(mode="json"), traceback=exc_traceback ) - raise Ignore from exc + raise Ignore from exc # ignore doing state updates return wrapper From 65a07e012c2bada41f3ad1405b9d744428306e94 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 6 Mar 2025 15:35:41 +0100 Subject: [PATCH 118/136] remove unused --- .../src/simcore_service_storage/modules/celery/client.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index 60576fcd2a8e..9293939fd5d9 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -68,11 +68,6 @@ def send_task( self._celery_app.send_task(task_name, task_id=task_id, kwargs=task_params) return task_uuid - @make_async() - def get_task(self, task_context: TaskContext, task_uuid: TaskUUID) -> Any: - task_id = _build_task_id(task_context, task_uuid) - return self._celery_app.tasks(task_id) - @make_async() def abort_task( # pylint: disable=R6301 self, task_context: TaskContext, task_uuid: TaskUUID From 327c8f1f52ef9fdf98ed056fe6d7588580b3afb3 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 7 Mar 2025 09:38:06 +0100 Subject: [PATCH 119/136] type hinting --- .../src/simcore_service_storage/modules/celery/_common.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/_common.py b/services/storage/src/simcore_service_storage/modules/celery/_common.py index fea06be05948..baaa0ffbba80 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/_common.py +++ b/services/storage/src/simcore_service_storage/modules/celery/_common.py @@ -2,6 +2,7 @@ from functools import wraps import logging import traceback +from typing import Any from celery import Celery, Task from celery.exceptions import Ignore @@ -30,9 +31,9 @@ def create_app(celery_settings: CelerySettings) -> Celery: return app -def error_handling(func: Callable): +def error_handling(func: Callable[..., Any]) -> Callable[..., Any]: @wraps(func) - def wrapper(task: Task, *args, **kwargs): + def wrapper(task: Task, *args: Any, **kwargs: Any) -> Any: try: return func(task, *args, **kwargs) except Exception as exc: From d0774721d78597030e8182e7ce047eb509aaec75 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 7 Mar 2025 10:49:52 +0100 Subject: [PATCH 120/136] remove taskresult --- .../src/simcore_service_storage/api/rpc/_data_export.py | 2 +- .../src/simcore_service_storage/modules/celery/models.py | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/services/storage/src/simcore_service_storage/api/rpc/_data_export.py b/services/storage/src/simcore_service_storage/api/rpc/_data_export.py index aab9d7339f62..424fbc2f0d0d 100644 --- a/services/storage/src/simcore_service_storage/api/rpc/_data_export.py +++ b/services/storage/src/simcore_service_storage/api/rpc/_data_export.py @@ -52,7 +52,7 @@ async def start_data_export( ) from err task_uuid = await get_celery_client(app).send_task( - "export_data", + "export_data_with_error", task_context=job_id_data.model_dump(), files=data_export_start.file_and_folder_ids, # ANE: adapt here your signature ) diff --git a/services/storage/src/simcore_service_storage/modules/celery/models.py b/services/storage/src/simcore_service_storage/modules/celery/models.py index b8fa7db6cd88..c3566cc92313 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/models.py +++ b/services/storage/src/simcore_service_storage/modules/celery/models.py @@ -55,8 +55,3 @@ def _check_consistency(self) -> Self: class TaskError(BaseModel): exc_type: str exc_msg: str - - -TaskResult: TypeAlias = Annotated[ - TaskError | Any, Field(union_mode="left_to_right") -] From 66e51d938ea9adc3dd18a4b17b0db8c08813b187 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 7 Mar 2025 11:28:12 +0100 Subject: [PATCH 121/136] add celery env --- .env-devel | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.env-devel b/.env-devel index e4f191a0c1ec..c97d8829ba79 100644 --- a/.env-devel +++ b/.env-devel @@ -49,6 +49,8 @@ CATALOG_SERVICES_DEFAULT_RESOURCES='{"CPU": {"limit": 0.1, "reservation": 0.1}, CATALOG_SERVICES_DEFAULT_SPECIFICATIONS='{}' CATALOG_TRACING=null +CELERY_RESULT_EXPIRES=P7D + CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH='{"type":"tls","tls_ca_file":"/home/scu/.dask/dask-crt.pem","tls_client_cert":"/home/scu/.dask/dask-crt.pem","tls_client_key":"/home/scu/.dask/dask-key.pem"}' CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DOCKER_IMAGE_TAG=master-github-latest CLUSTERS_KEEPER_DASK_NTHREADS=0 From 3c86023f16773d636a4ec37e7f682bb2333b5955 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 7 Mar 2025 15:08:11 +0100 Subject: [PATCH 122/136] get hostname --- services/storage/docker/healthcheck.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/storage/docker/healthcheck.py b/services/storage/docker/healthcheck.py index 9cb4f2421d7a..2dc74409d179 100755 --- a/services/storage/docker/healthcheck.py +++ b/services/storage/docker/healthcheck.py @@ -20,6 +20,7 @@ import os +import socket import subprocess import sys from urllib.request import urlopen @@ -50,7 +51,7 @@ def _is_celery_worker_healthy(): "inspect", "ping", "--destination", - "celery@" + os.getenv("HOSTNAME", ""), + "celery@" + socket.gethostname(), ], capture_output=True, text=True, From 8ea33e6e3f816d3755e1bc4b5d8a7917e7c61e6a Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 10 Mar 2025 09:25:30 +0100 Subject: [PATCH 123/136] fix imports --- .../simcore_service_storage/modules/celery/models.py | 2 +- .../simcore_service_storage/modules/celery/signals.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/models.py b/services/storage/src/simcore_service_storage/modules/celery/models.py index c3566cc92313..b2e76fafa39c 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/models.py +++ b/services/storage/src/simcore_service_storage/modules/celery/models.py @@ -1,5 +1,5 @@ from enum import StrEnum, auto -from typing import Annotated, Any, Final, Self, TypeAlias +from typing import Any, Final, Self, TypeAlias from uuid import UUID from models_library.progress_bar import ProgressReport diff --git a/services/storage/src/simcore_service_storage/modules/celery/signals.py b/services/storage/src/simcore_service_storage/modules/celery/signals.py index c10bb71dab89..9a82f80a477d 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/signals.py +++ b/services/storage/src/simcore_service_storage/modules/celery/signals.py @@ -7,15 +7,15 @@ from celery import Celery from fastapi import FastAPI from servicelib.async_utils import cancel_wait_task -from simcore_service_storage.core.application import create_app -from simcore_service_storage.core.settings import ApplicationSettings -from simcore_service_storage.modules.celery import get_event_loop, set_event_loop -from simcore_service_storage.modules.celery.utils import ( +from ...core.application import create_app +from ...core.settings import ApplicationSettings +from ...modules.celery import get_event_loop, set_event_loop +from ...modules.celery.utils import ( get_fastapi_app, set_celery_worker, set_fastapi_app, ) -from simcore_service_storage.modules.celery.worker import CeleryTaskQueueWorker +from ...modules.celery.worker import CeleryTaskQueueWorker _logger = logging.getLogger(__name__) From c76cbd338155b2a8f5f01aa2ff9daf05cae83299 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 10 Mar 2025 10:09:13 +0100 Subject: [PATCH 124/136] fix settings --- .../simcore_service_storage/core/settings.py | 44 +++++++++++-------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/services/storage/src/simcore_service_storage/core/settings.py b/services/storage/src/simcore_service_storage/core/settings.py index 11086f5baa58..a861085503bd 100644 --- a/services/storage/src/simcore_service_storage/core/settings.py +++ b/services/storage/src/simcore_service_storage/core/settings.py @@ -71,29 +71,35 @@ class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): Field( 30, description="Interval in seconds when task cleaning pending uploads runs. setting to NULL disables the cleaner.", + ), + ] + + STORAGE_RABBITMQ: Annotated[ + RabbitSettings | None, + Field( + json_schema_extra={"auto_default_from_env": True}, + ) + ] - STORAGE_CLEANER_INTERVAL_S: int | None = Field( - 30, - description="Interval in seconds when task cleaning pending uploads runs. setting to NULL disables the cleaner.", - ) - - STORAGE_RABBITMQ: RabbitSettings | None = Field( - json_schema_extra={"auto_default_from_env": True}, - ) - - STORAGE_S3_CLIENT_MAX_TRANSFER_CONCURRENCY: int = Field( - 4, - description="Maximal amount of threads used by underlying S3 client to transfer data to S3 backend", - ) - - STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field( - default=False, - validation_alias=AliasChoices( - "STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED", - "LOG_FORMAT_LOCAL_DEV_ENABLED", + STORAGE_S3_CLIENT_MAX_TRANSFER_CONCURRENCY: Annotated[ + int, + Field( + 4, + description="Maximal amount of threads used by underlying S3 client to transfer data to S3 backend", ), ] + STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED: Annotated[ + bool, + Field( + default=False, + validation_alias=AliasChoices( + "STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED", + "LOG_FORMAT_LOCAL_DEV_ENABLED", + ), + ) + ] + STORAGE_RABBITMQ: Annotated[ RabbitSettings | None, Field( From ed8f92c29850cd0ac76f9bf86900347e016f370b Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 10 Mar 2025 10:11:42 +0100 Subject: [PATCH 125/136] fix --- .../simcore_service_storage/modules/celery/tasks.py | 10 ---------- .../modules/celery/worker_main.py | 3 +-- .../tests/unit/modules/celery/test_celery.py | 13 +++++++++---- 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/tasks.py b/services/storage/src/simcore_service_storage/modules/celery/tasks.py index 3237dd5a587f..130ae6e9007d 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/tasks.py +++ b/services/storage/src/simcore_service_storage/modules/celery/tasks.py @@ -3,7 +3,6 @@ from celery import Task -from common_library.errors_classes import OsparcErrorMixin from models_library.progress_bar import ProgressReport from models_library.projects_nodes_io import StorageFileID from servicelib.logging_utils import log_context @@ -29,12 +28,3 @@ def export_data(task: Task, files: list[StorageFileID]): ) time.sleep(10) return "done" - - -class MyError(OsparcErrorMixin, Exception): - msg_template = "Something strange happened: {msg}" - - -def export_data_with_error(task: Task, files: list[StorageFileID]): - msg = "BOOM!" - raise MyError(msg=msg) diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker_main.py b/services/storage/src/simcore_service_storage/modules/celery/worker_main.py index fc4bd68be381..954bfbaa6afc 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker_main.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker_main.py @@ -11,7 +11,7 @@ from ...core.settings import ApplicationSettings from ._common import create_app as create_celery_app, define_task -from .tasks import export_data, export_data_with_error +from .tasks import export_data _settings = ApplicationSettings.create_from_envs() @@ -31,4 +31,3 @@ worker_shutdown.connect(on_worker_shutdown) define_task(app, export_data) -define_task(app, export_data_with_error) diff --git a/services/storage/tests/unit/modules/celery/test_celery.py b/services/storage/tests/unit/modules/celery/test_celery.py index 0d5edf7789c2..99c3cc34263a 100644 --- a/services/storage/tests/unit/modules/celery/test_celery.py +++ b/services/storage/tests/unit/modules/celery/test_celery.py @@ -8,6 +8,7 @@ import pytest from celery import Celery, Task from celery.contrib.abortable import AbortableTask +from common_library.errors_classes import OsparcErrorMixin from models_library.progress_bar import ProgressReport from servicelib.logging_utils import log_context from simcore_service_storage.modules.celery import get_event_loop @@ -52,9 +53,13 @@ def sync_archive(task: Task, files: list[str]) -> str: ).result() -def failure_task(task: Task) -> str: - msg = "my error here" - raise ValueError(msg) +class MyError(OsparcErrorMixin, Exception): + msg_template = "Something strange happened: {msg}" + + +def failure_task(task: Task): + msg = "BOOM!" + raise MyError(msg=msg) def dreamer_task(task: AbortableTask) -> list[int]: @@ -130,7 +135,7 @@ async def test_submitting_task_with_failure_results_with_error( ).task_state == TaskState.ERROR raw_result = await celery_client.get_task_result(task_context, task_uuid) result = TypeAdapter(TaskError).validate_python(raw_result) - assert f"{result.exc_msg}" == "my error here" + assert f"{result.exc_msg}" == "Something strange happened: BOOM!" @pytest.mark.usefixtures("celery_worker") From 7f09d30765f340fbb459419aef9357df8bb21886 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 10 Mar 2025 10:46:07 +0100 Subject: [PATCH 126/136] add comment --- .../storage/src/simcore_service_storage/modules/celery/client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index 9293939fd5d9..eb072c888887 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -89,6 +89,7 @@ def _get_progress_report( state = self._get_state(task_context, task_uuid) if result and state == TaskState.RUNNING: with contextlib.suppress(ValidationError): + # avoids exception if result is not a ProgressReport (or overwritten by a Celery's state update) return ProgressReport.model_validate(result) if state in ( TaskState.ABORTED, From 5b1f97a5cb5cf08c8a565ccca6c092f653c1b813 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 10 Mar 2025 11:03:42 +0100 Subject: [PATCH 127/136] fix docker compose --- services/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 3555eaec3292..77520234af41 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -1152,7 +1152,7 @@ services: image: ${DOCKER_REGISTRY:-itisfoundation}/storage:${DOCKER_IMAGE_TAG:-latest} init: true hostname: "sto-{{.Node.Hostname}}-{{.Task.Slot}}" - environment: + environment: &storage_environment DATCORE_ADAPTER_HOST: ${DATCORE_ADAPTER_HOST:-datcore-adapter} LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} LOG_FILTER_MAPPING : ${LOG_FILTER_MAPPING} From 9e6284b67a724fcd5dd80403f0bd1d8ebf84bb00 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 10 Mar 2025 11:06:05 +0100 Subject: [PATCH 128/136] use settings --- services/storage/docker/healthcheck.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/services/storage/docker/healthcheck.py b/services/storage/docker/healthcheck.py index 2dc74409d179..d938c860dabf 100755 --- a/services/storage/docker/healthcheck.py +++ b/services/storage/docker/healthcheck.py @@ -25,7 +25,7 @@ import sys from urllib.request import urlopen -from simcore_service_storage.main import ApplicationSettings +from simcore_service_storage.core.settings import ApplicationSettings SUCCESS, UNHEALTHY = 0, 1 @@ -35,10 +35,9 @@ # Queries host # pylint: disable=consider-using-with +app_settings = ApplicationSettings.create_from_envs() def _is_celery_worker_healthy(): - app_settings = ApplicationSettings.create_from_envs() - assert app_settings.STORAGE_CELERY broker_url = app_settings.STORAGE_CELERY.CELERY_RABBIT_BROKER.dsn @@ -64,7 +63,7 @@ def _is_celery_worker_healthy(): ok = ( ok - or (bool(os.getenv("STORAGE_WORKER_MODE", "") and _is_celery_worker_healthy())) + or (app_settings.STORAGE_WORKER_MODE and _is_celery_worker_healthy()) or urlopen( "{host}{baseurl}".format( host=sys.argv[1], baseurl=os.environ.get("SIMCORE_NODE_BASEPATH", "") From 99ba805c090ca9fdb68c69fedef70577188862d0 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 10 Mar 2025 11:33:34 +0100 Subject: [PATCH 129/136] fix typecheck --- .../src/simcore_service_storage/modules/celery/models.py | 2 +- services/storage/tests/unit/test_db_data_export.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/models.py b/services/storage/src/simcore_service_storage/modules/celery/models.py index b2e76fafa39c..2f04c5b81329 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/models.py +++ b/services/storage/src/simcore_service_storage/modules/celery/models.py @@ -3,7 +3,7 @@ from uuid import UUID from models_library.progress_bar import ProgressReport -from pydantic import BaseModel, Field, model_validator +from pydantic import BaseModel, model_validator TaskContext: TypeAlias = dict[str, Any] TaskID: TypeAlias = str diff --git a/services/storage/tests/unit/test_db_data_export.py b/services/storage/tests/unit/test_db_data_export.py index cc73c3b40601..0bf9db1a2edb 100644 --- a/services/storage/tests/unit/test_db_data_export.py +++ b/services/storage/tests/unit/test_db_data_export.py @@ -1,5 +1,6 @@ # pylint: disable=W0621 # pylint: disable=W0613 +# pylint: disable=R6301 from collections.abc import Awaitable, Callable from pathlib import Path from typing import Any, Literal, NamedTuple @@ -66,7 +67,7 @@ async def get_task_status(self, *args, **kwargs) -> TaskStatus: progress_report=ProgressReport(actual_value=42.0), ) - async def get_result(self, *args, **kwargs) -> Any: + async def get_task_result(self, *args, **kwargs) -> Any: return {} async def get_task_uuids(self, *args, **kwargs) -> set[TaskUUID]: From 8f95fd2969a01d89f234b36e6c4299278ecce868 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 10 Mar 2025 14:08:53 +0100 Subject: [PATCH 130/136] fix envs --- .../simcore_service_storage/core/settings.py | 27 +------------------ 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/services/storage/src/simcore_service_storage/core/settings.py b/services/storage/src/simcore_service_storage/core/settings.py index a861085503bd..66142a82b3ee 100644 --- a/services/storage/src/simcore_service_storage/core/settings.py +++ b/services/storage/src/simcore_service_storage/core/settings.py @@ -89,32 +89,6 @@ class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): ), ] - STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED: Annotated[ - bool, - Field( - default=False, - validation_alias=AliasChoices( - "STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED", - "LOG_FORMAT_LOCAL_DEV_ENABLED", - ), - ) - ] - - STORAGE_RABBITMQ: Annotated[ - RabbitSettings | None, - Field( - json_schema_extra={"auto_default_from_env": True}, - ), - ] - - STORAGE_S3_CLIENT_MAX_TRANSFER_CONCURRENCY: Annotated[ - int, - Field( - 4, - description="Maximal amount of threads used by underlying S3 client to transfer data to S3 backend", - ), - ] - STORAGE_LOG_FORMAT_LOCAL_DEV_ENABLED: Annotated[ bool, Field( @@ -126,6 +100,7 @@ class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): description="Enables local development _logger format. WARNING: make sure it is disabled if you want to have structured logs!", ), ] + STORAGE_LOG_FILTER_MAPPING: Annotated[ dict[LoggerName, list[MessageSubstring]], Field( From ec4bb0017cbd7a9314b77d0440ebb7e9e02ce671 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 10 Mar 2025 14:42:15 +0100 Subject: [PATCH 131/136] fix mypy --- mypy.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/mypy.ini b/mypy.ini index 9df50ed1d52f..8cefa3474452 100644 --- a/mypy.ini +++ b/mypy.ini @@ -5,6 +5,7 @@ disallow_any_generics = False # disallow_untyped_defs: if True, it enforces things like `def __init__(self) -> CLASSNAME` or `def test_() -> None` which does not worth the effort disallow_untyped_defs = False follow_imports = silent +follow_untyped_imports = True # ignore_missing_imports: setting this to True ignores issues from imported libraries, so do not set it!! ignore_missing_imports = False namespace_packages = True From 76e7fc328c0c6a53930b58371972821987221a2b Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 10 Mar 2025 15:23:40 +0100 Subject: [PATCH 132/136] fix env --- services/storage/.env-devel | 20 ++++++++++++++++++++ services/storage/Makefile | 5 +---- 2 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 services/storage/.env-devel diff --git a/services/storage/.env-devel b/services/storage/.env-devel new file mode 100644 index 000000000000..96d465822d39 --- /dev/null +++ b/services/storage/.env-devel @@ -0,0 +1,20 @@ +CELERY_RESULT_EXPIRES=P7D + +RABBIT_HOST=rabbit +RABBIT_PASSWORD=adminadmin +RABBIT_PORT=5672 +RABBIT_SECURE=false +RABBIT_USER=admin + +REDIS_HOST=redis +REDIS_PORT=6379 +REDIS_PASSWORD=adminadmin +REDIS_SECURE=false +REDIS_USER=null + +STORAGE_ENDPOINT=storage:8080 +STORAGE_HOST=storage +STORAGE_LOGLEVEL=INFO +STORAGE_PORT=8080 +STORAGE_PROFILING=1 +STORAGE_TRACING=null diff --git a/services/storage/Makefile b/services/storage/Makefile index 194e144f4cfb..ef350cae0917 100644 --- a/services/storage/Makefile +++ b/services/storage/Makefile @@ -5,12 +5,9 @@ include ../../scripts/common.Makefile include ../../scripts/common-service.Makefile -.env-ignore: - $(APP_CLI_NAME) echo-dotenv > $@ - .PHONY: openapi.json openapi-specs: openapi.json -openapi.json: .env-ignore +openapi.json: .env # generating openapi specs file (need to have the environment set for this) @set -o allexport; \ source $<; \ From 3cf012ee5d16be83f385f210aecf95f3098a7839 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 10 Mar 2025 16:13:41 +0100 Subject: [PATCH 133/136] revert mypy setting --- mypy.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/mypy.ini b/mypy.ini index 8cefa3474452..9df50ed1d52f 100644 --- a/mypy.ini +++ b/mypy.ini @@ -5,7 +5,6 @@ disallow_any_generics = False # disallow_untyped_defs: if True, it enforces things like `def __init__(self) -> CLASSNAME` or `def test_() -> None` which does not worth the effort disallow_untyped_defs = False follow_imports = silent -follow_untyped_imports = True # ignore_missing_imports: setting this to True ignores issues from imported libraries, so do not set it!! ignore_missing_imports = False namespace_packages = True From cf96a045dfb17522370b2aeace03ed94299a7748 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 10 Mar 2025 16:16:38 +0100 Subject: [PATCH 134/136] typecheck --- .../src/simcore_service_storage/modules/celery/_common.py | 6 +++--- .../src/simcore_service_storage/modules/celery/client.py | 4 ++-- .../src/simcore_service_storage/modules/celery/signals.py | 2 +- .../src/simcore_service_storage/modules/celery/tasks.py | 2 +- .../src/simcore_service_storage/modules/celery/utils.py | 2 +- .../src/simcore_service_storage/modules/celery/worker.py | 2 +- .../simcore_service_storage/modules/celery/worker_main.py | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/services/storage/src/simcore_service_storage/modules/celery/_common.py b/services/storage/src/simcore_service_storage/modules/celery/_common.py index baaa0ffbba80..ae5979d4f1fb 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/_common.py +++ b/services/storage/src/simcore_service_storage/modules/celery/_common.py @@ -4,9 +4,9 @@ import traceback from typing import Any -from celery import Celery, Task -from celery.exceptions import Ignore -from celery.contrib.abortable import AbortableTask +from celery import Celery, Task # type: ignore[import-untyped] +from celery.exceptions import Ignore # type: ignore[import-untyped] +from celery.contrib.abortable import AbortableTask # type: ignore[import-untyped] from settings_library.celery import CelerySettings from settings_library.redis import RedisDatabase diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/services/storage/src/simcore_service_storage/modules/celery/client.py index eb072c888887..300aadb66b2c 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/services/storage/src/simcore_service_storage/modules/celery/client.py @@ -3,8 +3,8 @@ from typing import Any, Final from uuid import uuid4 -from celery import Celery -from celery.contrib.abortable import AbortableAsyncResult +from celery import Celery # type: ignore[import-untyped] +from celery.contrib.abortable import AbortableAsyncResult # type: ignore[import-untyped] from common_library.async_tools import make_async from models_library.progress_bar import ProgressReport from pydantic import ValidationError diff --git a/services/storage/src/simcore_service_storage/modules/celery/signals.py b/services/storage/src/simcore_service_storage/modules/celery/signals.py index 9a82f80a477d..f2186c519729 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/signals.py +++ b/services/storage/src/simcore_service_storage/modules/celery/signals.py @@ -4,7 +4,7 @@ from typing import Final from asgi_lifespan import LifespanManager -from celery import Celery +from celery import Celery # type: ignore[import-untyped] from fastapi import FastAPI from servicelib.async_utils import cancel_wait_task from ...core.application import create_app diff --git a/services/storage/src/simcore_service_storage/modules/celery/tasks.py b/services/storage/src/simcore_service_storage/modules/celery/tasks.py index 130ae6e9007d..b58a09a69361 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/tasks.py +++ b/services/storage/src/simcore_service_storage/modules/celery/tasks.py @@ -2,7 +2,7 @@ import time -from celery import Task +from celery import Task # type: ignore[import-untyped] from models_library.progress_bar import ProgressReport from models_library.projects_nodes_io import StorageFileID from servicelib.logging_utils import log_context diff --git a/services/storage/src/simcore_service_storage/modules/celery/utils.py b/services/storage/src/simcore_service_storage/modules/celery/utils.py index 6eff8a9b081f..5f5186548204 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/utils.py +++ b/services/storage/src/simcore_service_storage/modules/celery/utils.py @@ -1,4 +1,4 @@ -from celery import Celery +from celery import Celery # type: ignore[import-untyped] from fastapi import FastAPI from .worker import CeleryTaskQueueWorker diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker.py b/services/storage/src/simcore_service_storage/modules/celery/worker.py index 65aa341bc4a6..36456d887b5a 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker.py @@ -1,6 +1,6 @@ import logging -from celery import Celery +from celery import Celery # type: ignore[import-untyped] from models_library.progress_bar import ProgressReport from servicelib.logging_utils import log_context diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker_main.py b/services/storage/src/simcore_service_storage/modules/celery/worker_main.py index 954bfbaa6afc..16bc216b74ad 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker_main.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker_main.py @@ -2,7 +2,7 @@ import logging -from celery.signals import worker_init, worker_shutdown +from celery.signals import worker_init, worker_shutdown # type: ignore[import-untyped] from servicelib.logging_utils import config_all_loggers from simcore_service_storage.modules.celery.signals import ( on_worker_init, From 31337584c14e4803474474d47445c0ef2725bac6 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 11 Mar 2025 10:21:11 +0100 Subject: [PATCH 135/136] exclude service --- services/web/server/tests/integration/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/server/tests/integration/conftest.py b/services/web/server/tests/integration/conftest.py index c6575d80e21f..a66e1e4bec65 100644 --- a/services/web/server/tests/integration/conftest.py +++ b/services/web/server/tests/integration/conftest.py @@ -62,7 +62,7 @@ def webserver_environ( # version tha loads only the subsystems under test. For that reason, # the test webserver is built-up in webserver_service fixture that runs # on the host. - EXCLUDED_SERVICES = ["dask-scheduler", "director"] + EXCLUDED_SERVICES = ["dask-scheduler", "director", "sto-worker"] services_with_published_ports = [ name for name in core_services From f6232ff029e3050f9a4f971cd7cd01888adc40dd Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 11 Mar 2025 11:00:34 +0100 Subject: [PATCH 136/136] exclude sto-worker --- packages/pytest-simcore/src/pytest_simcore/simcore_services.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/pytest-simcore/src/pytest_simcore/simcore_services.py b/packages/pytest-simcore/src/pytest_simcore/simcore_services.py index 88dba2e4543c..749bbc042300 100644 --- a/packages/pytest-simcore/src/pytest_simcore/simcore_services.py +++ b/packages/pytest-simcore/src/pytest_simcore/simcore_services.py @@ -37,6 +37,7 @@ "static-webserver", "traefik", "whoami", + "sto-worker", } # TODO: unify healthcheck policies see https://github.com/ITISFoundation/osparc-simcore/pull/2281 SERVICE_PUBLISHED_PORT = {}