From 38621054acc259c393fe96bbce4478ec8758c3b5 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 30 Jun 2025 11:56:53 +0200 Subject: [PATCH 01/69] refactor: move routes modules --- .../api/rest/{routing.py => routes.py} | 0 .../api/rpc/{routing.py => routes.py} | 0 .../src/simcore_service_notifications/core/application.py | 2 +- .../src/simcore_service_notifications/core/events.py | 2 +- 4 files changed, 2 insertions(+), 2 deletions(-) rename services/notifications/src/simcore_service_notifications/api/rest/{routing.py => routes.py} (100%) rename services/notifications/src/simcore_service_notifications/api/rpc/{routing.py => routes.py} (100%) diff --git a/services/notifications/src/simcore_service_notifications/api/rest/routing.py b/services/notifications/src/simcore_service_notifications/api/rest/routes.py similarity index 100% rename from services/notifications/src/simcore_service_notifications/api/rest/routing.py rename to services/notifications/src/simcore_service_notifications/api/rest/routes.py diff --git a/services/notifications/src/simcore_service_notifications/api/rpc/routing.py b/services/notifications/src/simcore_service_notifications/api/rpc/routes.py similarity index 100% rename from services/notifications/src/simcore_service_notifications/api/rpc/routing.py rename to services/notifications/src/simcore_service_notifications/api/rpc/routes.py diff --git a/services/notifications/src/simcore_service_notifications/core/application.py b/services/notifications/src/simcore_service_notifications/core/application.py index 5f3245d9d522..7187145e78c2 100644 --- a/services/notifications/src/simcore_service_notifications/core/application.py +++ b/services/notifications/src/simcore_service_notifications/core/application.py @@ -15,7 +15,7 @@ from servicelib.logging_utils import config_all_loggers from .._meta import API_VTAG, APP_NAME, SUMMARY, VERSION -from ..api.rest.routing import initialize_rest_api +from ..api.rest.routes import initialize_rest_api from . import events from .settings import ApplicationSettings diff --git a/services/notifications/src/simcore_service_notifications/core/events.py b/services/notifications/src/simcore_service_notifications/core/events.py index 879582575c0e..088c2dfb9045 100644 --- a/services/notifications/src/simcore_service_notifications/core/events.py +++ b/services/notifications/src/simcore_service_notifications/core/events.py @@ -12,7 +12,7 @@ ) from .._meta import APP_FINISHED_BANNER_MSG, APP_STARTED_BANNER_MSG -from ..api.rpc.routing import rpc_api_routes_lifespan +from ..api.rpc.routes import rpc_api_routes_lifespan from ..clients.postgres import postgres_lifespan from ..clients.rabbitmq import rabbitmq_lifespan from .settings import ApplicationSettings From b9b64cd4e155442bbc877a6f0b23089ed1692248 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 2 Jul 2025 16:01:33 +0200 Subject: [PATCH 02/69] feat: add skeletons and schemas --- .../api/rpc/_notifications.py | 16 +++++++++++++++ .../api/rpc/routes.py | 5 ++--- .../models/schemas.py | 20 +++++++++++++++++++ .../services/notifications_service.py | 4 ++++ 4 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 services/notifications/src/simcore_service_notifications/api/rpc/_notifications.py create mode 100644 services/notifications/src/simcore_service_notifications/models/schemas.py create mode 100644 services/notifications/src/simcore_service_notifications/services/notifications_service.py diff --git a/services/notifications/src/simcore_service_notifications/api/rpc/_notifications.py b/services/notifications/src/simcore_service_notifications/api/rpc/_notifications.py new file mode 100644 index 000000000000..9d5b328a958e --- /dev/null +++ b/services/notifications/src/simcore_service_notifications/api/rpc/_notifications.py @@ -0,0 +1,16 @@ +from fastapi import FastAPI +from servicelib.rabbitmq import RPCRouter + +from ...models.schemas import NotificationMessage +from ...services import notifications_service + +router = RPCRouter() + + +@router.expose(reraise_if_error_type=()) +async def send_notification_message( + app: FastAPI, + *, + message: NotificationMessage, +) -> None: + await notifications_service.send_notification_message(message=message) diff --git a/services/notifications/src/simcore_service_notifications/api/rpc/routes.py b/services/notifications/src/simcore_service_notifications/api/rpc/routes.py index c43bcdb7c05a..9b31bf9de84e 100644 --- a/services/notifications/src/simcore_service_notifications/api/rpc/routes.py +++ b/services/notifications/src/simcore_service_notifications/api/rpc/routes.py @@ -6,10 +6,9 @@ from servicelib.rabbitmq import RPCRouter from ...clients.rabbitmq import get_rabbitmq_rpc_server +from . import _notifications -ROUTERS: list[RPCRouter] = [ - # import and use all routers here -] +ROUTERS: list[RPCRouter] = [_notifications.router] async def rpc_api_routes_lifespan(app: FastAPI) -> AsyncIterator[State]: diff --git a/services/notifications/src/simcore_service_notifications/models/schemas.py b/services/notifications/src/simcore_service_notifications/models/schemas.py new file mode 100644 index 000000000000..084c4496e891 --- /dev/null +++ b/services/notifications/src/simcore_service_notifications/models/schemas.py @@ -0,0 +1,20 @@ +from typing import Any, TypeAlias + +from pydantic import BaseModel + + +class SMSRecipient(BaseModel): + phone_number: str + + +class EmailRecipient(BaseModel): + email: str + + +Recipient: TypeAlias = SMSRecipient | EmailRecipient + + +class NotificationMessage(BaseModel): + recipients: list[Recipient] + event: str + context: dict[str, Any] | None = None diff --git a/services/notifications/src/simcore_service_notifications/services/notifications_service.py b/services/notifications/src/simcore_service_notifications/services/notifications_service.py new file mode 100644 index 000000000000..683ca0f9dc7b --- /dev/null +++ b/services/notifications/src/simcore_service_notifications/services/notifications_service.py @@ -0,0 +1,4 @@ +from ..models.schemas import NotificationMessage + + +async def send_notification_message(message: NotificationMessage) -> None: ... From cc93001384b6428f4f95f7de22bd0fe89763e422 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 2 Jul 2025 16:32:18 +0200 Subject: [PATCH 03/69] feat: add reqs --- services/notifications/requirements/_base.in | 1 + services/notifications/requirements/_base.txt | 316 +++++++++++++++++- services/notifications/requirements/_test.txt | 4 +- 3 files changed, 302 insertions(+), 19 deletions(-) diff --git a/services/notifications/requirements/_base.in b/services/notifications/requirements/_base.in index 77bb3fd4051a..c199782b440d 100644 --- a/services/notifications/requirements/_base.in +++ b/services/notifications/requirements/_base.in @@ -6,6 +6,7 @@ --constraint ./constraints.txt # intra-repo required dependencies +--requirement ../../../packages/celery-library/requirements/_base.in --requirement ../../../packages/common-library/requirements/_base.in --requirement ../../../packages/models-library/requirements/_base.in --requirement ../../../packages/settings-library/requirements/_base.in diff --git a/services/notifications/requirements/_base.txt b/services/notifications/requirements/_base.txt index 9a93d41335fe..ed4b541aa214 100644 --- a/services/notifications/requirements/_base.txt +++ b/services/notifications/requirements/_base.txt @@ -1,17 +1,39 @@ aio-pika==9.5.5 - # via -r requirements/../../../packages/service-library/requirements/_base.in + # via + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in aiocache==0.12.3 - # via -r requirements/../../../packages/service-library/requirements/_base.in + # via + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in aiodebug==2.3.0 - # via -r requirements/../../../packages/service-library/requirements/_base.in + # via + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in aiodocker==0.24.0 - # via -r requirements/../../../packages/service-library/requirements/_base.in + # via + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in aiofiles==24.1.0 - # via -r requirements/../../../packages/service-library/requirements/_base.in + # via + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in aiohappyeyeballs==2.6.1 # via aiohttp aiohttp==3.12.12 # via + # -c requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -33,6 +55,8 @@ aiosignal==1.3.2 # via aiohttp alembic==1.15.1 # via -r requirements/../../../packages/postgres-database/requirements/_base.in +amqp==5.3.1 + # via kombu annotated-types==0.7.0 # via pydantic anyio==4.9.0 @@ -44,6 +68,9 @@ anyio==4.9.0 # watchfiles arrow==1.3.0 # via + # -r requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in @@ -56,8 +83,24 @@ attrs==25.3.0 # aiohttp # jsonschema # referencing +billiard==4.2.1 + # via celery +celery==5.5.3 + # via -r requirements/../../../packages/celery-library/requirements/_base.in certifi==2025.1.31 # via + # -c requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -79,9 +122,19 @@ charset-normalizer==3.4.1 # via requests click==8.1.8 # via + # celery + # click-didyoumean + # click-plugins + # click-repl # rich-toolkit # typer # uvicorn +click-didyoumean==0.3.1 + # via celery +click-plugins==1.1.1.2 + # via celery +click-repl==0.3.0 + # via celery deprecated==1.2.18 # via # opentelemetry-api @@ -107,7 +160,9 @@ fastapi-cli==0.0.7 fastapi-lifespan-manager==0.1.4 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in faststream==0.5.37 - # via -r requirements/../../../packages/service-library/requirements/_base.in + # via + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in frozenlist==1.5.0 # via # aiohttp @@ -134,6 +189,18 @@ httptools==0.6.4 # via uvicorn httpx==0.28.1 # via + # -c requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -163,6 +230,18 @@ importlib-metadata==8.6.1 # via opentelemetry-api jinja2==3.1.6 # via + # -c requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -180,12 +259,28 @@ jinja2==3.1.6 # fastapi jsonschema==4.23.0 # via + # -r requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in jsonschema-specifications==2024.10.1 # via jsonschema +kombu==5.5.4 + # via celery mako==1.3.9 # via + # -c requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -215,6 +310,7 @@ multidict==6.2.0 # yarl opentelemetry-api==1.31.1 # via + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http @@ -230,7 +326,9 @@ opentelemetry-api==1.31.1 # opentelemetry-sdk # opentelemetry-semantic-conventions opentelemetry-exporter-otlp==1.31.1 - # via -r requirements/../../../packages/service-library/requirements/_base.in + # via + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-exporter-otlp-proto-common==1.31.1 # via # opentelemetry-exporter-otlp-proto-grpc @@ -250,7 +348,9 @@ opentelemetry-instrumentation==0.52b1 # opentelemetry-instrumentation-redis # opentelemetry-instrumentation-requests opentelemetry-instrumentation-aio-pika==0.52b1 - # via -r requirements/../../../packages/service-library/requirements/_base.in + # via + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-asgi==0.52b1 # via opentelemetry-instrumentation-fastapi opentelemetry-instrumentation-asyncpg==0.52b1 @@ -260,11 +360,17 @@ opentelemetry-instrumentation-fastapi==0.52b1 opentelemetry-instrumentation-httpx==0.52b1 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in opentelemetry-instrumentation-logging==0.52b1 - # via -r requirements/../../../packages/service-library/requirements/_base.in + # via + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-redis==0.52b1 - # via -r requirements/../../../packages/service-library/requirements/_base.in + # via + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-requests==0.52b1 - # via -r requirements/../../../packages/service-library/requirements/_base.in + # via + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-proto==1.31.1 # via # opentelemetry-exporter-otlp-proto-common @@ -272,6 +378,7 @@ opentelemetry-proto==1.31.1 # opentelemetry-exporter-otlp-proto-http opentelemetry-sdk==1.31.1 # via + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http @@ -293,6 +400,18 @@ opentelemetry-util-http==0.52b1 # opentelemetry-instrumentation-requests orjson==3.10.16 # via + # -c requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -307,6 +426,14 @@ orjson==3.10.16 # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in @@ -319,11 +446,14 @@ orjson==3.10.16 packaging==24.2 # via # -r requirements/_base.in + # kombu # opentelemetry-instrumentation pamqp==3.3.0 # via aiormq prometheus-client==0.21.1 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in +prompt-toolkit==3.0.51 + # via click-repl propcache==0.3.1 # via # aiohttp @@ -333,13 +463,27 @@ protobuf==5.29.4 # googleapis-common-protos # opentelemetry-proto psutil==7.0.0 - # via -r requirements/../../../packages/service-library/requirements/_base.in + # via + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in psycopg2-binary==2.9.10 # via sqlalchemy pycryptodome==3.22.0 # via stream-zip pydantic==2.11.0 # via + # -c requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -354,6 +498,17 @@ pydantic==2.11.0 # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in @@ -376,6 +531,14 @@ pydantic-core==2.33.0 # via pydantic pydantic-extra-types==2.10.3 # via + # -r requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in @@ -387,6 +550,18 @@ pydantic-extra-types==2.10.3 # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in pydantic-settings==2.7.0 # via + # -c requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -401,6 +576,10 @@ pydantic-settings==2.7.0 # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in @@ -408,9 +587,13 @@ pydantic-settings==2.7.0 pygments==2.19.1 # via rich pyinstrument==5.0.1 - # via -r requirements/../../../packages/service-library/requirements/_base.in + # via + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in python-dateutil==2.9.0.post0 - # via arrow + # via + # arrow + # celery python-dotenv==1.1.0 # via # pydantic-settings @@ -419,6 +602,18 @@ python-multipart==0.0.20 # via fastapi pyyaml==6.0.2 # via + # -c requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -433,10 +628,23 @@ pyyaml==6.0.2 # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # uvicorn redis==5.2.1 # via + # -c requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -451,9 +659,23 @@ redis==5.2.1 # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in + # kombu referencing==0.35.1 # via + # -c requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -474,6 +696,8 @@ requests==2.32.3 # via opentelemetry-exporter-otlp-proto-http rich==13.9.4 # via + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # rich-toolkit @@ -492,6 +716,18 @@ sniffio==1.3.1 # via anyio sqlalchemy==1.4.54 # via + # -c requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -510,6 +746,18 @@ sqlalchemy==1.4.54 # alembic starlette==0.46.1 # via + # -c requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -526,15 +774,25 @@ starlette==0.46.1 # -c requirements/../../../requirements/constraints.txt # fastapi stream-zip==0.0.83 - # via -r requirements/../../../packages/service-library/requirements/_base.in + # via + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in tenacity==9.0.0 - # via -r requirements/../../../packages/service-library/requirements/_base.in + # via + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in toolz==1.0.0 - # via -r requirements/../../../packages/service-library/requirements/_base.in + # via + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in tqdm==4.67.1 - # via -r requirements/../../../packages/service-library/requirements/_base.in + # via + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in typer==0.15.2 # via + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # fastapi-cli @@ -556,8 +814,22 @@ typing-extensions==4.13.0 # typing-inspection typing-inspection==0.4.0 # via pydantic +tzdata==2025.2 + # via kombu urllib3==2.3.0 # via + # -c requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -579,8 +851,15 @@ uvicorn==0.34.2 # fastapi-cli uvloop==0.21.0 # via uvicorn +vine==5.1.0 + # via + # amqp + # celery + # kombu watchfiles==1.0.5 # via uvicorn +wcwidth==0.2.13 + # via prompt-toolkit websockets==15.0.1 # via uvicorn wrapt==1.17.2 @@ -592,6 +871,7 @@ wrapt==1.17.2 # opentelemetry-instrumentation-redis yarl==1.18.3 # via + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/postgres-database/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # aio-pika diff --git a/services/notifications/requirements/_test.txt b/services/notifications/requirements/_test.txt index 483fca1f9a3a..b6700d41494d 100644 --- a/services/notifications/requirements/_test.txt +++ b/services/notifications/requirements/_test.txt @@ -82,7 +82,9 @@ typing-extensions==4.13.0 # -c requirements/_base.txt # anyio tzdata==2025.2 - # via faker + # via + # -c requirements/_base.txt + # faker urllib3==2.3.0 # via # -c requirements/../../../requirements/constraints.txt From f196f2e7f5185239449749293f32aef37b6f5737 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 2 Jul 2025 16:49:46 +0200 Subject: [PATCH 04/69] fix: settings --- .../core/settings.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/services/notifications/src/simcore_service_notifications/core/settings.py b/services/notifications/src/simcore_service_notifications/core/settings.py index 6f7e13a546e3..73803dcaff00 100644 --- a/services/notifications/src/simcore_service_notifications/core/settings.py +++ b/services/notifications/src/simcore_service_notifications/core/settings.py @@ -5,6 +5,7 @@ from pydantic import AliasChoices, Field, field_validator from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.base import BaseCustomSettings +from settings_library.celery import CelerySettings from settings_library.postgres import PostgresSettings from settings_library.rabbit import RabbitSettings from settings_library.tracing import TracingSettings @@ -25,11 +26,11 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): SC_BOOT_MODE: BootModeEnum | None - NOTIFICATIONS_VOLUMES_LOG_FORMAT_LOCAL_DEV_ENABLED: Annotated[ + NOTIFICATIONS_LOG_FORMAT_LOCAL_DEV_ENABLED: Annotated[ bool, Field( validation_alias=AliasChoices( - "NOTIFICATIONS_VOLUMES_LOG_FORMAT_LOCAL_DEV_ENABLED", + "NOTIFICATIONS_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED", ), description=( @@ -39,12 +40,12 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): ), ] = False - NOTIFICATIONS_VOLUMES_LOG_FILTER_MAPPING: Annotated[ + NOTIFICATIONS_LOG_FILTER_MAPPING: Annotated[ dict[LoggerName, list[MessageSubstring]], Field( default_factory=dict, validation_alias=AliasChoices( - "NOTIFICATIONS_VOLUMES_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING" + "NOTIFICATIONS_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING" ), description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", ), @@ -58,6 +59,14 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): ), ] + NOTIFICATIONS_CELERY: Annotated[ + CelerySettings, + Field( + description="settings for service/celery", + json_schema_extra={"auto_default_from_env": True}, + ), + ] + NOTIFICATIONS_POSTGRES: Annotated[ PostgresSettings, Field( From 0e890d768ea7d3ec98ccae28e8831159e1233b8d Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 2 Jul 2025 16:50:25 +0200 Subject: [PATCH 05/69] fix: add Celery worker setup --- .../modules/__init__.py | 0 .../modules/celery/__init__.py | 0 .../modules/celery/tasks.py | 18 ++++++++ .../modules/celery/worker_main.py | 44 +++++++++++++++++++ 4 files changed, 62 insertions(+) create mode 100644 services/notifications/src/simcore_service_notifications/modules/__init__.py create mode 100644 services/notifications/src/simcore_service_notifications/modules/celery/__init__.py create mode 100644 services/notifications/src/simcore_service_notifications/modules/celery/tasks.py create mode 100644 services/notifications/src/simcore_service_notifications/modules/celery/worker_main.py diff --git a/services/notifications/src/simcore_service_notifications/modules/__init__.py b/services/notifications/src/simcore_service_notifications/modules/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/services/notifications/src/simcore_service_notifications/modules/celery/__init__.py b/services/notifications/src/simcore_service_notifications/modules/celery/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py b/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py new file mode 100644 index 000000000000..2b985dcfe2a5 --- /dev/null +++ b/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py @@ -0,0 +1,18 @@ +import logging + +from celery import Celery # type: ignore[import-untyped] +from celery_library.types import register_celery_types +from servicelib.logging_utils import log_context + +_logger = logging.getLogger(__name__) + + +def setup_worker_tasks(app: Celery) -> None: + register_celery_types() + # TODO: add more types as needed + # register_pydantic_types(FileUploadCompletionBody, FileMetaData, FoldersBody) + + with log_context(_logger, logging.INFO, msg="worker tasks registration"): + ... + # TODO: register tasks here + # register_task(app, send_email_notification) diff --git a/services/notifications/src/simcore_service_notifications/modules/celery/worker_main.py b/services/notifications/src/simcore_service_notifications/modules/celery/worker_main.py new file mode 100644 index 000000000000..8a75aaaaafe5 --- /dev/null +++ b/services/notifications/src/simcore_service_notifications/modules/celery/worker_main.py @@ -0,0 +1,44 @@ +import logging +from functools import partial + +from celery.signals import worker_init, worker_shutdown # type: ignore[import-untyped] +from celery_library.common import create_app as create_celery_app +from celery_library.signals import ( + on_worker_init, + on_worker_shutdown, +) +from servicelib.fastapi.celery.app_server import FastAPIAppServer +from servicelib.logging_utils import config_all_loggers + +from ...core.application import create_app +from ...core.settings import ApplicationSettings +from .tasks import setup_worker_tasks + +_settings = ApplicationSettings.create_from_envs() + +logging.basicConfig(level=_settings.log_level) # NOSONAR +logging.root.setLevel(_settings.log_level) +config_all_loggers( + log_format_local_dev_enabled=_settings.NOTIFICATIONS_LOG_FORMAT_LOCAL_DEV_ENABLED, + logger_filter_mapping=_settings.NOTIFICATIONS_LOG_FILTER_MAPPING, + tracing_settings=_settings.NOTIFICATIONS_TRACING, +) + + +assert _settings.NOTIFICATIONS_CELERY # nosec +app = create_celery_app(_settings.NOTIFICATIONS_CELERY) + +app_server = FastAPIAppServer(app=create_app()) + + +def worker_init_wrapper(sender, **_kwargs): + assert _settings.NOTIFICATIONS_CELERY # nosec + return partial(on_worker_init, app_server, _settings.NOTIFICATIONS_CELERY)( + sender, **_kwargs + ) + + +worker_init.connect(worker_init_wrapper) +worker_shutdown.connect(on_worker_shutdown) + +setup_worker_tasks(app) From 217d6a552006fd515b72c85deb30e70b7ad6be0c Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 3 Jul 2025 10:22:28 +0200 Subject: [PATCH 06/69] feat: docker boot --- services/docker-compose.yml | 17 +++++++- services/notifications/docker/boot.sh | 58 +++++++++++++++++++-------- 2 files changed, 58 insertions(+), 17 deletions(-) diff --git a/services/docker-compose.yml b/services/docker-compose.yml index cb92ec8b43c5..c17ff09be6b0 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -1140,7 +1140,7 @@ services: init: true hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" - environment: + environment: ¬ifications_environment LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} @@ -1163,6 +1163,21 @@ services: TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT: ${TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT} TRACING_OPENTELEMETRY_COLLECTOR_PORT: ${TRACING_OPENTELEMETRY_COLLECTOR_PORT} + no-worker: + image: ${DOCKER_REGISTRY:-itisfoundation}/notifications:${DOCKER_IMAGE_TAG:-master-github-latest} + init: true + hostname: "no-worker-{{.Node.Hostname}}-{{.Task.Slot}}" + environment: + <<: *notifications_environment + REDIS_HOST: ${REDIS_HOST} + REDIS_PORT: ${REDIS_PORT} + REDIS_SECURE: ${REDIS_SECURE} + REDIS_USER: ${REDIS_USER} + REDIS_PASSWORD: ${REDIS_PASSWORD} + NOTIFICATIONS_WORKER_NAME: "no-worker-{{.Node.Hostname}}-{{.Task.Slot}}-{{.Task.ID}}" + NOTIFICATIONS_WORKER_MODE: "true" + CELERY_CONCURRENCY: 100 + dask-sidecar: image: ${DOCKER_REGISTRY:-itisfoundation}/dask-sidecar:${DOCKER_IMAGE_TAG:-latest} init: true diff --git a/services/notifications/docker/boot.sh b/services/notifications/docker/boot.sh index 8d079d9bc1be..6499d50d2f4f 100755 --- a/services/notifications/docker/boot.sh +++ b/services/notifications/docker/boot.sh @@ -47,23 +47,49 @@ APP_LOG_LEVEL=${LOGLEVEL:-${LOG_LEVEL:-${LOGLEVEL:-INFO}}} SERVER_LOG_LEVEL=$(echo "${APP_LOG_LEVEL}" | tr '[:upper:]' '[:lower:]') echo "$INFO" "Log-level app/server: $APP_LOG_LEVEL/$SERVER_LOG_LEVEL" -if [ "${SC_BOOT_MODE}" = "debug" ]; then - reload_dir_packages=$(find /devel/packages -maxdepth 3 -type d -path "*/src/*" ! -path "*.*" -exec echo '--reload-dir {} \' \;) +if [ "${NOTIFICATIONS_WORKER_MODE}" = "true" ]; then + if [ "${SC_BOOT_MODE}" = "debug" ]; then + exec watchmedo auto-restart \ + --directory /devel/packages \ + --directory services/notifications \ + --pattern "*.py" \ + --recursive \ + -- \ + celery \ + --app=simcore_service_notifications.modules.celery.worker_main:the_celery_app \ + worker --pool=threads \ + --loglevel="${SERVER_LOG_LEVEL}" \ + --concurrency="${CELERY_CONCURRENCY}" \ + --hostname="${NOTIFICATIONS_WORKER_NAME}" \ + --queues="${CELERY_QUEUES:-default}" + else + exec celery \ + --app=simcore_service_notifications.modules.celery.worker_main:the_celery_app \ + worker --pool=threads \ + --loglevel="${SERVER_LOG_LEVEL}" \ + --concurrency="${CELERY_CONCURRENCY}" \ + --hostname="${NOTIFICATIONS_WORKER_NAME}" \ + --queues="${CELERY_QUEUES:-default}" + fi +else + if [ "${SC_BOOT_MODE}" = "debug" ]; then + reload_dir_packages=$(find /devel/packages -maxdepth 3 -type d -path "*/src/*" ! -path "*.*" -exec echo '--reload-dir {} \' \;) - exec sh -c " - cd services/notifications/src/simcore_service_notifications && \ - python -Xfrozen_modules=off -m debugpy --listen 0.0.0.0:${NOTIFICATIONS_REMOTE_DEBUGGING_PORT} -m uvicorn main:the_app \ + exec sh -c " + cd services/notifications/src/simcore_service_notifications && \ + python -Xfrozen_modules=off -m debugpy --listen 0.0.0.0:${NOTIFICATIONS_REMOTE_DEBUGGING_PORT} -m uvicorn main:the_app \ + --host 0.0.0.0 \ + --port 8000 \ + --reload \ + $reload_dir_packages + --reload-dir . \ + --log-level \"${SERVER_LOG_LEVEL}\" + " + else + exec uvicorn simcore_service_notifications.main:the_app \ --host 0.0.0.0 \ --port 8000 \ - --reload \ - $reload_dir_packages - --reload-dir . \ - --log-level \"${SERVER_LOG_LEVEL}\" - " -else - exec uvicorn simcore_service_notifications.main:the_app \ - --host 0.0.0.0 \ - --port 8000 \ - --log-level "${SERVER_LOG_LEVEL}" \ - --no-access-log + --log-level "${SERVER_LOG_LEVEL}" \ + --no-access-log + fi fi From 02be423f5be14627bacbd9c6bd8568cc3e371e34 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 3 Jul 2025 10:23:18 +0200 Subject: [PATCH 07/69] feat: refactor models --- .../src/simcore_service_notifications/models/schemas.py | 1 - .../modules/celery/_email_tasks.py | 7 +++++++ .../modules/celery/worker_main.py | 4 ++-- .../services/notifications_service.py | 6 ++++-- 4 files changed, 13 insertions(+), 5 deletions(-) create mode 100644 services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py diff --git a/services/notifications/src/simcore_service_notifications/models/schemas.py b/services/notifications/src/simcore_service_notifications/models/schemas.py index 084c4496e891..2eaf4e0983b5 100644 --- a/services/notifications/src/simcore_service_notifications/models/schemas.py +++ b/services/notifications/src/simcore_service_notifications/models/schemas.py @@ -15,6 +15,5 @@ class EmailRecipient(BaseModel): class NotificationMessage(BaseModel): - recipients: list[Recipient] event: str context: dict[str, Any] | None = None diff --git a/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py b/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py new file mode 100644 index 000000000000..4c1b61dea603 --- /dev/null +++ b/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py @@ -0,0 +1,7 @@ +from ...models.schemas import EmailRecipient, NotificationMessage + + +async def send_email_notification( + message: NotificationMessage, recipient: EmailRecipient +) -> None: + pass diff --git a/services/notifications/src/simcore_service_notifications/modules/celery/worker_main.py b/services/notifications/src/simcore_service_notifications/modules/celery/worker_main.py index 8a75aaaaafe5..b312fddb026b 100644 --- a/services/notifications/src/simcore_service_notifications/modules/celery/worker_main.py +++ b/services/notifications/src/simcore_service_notifications/modules/celery/worker_main.py @@ -26,7 +26,7 @@ assert _settings.NOTIFICATIONS_CELERY # nosec -app = create_celery_app(_settings.NOTIFICATIONS_CELERY) +the_celery_app = create_celery_app(_settings.NOTIFICATIONS_CELERY) app_server = FastAPIAppServer(app=create_app()) @@ -41,4 +41,4 @@ def worker_init_wrapper(sender, **_kwargs): worker_init.connect(worker_init_wrapper) worker_shutdown.connect(on_worker_shutdown) -setup_worker_tasks(app) +setup_worker_tasks(the_celery_app) diff --git a/services/notifications/src/simcore_service_notifications/services/notifications_service.py b/services/notifications/src/simcore_service_notifications/services/notifications_service.py index 683ca0f9dc7b..1af8be12c50a 100644 --- a/services/notifications/src/simcore_service_notifications/services/notifications_service.py +++ b/services/notifications/src/simcore_service_notifications/services/notifications_service.py @@ -1,4 +1,6 @@ -from ..models.schemas import NotificationMessage +from ..models.schemas import NotificationMessage, Recipient -async def send_notification_message(message: NotificationMessage) -> None: ... +async def send_notification( + message: NotificationMessage, *recipients: list[Recipient] +) -> None: ... From 0ba8ce4dceff8ee016cc5b878cdb8e8c8e653926 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 3 Jul 2025 11:12:20 +0200 Subject: [PATCH 08/69] fix: req deps --- packages/service-library/requirements/_fastapi.in | 2 +- packages/service-library/requirements/_fastapi.txt | 6 +++++- services/notifications/requirements/_base.txt | 6 +++++- services/notifications/requirements/_test.txt | 4 +++- services/notifications/requirements/ci.txt | 1 + services/notifications/requirements/dev.txt | 1 + services/notifications/requirements/prod.txt | 1 + 7 files changed, 17 insertions(+), 4 deletions(-) diff --git a/packages/service-library/requirements/_fastapi.in b/packages/service-library/requirements/_fastapi.in index 3303e6043afa..940a289b3c55 100644 --- a/packages/service-library/requirements/_fastapi.in +++ b/packages/service-library/requirements/_fastapi.in @@ -3,7 +3,7 @@ # # - +asgi-lifespan fastapi[standard] fastapi-lifespan-manager httpx[http2] diff --git a/packages/service-library/requirements/_fastapi.txt b/packages/service-library/requirements/_fastapi.txt index c6e5a29f597a..e77b62363e17 100644 --- a/packages/service-library/requirements/_fastapi.txt +++ b/packages/service-library/requirements/_fastapi.txt @@ -5,6 +5,8 @@ anyio==4.8.0 # httpx # starlette # watchfiles +asgi-lifespan==2.1.0 + # via -r requirements/_fastapi.in asgiref==3.8.1 # via opentelemetry-instrumentation-asgi certifi==2025.1.31 @@ -119,7 +121,9 @@ rich-toolkit==0.14.7 shellingham==1.5.4 # via typer sniffio==1.3.1 - # via anyio + # via + # anyio + # asgi-lifespan starlette==0.46.0 # via fastapi typer==0.16.0 diff --git a/services/notifications/requirements/_base.txt b/services/notifications/requirements/_base.txt index ed4b541aa214..bf7da3ac1369 100644 --- a/services/notifications/requirements/_base.txt +++ b/services/notifications/requirements/_base.txt @@ -74,6 +74,8 @@ arrow==1.3.0 # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in +asgi-lifespan==2.1.0 + # via -r requirements/../../../packages/service-library/requirements/_fastapi.in asgiref==3.8.1 # via opentelemetry-instrumentation-asgi asyncpg==0.30.0 @@ -713,7 +715,9 @@ shellingham==1.5.4 six==1.17.0 # via python-dateutil sniffio==1.3.1 - # via anyio + # via + # anyio + # asgi-lifespan sqlalchemy==1.4.54 # via # -c requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt diff --git a/services/notifications/requirements/_test.txt b/services/notifications/requirements/_test.txt index b6700d41494d..a5d1a27dbd34 100644 --- a/services/notifications/requirements/_test.txt +++ b/services/notifications/requirements/_test.txt @@ -3,7 +3,9 @@ anyio==4.9.0 # -c requirements/_base.txt # httpx asgi-lifespan==2.1.0 - # via -r requirements/_test.in + # via + # -c requirements/_base.txt + # -r requirements/_test.in certifi==2025.1.31 # via # -c requirements/../../../requirements/constraints.txt diff --git a/services/notifications/requirements/ci.txt b/services/notifications/requirements/ci.txt index 21975753559d..0bf441d418f1 100644 --- a/services/notifications/requirements/ci.txt +++ b/services/notifications/requirements/ci.txt @@ -12,6 +12,7 @@ --requirement _tools.txt # installs this repo's packages +simcore-celery-library @ ../../packages/celery-library/ simcore-common-library @ ../../packages/common-library/ simcore-models-library @ ../../packages/models-library/ simcore-postgres-database @ ../../packages/postgres-database/ diff --git a/services/notifications/requirements/dev.txt b/services/notifications/requirements/dev.txt index 4e73fc7a83a3..f34ee2bd24eb 100644 --- a/services/notifications/requirements/dev.txt +++ b/services/notifications/requirements/dev.txt @@ -12,6 +12,7 @@ --requirement _tools.txt # installs this repo's packages +--editable ../../packages/celery-library --editable ../../packages/common-library --editable ../../packages/models-library --editable ../../packages/postgres-database diff --git a/services/notifications/requirements/prod.txt b/services/notifications/requirements/prod.txt index f203156b59cd..76ada91ee6f4 100644 --- a/services/notifications/requirements/prod.txt +++ b/services/notifications/requirements/prod.txt @@ -10,6 +10,7 @@ --requirement _base.txt # installs this repo's packages +simcore-celery-library @ ../../packages/celery-library/ simcore-common-library @ ../../packages/common-library/ simcore-models-library @ ../../packages/models-library/ simcore-postgres-database @ ../../packages/postgres-database/ From 31b06bd204a8a7f00f79004ff060aa96f3b99779 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 3 Jul 2025 11:12:35 +0200 Subject: [PATCH 09/69] fix: docker scripts --- services/notifications/docker/boot.sh | 2 +- services/notifications/docker/healthcheck.py | 36 ++++++++++++++++++-- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/services/notifications/docker/boot.sh b/services/notifications/docker/boot.sh index 6499d50d2f4f..ef0fe364954a 100755 --- a/services/notifications/docker/boot.sh +++ b/services/notifications/docker/boot.sh @@ -47,7 +47,7 @@ APP_LOG_LEVEL=${LOGLEVEL:-${LOG_LEVEL:-${LOGLEVEL:-INFO}}} SERVER_LOG_LEVEL=$(echo "${APP_LOG_LEVEL}" | tr '[:upper:]' '[:lower:]') echo "$INFO" "Log-level app/server: $APP_LOG_LEVEL/$SERVER_LOG_LEVEL" -if [ "${NOTIFICATIONS_WORKER_MODE}" = "true" ]; then +if [ "${NOTIFICATIONS_WORKER_MODE:-}" = "true" ]; then if [ "${SC_BOOT_MODE}" = "debug" ]; then exec watchmedo auto-restart \ --directory /devel/packages \ diff --git a/services/notifications/docker/healthcheck.py b/services/notifications/docker/healthcheck.py index 9e3f3274a292..6861107e4f37 100755 --- a/services/notifications/docker/healthcheck.py +++ b/services/notifications/docker/healthcheck.py @@ -16,18 +16,50 @@ - SEE https://blog.sixeyed.com/docker-healthchecks-why-not-to-use-curl-or-iwr/ """ import os +import subprocess import sys from urllib.request import urlopen +from simcore_service_notifications.core.application import ApplicationSettings + SUCCESS, UNHEALTHY = 0, 1 -# Disabled if boots with debugger (e.g. debug, pdb-debug, debug-ptvsd, debugpy, etc) -ok = "debug" in os.environ.get("SC_BOOT_MODE", "").lower() +# Disabled if boots with debugger +ok = os.getenv("SC_BOOT_MODE", "").lower() == "debug" # Queries host # pylint: disable=consider-using-with + +app_settings = ApplicationSettings.create_from_envs() + + +def _is_celery_worker_healthy(): + assert app_settings.NOTIFICATIONS_CELERY + broker_url = app_settings.NOTIFICATIONS_CELERY.CELERY_RABBIT_BROKER.dsn + + try: + result = subprocess.run( + [ + "celery", + "--broker", + broker_url, + "inspect", + "ping", + "--destination", + "celery@" + os.getenv("NOTIFICATIONS_WORKER_NAME", "worker"), + ], + capture_output=True, + text=True, + check=True, + ) + return "pong" in result.stdout + except subprocess.CalledProcessError: + return False + + ok = ( ok + or (app_settings.NOTIFICATIONS_WORKER_MODE and _is_celery_worker_healthy()) or urlopen( "{host}{baseurl}".format( host=sys.argv[1], baseurl=os.environ.get("SIMCORE_NODE_BASEPATH", "") From 19a467a1b07b328b3766e8bdb09b0620bea80e6d Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 3 Jul 2025 11:14:52 +0200 Subject: [PATCH 10/69] fix: add package --- .../src/simcore_service_notifications/models/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 services/notifications/src/simcore_service_notifications/models/__init__.py diff --git a/services/notifications/src/simcore_service_notifications/models/__init__.py b/services/notifications/src/simcore_service_notifications/models/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 From e153e84f8e734157518dc7845194dda4d9d789c3 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 3 Jul 2025 11:15:11 +0200 Subject: [PATCH 11/69] fix: settings --- .../src/simcore_service_notifications/core/application.py | 4 ++-- .../src/simcore_service_notifications/core/settings.py | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/services/notifications/src/simcore_service_notifications/core/application.py b/services/notifications/src/simcore_service_notifications/core/application.py index 7187145e78c2..39a69e3ee6ef 100644 --- a/services/notifications/src/simcore_service_notifications/core/application.py +++ b/services/notifications/src/simcore_service_notifications/core/application.py @@ -27,8 +27,8 @@ def _initialise_logger(settings: ApplicationSettings): logging.basicConfig(level=settings.LOG_LEVEL.value) # NOSONAR logging.root.setLevel(settings.LOG_LEVEL.value) config_all_loggers( - log_format_local_dev_enabled=settings.NOTIFICATIONS_VOLUMES_LOG_FORMAT_LOCAL_DEV_ENABLED, - logger_filter_mapping=settings.NOTIFICATIONS_VOLUMES_LOG_FILTER_MAPPING, + log_format_local_dev_enabled=settings.NOTIFICATIONS_LOG_FORMAT_LOCAL_DEV_ENABLED, + logger_filter_mapping=settings.NOTIFICATIONS_LOG_FILTER_MAPPING, tracing_settings=settings.NOTIFICATIONS_TRACING, ) diff --git a/services/notifications/src/simcore_service_notifications/core/settings.py b/services/notifications/src/simcore_service_notifications/core/settings.py index 73803dcaff00..508ccc09f0d7 100644 --- a/services/notifications/src/simcore_service_notifications/core/settings.py +++ b/services/notifications/src/simcore_service_notifications/core/settings.py @@ -67,6 +67,10 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): ), ] + NOTIFICATIONS_WORKER_MODE: Annotated[ + bool, Field(description="If True, run as a worker") + ] = False + NOTIFICATIONS_POSTGRES: Annotated[ PostgresSettings, Field( From 6d5c30e9b0813ec1d7dab9ad4c79324fa6feafa9 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 3 Jul 2025 11:34:09 +0200 Subject: [PATCH 12/69] fix: set celery queue --- services/docker-compose.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/services/docker-compose.yml b/services/docker-compose.yml index c17ff09be6b0..d9d821100280 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -1163,10 +1163,10 @@ services: TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT: ${TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT} TRACING_OPENTELEMETRY_COLLECTOR_PORT: ${TRACING_OPENTELEMETRY_COLLECTOR_PORT} - no-worker: + not-worker: image: ${DOCKER_REGISTRY:-itisfoundation}/notifications:${DOCKER_IMAGE_TAG:-master-github-latest} init: true - hostname: "no-worker-{{.Node.Hostname}}-{{.Task.Slot}}" + hostname: "not-worker-{{.Node.Hostname}}-{{.Task.Slot}}" environment: <<: *notifications_environment REDIS_HOST: ${REDIS_HOST} @@ -1174,9 +1174,10 @@ services: REDIS_SECURE: ${REDIS_SECURE} REDIS_USER: ${REDIS_USER} REDIS_PASSWORD: ${REDIS_PASSWORD} - NOTIFICATIONS_WORKER_NAME: "no-worker-{{.Node.Hostname}}-{{.Task.Slot}}-{{.Task.ID}}" + NOTIFICATIONS_WORKER_NAME: "not-worker-{{.Node.Hostname}}-{{.Task.Slot}}-{{.Task.ID}}" NOTIFICATIONS_WORKER_MODE: "true" CELERY_CONCURRENCY: 100 + CELERY_QUEUES: "notifications.default" dask-sidecar: image: ${DOCKER_REGISTRY:-itisfoundation}/dask-sidecar:${DOCKER_IMAGE_TAG:-latest} From 92b27141a91b48131d2fab6dd2593b4198889d48 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 3 Jul 2025 13:08:32 +0200 Subject: [PATCH 13/69] fix: env vars --- services/docker-compose.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/services/docker-compose.yml b/services/docker-compose.yml index d9d821100280..9c261c4e0fdc 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -1160,6 +1160,12 @@ services: RABBIT_SECURE: ${RABBIT_SECURE} RABBIT_USER: ${RABBIT_USER} + REDIS_HOST: ${REDIS_HOST} + REDIS_PORT: ${REDIS_PORT} + REDIS_SECURE: ${REDIS_SECURE} + REDIS_USER: ${REDIS_USER} + REDIS_PASSWORD: ${REDIS_PASSWORD} + TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT: ${TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT} TRACING_OPENTELEMETRY_COLLECTOR_PORT: ${TRACING_OPENTELEMETRY_COLLECTOR_PORT} @@ -1169,11 +1175,6 @@ services: hostname: "not-worker-{{.Node.Hostname}}-{{.Task.Slot}}" environment: <<: *notifications_environment - REDIS_HOST: ${REDIS_HOST} - REDIS_PORT: ${REDIS_PORT} - REDIS_SECURE: ${REDIS_SECURE} - REDIS_USER: ${REDIS_USER} - REDIS_PASSWORD: ${REDIS_PASSWORD} NOTIFICATIONS_WORKER_NAME: "not-worker-{{.Node.Hostname}}-{{.Task.Slot}}-{{.Task.ID}}" NOTIFICATIONS_WORKER_MODE: "true" CELERY_CONCURRENCY: 100 From ca423cf423c8bd0b22652a78d269b7627bc676b5 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 3 Jul 2025 13:09:26 +0200 Subject: [PATCH 14/69] fix: startup --- .../api/rpc/_notifications.py | 11 +++--- .../api/rpc/routes.py | 5 +-- .../clients/celery.py | 35 +++++++++++++++++++ .../clients/rabbitmq.py | 2 +- .../core/events.py | 4 +++ .../modules/celery/_email_tasks.py | 16 +++++++-- .../modules/celery/tasks.py | 13 +++---- .../services/notifications_service.py | 7 +++- 8 files changed, 76 insertions(+), 17 deletions(-) create mode 100644 services/notifications/src/simcore_service_notifications/clients/celery.py diff --git a/services/notifications/src/simcore_service_notifications/api/rpc/_notifications.py b/services/notifications/src/simcore_service_notifications/api/rpc/_notifications.py index 9d5b328a958e..a5090db8d04d 100644 --- a/services/notifications/src/simcore_service_notifications/api/rpc/_notifications.py +++ b/services/notifications/src/simcore_service_notifications/api/rpc/_notifications.py @@ -1,7 +1,7 @@ -from fastapi import FastAPI +from servicelib.celery.task_manager import TaskManager from servicelib.rabbitmq import RPCRouter -from ...models.schemas import NotificationMessage +from ...models.schemas import NotificationMessage, Recipient from ...services import notifications_service router = RPCRouter() @@ -9,8 +9,11 @@ @router.expose(reraise_if_error_type=()) async def send_notification_message( - app: FastAPI, + task_manager: TaskManager, *, message: NotificationMessage, + recipients: list[Recipient], ) -> None: - await notifications_service.send_notification_message(message=message) + await notifications_service.send_notification( + task_manager, message=message, recipients=recipients + ) diff --git a/services/notifications/src/simcore_service_notifications/api/rpc/routes.py b/services/notifications/src/simcore_service_notifications/api/rpc/routes.py index 9b31bf9de84e..9968a5a4eff2 100644 --- a/services/notifications/src/simcore_service_notifications/api/rpc/routes.py +++ b/services/notifications/src/simcore_service_notifications/api/rpc/routes.py @@ -5,6 +5,7 @@ from models_library.api_schemas_notifications import NOTIFICATIONS_RPC_NAMESPACE from servicelib.rabbitmq import RPCRouter +from ...clients.celery import get_task_manager_from_app from ...clients.rabbitmq import get_rabbitmq_rpc_server from . import _notifications @@ -13,10 +14,10 @@ async def rpc_api_routes_lifespan(app: FastAPI) -> AsyncIterator[State]: rpc_server = get_rabbitmq_rpc_server(app) - + task_manager = get_task_manager_from_app(app) for router in ROUTERS: await rpc_server.register_router( - router, NOTIFICATIONS_RPC_NAMESPACE, app + router, NOTIFICATIONS_RPC_NAMESPACE, task_manager=task_manager ) # pragma: no cover yield {} diff --git a/services/notifications/src/simcore_service_notifications/clients/celery.py b/services/notifications/src/simcore_service_notifications/clients/celery.py new file mode 100644 index 000000000000..691f108d6031 --- /dev/null +++ b/services/notifications/src/simcore_service_notifications/clients/celery.py @@ -0,0 +1,35 @@ +import logging +from collections.abc import AsyncIterator + +from celery_library.common import create_app, create_task_manager +from celery_library.task_manager import CeleryTaskManager +from celery_library.types import register_celery_types, register_pydantic_types +from fastapi import FastAPI +from fastapi_lifespan_manager import State +from settings_library.celery import CelerySettings + +from ..core.settings import ApplicationSettings +from ..models.schemas import EmailRecipient, NotificationMessage, SMSRecipient + +_logger = logging.getLogger(__name__) + + +async def celery_lifespan(app: FastAPI) -> AsyncIterator[State]: + settings: ApplicationSettings = app.state.settings + if settings.NOTIFICATIONS_CELERY and not settings.NOTIFICATIONS_WORKER_MODE: + celery_settings: CelerySettings = settings.NOTIFICATIONS_CELERY + + app.state.task_manager = await create_task_manager( + create_app(celery_settings), celery_settings + ) + + register_celery_types() + register_pydantic_types(NotificationMessage, EmailRecipient, SMSRecipient) + yield {} + + +def get_task_manager_from_app(app: FastAPI) -> CeleryTaskManager: + assert hasattr(app.state, "task_manager") # nosec + task_manager = app.state.task_manager + assert isinstance(task_manager, CeleryTaskManager) # nosec + return task_manager diff --git a/services/notifications/src/simcore_service_notifications/clients/rabbitmq.py b/services/notifications/src/simcore_service_notifications/clients/rabbitmq.py index 3c205c401621..3594bb1af234 100644 --- a/services/notifications/src/simcore_service_notifications/clients/rabbitmq.py +++ b/services/notifications/src/simcore_service_notifications/clients/rabbitmq.py @@ -17,7 +17,7 @@ async def rabbitmq_lifespan(app: FastAPI) -> AsyncIterator[State]: await wait_till_rabbitmq_responsive(rabbit_settings.dsn) app.state.rabbitmq_rpc_server = await RabbitMQRPCClient.create( - client_name="dynamic_scheduler_rpc_server", settings=rabbit_settings + client_name="notifications_rpc_server", settings=rabbit_settings ) yield {} diff --git a/services/notifications/src/simcore_service_notifications/core/events.py b/services/notifications/src/simcore_service_notifications/core/events.py index 088c2dfb9045..9ece19f83775 100644 --- a/services/notifications/src/simcore_service_notifications/core/events.py +++ b/services/notifications/src/simcore_service_notifications/core/events.py @@ -13,6 +13,7 @@ from .._meta import APP_FINISHED_BANNER_MSG, APP_STARTED_BANNER_MSG from ..api.rpc.routes import rpc_api_routes_lifespan +from ..clients.celery import celery_lifespan from ..clients.postgres import postgres_lifespan from ..clients.rabbitmq import rabbitmq_lifespan from .settings import ApplicationSettings @@ -47,6 +48,9 @@ def create_app_lifespan(): # - rabbitmq app_lifespan.add(rabbitmq_lifespan) + # - celery + app_lifespan.add(celery_lifespan) + # - rpc api routes app_lifespan.add(rpc_api_routes_lifespan) diff --git a/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py b/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py index 4c1b61dea603..f46b6f0daadd 100644 --- a/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py +++ b/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py @@ -1,7 +1,17 @@ +import logging + +from celery import Task # type: ignore[import-untyped] + from ...models.schemas import EmailRecipient, NotificationMessage +_logger = logging.getLogger(__name__) + -async def send_email_notification( - message: NotificationMessage, recipient: EmailRecipient +async def send_email( + task: Task, + message: NotificationMessage, + recipient: EmailRecipient, ) -> None: - pass + # TODO: render email template with message and recipient details + # and send the email using an email service + _logger.info(f"Sending email notification to {recipient.email}") diff --git a/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py b/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py index 2b985dcfe2a5..d08f513d064f 100644 --- a/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py +++ b/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py @@ -1,18 +1,19 @@ import logging from celery import Celery # type: ignore[import-untyped] -from celery_library.types import register_celery_types +from celery_library.task import register_task +from celery_library.types import register_celery_types, register_pydantic_types from servicelib.logging_utils import log_context +from ...models.schemas import NotificationMessage, SMSRecipient +from ...modules.celery._email_tasks import EmailRecipient, send_email + _logger = logging.getLogger(__name__) def setup_worker_tasks(app: Celery) -> None: register_celery_types() - # TODO: add more types as needed - # register_pydantic_types(FileUploadCompletionBody, FileMetaData, FoldersBody) + register_pydantic_types(NotificationMessage, EmailRecipient, SMSRecipient) with log_context(_logger, logging.INFO, msg="worker tasks registration"): - ... - # TODO: register tasks here - # register_task(app, send_email_notification) + register_task(app, send_email) diff --git a/services/notifications/src/simcore_service_notifications/services/notifications_service.py b/services/notifications/src/simcore_service_notifications/services/notifications_service.py index 1af8be12c50a..85d5eb718065 100644 --- a/services/notifications/src/simcore_service_notifications/services/notifications_service.py +++ b/services/notifications/src/simcore_service_notifications/services/notifications_service.py @@ -1,6 +1,11 @@ +from servicelib.celery.task_manager import TaskManager + from ..models.schemas import NotificationMessage, Recipient async def send_notification( - message: NotificationMessage, *recipients: list[Recipient] + task_manager: TaskManager, + *, + message: NotificationMessage, + recipients: list[Recipient], ) -> None: ... From abbeaa4bdc1a1d1aac041140ac39346f325fde05 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 3 Jul 2025 13:29:09 +0200 Subject: [PATCH 15/69] fix: lifespan --- .../core/application.py | 2 +- .../simcore_service_notifications/core/events.py | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/services/notifications/src/simcore_service_notifications/core/application.py b/services/notifications/src/simcore_service_notifications/core/application.py index 39a69e3ee6ef..9b326a41dd1f 100644 --- a/services/notifications/src/simcore_service_notifications/core/application.py +++ b/services/notifications/src/simcore_service_notifications/core/application.py @@ -46,7 +46,7 @@ def create_app() -> FastAPI: description=SUMMARY, version=f"{VERSION}", openapi_url=f"/api/{API_VTAG}/openapi.json", - lifespan=events.create_app_lifespan(), + lifespan=events.create_app_lifespan(settings), **get_common_oas_options(is_devel_mode=settings.SC_BOOT_MODE.is_devel_mode()), ) override_fastapi_openapi_method(app) diff --git a/services/notifications/src/simcore_service_notifications/core/events.py b/services/notifications/src/simcore_service_notifications/core/events.py index 9ece19f83775..11c7ec6dab64 100644 --- a/services/notifications/src/simcore_service_notifications/core/events.py +++ b/services/notifications/src/simcore_service_notifications/core/events.py @@ -36,7 +36,7 @@ async def _settings_lifespan(app: FastAPI) -> AsyncIterator[State]: } -def create_app_lifespan(): +def create_app_lifespan(settings: ApplicationSettings) -> LifespanManager: # WARNING: order matters app_lifespan = LifespanManager() app_lifespan.add(_settings_lifespan) @@ -45,14 +45,15 @@ def create_app_lifespan(): app_lifespan.add(postgres_database_lifespan) app_lifespan.add(postgres_lifespan) - # - rabbitmq - app_lifespan.add(rabbitmq_lifespan) + if settings.NOTIFICATIONS_CELERY and not settings.NOTIFICATIONS_WORKER_MODE: + # - rabbitmq + app_lifespan.add(rabbitmq_lifespan) - # - celery - app_lifespan.add(celery_lifespan) + # - celery + app_lifespan.add(celery_lifespan) - # - rpc api routes - app_lifespan.add(rpc_api_routes_lifespan) + # - rpc api routes + app_lifespan.add(rpc_api_routes_lifespan) # - prometheus instrumentation app_lifespan.add(prometheus_instrumentation_lifespan) From 1c50f232227b16bcc57dbf2358c1fb5c1e22fef1 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 3 Jul 2025 14:31:34 +0200 Subject: [PATCH 16/69] refactor: queue names --- .../tests/unit/test_async_jobs.py | 4 ++-- .../src/servicelib/celery/models.py | 18 ++++++++++------ .../src/servicelib/celery/task_manager.py | 4 ++-- services/docker-compose.yml | 2 +- .../modules/celery/tasks.py | 8 +++++++ .../services/notifications_service.py | 21 ++++++++++++++++++- .../api/rest/_files.py | 2 +- .../simcore_service_storage/api/rpc/_paths.py | 4 ++-- .../api/rpc/_simcore_s3.py | 9 ++++---- .../modules/celery/tasks.py | 8 +++++++ services/storage/tests/conftest.py | 3 ++- 11 files changed, 63 insertions(+), 20 deletions(-) create mode 100644 services/storage/src/simcore_service_storage/modules/celery/tasks.py diff --git a/packages/celery-library/tests/unit/test_async_jobs.py b/packages/celery-library/tests/unit/test_async_jobs.py index 02c8362c1aae..7772c6812d67 100644 --- a/packages/celery-library/tests/unit/test_async_jobs.py +++ b/packages/celery-library/tests/unit/test_async_jobs.py @@ -82,7 +82,7 @@ async def rpc_sync_job( task_manager: TaskManager, *, job_id_data: AsyncJobNameData, **kwargs: Any ) -> AsyncJobGet: task_name = sync_job.__name__ - task_uuid = await task_manager.submit_task( + task_uuid = await task_manager.send_task( TaskMetadata(name=task_name), task_context=job_id_data.model_dump(), **kwargs ) @@ -94,7 +94,7 @@ async def rpc_async_job( task_manager: TaskManager, *, job_id_data: AsyncJobNameData, **kwargs: Any ) -> AsyncJobGet: task_name = async_job.__name__ - task_uuid = await task_manager.submit_task( + task_uuid = await task_manager.send_task( TaskMetadata(name=task_name), task_context=job_id_data.model_dump(), **kwargs ) diff --git a/packages/service-library/src/servicelib/celery/models.py b/packages/service-library/src/servicelib/celery/models.py index 8bc744fcb3eb..019a49d800ad 100644 --- a/packages/service-library/src/servicelib/celery/models.py +++ b/packages/service-library/src/servicelib/celery/models.py @@ -11,6 +11,17 @@ TaskName: TypeAlias = Annotated[ str, StringConstraints(strip_whitespace=True, min_length=1) ] + +TaskQueue: TypeAlias = Annotated[ + str, + StringConstraints( + strip_whitespace=True, + min_length=1, + max_length=64, + ), +] +TASK_QUEUE_DEFAULT: TaskQueue = "default" + TaskUUID: TypeAlias = UUID @@ -23,15 +34,10 @@ class TaskState(StrEnum): ABORTED = "ABORTED" -class TasksQueue(StrEnum): - CPU_BOUND = "cpu_bound" - DEFAULT = "default" - - class TaskMetadata(BaseModel): name: TaskName ephemeral: bool = True - queue: TasksQueue = TasksQueue.DEFAULT + queue: TaskQueue = TASK_QUEUE_DEFAULT class Task(BaseModel): diff --git a/packages/service-library/src/servicelib/celery/task_manager.py b/packages/service-library/src/servicelib/celery/task_manager.py index f8e178348c06..172183e44413 100644 --- a/packages/service-library/src/servicelib/celery/task_manager.py +++ b/packages/service-library/src/servicelib/celery/task_manager.py @@ -13,8 +13,8 @@ class TaskManager(Protocol): - async def submit_task( - self, task_metadata: TaskMetadata, *, task_context: TaskContext, **task_param + async def send_task( + self, task_metadata: TaskMetadata, *, task_context: TaskContext, **task_params ) -> TaskUUID: ... async def cancel_task( diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 9c261c4e0fdc..4b0aca2969f6 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -1299,7 +1299,7 @@ services: STORAGE_WORKER_NAME: "sto-worker-cpu-bound-{{.Node.Hostname}}-{{.Task.Slot}}-{{.Task.ID}}" STORAGE_WORKER_MODE: "true" CELERY_CONCURRENCY: 1 - CELERY_QUEUES: "cpu_bound" + CELERY_QUEUES: "storage.cpu_bound" networks: *storage_networks rabbit: diff --git a/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py b/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py index d08f513d064f..351dd5677097 100644 --- a/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py +++ b/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py @@ -1,4 +1,5 @@ import logging +from enum import StrEnum from celery import Celery # type: ignore[import-untyped] from celery_library.task import register_task @@ -11,6 +12,13 @@ _logger = logging.getLogger(__name__) +_TASK_QUEUE_PREFIX: str = "notifications" + + +class TaskQueue(StrEnum): + DEFAULT = f"{_TASK_QUEUE_PREFIX}.default" + + def setup_worker_tasks(app: Celery) -> None: register_celery_types() register_pydantic_types(NotificationMessage, EmailRecipient, SMSRecipient) diff --git a/services/notifications/src/simcore_service_notifications/services/notifications_service.py b/services/notifications/src/simcore_service_notifications/services/notifications_service.py index 85d5eb718065..0a2a22ff4bce 100644 --- a/services/notifications/src/simcore_service_notifications/services/notifications_service.py +++ b/services/notifications/src/simcore_service_notifications/services/notifications_service.py @@ -1,11 +1,30 @@ +from enum import StrEnum + +from servicelib.celery.models import TaskContext, TaskMetadata from servicelib.celery.task_manager import TaskManager from ..models.schemas import NotificationMessage, Recipient +CHANNELS = { + "email": "notifications.email", +} + + +class TaskQueues(StrEnum): + DEFAULT = "notifications.default" + async def send_notification( task_manager: TaskManager, *, message: NotificationMessage, recipients: list[Recipient], -) -> None: ... +) -> None: + for recipient in recipients: + await task_manager.send_task( + TaskMetadata( + name="notifications.send_email", + queue=TaskQueues.DEFAULT, + ), + task_context=TaskContext(), + ) diff --git a/services/storage/src/simcore_service_storage/api/rest/_files.py b/services/storage/src/simcore_service_storage/api/rest/_files.py index 9092d8da0115..f0f6c39529ad 100644 --- a/services/storage/src/simcore_service_storage/api/rest/_files.py +++ b/services/storage/src/simcore_service_storage/api/rest/_files.py @@ -288,7 +288,7 @@ async def complete_upload_file( user_id=query_params.user_id, product_name=_UNDEFINED_PRODUCT_NAME_FOR_WORKER_TASKS, # NOTE: I would need to change the API here ) - task_uuid = await task_manager.submit_task( + task_uuid = await task_manager.send_task( TaskMetadata( name=remote_complete_upload_file.__name__, ), diff --git a/services/storage/src/simcore_service_storage/api/rpc/_paths.py b/services/storage/src/simcore_service_storage/api/rpc/_paths.py index f4b0eae297db..bb7594ea1c1b 100644 --- a/services/storage/src/simcore_service_storage/api/rpc/_paths.py +++ b/services/storage/src/simcore_service_storage/api/rpc/_paths.py @@ -25,7 +25,7 @@ async def compute_path_size( path: Path, ) -> AsyncJobGet: task_name = remote_compute_path_size.__name__ - task_uuid = await task_manager.submit_task( + task_uuid = await task_manager.send_task( task_metadata=TaskMetadata( name=task_name, ), @@ -46,7 +46,7 @@ async def delete_paths( paths: set[Path], ) -> AsyncJobGet: task_name = remote_delete_paths.__name__ - task_uuid = await task_manager.submit_task( + task_uuid = await task_manager.send_task( task_metadata=TaskMetadata( name=task_name, ), diff --git a/services/storage/src/simcore_service_storage/api/rpc/_simcore_s3.py b/services/storage/src/simcore_service_storage/api/rpc/_simcore_s3.py index bd144179cd23..fab1fee7b47b 100644 --- a/services/storage/src/simcore_service_storage/api/rpc/_simcore_s3.py +++ b/services/storage/src/simcore_service_storage/api/rpc/_simcore_s3.py @@ -4,10 +4,11 @@ ) from models_library.api_schemas_storage.storage_schemas import FoldersBody from models_library.api_schemas_webserver.storage import PathToExport -from servicelib.celery.models import TaskMetadata, TasksQueue +from servicelib.celery.models import TaskMetadata from servicelib.celery.task_manager import TaskManager from servicelib.rabbitmq import RPCRouter +from ...modules.celery.tasks import TaskQueue from .._worker_tasks._simcore_s3 import deep_copy_files_from_project, export_data router = RPCRouter() @@ -20,7 +21,7 @@ async def copy_folders_from_project( body: FoldersBody, ) -> AsyncJobGet: task_name = deep_copy_files_from_project.__name__ - task_uuid = await task_manager.submit_task( + task_uuid = await task_manager.send_task( task_metadata=TaskMetadata( name=task_name, ), @@ -39,11 +40,11 @@ async def start_export_data( paths_to_export: list[PathToExport], ) -> AsyncJobGet: task_name = export_data.__name__ - task_uuid = await task_manager.submit_task( + task_uuid = await task_manager.send_task( task_metadata=TaskMetadata( name=task_name, ephemeral=False, - queue=TasksQueue.CPU_BOUND, + queue=TaskQueue.CPU_BOUND, ), task_context=job_id_data.model_dump(), user_id=job_id_data.user_id, diff --git a/services/storage/src/simcore_service_storage/modules/celery/tasks.py b/services/storage/src/simcore_service_storage/modules/celery/tasks.py new file mode 100644 index 000000000000..6882551c052a --- /dev/null +++ b/services/storage/src/simcore_service_storage/modules/celery/tasks.py @@ -0,0 +1,8 @@ +from enum import StrEnum + +_TASK_QUEUE_PREFIX: str = "storage." + + +class TaskQueue(StrEnum): + DEFAULT = f"{_TASK_QUEUE_PREFIX}.default" + CPU_BOUND = f"{_TASK_QUEUE_PREFIX}.cpu_bound" diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index ad37ba752e66..2ae6dd96d4d2 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -75,6 +75,7 @@ from simcore_service_storage.datcore_dsm import DatCoreDataManager from simcore_service_storage.dsm import get_dsm_provider from simcore_service_storage.models import FileMetaData, FileMetaDataAtDB, S3BucketName +from simcore_service_storage.modules.celery.tasks import TaskQueue from simcore_service_storage.modules.s3 import get_s3_client from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager from sqlalchemy import literal_column @@ -1021,7 +1022,7 @@ def _on_worker_init_wrapper(sender: WorkController, **_kwargs): concurrency=1, loglevel="info", perform_ping_check=False, - queues="default,cpu_bound", + queues=",".join(queue.value for queue in TaskQueue), ) as worker: yield worker From 610906c235aa1a2073b7bda9dfec6cf9174d7cd9 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 3 Jul 2025 14:44:50 +0200 Subject: [PATCH 17/69] fix: recipient --- .../models/schemas.py | 20 +++++++++++++------ .../services/notifications_service.py | 8 +++----- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/services/notifications/src/simcore_service_notifications/models/schemas.py b/services/notifications/src/simcore_service_notifications/models/schemas.py index 2eaf4e0983b5..b8b895dba405 100644 --- a/services/notifications/src/simcore_service_notifications/models/schemas.py +++ b/services/notifications/src/simcore_service_notifications/models/schemas.py @@ -1,17 +1,25 @@ -from typing import Any, TypeAlias +from typing import Annotated, Any, Literal, TypeAlias -from pydantic import BaseModel +from pydantic import BaseModel, Field -class SMSRecipient(BaseModel): +class _BaseRecipient(BaseModel): + type: str + + +class SMSRecipient(_BaseRecipient): + type: Literal["sms"] phone_number: str -class EmailRecipient(BaseModel): - email: str +class EmailRecipient(_BaseRecipient): + type: Literal["email"] + address: str -Recipient: TypeAlias = SMSRecipient | EmailRecipient +Recipient: TypeAlias = Annotated[ + EmailRecipient | SMSRecipient, Field(discriminator="type") +] class NotificationMessage(BaseModel): diff --git a/services/notifications/src/simcore_service_notifications/services/notifications_service.py b/services/notifications/src/simcore_service_notifications/services/notifications_service.py index 0a2a22ff4bce..1460c687a28b 100644 --- a/services/notifications/src/simcore_service_notifications/services/notifications_service.py +++ b/services/notifications/src/simcore_service_notifications/services/notifications_service.py @@ -5,10 +5,6 @@ from ..models.schemas import NotificationMessage, Recipient -CHANNELS = { - "email": "notifications.email", -} - class TaskQueues(StrEnum): DEFAULT = "notifications.default" @@ -23,8 +19,10 @@ async def send_notification( for recipient in recipients: await task_manager.send_task( TaskMetadata( - name="notifications.send_email", + name=f"notifications.{recipient.type}", queue=TaskQueues.DEFAULT, ), task_context=TaskContext(), + message=message, + recipient=recipient, ) From cf92a77951da6ff4557051bd9e40213507ce1a60 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 3 Jul 2025 14:49:07 +0200 Subject: [PATCH 18/69] fix: log --- .../modules/celery/_email_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py b/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py index f46b6f0daadd..13e8a1658dda 100644 --- a/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py +++ b/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py @@ -14,4 +14,4 @@ async def send_email( ) -> None: # TODO: render email template with message and recipient details # and send the email using an email service - _logger.info(f"Sending email notification to {recipient.email}") + _logger.info(f"Sending email notification to {recipient.address}") From 44d76ff0a4e7ad8828a3b611246e1c6ad3fe3872 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 3 Jul 2025 14:51:21 +0200 Subject: [PATCH 19/69] refactor: change name --- packages/celery-library/src/celery_library/task_manager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/celery-library/src/celery_library/task_manager.py b/packages/celery-library/src/celery_library/task_manager.py index 7f14d4ddd34f..2b266b0ea001 100644 --- a/packages/celery-library/src/celery_library/task_manager.py +++ b/packages/celery-library/src/celery_library/task_manager.py @@ -37,7 +37,7 @@ class CeleryTaskManager: _celery_settings: CelerySettings _task_info_store: TaskInfoStore - async def submit_task( + async def send_task( self, task_metadata: TaskMetadata, *, @@ -47,7 +47,7 @@ async def submit_task( with log_context( _logger, logging.DEBUG, - msg=f"Submit {task_metadata.name=}: {task_context=} {task_params=}", + msg=f"Send {task_metadata.name=}: {task_context=} {task_params=}", ): task_uuid = uuid4() task_id = build_task_id(task_context, task_uuid) @@ -55,7 +55,7 @@ async def submit_task( task_metadata.name, task_id=task_id, kwargs={"task_id": task_id} | task_params, - queue=task_metadata.queue.value, + queue=task_metadata.queue, ) expiry = ( From a41e75697bbc48750a199e133be16b73ec4f3457 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 3 Jul 2025 15:01:47 +0200 Subject: [PATCH 20/69] fix: task name --- .../modules/celery/_email_tasks.py | 4 +++- .../simcore_service_notifications/modules/celery/tasks.py | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py b/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py index 13e8a1658dda..a67d3ea07eb0 100644 --- a/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py +++ b/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py @@ -6,6 +6,8 @@ _logger = logging.getLogger(__name__) +EMAIL_CHANNEL_NAME = "email" + async def send_email( task: Task, @@ -14,4 +16,4 @@ async def send_email( ) -> None: # TODO: render email template with message and recipient details # and send the email using an email service - _logger.info(f"Sending email notification to {recipient.address}") + _logger.info("Sending email notification to %s", recipient.address) diff --git a/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py b/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py index 351dd5677097..7978db6f99ea 100644 --- a/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py +++ b/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py @@ -6,8 +6,8 @@ from celery_library.types import register_celery_types, register_pydantic_types from servicelib.logging_utils import log_context -from ...models.schemas import NotificationMessage, SMSRecipient -from ...modules.celery._email_tasks import EmailRecipient, send_email +from ...models.schemas import EmailRecipient, NotificationMessage, SMSRecipient +from ...modules.celery._email_tasks import EMAIL_CHANNEL_NAME, send_email _logger = logging.getLogger(__name__) @@ -24,4 +24,4 @@ def setup_worker_tasks(app: Celery) -> None: register_pydantic_types(NotificationMessage, EmailRecipient, SMSRecipient) with log_context(_logger, logging.INFO, msg="worker tasks registration"): - register_task(app, send_email) + register_task(app, send_email, EMAIL_CHANNEL_NAME) From bbc28597e5d1263def9bdf42b1b82bb39f8ab31f Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 3 Jul 2025 15:11:29 +0200 Subject: [PATCH 21/69] tests: add pytest-celery --- services/notifications/requirements/_test.in | 1 + services/notifications/requirements/_test.txt | 82 ++++++++++++++++++- .../notifications/requirements/_tools.txt | 7 +- 3 files changed, 87 insertions(+), 3 deletions(-) diff --git a/services/notifications/requirements/_test.in b/services/notifications/requirements/_test.in index 0269b0db4202..f27c540aaa50 100644 --- a/services/notifications/requirements/_test.in +++ b/services/notifications/requirements/_test.in @@ -18,6 +18,7 @@ faker httpx pytest pytest-asyncio +pytest-celery pytest-cov pytest-mock pytest-runner diff --git a/services/notifications/requirements/_test.txt b/services/notifications/requirements/_test.txt index a5d1a27dbd34..08abcce08b97 100644 --- a/services/notifications/requirements/_test.txt +++ b/services/notifications/requirements/_test.txt @@ -1,3 +1,7 @@ +amqp==5.3.1 + # via + # -c requirements/_base.txt + # kombu anyio==4.9.0 # via # -c requirements/_base.txt @@ -6,6 +10,14 @@ asgi-lifespan==2.1.0 # via # -c requirements/_base.txt # -r requirements/_test.in +billiard==4.2.1 + # via + # -c requirements/_base.txt + # celery +celery==5.5.3 + # via + # -c requirements/_base.txt + # pytest-celery certifi==2025.1.31 # via # -c requirements/../../../requirements/constraints.txt @@ -17,12 +29,36 @@ charset-normalizer==3.4.1 # via # -c requirements/_base.txt # requests +click==8.1.8 + # via + # -c requirements/_base.txt + # celery + # click-didyoumean + # click-plugins + # click-repl +click-didyoumean==0.3.1 + # via + # -c requirements/_base.txt + # celery +click-plugins==1.1.1.2 + # via + # -c requirements/_base.txt + # celery +click-repl==0.3.0 + # via + # -c requirements/_base.txt + # celery coverage==7.7.1 # via # -r requirements/_test.in # pytest-cov +debugpy==1.8.14 + # via pytest-celery docker==7.1.0 - # via -r requirements/_test.in + # via + # -r requirements/_test.in + # pytest-celery + # pytest-docker-tools faker==37.1.0 # via -r requirements/_test.in h11==0.14.0 @@ -46,26 +82,49 @@ idna==3.10 # requests iniconfig==2.1.0 # via pytest +kombu==5.5.4 + # via + # -c requirements/_base.txt + # celery + # pytest-celery packaging==24.2 # via # -c requirements/_base.txt + # kombu # pytest pluggy==1.5.0 # via pytest +prompt-toolkit==3.0.51 + # via + # -c requirements/_base.txt + # click-repl +psutil==7.0.0 + # via + # -c requirements/_base.txt + # pytest-celery pytest==8.3.5 # via # -r requirements/_test.in # pytest-asyncio # pytest-cov + # pytest-docker-tools # pytest-mock pytest-asyncio==0.26.0 # via -r requirements/_test.in +pytest-celery==1.2.0 + # via -r requirements/_test.in pytest-cov==6.0.0 # via -r requirements/_test.in +pytest-docker-tools==3.1.9 + # via pytest-celery pytest-mock==3.14.0 # via -r requirements/_test.in pytest-runner==6.0.1 # via -r requirements/_test.in +python-dateutil==2.9.0.post0 + # via + # -c requirements/_base.txt + # celery python-dotenv==1.1.0 # via # -c requirements/_base.txt @@ -74,11 +133,21 @@ requests==2.32.3 # via # -c requirements/_base.txt # docker +setuptools==80.9.0 + # via pytest-celery +six==1.17.0 + # via + # -c requirements/_base.txt + # python-dateutil sniffio==1.3.1 # via # -c requirements/_base.txt # anyio # asgi-lifespan +tenacity==9.0.0 + # via + # -c requirements/_base.txt + # pytest-celery typing-extensions==4.13.0 # via # -c requirements/_base.txt @@ -87,9 +156,20 @@ tzdata==2025.2 # via # -c requirements/_base.txt # faker + # kombu urllib3==2.3.0 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # docker # requests +vine==5.1.0 + # via + # -c requirements/_base.txt + # amqp + # celery + # kombu +wcwidth==0.2.13 + # via + # -c requirements/_base.txt + # prompt-toolkit diff --git a/services/notifications/requirements/_tools.txt b/services/notifications/requirements/_tools.txt index 4deff3bbf27d..a55ed9ed324b 100644 --- a/services/notifications/requirements/_tools.txt +++ b/services/notifications/requirements/_tools.txt @@ -11,6 +11,7 @@ cfgv==3.4.0 click==8.1.8 # via # -c requirements/_base.txt + # -c requirements/_test.txt # black # pip-tools dill==0.3.9 @@ -67,8 +68,10 @@ pyyaml==6.0.2 # pre-commit ruff==0.11.2 # via -r requirements/../../../requirements/devenv.txt -setuptools==78.1.0 - # via pip-tools +setuptools==80.9.0 + # via + # -c requirements/_test.txt + # pip-tools tomlkit==0.13.2 # via pylint typing-extensions==4.13.0 From 7938e6f1f9c82eb11196ec81cffc4fd3565062ff Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 3 Jul 2025 15:30:45 +0200 Subject: [PATCH 22/69] tests: add mocks --- .../core/application.py | 3 +- .../src/simcore_service_notifications/main.py | 5 +- .../modules/celery/worker_main.py | 2 +- services/notifications/tests/conftest.py | 72 +++++++++++++++++++ services/notifications/tests/unit/conftest.py | 3 +- 5 files changed, 80 insertions(+), 5 deletions(-) diff --git a/services/notifications/src/simcore_service_notifications/core/application.py b/services/notifications/src/simcore_service_notifications/core/application.py index 9b326a41dd1f..a8ea3bab6f92 100644 --- a/services/notifications/src/simcore_service_notifications/core/application.py +++ b/services/notifications/src/simcore_service_notifications/core/application.py @@ -33,8 +33,7 @@ def _initialise_logger(settings: ApplicationSettings): ) -def create_app() -> FastAPI: - settings = ApplicationSettings.create_from_envs() +def create_app(settings: ApplicationSettings) -> FastAPI: _logger.debug(settings.model_dump_json(indent=2)) _initialise_logger(settings) diff --git a/services/notifications/src/simcore_service_notifications/main.py b/services/notifications/src/simcore_service_notifications/main.py index 8b2e0ed31966..6876333cbf0c 100644 --- a/services/notifications/src/simcore_service_notifications/main.py +++ b/services/notifications/src/simcore_service_notifications/main.py @@ -1,3 +1,6 @@ from simcore_service_notifications.core.application import create_app +from simcore_service_notifications.core.settings import ApplicationSettings -the_app = create_app() +_settings = ApplicationSettings.create_from_envs() + +the_app = create_app(_settings) diff --git a/services/notifications/src/simcore_service_notifications/modules/celery/worker_main.py b/services/notifications/src/simcore_service_notifications/modules/celery/worker_main.py index b312fddb026b..904313a94af2 100644 --- a/services/notifications/src/simcore_service_notifications/modules/celery/worker_main.py +++ b/services/notifications/src/simcore_service_notifications/modules/celery/worker_main.py @@ -28,7 +28,7 @@ assert _settings.NOTIFICATIONS_CELERY # nosec the_celery_app = create_celery_app(_settings.NOTIFICATIONS_CELERY) -app_server = FastAPIAppServer(app=create_app()) +app_server = FastAPIAppServer(app=create_app(_settings)) def worker_init_wrapper(sender, **_kwargs): diff --git a/services/notifications/tests/conftest.py b/services/notifications/tests/conftest.py index ba95f08d2e1f..c84938b8c35a 100644 --- a/services/notifications/tests/conftest.py +++ b/services/notifications/tests/conftest.py @@ -2,11 +2,32 @@ # pylint: disable=unused-argument +import datetime +from collections.abc import AsyncIterator +from functools import partial from pathlib import Path +from typing import Any import pytest +from celery import Celery # type: ignore[import-untyped] +from celery.contrib.testing.worker import ( # type: ignore[import-untyped] + TestWorkController, + start_worker, +) +from celery.signals import worker_init, worker_shutdown # type: ignore[import-untyped] +from celery.worker.worker import WorkController # type: ignore[import-untyped] +from celery_library.signals import on_worker_init, on_worker_shutdown from models_library.basic_types import BootModeEnum from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict +from servicelib.fastapi.celery.app_server import FastAPIAppServer +from simcore_service_notifications.core.application import create_app +from simcore_service_notifications.core.settings import ApplicationSettings +from simcore_service_notifications.modules.celery.tasks import ( + TaskQueue, # type: ignore[import-untyped] +) +from simcore_service_notifications.modules.celery.tasks import ( + setup_worker_tasks, +) pytest_plugins = [ "pytest_simcore.docker_compose", @@ -14,6 +35,7 @@ "pytest_simcore.environment_configs", "pytest_simcore.postgres_service", "pytest_simcore.rabbit_service", + "pytest_simcore.redis_service", "pytest_simcore.repository_paths", ] @@ -40,3 +62,53 @@ def mock_environment( "SC_BOOT_MODE": BootModeEnum.DEBUG, }, ) + + +@pytest.fixture(scope="session") +def celery_config() -> dict[str, Any]: + return { + "broker_connection_retry_on_startup": True, + "broker_url": "memory://localhost//", + "result_backend": "cache+memory://localhost//", + "result_expires": datetime.timedelta(days=7), + "result_extended": True, + "pool": "threads", + "task_default_queue": "default", + "task_send_sent_event": True, + "task_track_started": True, + "worker_send_task_events": True, + } + + +@pytest.fixture +async def with_celery_worker( + mock_environment: EnvVarsDict, + celery_app: Celery, + monkeypatch: pytest.MonkeyPatch, +) -> AsyncIterator[TestWorkController]: + # Signals must be explicitily connected + monkeypatch.setenv("NOTIFICATIONS_WORKER_MODE", "true") + app_settings = ApplicationSettings.create_from_envs() + + def _on_worker_init_wrapper(sender: WorkController, **_kwargs): + assert app_settings.NOTIFICATIONS_CELERY # nosec + return partial( + on_worker_init, + FastAPIAppServer(app=create_app(app_settings)), + app_settings.NOTIFICATIONS_CELERY, + )(sender, **_kwargs) + + worker_init.connect(_on_worker_init_wrapper) + worker_shutdown.connect(on_worker_shutdown) + + setup_worker_tasks(celery_app) + + with start_worker( + celery_app, + pool="threads", + concurrency=1, + loglevel="info", + perform_ping_check=False, + queues=",".join(queue.value for queue in TaskQueue), + ) as worker: + yield worker diff --git a/services/notifications/tests/unit/conftest.py b/services/notifications/tests/unit/conftest.py index 5f785451f12a..f93d9c428a27 100644 --- a/services/notifications/tests/unit/conftest.py +++ b/services/notifications/tests/unit/conftest.py @@ -11,6 +11,7 @@ from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict from settings_library.rabbit import RabbitSettings from simcore_service_notifications.core.application import create_app +from simcore_service_notifications.core.settings import ApplicationSettings @pytest.fixture @@ -37,7 +38,7 @@ def app_environment( @pytest.fixture async def initialized_app(app_environment: EnvVarsDict) -> AsyncIterator[FastAPI]: - app: FastAPI = create_app() + app: FastAPI = create_app(ApplicationSettings.create_from_envs()) async with LifespanManager(app, startup_timeout=30, shutdown_timeout=30): yield app From 77b398a4c5819c80b200940f4dacff60432899d5 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 3 Jul 2025 15:38:02 +0200 Subject: [PATCH 23/69] tests: fix name --- packages/celery-library/tests/unit/test_tasks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/celery-library/tests/unit/test_tasks.py b/packages/celery-library/tests/unit/test_tasks.py index 4270efcc0655..6a6df578c0af 100644 --- a/packages/celery-library/tests/unit/test_tasks.py +++ b/packages/celery-library/tests/unit/test_tasks.py @@ -95,7 +95,7 @@ async def test_submitting_task_calling_async_function_results_with_success_state ): task_context = TaskContext(user_id=42) - task_uuid = await celery_task_manager.submit_task( + task_uuid = await celery_task_manager.send_task( TaskMetadata( name=fake_file_processor.__name__, ), @@ -125,7 +125,7 @@ async def test_submitting_task_with_failure_results_with_error( ): task_context = TaskContext(user_id=42) - task_uuid = await celery_task_manager.submit_task( + task_uuid = await celery_task_manager.send_task( TaskMetadata( name=failure_task.__name__, ), @@ -153,7 +153,7 @@ async def test_cancelling_a_running_task_aborts_and_deletes( ): task_context = TaskContext(user_id=42) - task_uuid = await celery_task_manager.submit_task( + task_uuid = await celery_task_manager.send_task( TaskMetadata( name=dreamer_task.__name__, ), @@ -187,7 +187,7 @@ async def test_listing_task_uuids_contains_submitted_task( ): task_context = TaskContext(user_id=42) - task_uuid = await celery_task_manager.submit_task( + task_uuid = await celery_task_manager.send_task( TaskMetadata( name=dreamer_task.__name__, ), From 79dcafa065d87780fb0ab0346f02a1871153785e Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 3 Jul 2025 15:46:05 +0200 Subject: [PATCH 24/69] tests: linting --- .../modules/celery/_email_tasks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py b/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py index a67d3ea07eb0..1ffaa3e8ec53 100644 --- a/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py +++ b/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py @@ -1,3 +1,5 @@ +# pylint: disable=unused-argument + import logging from celery import Task # type: ignore[import-untyped] From 95991af954efe1b3c6732d2d2b12f8144db6ae9c Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 4 Jul 2025 09:49:06 +0200 Subject: [PATCH 25/69] refactor: move models --- .../src/models_library/rpc/notifications}/__init__.py | 0 .../src/models_library/rpc/notifications/notifications.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {services/notifications/src/simcore_service_notifications/models => packages/models-library/src/models_library/rpc/notifications}/__init__.py (100%) rename services/notifications/src/simcore_service_notifications/models/schemas.py => packages/models-library/src/models_library/rpc/notifications/notifications.py (100%) diff --git a/services/notifications/src/simcore_service_notifications/models/__init__.py b/packages/models-library/src/models_library/rpc/notifications/__init__.py similarity index 100% rename from services/notifications/src/simcore_service_notifications/models/__init__.py rename to packages/models-library/src/models_library/rpc/notifications/__init__.py diff --git a/services/notifications/src/simcore_service_notifications/models/schemas.py b/packages/models-library/src/models_library/rpc/notifications/notifications.py similarity index 100% rename from services/notifications/src/simcore_service_notifications/models/schemas.py rename to packages/models-library/src/models_library/rpc/notifications/notifications.py From 50ea6f9f013207ec84ec18abec93a9f97a6c09c0 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 4 Jul 2025 10:28:29 +0200 Subject: [PATCH 26/69] refactor: models --- .../rpc/notifications/{notifications.py => messages.py} | 7 ++++--- .../rabbitmq/rpc_interfaces/notifications/messages.py | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) rename packages/models-library/src/models_library/rpc/notifications/{notifications.py => messages.py} (75%) create mode 100644 packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/messages.py diff --git a/packages/models-library/src/models_library/rpc/notifications/notifications.py b/packages/models-library/src/models_library/rpc/notifications/messages.py similarity index 75% rename from packages/models-library/src/models_library/rpc/notifications/notifications.py rename to packages/models-library/src/models_library/rpc/notifications/messages.py index b8b895dba405..0e5fcb81cc7e 100644 --- a/packages/models-library/src/models_library/rpc/notifications/notifications.py +++ b/packages/models-library/src/models_library/rpc/notifications/messages.py @@ -1,18 +1,19 @@ +from abc import ABC from typing import Annotated, Any, Literal, TypeAlias from pydantic import BaseModel, Field -class _BaseRecipient(BaseModel): +class BaseRecipient(BaseModel, ABC): type: str -class SMSRecipient(_BaseRecipient): +class SMSRecipient(BaseRecipient): type: Literal["sms"] phone_number: str -class EmailRecipient(_BaseRecipient): +class EmailRecipient(BaseRecipient): type: Literal["email"] address: str diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/messages.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/messages.py new file mode 100644 index 000000000000..b9f71b1d5bf3 --- /dev/null +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/messages.py @@ -0,0 +1,2 @@ +async def send_notification_message(): + pass From 60db2ca65b36fbd9990fd22d9bd45b58ea551e7d Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 4 Jul 2025 10:33:37 +0200 Subject: [PATCH 27/69] tests: add rabbitmq client --- services/notifications/tests/conftest.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/services/notifications/tests/conftest.py b/services/notifications/tests/conftest.py index c84938b8c35a..67d9e2a51aea 100644 --- a/services/notifications/tests/conftest.py +++ b/services/notifications/tests/conftest.py @@ -3,7 +3,7 @@ import datetime -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Awaitable, Callable from functools import partial from pathlib import Path from typing import Any @@ -20,6 +20,7 @@ from models_library.basic_types import BootModeEnum from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict from servicelib.fastapi.celery.app_server import FastAPIAppServer +from servicelib.rabbitmq import RabbitMQRPCClient from simcore_service_notifications.core.application import create_app from simcore_service_notifications.core.settings import ApplicationSettings from simcore_service_notifications.modules.celery.tasks import ( @@ -112,3 +113,12 @@ def _on_worker_init_wrapper(sender: WorkController, **_kwargs): queues=",".join(queue.value for queue in TaskQueue), ) as worker: yield worker + + +@pytest.fixture +async def notifications_rabbitmq_rpc_client( + rabbitmq_rpc_client: Callable[[str], Awaitable[RabbitMQRPCClient]], +) -> RabbitMQRPCClient: + rpc_client = await rabbitmq_rpc_client("pytest_notifications_rpc_client") + assert rpc_client + return rpc_client From 421d5f8dc230865e7f03f0c72f5106b5d7f99b70 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 4 Jul 2025 11:11:11 +0200 Subject: [PATCH 28/69] fix: imports --- .../src/simcore_service_notifications/clients/celery.py | 6 +++++- .../modules/celery/_email_tasks.py | 6 ++++-- .../simcore_service_notifications/modules/celery/tasks.py | 6 +++++- .../services/notifications_service.py | 3 +-- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/services/notifications/src/simcore_service_notifications/clients/celery.py b/services/notifications/src/simcore_service_notifications/clients/celery.py index 691f108d6031..24ba2a19765c 100644 --- a/services/notifications/src/simcore_service_notifications/clients/celery.py +++ b/services/notifications/src/simcore_service_notifications/clients/celery.py @@ -6,10 +6,14 @@ from celery_library.types import register_celery_types, register_pydantic_types from fastapi import FastAPI from fastapi_lifespan_manager import State +from models_library.rpc.notifications.messages import ( + EmailRecipient, + NotificationMessage, + SMSRecipient, +) from settings_library.celery import CelerySettings from ..core.settings import ApplicationSettings -from ..models.schemas import EmailRecipient, NotificationMessage, SMSRecipient _logger = logging.getLogger(__name__) diff --git a/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py b/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py index 1ffaa3e8ec53..82592827ba98 100644 --- a/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py +++ b/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py @@ -3,8 +3,10 @@ import logging from celery import Task # type: ignore[import-untyped] - -from ...models.schemas import EmailRecipient, NotificationMessage +from models_library.rpc.notifications.messages import ( + EmailRecipient, + NotificationMessage, +) _logger = logging.getLogger(__name__) diff --git a/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py b/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py index 7978db6f99ea..df4fca4059d0 100644 --- a/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py +++ b/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py @@ -4,9 +4,13 @@ from celery import Celery # type: ignore[import-untyped] from celery_library.task import register_task from celery_library.types import register_celery_types, register_pydantic_types +from models_library.rpc.notifications.messages import ( + EmailRecipient, + NotificationMessage, + SMSRecipient, +) from servicelib.logging_utils import log_context -from ...models.schemas import EmailRecipient, NotificationMessage, SMSRecipient from ...modules.celery._email_tasks import EMAIL_CHANNEL_NAME, send_email _logger = logging.getLogger(__name__) diff --git a/services/notifications/src/simcore_service_notifications/services/notifications_service.py b/services/notifications/src/simcore_service_notifications/services/notifications_service.py index 1460c687a28b..27abc2dbf150 100644 --- a/services/notifications/src/simcore_service_notifications/services/notifications_service.py +++ b/services/notifications/src/simcore_service_notifications/services/notifications_service.py @@ -1,10 +1,9 @@ from enum import StrEnum +from models_library.rpc.notifications.messages import NotificationMessage, Recipient from servicelib.celery.models import TaskContext, TaskMetadata from servicelib.celery.task_manager import TaskManager -from ..models.schemas import NotificationMessage, Recipient - class TaskQueues(StrEnum): DEFAULT = "notifications.default" From 2bda0d8e2f43b4f537321ba15410b119475e7eaf Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 4 Jul 2025 11:16:50 +0200 Subject: [PATCH 29/69] fix: update import --- .../src/simcore_service_notifications/api/rpc/_notifications.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/notifications/src/simcore_service_notifications/api/rpc/_notifications.py b/services/notifications/src/simcore_service_notifications/api/rpc/_notifications.py index a5090db8d04d..bead32a8a827 100644 --- a/services/notifications/src/simcore_service_notifications/api/rpc/_notifications.py +++ b/services/notifications/src/simcore_service_notifications/api/rpc/_notifications.py @@ -1,7 +1,7 @@ +from models_library.rpc.notifications.messages import NotificationMessage, Recipient from servicelib.celery.task_manager import TaskManager from servicelib.rabbitmq import RPCRouter -from ...models.schemas import NotificationMessage, Recipient from ...services import notifications_service router = RPCRouter() From e23cc2ea6a508b02d8047c8212c133fce33ce01b Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 4 Jul 2025 16:07:45 +0200 Subject: [PATCH 30/69] refactor: move task metadata --- .../src/celery_library/backends/_redis.py | 4 +- .../src/celery_library/rpc/_async_jobs.py | 10 +-- .../src/celery_library/task_manager.py | 73 +++++++++++-------- packages/celery-library/tests/conftest.py | 2 +- .../tests/unit/test_async_jobs.py | 16 ++-- .../celery-library/tests/unit/test_tasks.py | 67 +++++++---------- .../src/servicelib/celery/task_manager.py | 22 ++++-- .../services/notifications_service.py | 10 +-- .../api/rest/_files.py | 15 ++-- .../simcore_service_storage/api/rpc/_paths.py | 13 +--- .../api/rpc/_simcore_s3.py | 17 ++--- 11 files changed, 121 insertions(+), 128 deletions(-) diff --git a/packages/celery-library/src/celery_library/backends/_redis.py b/packages/celery-library/src/celery_library/backends/_redis.py index 37a9a415cd5d..ddfd2a8e4bb8 100644 --- a/packages/celery-library/src/celery_library/backends/_redis.py +++ b/packages/celery-library/src/celery_library/backends/_redis.py @@ -82,10 +82,10 @@ async def get_task_progress(self, task_id: TaskID) -> ProgressReport | None: ) return None - async def list_tasks(self, task_context: TaskContext) -> list[Task]: + async def list_tasks(self, context: TaskContext) -> list[Task]: search_key = ( _CELERY_TASK_INFO_PREFIX - + build_task_id_prefix(task_context) + + build_task_id_prefix(context) + _CELERY_TASK_ID_KEY_SEPARATOR ) search_key_len = len(search_key) diff --git a/packages/celery-library/src/celery_library/rpc/_async_jobs.py b/packages/celery-library/src/celery_library/rpc/_async_jobs.py index 4972142a457f..65d63461a82e 100644 --- a/packages/celery-library/src/celery_library/rpc/_async_jobs.py +++ b/packages/celery-library/src/celery_library/rpc/_async_jobs.py @@ -38,7 +38,7 @@ async def cancel( assert job_id_data # nosec try: await task_manager.cancel_task( - task_context=job_id_data.model_dump(), + context=job_id_data.model_dump(), task_uuid=job_id, ) except CeleryError as exc: @@ -54,7 +54,7 @@ async def status( try: task_status = await task_manager.get_task_status( - task_context=job_id_data.model_dump(), + context=job_id_data.model_dump(), task_uuid=job_id, ) except CeleryError as exc: @@ -84,13 +84,13 @@ async def result( try: _status = await task_manager.get_task_status( - task_context=job_id_data.model_dump(), + context=job_id_data.model_dump(), task_uuid=job_id, ) if not _status.is_done: raise JobNotDoneError(job_id=job_id) _result = await task_manager.get_task_result( - task_context=job_id_data.model_dump(), + context=job_id_data.model_dump(), task_uuid=job_id, ) except CeleryError as exc: @@ -129,7 +129,7 @@ async def list_jobs( assert task_manager # nosec try: tasks = await task_manager.list_tasks( - task_context=job_id_data.model_dump(), + context=job_id_data.model_dump(), ) except CeleryError as exc: raise JobSchedulerError(exc=f"{exc}") from exc diff --git a/packages/celery-library/src/celery_library/task_manager.py b/packages/celery-library/src/celery_library/task_manager.py index 2b266b0ea001..c8999efe069e 100644 --- a/packages/celery-library/src/celery_library/task_manager.py +++ b/packages/celery-library/src/celery_library/task_manager.py @@ -10,11 +10,14 @@ from common_library.async_tools import make_async from models_library.progress_bar import ProgressReport from servicelib.celery.models import ( + TASK_QUEUE_DEFAULT, Task, TaskContext, TaskID, TaskInfoStore, TaskMetadata, + TaskName, + TaskQueue, TaskState, TaskStatus, TaskUUID, @@ -39,32 +42,40 @@ class CeleryTaskManager: async def send_task( self, - task_metadata: TaskMetadata, + name: TaskName, + context: TaskContext, *, - task_context: TaskContext, - **task_params, + is_ephemeral: bool = False, + queue: TaskQueue = TASK_QUEUE_DEFAULT, + **params, ) -> TaskUUID: with log_context( _logger, logging.DEBUG, - msg=f"Send {task_metadata.name=}: {task_context=} {task_params=}", + msg=f"Send {name=}: {context=} {params=}", ): task_uuid = uuid4() - task_id = build_task_id(task_context, task_uuid) + task_id = build_task_id(context, task_uuid) self._celery_app.send_task( - task_metadata.name, + name, task_id=task_id, - kwargs={"task_id": task_id} | task_params, - queue=task_metadata.queue, + kwargs={"task_id": task_id} | params, + queue=queue, ) expiry = ( self._celery_settings.CELERY_EPHEMERAL_RESULT_EXPIRES - if task_metadata.ephemeral + if is_ephemeral else self._celery_settings.CELERY_RESULT_EXPIRES ) await self._task_info_store.create_task( - task_id, task_metadata, expiry=expiry + task_id, + TaskMetadata( + name=name, + ephemeral=is_ephemeral, + queue=queue, + ), + expiry=expiry, ) return task_uuid @@ -72,14 +83,14 @@ async def send_task( def _abort_task(self, task_id: TaskID) -> None: AbortableAsyncResult(task_id, app=self._celery_app).abort() - async def cancel_task(self, task_context: TaskContext, task_uuid: TaskUUID) -> None: + async def cancel_task(self, context: TaskContext, task_uuid: TaskUUID) -> None: with log_context( _logger, logging.DEBUG, - msg=f"task cancellation: {task_context=} {task_uuid=}", + msg=f"task cancellation: {context=} {task_uuid=}", ): - task_id = build_task_id(task_context, task_uuid) - if not (await self.get_task_status(task_context, task_uuid)).is_done: + task_id = build_task_id(context, task_uuid) + if not (await self.get_task_status(context, task_uuid)).is_done: await self._abort_task(task_id) await self._task_info_store.remove_task(task_id) @@ -87,15 +98,13 @@ async def cancel_task(self, task_context: TaskContext, task_uuid: TaskUUID) -> N def _forget_task(self, task_id: TaskID) -> None: AbortableAsyncResult(task_id, app=self._celery_app).forget() - async def get_task_result( - self, task_context: TaskContext, task_uuid: TaskUUID - ) -> Any: + async def get_task_result(self, context: TaskContext, task_uuid: TaskUUID) -> Any: with log_context( _logger, logging.DEBUG, - msg=f"Get task result: {task_context=} {task_uuid=}", + msg=f"Get task result: {context=} {task_uuid=}", ): - task_id = build_task_id(task_context, task_uuid) + task_id = build_task_id(context, task_uuid) async_result = self._celery_app.AsyncResult(task_id) result = async_result.result if async_result.ready(): @@ -105,15 +114,15 @@ async def get_task_result( await self._task_info_store.remove_task(task_id) return result - async def _get_task_progress_report( - self, task_context: TaskContext, task_uuid: TaskUUID, task_state: TaskState + async def _get_progress_report( + self, context: TaskContext, task_uuid: TaskUUID, state: TaskState ) -> ProgressReport: - if task_state in (TaskState.STARTED, TaskState.RETRY, TaskState.ABORTED): - task_id = build_task_id(task_context, task_uuid) + if state in (TaskState.STARTED, TaskState.RETRY, TaskState.ABORTED): + task_id = build_task_id(context, task_uuid) progress = await self._task_info_store.get_task_progress(task_id) if progress is not None: return progress - if task_state in ( + if state in ( TaskState.SUCCESS, TaskState.FAILURE, ): @@ -131,30 +140,30 @@ def _get_task_celery_state(self, task_id: TaskID) -> TaskState: return TaskState(self._celery_app.AsyncResult(task_id).state) async def get_task_status( - self, task_context: TaskContext, task_uuid: TaskUUID + self, context: TaskContext, task_uuid: TaskUUID ) -> TaskStatus: with log_context( _logger, logging.DEBUG, - msg=f"Getting task status: {task_context=} {task_uuid=}", + msg=f"Getting task status: {context=} {task_uuid=}", ): - task_id = build_task_id(task_context, task_uuid) + task_id = build_task_id(context, task_uuid) task_state = await self._get_task_celery_state(task_id) return TaskStatus( task_uuid=task_uuid, task_state=task_state, - progress_report=await self._get_task_progress_report( - task_context, task_uuid, task_state + progress_report=await self._get_progress_report( + context, task_uuid, task_state ), ) - async def list_tasks(self, task_context: TaskContext) -> list[Task]: + async def list_tasks(self, context: TaskContext) -> list[Task]: with log_context( _logger, logging.DEBUG, - msg=f"Listing tasks: {task_context=}", + msg=f"Listing tasks: {context=}", ): - return await self._task_info_store.list_tasks(task_context) + return await self._task_info_store.list_tasks(context) async def set_task_progress(self, task_id: TaskID, report: ProgressReport) -> None: await self._task_info_store.set_task_progress( diff --git a/packages/celery-library/tests/conftest.py b/packages/celery-library/tests/conftest.py index 2553d9df6d77..394304f64346 100644 --- a/packages/celery-library/tests/conftest.py +++ b/packages/celery-library/tests/conftest.py @@ -120,7 +120,7 @@ def _on_worker_init_wrapper(sender: WorkController, **_kwargs): @pytest.fixture -async def celery_task_manager( +async def task_manager( celery_app: Celery, celery_settings: CelerySettings, with_celery_worker: TestWorkController, diff --git a/packages/celery-library/tests/unit/test_async_jobs.py b/packages/celery-library/tests/unit/test_async_jobs.py index 7772c6812d67..7f5bacedf034 100644 --- a/packages/celery-library/tests/unit/test_async_jobs.py +++ b/packages/celery-library/tests/unit/test_async_jobs.py @@ -27,7 +27,7 @@ from models_library.rabbitmq_basic_types import RPCNamespace from models_library.users import UserID from pydantic import TypeAdapter -from servicelib.celery.models import TaskID, TaskMetadata +from servicelib.celery.models import TaskID from servicelib.celery.task_manager import TaskManager from servicelib.rabbitmq import RabbitMQRPCClient, RPCRouter from servicelib.rabbitmq.rpc_interfaces.async_jobs import async_jobs @@ -83,7 +83,9 @@ async def rpc_sync_job( ) -> AsyncJobGet: task_name = sync_job.__name__ task_uuid = await task_manager.send_task( - TaskMetadata(name=task_name), task_context=job_id_data.model_dump(), **kwargs + name=task_name, + context=job_id_data.model_dump(), + **kwargs, ) return AsyncJobGet(job_id=task_uuid, job_name=task_name) @@ -95,7 +97,9 @@ async def rpc_async_job( ) -> AsyncJobGet: task_name = async_job.__name__ task_uuid = await task_manager.send_task( - TaskMetadata(name=task_name), task_context=job_id_data.model_dump(), **kwargs + name=task_name, + context=job_id_data.model_dump(), + **kwargs, ) return AsyncJobGet(job_id=task_uuid, job_name=task_name) @@ -139,13 +143,13 @@ async def async_job(task: Task, task_id: TaskID, action: Action, payload: Any) - @pytest.fixture async def register_rpc_routes( - async_jobs_rabbitmq_rpc_client: RabbitMQRPCClient, celery_task_manager: TaskManager + async_jobs_rabbitmq_rpc_client: RabbitMQRPCClient, task_manager: TaskManager ) -> None: await async_jobs_rabbitmq_rpc_client.register_router( - _async_jobs.router, ASYNC_JOBS_RPC_NAMESPACE, task_manager=celery_task_manager + _async_jobs.router, ASYNC_JOBS_RPC_NAMESPACE, task_manager=task_manager ) await async_jobs_rabbitmq_rpc_client.register_router( - router, ASYNC_JOBS_RPC_NAMESPACE, task_manager=celery_task_manager + router, ASYNC_JOBS_RPC_NAMESPACE, task_manager=task_manager ) diff --git a/packages/celery-library/tests/unit/test_tasks.py b/packages/celery-library/tests/unit/test_tasks.py index 6a6df578c0af..251d8be2d5e5 100644 --- a/packages/celery-library/tests/unit/test_tasks.py +++ b/packages/celery-library/tests/unit/test_tasks.py @@ -22,7 +22,6 @@ from servicelib.celery.models import ( TaskContext, TaskID, - TaskMetadata, TaskState, ) from servicelib.logging_utils import log_context @@ -91,15 +90,13 @@ def _(celery_app: Celery) -> None: async def test_submitting_task_calling_async_function_results_with_success_state( - celery_task_manager: CeleryTaskManager, + task_manager: CeleryTaskManager, ): task_context = TaskContext(user_id=42) - task_uuid = await celery_task_manager.send_task( - TaskMetadata( - name=fake_file_processor.__name__, - ), - task_context=task_context, + task_uuid = await task_manager.send_task( + name=fake_file_processor.__name__, + context=task_context, files=[f"file{n}" for n in range(5)], ) @@ -109,27 +106,25 @@ async def test_submitting_task_calling_async_function_results_with_success_state stop=stop_after_delay(30), ): with attempt: - status = await celery_task_manager.get_task_status(task_context, task_uuid) + status = await task_manager.get_task_status(task_context, task_uuid) assert status.task_state == TaskState.SUCCESS assert ( - await celery_task_manager.get_task_status(task_context, task_uuid) + await task_manager.get_task_status(task_context, task_uuid) ).task_state == TaskState.SUCCESS assert ( - await celery_task_manager.get_task_result(task_context, task_uuid) + await task_manager.get_task_result(task_context, task_uuid) ) == "archive.zip" async def test_submitting_task_with_failure_results_with_error( - celery_task_manager: CeleryTaskManager, + task_manager: CeleryTaskManager, ): task_context = TaskContext(user_id=42) - task_uuid = await celery_task_manager.send_task( - TaskMetadata( - name=failure_task.__name__, - ), - task_context=task_context, + task_uuid = await task_manager.send_task( + name=failure_task.__name__, + context=task_context, ) for attempt in Retrying( @@ -139,30 +134,26 @@ async def test_submitting_task_with_failure_results_with_error( ): with attempt: - raw_result = await celery_task_manager.get_task_result( - task_context, task_uuid - ) + raw_result = await task_manager.get_task_result(task_context, task_uuid) assert isinstance(raw_result, TransferrableCeleryError) - raw_result = await celery_task_manager.get_task_result(task_context, task_uuid) + raw_result = await task_manager.get_task_result(task_context, task_uuid) assert f"{raw_result}" == "Something strange happened: BOOM!" async def test_cancelling_a_running_task_aborts_and_deletes( - celery_task_manager: CeleryTaskManager, + task_manager: CeleryTaskManager, ): task_context = TaskContext(user_id=42) - task_uuid = await celery_task_manager.send_task( - TaskMetadata( - name=dreamer_task.__name__, - ), - task_context=task_context, + task_uuid = await task_manager.send_task( + name=dreamer_task.__name__, + context=task_context, ) await asyncio.sleep(3.0) - await celery_task_manager.cancel_task(task_context, task_uuid) + await task_manager.cancel_task(task_context, task_uuid) for attempt in Retrying( retry=retry_if_exception_type(AssertionError), @@ -170,28 +161,24 @@ async def test_cancelling_a_running_task_aborts_and_deletes( stop=stop_after_delay(30), ): with attempt: - progress = await celery_task_manager.get_task_status( - task_context, task_uuid - ) + progress = await task_manager.get_task_status(task_context, task_uuid) assert progress.task_state == TaskState.ABORTED assert ( - await celery_task_manager.get_task_status(task_context, task_uuid) + await task_manager.get_task_status(task_context, task_uuid) ).task_state == TaskState.ABORTED - assert task_uuid not in await celery_task_manager.list_tasks(task_context) + assert task_uuid not in await task_manager.list_tasks(task_context) async def test_listing_task_uuids_contains_submitted_task( - celery_task_manager: CeleryTaskManager, + task_manager: CeleryTaskManager, ): task_context = TaskContext(user_id=42) - task_uuid = await celery_task_manager.send_task( - TaskMetadata( - name=dreamer_task.__name__, - ), - task_context=task_context, + task_uuid = await task_manager.send_task( + name=dreamer_task.__name__, + context=task_context, ) for attempt in Retrying( @@ -200,8 +187,8 @@ async def test_listing_task_uuids_contains_submitted_task( stop=stop_after_delay(10), ): with attempt: - tasks = await celery_task_manager.list_tasks(task_context) + tasks = await task_manager.list_tasks(task_context) assert any(task.uuid == task_uuid for task in tasks) - tasks = await celery_task_manager.list_tasks(task_context) + tasks = await task_manager.list_tasks(task_context) assert any(task.uuid == task_uuid for task in tasks) diff --git a/packages/service-library/src/servicelib/celery/task_manager.py b/packages/service-library/src/servicelib/celery/task_manager.py index 172183e44413..eeb06769e96d 100644 --- a/packages/service-library/src/servicelib/celery/task_manager.py +++ b/packages/service-library/src/servicelib/celery/task_manager.py @@ -3,10 +3,12 @@ from models_library.progress_bar import ProgressReport from ..celery.models import ( + TASK_QUEUE_DEFAULT, Task, TaskContext, TaskID, - TaskMetadata, + TaskName, + TaskQueue, TaskStatus, TaskUUID, ) @@ -14,22 +16,26 @@ class TaskManager(Protocol): async def send_task( - self, task_metadata: TaskMetadata, *, task_context: TaskContext, **task_params + self, + name: TaskName, + context: TaskContext, + *, + ephemeral: bool = True, + queue: TaskQueue = TASK_QUEUE_DEFAULT, + **params, ) -> TaskUUID: ... - async def cancel_task( - self, task_context: TaskContext, task_uuid: TaskUUID - ) -> None: ... + async def cancel_task(self, context: TaskContext, task_uuid: TaskUUID) -> None: ... async def get_task_result( - self, task_context: TaskContext, task_uuid: TaskUUID + self, context: TaskContext, task_uuid: TaskUUID ) -> Any: ... async def get_task_status( - self, task_context: TaskContext, task_uuid: TaskUUID + self, context: TaskContext, task_uuid: TaskUUID ) -> TaskStatus: ... - async def list_tasks(self, task_context: TaskContext) -> list[Task]: ... + async def list_tasks(self, context: TaskContext) -> list[Task]: ... async def set_task_progress( self, task_id: TaskID, report: ProgressReport diff --git a/services/notifications/src/simcore_service_notifications/services/notifications_service.py b/services/notifications/src/simcore_service_notifications/services/notifications_service.py index 27abc2dbf150..735d98edeaee 100644 --- a/services/notifications/src/simcore_service_notifications/services/notifications_service.py +++ b/services/notifications/src/simcore_service_notifications/services/notifications_service.py @@ -1,7 +1,7 @@ from enum import StrEnum from models_library.rpc.notifications.messages import NotificationMessage, Recipient -from servicelib.celery.models import TaskContext, TaskMetadata +from servicelib.celery.models import TaskContext from servicelib.celery.task_manager import TaskManager @@ -17,11 +17,9 @@ async def send_notification( ) -> None: for recipient in recipients: await task_manager.send_task( - TaskMetadata( - name=f"notifications.{recipient.type}", - queue=TaskQueues.DEFAULT, - ), - task_context=TaskContext(), + name=f"notifications.{recipient.type}", + context=TaskContext(), + queue=TaskQueues.DEFAULT, message=message, recipient=recipient, ) diff --git a/services/storage/src/simcore_service_storage/api/rest/_files.py b/services/storage/src/simcore_service_storage/api/rest/_files.py index f0f6c39529ad..e7ab6364c794 100644 --- a/services/storage/src/simcore_service_storage/api/rest/_files.py +++ b/services/storage/src/simcore_service_storage/api/rest/_files.py @@ -20,7 +20,7 @@ from models_library.projects_nodes_io import LocationID, StorageFileID from pydantic import AnyUrl, ByteSize, TypeAdapter from servicelib.aiohttp import status -from servicelib.celery.models import TaskMetadata, TaskUUID +from servicelib.celery.models import TaskUUID from servicelib.celery.task_manager import TaskManager from servicelib.logging_utils import log_context from yarl import URL @@ -289,10 +289,8 @@ async def complete_upload_file( product_name=_UNDEFINED_PRODUCT_NAME_FOR_WORKER_TASKS, # NOTE: I would need to change the API here ) task_uuid = await task_manager.send_task( - TaskMetadata( - name=remote_complete_upload_file.__name__, - ), - task_context=async_job_name_data.model_dump(), + name=remote_complete_upload_file.__name__, + context=async_job_name_data.model_dump(), user_id=async_job_name_data.user_id, location_id=location_id, file_id=file_id, @@ -345,13 +343,14 @@ async def is_completed_upload_file( product_name=_UNDEFINED_PRODUCT_NAME_FOR_WORKER_TASKS, # NOTE: I would need to change the API here ) task_status = await task_manager.get_task_status( - task_context=async_job_name_data.model_dump(), task_uuid=TaskUUID(future_id) + context=async_job_name_data.model_dump(), + task_uuid=TaskUUID(future_id), ) # first check if the task is in the app if task_status.is_done: task_result = TypeAdapter(FileMetaData).validate_python( - await task_manager.get_task_result( - task_context=async_job_name_data.model_dump(), + await task_manager.get_result( + context=async_job_name_data.model_dump(), task_uuid=TaskUUID(future_id), ) ) diff --git a/services/storage/src/simcore_service_storage/api/rpc/_paths.py b/services/storage/src/simcore_service_storage/api/rpc/_paths.py index bb7594ea1c1b..e6c5017145ed 100644 --- a/services/storage/src/simcore_service_storage/api/rpc/_paths.py +++ b/services/storage/src/simcore_service_storage/api/rpc/_paths.py @@ -6,7 +6,6 @@ AsyncJobNameData, ) from models_library.projects_nodes_io import LocationID -from servicelib.celery.models import TaskMetadata from servicelib.celery.task_manager import TaskManager from servicelib.rabbitmq import RPCRouter @@ -26,10 +25,8 @@ async def compute_path_size( ) -> AsyncJobGet: task_name = remote_compute_path_size.__name__ task_uuid = await task_manager.send_task( - task_metadata=TaskMetadata( - name=task_name, - ), - task_context=job_id_data.model_dump(), + name=task_name, + context=job_id_data.model_dump(), user_id=job_id_data.user_id, location_id=location_id, path=path, @@ -47,10 +44,8 @@ async def delete_paths( ) -> AsyncJobGet: task_name = remote_delete_paths.__name__ task_uuid = await task_manager.send_task( - task_metadata=TaskMetadata( - name=task_name, - ), - task_context=job_id_data.model_dump(), + name=task_name, + context=job_id_data.model_dump(), user_id=job_id_data.user_id, location_id=location_id, paths=paths, diff --git a/services/storage/src/simcore_service_storage/api/rpc/_simcore_s3.py b/services/storage/src/simcore_service_storage/api/rpc/_simcore_s3.py index fab1fee7b47b..e9dc58a6a11d 100644 --- a/services/storage/src/simcore_service_storage/api/rpc/_simcore_s3.py +++ b/services/storage/src/simcore_service_storage/api/rpc/_simcore_s3.py @@ -4,7 +4,6 @@ ) from models_library.api_schemas_storage.storage_schemas import FoldersBody from models_library.api_schemas_webserver.storage import PathToExport -from servicelib.celery.models import TaskMetadata from servicelib.celery.task_manager import TaskManager from servicelib.rabbitmq import RPCRouter @@ -22,10 +21,8 @@ async def copy_folders_from_project( ) -> AsyncJobGet: task_name = deep_copy_files_from_project.__name__ task_uuid = await task_manager.send_task( - task_metadata=TaskMetadata( - name=task_name, - ), - task_context=job_id_data.model_dump(), + name=task_name, + context=job_id_data.model_dump(), user_id=job_id_data.user_id, body=body, ) @@ -41,12 +38,10 @@ async def start_export_data( ) -> AsyncJobGet: task_name = export_data.__name__ task_uuid = await task_manager.send_task( - task_metadata=TaskMetadata( - name=task_name, - ephemeral=False, - queue=TaskQueue.CPU_BOUND, - ), - task_context=job_id_data.model_dump(), + name=task_name, + ephemeral=False, + queue=TaskQueue.CPU_BOUND, + context=job_id_data.model_dump(), user_id=job_id_data.user_id, paths_to_export=paths_to_export, ) From 380820ecfada2813ecbef262645e4498a2f6dbf1 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 4 Jul 2025 16:14:33 +0200 Subject: [PATCH 31/69] fix: param name --- packages/celery-library/src/celery_library/task_manager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/celery-library/src/celery_library/task_manager.py b/packages/celery-library/src/celery_library/task_manager.py index c8999efe069e..d9e35f702ec8 100644 --- a/packages/celery-library/src/celery_library/task_manager.py +++ b/packages/celery-library/src/celery_library/task_manager.py @@ -45,7 +45,7 @@ async def send_task( name: TaskName, context: TaskContext, *, - is_ephemeral: bool = False, + ephemeral: bool = True, queue: TaskQueue = TASK_QUEUE_DEFAULT, **params, ) -> TaskUUID: @@ -65,14 +65,14 @@ async def send_task( expiry = ( self._celery_settings.CELERY_EPHEMERAL_RESULT_EXPIRES - if is_ephemeral + if ephemeral else self._celery_settings.CELERY_RESULT_EXPIRES ) await self._task_info_store.create_task( task_id, TaskMetadata( name=name, - ephemeral=is_ephemeral, + ephemeral=ephemeral, queue=queue, ), expiry=expiry, From accab180e3c9358084fc37f9489645b6802ad59b Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 7 Jul 2025 09:53:54 +0200 Subject: [PATCH 32/69] feat: add rabbitmq call --- .../rpc_interfaces/notifications/messages.py | 27 +++++++++++++++++-- services/notifications/tests/conftest.py | 4 +-- .../notifications/tests/unit/test_tasks.py | 9 +++++++ 3 files changed, 35 insertions(+), 5 deletions(-) create mode 100644 services/notifications/tests/unit/test_tasks.py diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/messages.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/messages.py index b9f71b1d5bf3..7434396532d9 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/messages.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/messages.py @@ -1,2 +1,25 @@ -async def send_notification_message(): - pass +from typing import Final + +from models_library.api_schemas_notifications import NOTIFICATIONS_RPC_NAMESPACE +from models_library.rabbitmq_basic_types import RPCMethodName +from models_library.rpc.notifications.messages import NotificationMessage, Recipient +from pydantic import NonNegativeInt, TypeAdapter + +from ... import RabbitMQRPCClient + +_DEFAULT_TIMEOUT_S: Final[NonNegativeInt] = 30 + + +async def send_notification_message( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + message: NotificationMessage, + recipients: list[Recipient], +) -> None: + await rabbitmq_rpc_client.request( + NOTIFICATIONS_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python("send_notification_message"), + timeout_s=_DEFAULT_TIMEOUT_S, + message=message, + recipients=recipients, + ) diff --git a/services/notifications/tests/conftest.py b/services/notifications/tests/conftest.py index 67d9e2a51aea..66eb051d8d9b 100644 --- a/services/notifications/tests/conftest.py +++ b/services/notifications/tests/conftest.py @@ -24,9 +24,7 @@ from simcore_service_notifications.core.application import create_app from simcore_service_notifications.core.settings import ApplicationSettings from simcore_service_notifications.modules.celery.tasks import ( - TaskQueue, # type: ignore[import-untyped] -) -from simcore_service_notifications.modules.celery.tasks import ( + TaskQueue, setup_worker_tasks, ) diff --git a/services/notifications/tests/unit/test_tasks.py b/services/notifications/tests/unit/test_tasks.py new file mode 100644 index 000000000000..c3d9a751c609 --- /dev/null +++ b/services/notifications/tests/unit/test_tasks.py @@ -0,0 +1,9 @@ +from celery.contrib.testing.worker import TestWorkController +from servicelib.rabbitmq import RabbitMQRPCClient + + +async def test_send_email( + notifications_rabbitmq_rpc_client: RabbitMQRPCClient, + with_celery_worker: TestWorkController, +): + pass From a26d82c683f7449326fc07988d2730db20530f77 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 7 Jul 2025 11:36:20 +0200 Subject: [PATCH 33/69] tests: add test --- .../rpc/notifications/messages.py | 6 +-- .../rpc_interfaces/notifications/messages.py | 4 +- .../api/rest/_health.py | 8 +--- .../api/rest/dependencies.py | 9 ---- .../clients/postgres/__init__.py | 33 -------------- .../clients/postgres/_liveness.py | 43 ------------------- .../core/events.py | 6 --- services/notifications/tests/conftest.py | 5 +-- services/notifications/tests/unit/conftest.py | 43 +++++++++++++------ .../notifications/tests/unit/test_tasks.py | 21 ++++++++- 10 files changed, 59 insertions(+), 119 deletions(-) delete mode 100644 services/notifications/src/simcore_service_notifications/clients/postgres/__init__.py delete mode 100644 services/notifications/src/simcore_service_notifications/clients/postgres/_liveness.py diff --git a/packages/models-library/src/models_library/rpc/notifications/messages.py b/packages/models-library/src/models_library/rpc/notifications/messages.py index 0e5fcb81cc7e..8c51a7a9ba34 100644 --- a/packages/models-library/src/models_library/rpc/notifications/messages.py +++ b/packages/models-library/src/models_library/rpc/notifications/messages.py @@ -1,5 +1,5 @@ from abc import ABC -from typing import Annotated, Any, Literal, TypeAlias +from typing import Annotated, Any, TypeAlias from pydantic import BaseModel, Field @@ -9,12 +9,12 @@ class BaseRecipient(BaseModel, ABC): class SMSRecipient(BaseRecipient): - type: Literal["sms"] + type: Annotated[str, Field(frozen=True)] = "sms" phone_number: str class EmailRecipient(BaseRecipient): - type: Literal["email"] + type: Annotated[str, Field(frozen=True)] = "email" address: str diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/messages.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/messages.py index 7434396532d9..3e8df031b7fa 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/messages.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/messages.py @@ -2,7 +2,7 @@ from models_library.api_schemas_notifications import NOTIFICATIONS_RPC_NAMESPACE from models_library.rabbitmq_basic_types import RPCMethodName -from models_library.rpc.notifications.messages import NotificationMessage, Recipient +from models_library.rpc.notifications.messages import BaseRecipient, NotificationMessage from pydantic import NonNegativeInt, TypeAdapter from ... import RabbitMQRPCClient @@ -14,7 +14,7 @@ async def send_notification_message( rabbitmq_rpc_client: RabbitMQRPCClient, *, message: NotificationMessage, - recipients: list[Recipient], + recipients: list[BaseRecipient], ) -> None: await rabbitmq_rpc_client.request( NOTIFICATIONS_RPC_NAMESPACE, diff --git a/services/notifications/src/simcore_service_notifications/api/rest/_health.py b/services/notifications/src/simcore_service_notifications/api/rest/_health.py index 5f38f21d5e03..80e4fccf0c39 100644 --- a/services/notifications/src/simcore_service_notifications/api/rest/_health.py +++ b/services/notifications/src/simcore_service_notifications/api/rest/_health.py @@ -4,13 +4,11 @@ from fastapi import APIRouter, Depends from models_library.api_schemas__common.health import HealthCheckGet from models_library.errors import ( - POSRGRES_DATABASE_UNHEALTHY_MSG, RABBITMQ_CLIENT_UNHEALTHY_MSG, ) from servicelib.rabbitmq import RabbitMQClient -from ...clients.postgres import PostgresLiveness -from .dependencies import get_postgres_liveness, get_rabbitmq_client +from .dependencies import get_rabbitmq_client router = APIRouter() @@ -22,12 +20,8 @@ class HealthCheckError(RuntimeError): @router.get("/", response_model=HealthCheckGet) async def check_service_health( rabbitmq_client: Annotated[RabbitMQClient, Depends(get_rabbitmq_client)], - postgres_liveness: Annotated[PostgresLiveness, Depends(get_postgres_liveness)], ): if not rabbitmq_client.healthy: raise HealthCheckError(RABBITMQ_CLIENT_UNHEALTHY_MSG) - if not postgres_liveness.is_responsive: - raise HealthCheckError(POSRGRES_DATABASE_UNHEALTHY_MSG) - return HealthCheckGet(timestamp=f"{__name__}@{arrow.utcnow().datetime.isoformat()}") diff --git a/services/notifications/src/simcore_service_notifications/api/rest/dependencies.py b/services/notifications/src/simcore_service_notifications/api/rest/dependencies.py index 962154ea9f77..800efcfe15d8 100644 --- a/services/notifications/src/simcore_service_notifications/api/rest/dependencies.py +++ b/services/notifications/src/simcore_service_notifications/api/rest/dependencies.py @@ -5,9 +5,6 @@ from fastapi import Depends, FastAPI, Request from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient -from ...clients.postgres import PostgresLiveness -from ...clients.postgres import get_postgres_liveness as get_postgress_db_liveness - def get_application(request: Request) -> FastAPI: return cast(FastAPI, request.app) @@ -18,9 +15,3 @@ def get_rabbitmq_client( ) -> RabbitMQRPCClient: assert isinstance(app.state.rabbitmq_rpc_server, RabbitMQRPCClient) # nosec return app.state.rabbitmq_rpc_server - - -def get_postgres_liveness( - app: Annotated[FastAPI, Depends(get_application)], -) -> PostgresLiveness: - return get_postgress_db_liveness(app) diff --git a/services/notifications/src/simcore_service_notifications/clients/postgres/__init__.py b/services/notifications/src/simcore_service_notifications/clients/postgres/__init__.py deleted file mode 100644 index e0883fba1ced..000000000000 --- a/services/notifications/src/simcore_service_notifications/clients/postgres/__init__.py +++ /dev/null @@ -1,33 +0,0 @@ -import logging -from collections.abc import AsyncIterator - -from fastapi import FastAPI -from fastapi_lifespan_manager import State -from servicelib.fastapi.postgres_lifespan import PostgresLifespanState -from servicelib.logging_utils import log_context - -from ._liveness import PostgresLiveness - -_logger = logging.getLogger(__name__) - - -async def postgres_lifespan(app: FastAPI, state: State) -> AsyncIterator[State]: - app.state.engine = state[PostgresLifespanState.POSTGRES_ASYNC_ENGINE] - - app.state.postgres_liveness = PostgresLiveness(app) - - with log_context(_logger, logging.INFO, msg="setup postgres health"): - await app.state.postgres_liveness.setup() - - yield {} - - with log_context(_logger, logging.INFO, msg="teardown postgres health"): - await app.state.postgres_liveness.teardown() - - -def get_postgres_liveness(app: FastAPI) -> PostgresLiveness: - assert isinstance(app.state.postgres_liveness, PostgresLiveness) # nosec - return app.state.postgres_liveness - - -__all__: tuple[str, ...] = ("PostgresLiveness",) diff --git a/services/notifications/src/simcore_service_notifications/clients/postgres/_liveness.py b/services/notifications/src/simcore_service_notifications/clients/postgres/_liveness.py deleted file mode 100644 index 6d26fd83e939..000000000000 --- a/services/notifications/src/simcore_service_notifications/clients/postgres/_liveness.py +++ /dev/null @@ -1,43 +0,0 @@ -import logging -from asyncio import Task -from datetime import timedelta -from typing import Final - -from common_library.async_tools import cancel_wait_task -from fastapi import FastAPI -from models_library.healthchecks import IsResponsive, LivenessResult -from servicelib.background_task import create_periodic_task -from servicelib.db_asyncpg_utils import check_postgres_liveness -from servicelib.fastapi.db_asyncpg_engine import get_engine -from servicelib.logging_utils import log_catch - -_logger = logging.getLogger(__name__) - -_LVENESS_CHECK_INTERVAL: Final[timedelta] = timedelta(seconds=10) - - -class PostgresLiveness: - def __init__(self, app: FastAPI) -> None: - self.app = app - - self._liveness_result: LivenessResult = IsResponsive(elapsed=timedelta(0)) - self._task: Task | None = None - - async def _check_task(self) -> None: - self._liveness_result = await check_postgres_liveness(get_engine(self.app)) - - @property - def is_responsive(self) -> bool: - return isinstance(self._liveness_result, IsResponsive) - - async def setup(self) -> None: - self._task = create_periodic_task( - self._check_task, - interval=_LVENESS_CHECK_INTERVAL, - task_name="posgress_liveness_check", - ) - - async def teardown(self) -> None: - if self._task is not None: - with log_catch(_logger, reraise=False): - await cancel_wait_task(self._task, max_delay=5) diff --git a/services/notifications/src/simcore_service_notifications/core/events.py b/services/notifications/src/simcore_service_notifications/core/events.py index 11c7ec6dab64..e1e94b3b7588 100644 --- a/services/notifications/src/simcore_service_notifications/core/events.py +++ b/services/notifications/src/simcore_service_notifications/core/events.py @@ -8,13 +8,11 @@ ) from servicelib.fastapi.postgres_lifespan import ( create_postgres_database_input_state, - postgres_database_lifespan, ) from .._meta import APP_FINISHED_BANNER_MSG, APP_STARTED_BANNER_MSG from ..api.rpc.routes import rpc_api_routes_lifespan from ..clients.celery import celery_lifespan -from ..clients.postgres import postgres_lifespan from ..clients.rabbitmq import rabbitmq_lifespan from .settings import ApplicationSettings @@ -41,10 +39,6 @@ def create_app_lifespan(settings: ApplicationSettings) -> LifespanManager: app_lifespan = LifespanManager() app_lifespan.add(_settings_lifespan) - # - postgres - app_lifespan.add(postgres_database_lifespan) - app_lifespan.add(postgres_lifespan) - if settings.NOTIFICATIONS_CELERY and not settings.NOTIFICATIONS_WORKER_MODE: # - rabbitmq app_lifespan.add(rabbitmq_lifespan) diff --git a/services/notifications/tests/conftest.py b/services/notifications/tests/conftest.py index 66eb051d8d9b..331c4acab0a1 100644 --- a/services/notifications/tests/conftest.py +++ b/services/notifications/tests/conftest.py @@ -15,7 +15,7 @@ start_worker, ) from celery.signals import worker_init, worker_shutdown # type: ignore[import-untyped] -from celery.worker.worker import WorkController # type: ignore[import-untyped] +from celery.worker.worker import WorkController from celery_library.signals import on_worker_init, on_worker_shutdown from models_library.basic_types import BootModeEnum from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict @@ -32,7 +32,6 @@ "pytest_simcore.docker_compose", "pytest_simcore.docker_swarm", "pytest_simcore.environment_configs", - "pytest_simcore.postgres_service", "pytest_simcore.rabbit_service", "pytest_simcore.redis_service", "pytest_simcore.repository_paths", @@ -81,7 +80,7 @@ def celery_config() -> dict[str, Any]: @pytest.fixture async def with_celery_worker( - mock_environment: EnvVarsDict, + app_environment: EnvVarsDict, celery_app: Celery, monkeypatch: pytest.MonkeyPatch, ) -> AsyncIterator[TestWorkController]: diff --git a/services/notifications/tests/unit/conftest.py b/services/notifications/tests/unit/conftest.py index f93d9c428a27..30fb05727fb7 100644 --- a/services/notifications/tests/unit/conftest.py +++ b/services/notifications/tests/unit/conftest.py @@ -2,43 +2,62 @@ # pylint: disable=unused-argument from collections.abc import AsyncIterator +from typing import Final import pytest -import sqlalchemy as sa from asgi_lifespan import LifespanManager from fastapi import FastAPI from fastapi.testclient import TestClient from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict from settings_library.rabbit import RabbitSettings +from settings_library.redis import RedisSettings from simcore_service_notifications.core.application import create_app from simcore_service_notifications.core.settings import ApplicationSettings +_LIFESPAN_TIMEOUT: Final[int] = 30 + @pytest.fixture def app_environment( monkeypatch: pytest.MonkeyPatch, mock_environment: EnvVarsDict, - rabbit_service: RabbitSettings, - postgres_db: sa.engine.Engine, # waiting for postgres service to start - postgres_env_vars_dict: EnvVarsDict, ) -> EnvVarsDict: return setenvs_from_dict( monkeypatch, { **mock_environment, - "RABBIT_HOST": rabbit_service.RABBIT_HOST, - "RABBIT_PASSWORD": rabbit_service.RABBIT_PASSWORD.get_secret_value(), - "RABBIT_PORT": f"{rabbit_service.RABBIT_PORT}", - "RABBIT_SECURE": f"{rabbit_service.RABBIT_SECURE}", - "RABBIT_USER": rabbit_service.RABBIT_USER, - **postgres_env_vars_dict, }, ) @pytest.fixture -async def initialized_app(app_environment: EnvVarsDict) -> AsyncIterator[FastAPI]: - app: FastAPI = create_app(ApplicationSettings.create_from_envs()) +def enabled_rabbitmq( + app_environment: EnvVarsDict, rabbit_service: RabbitSettings +) -> RabbitSettings: + return rabbit_service + + +@pytest.fixture +def enabled_redis( + app_environment: EnvVarsDict, redis_service: RedisSettings +) -> RedisSettings: + return redis_service + + +@pytest.fixture +def app_settings( + app_environment: EnvVarsDict, + enabled_rabbitmq: RabbitSettings, + enabled_redis: RedisSettings, +) -> ApplicationSettings: + settings = ApplicationSettings.create_from_envs() + print(f"{settings.model_dump_json(indent=2)=}") + return settings + + +@pytest.fixture +async def initialized_app(app_settings: ApplicationSettings) -> AsyncIterator[FastAPI]: + app: FastAPI = create_app(app_settings) async with LifespanManager(app, startup_timeout=30, shutdown_timeout=30): yield app diff --git a/services/notifications/tests/unit/test_tasks.py b/services/notifications/tests/unit/test_tasks.py index c3d9a751c609..bd8729500773 100644 --- a/services/notifications/tests/unit/test_tasks.py +++ b/services/notifications/tests/unit/test_tasks.py @@ -1,9 +1,28 @@ from celery.contrib.testing.worker import TestWorkController +from fastapi import FastAPI +from models_library.rpc.notifications.messages import NotificationMessage from servicelib.rabbitmq import RabbitMQRPCClient +from servicelib.rabbitmq.rpc_interfaces.notifications.messages import ( + send_notification_message, +) +from simcore_service_notifications.clients.celery import EmailRecipient + +pytest_simcore_core_services_selection = [ + "rabbit", + "redis", +] async def test_send_email( + initialized_app: FastAPI, notifications_rabbitmq_rpc_client: RabbitMQRPCClient, with_celery_worker: TestWorkController, ): - pass + await send_notification_message( + notifications_rabbitmq_rpc_client, + message=NotificationMessage( + event="test_event", + context={"key": "value"}, + ), + recipients=[EmailRecipient(address="test@example.com")], + ) From 30ff8c9c7fc7a886202cc0462f75040e67028c35 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 7 Jul 2025 14:29:45 +0200 Subject: [PATCH 34/69] tests: fix initialization --- .../rpc_interfaces/notifications/__init__.py | 0 .../rpc_interfaces/notifications/messages.py | 4 +- .../modules/celery/_email_tasks.py | 2 + .../modules/celery/tasks.py | 8 +- .../services/notifications_service.py | 8 +- services/notifications/tests/conftest.py | 80 ----------------- services/notifications/tests/unit/conftest.py | 89 +++++++++++++++++-- .../notifications/tests/unit/test_tasks.py | 10 ++- 8 files changed, 102 insertions(+), 99 deletions(-) create mode 100644 packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/__init__.py diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/__init__.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/messages.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/messages.py index 3e8df031b7fa..7434396532d9 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/messages.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/messages.py @@ -2,7 +2,7 @@ from models_library.api_schemas_notifications import NOTIFICATIONS_RPC_NAMESPACE from models_library.rabbitmq_basic_types import RPCMethodName -from models_library.rpc.notifications.messages import BaseRecipient, NotificationMessage +from models_library.rpc.notifications.messages import NotificationMessage, Recipient from pydantic import NonNegativeInt, TypeAdapter from ... import RabbitMQRPCClient @@ -14,7 +14,7 @@ async def send_notification_message( rabbitmq_rpc_client: RabbitMQRPCClient, *, message: NotificationMessage, - recipients: list[BaseRecipient], + recipients: list[Recipient], ) -> None: await rabbitmq_rpc_client.request( NOTIFICATIONS_RPC_NAMESPACE, diff --git a/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py b/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py index 82592827ba98..086e4f9f3bb0 100644 --- a/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py +++ b/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py @@ -7,6 +7,7 @@ EmailRecipient, NotificationMessage, ) +from servicelib.celery.models import TaskID _logger = logging.getLogger(__name__) @@ -15,6 +16,7 @@ async def send_email( task: Task, + task_id: TaskID, message: NotificationMessage, recipient: EmailRecipient, ) -> None: diff --git a/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py b/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py index df4fca4059d0..3d5a55271e13 100644 --- a/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py +++ b/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py @@ -16,11 +16,11 @@ _logger = logging.getLogger(__name__) -_TASK_QUEUE_PREFIX: str = "notifications" +_NOTIFICATIONS_PREFIX: str = "notifications" class TaskQueue(StrEnum): - DEFAULT = f"{_TASK_QUEUE_PREFIX}.default" + DEFAULT = f"{_NOTIFICATIONS_PREFIX}.default" def setup_worker_tasks(app: Celery) -> None: @@ -28,4 +28,6 @@ def setup_worker_tasks(app: Celery) -> None: register_pydantic_types(NotificationMessage, EmailRecipient, SMSRecipient) with log_context(_logger, logging.INFO, msg="worker tasks registration"): - register_task(app, send_email, EMAIL_CHANNEL_NAME) + register_task( + app, send_email, ".".join((_NOTIFICATIONS_PREFIX, EMAIL_CHANNEL_NAME)) + ) diff --git a/services/notifications/src/simcore_service_notifications/services/notifications_service.py b/services/notifications/src/simcore_service_notifications/services/notifications_service.py index 735d98edeaee..12f2358beac2 100644 --- a/services/notifications/src/simcore_service_notifications/services/notifications_service.py +++ b/services/notifications/src/simcore_service_notifications/services/notifications_service.py @@ -1,12 +1,12 @@ -from enum import StrEnum +import logging from models_library.rpc.notifications.messages import NotificationMessage, Recipient from servicelib.celery.models import TaskContext from servicelib.celery.task_manager import TaskManager +from ..modules.celery.tasks import TaskQueue -class TaskQueues(StrEnum): - DEFAULT = "notifications.default" +_logger = logging.getLogger(__name__) async def send_notification( @@ -19,7 +19,7 @@ async def send_notification( await task_manager.send_task( name=f"notifications.{recipient.type}", context=TaskContext(), - queue=TaskQueues.DEFAULT, + queue=TaskQueue.DEFAULT, message=message, recipient=recipient, ) diff --git a/services/notifications/tests/conftest.py b/services/notifications/tests/conftest.py index 331c4acab0a1..81d50f1e2b1e 100644 --- a/services/notifications/tests/conftest.py +++ b/services/notifications/tests/conftest.py @@ -1,32 +1,11 @@ # pylint: disable=redefined-outer-name # pylint: disable=unused-argument - -import datetime -from collections.abc import AsyncIterator, Awaitable, Callable -from functools import partial from pathlib import Path -from typing import Any import pytest -from celery import Celery # type: ignore[import-untyped] -from celery.contrib.testing.worker import ( # type: ignore[import-untyped] - TestWorkController, - start_worker, -) -from celery.signals import worker_init, worker_shutdown # type: ignore[import-untyped] -from celery.worker.worker import WorkController -from celery_library.signals import on_worker_init, on_worker_shutdown from models_library.basic_types import BootModeEnum from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict -from servicelib.fastapi.celery.app_server import FastAPIAppServer -from servicelib.rabbitmq import RabbitMQRPCClient -from simcore_service_notifications.core.application import create_app -from simcore_service_notifications.core.settings import ApplicationSettings -from simcore_service_notifications.modules.celery.tasks import ( - TaskQueue, - setup_worker_tasks, -) pytest_plugins = [ "pytest_simcore.docker_compose", @@ -60,62 +39,3 @@ def mock_environment( "SC_BOOT_MODE": BootModeEnum.DEBUG, }, ) - - -@pytest.fixture(scope="session") -def celery_config() -> dict[str, Any]: - return { - "broker_connection_retry_on_startup": True, - "broker_url": "memory://localhost//", - "result_backend": "cache+memory://localhost//", - "result_expires": datetime.timedelta(days=7), - "result_extended": True, - "pool": "threads", - "task_default_queue": "default", - "task_send_sent_event": True, - "task_track_started": True, - "worker_send_task_events": True, - } - - -@pytest.fixture -async def with_celery_worker( - app_environment: EnvVarsDict, - celery_app: Celery, - monkeypatch: pytest.MonkeyPatch, -) -> AsyncIterator[TestWorkController]: - # Signals must be explicitily connected - monkeypatch.setenv("NOTIFICATIONS_WORKER_MODE", "true") - app_settings = ApplicationSettings.create_from_envs() - - def _on_worker_init_wrapper(sender: WorkController, **_kwargs): - assert app_settings.NOTIFICATIONS_CELERY # nosec - return partial( - on_worker_init, - FastAPIAppServer(app=create_app(app_settings)), - app_settings.NOTIFICATIONS_CELERY, - )(sender, **_kwargs) - - worker_init.connect(_on_worker_init_wrapper) - worker_shutdown.connect(on_worker_shutdown) - - setup_worker_tasks(celery_app) - - with start_worker( - celery_app, - pool="threads", - concurrency=1, - loglevel="info", - perform_ping_check=False, - queues=",".join(queue.value for queue in TaskQueue), - ) as worker: - yield worker - - -@pytest.fixture -async def notifications_rabbitmq_rpc_client( - rabbitmq_rpc_client: Callable[[str], Awaitable[RabbitMQRPCClient]], -) -> RabbitMQRPCClient: - rpc_client = await rabbitmq_rpc_client("pytest_notifications_rpc_client") - assert rpc_client - return rpc_client diff --git a/services/notifications/tests/unit/conftest.py b/services/notifications/tests/unit/conftest.py index 30fb05727fb7..439dccb53b5a 100644 --- a/services/notifications/tests/unit/conftest.py +++ b/services/notifications/tests/unit/conftest.py @@ -1,18 +1,31 @@ # pylint: disable=redefined-outer-name # pylint: disable=unused-argument -from collections.abc import AsyncIterator -from typing import Final +import datetime +from collections.abc import AsyncIterator, Awaitable, Callable +from functools import partial +from typing import Any, Final import pytest from asgi_lifespan import LifespanManager +from celery import Celery +from celery.contrib.testing.worker import start_worker +from celery.signals import worker_init, worker_shutdown +from celery.worker.worker import WorkController +from celery_library.signals import on_worker_init, on_worker_shutdown from fastapi import FastAPI -from fastapi.testclient import TestClient +from pytest_mock import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict +from servicelib.fastapi.celery.app_server import FastAPIAppServer +from servicelib.rabbitmq import RabbitMQRPCClient from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings from simcore_service_notifications.core.application import create_app from simcore_service_notifications.core.settings import ApplicationSettings +from simcore_service_notifications.modules.celery.tasks import ( + TaskQueue, + setup_worker_tasks, +) _LIFESPAN_TIMEOUT: Final[int] = 30 @@ -56,13 +69,77 @@ def app_settings( @pytest.fixture -async def initialized_app(app_settings: ApplicationSettings) -> AsyncIterator[FastAPI]: +async def fastapi_app(app_settings: ApplicationSettings) -> AsyncIterator[FastAPI]: app: FastAPI = create_app(app_settings) async with LifespanManager(app, startup_timeout=30, shutdown_timeout=30): yield app +@pytest.fixture(scope="session") +def celery_config() -> dict[str, Any]: + return { + "broker_connection_retry_on_startup": True, + "broker_url": "memory://localhost//", + "result_backend": "cache+memory://localhost//", + "result_expires": datetime.timedelta(days=7), + "result_extended": True, + "pool": "threads", + "task_default_queue": "default", + "task_send_sent_event": True, + "task_track_started": True, + "worker_send_task_events": True, + } + + +@pytest.fixture +def mock_celery_app(mocker: MockerFixture, celery_config: dict[str, Any]) -> Celery: + celery_app = Celery(**celery_config) + + for module in ("simcore_service_notifications.clients.celery.create_app",): + mocker.patch(module, return_value=celery_app) + + return celery_app + + +@pytest.fixture +async def mock_celery_worker( + app_environment: EnvVarsDict, + celery_app: Celery, + fastapi_app: FastAPI, + monkeypatch: pytest.MonkeyPatch, +) -> AsyncIterator[Any]: + monkeypatch.setenv("NOTIFICATIONS_WORKER_MODE", "true") + app_settings = ApplicationSettings.create_from_envs() + + def _on_worker_init_wrapper(sender: WorkController, **_kwargs): + assert app_settings.NOTIFICATIONS_CELERY # nosec + return partial( + on_worker_init, + FastAPIAppServer(app=fastapi_app), + app_settings.NOTIFICATIONS_CELERY, + )(sender, **_kwargs) + + worker_init.connect(_on_worker_init_wrapper) + worker_shutdown.connect(on_worker_shutdown) + + setup_worker_tasks(celery_app) + + with start_worker( + celery_app, + pool="threads", + concurrency=1, + loglevel="debug", + perform_ping_check=False, + queues=",".join(queue.value for queue in TaskQueue), + ) as worker: + yield worker + + @pytest.fixture -def test_client(initialized_app: FastAPI) -> TestClient: - return TestClient(initialized_app) +async def notifications_rabbitmq_rpc_client( + rabbitmq_rpc_client: Callable[[str], Awaitable[RabbitMQRPCClient]], +) -> RabbitMQRPCClient: + rpc_client = await rabbitmq_rpc_client("pytest_notifications_rpc_client") + assert rpc_client + return rpc_client diff --git a/services/notifications/tests/unit/test_tasks.py b/services/notifications/tests/unit/test_tasks.py index bd8729500773..4da88f490d28 100644 --- a/services/notifications/tests/unit/test_tasks.py +++ b/services/notifications/tests/unit/test_tasks.py @@ -1,5 +1,4 @@ -from celery.contrib.testing.worker import TestWorkController -from fastapi import FastAPI +import pytest from models_library.rpc.notifications.messages import NotificationMessage from servicelib.rabbitmq import RabbitMQRPCClient from servicelib.rabbitmq.rpc_interfaces.notifications.messages import ( @@ -13,10 +12,13 @@ ] +@pytest.mark.usefixtures( + "mock_celery_app", + "mock_celery_worker", + "fastapi_app", +) async def test_send_email( - initialized_app: FastAPI, notifications_rabbitmq_rpc_client: RabbitMQRPCClient, - with_celery_worker: TestWorkController, ): await send_notification_message( notifications_rabbitmq_rpc_client, From 116e3553bfa77e15a827fdbc129d58f1aaeb8969 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 7 Jul 2025 14:44:41 +0200 Subject: [PATCH 35/69] fix: add notification library dependency --- services/notifications/requirements/_base.in | 2 +- services/notifications/requirements/_base.txt | 179 ++++++++++++++---- services/notifications/requirements/ci.txt | 2 +- services/notifications/requirements/dev.txt | 2 +- services/notifications/requirements/prod.txt | 2 +- 5 files changed, 147 insertions(+), 40 deletions(-) diff --git a/services/notifications/requirements/_base.in b/services/notifications/requirements/_base.in index c199782b440d..7d5ff96d812a 100644 --- a/services/notifications/requirements/_base.in +++ b/services/notifications/requirements/_base.in @@ -8,9 +8,9 @@ # intra-repo required dependencies --requirement ../../../packages/celery-library/requirements/_base.in --requirement ../../../packages/common-library/requirements/_base.in +--requirement ../../../packages/notifications-library/requirements/_base.in --requirement ../../../packages/models-library/requirements/_base.in --requirement ../../../packages/settings-library/requirements/_base.in ---requirement ../../../packages/postgres-database/requirements/_base.in # service-library[fastapi] --requirement ../../../packages/service-library/requirements/_base.in --requirement ../../../packages/service-library/requirements/_fastapi.in diff --git a/services/notifications/requirements/_base.txt b/services/notifications/requirements/_base.txt index bf7da3ac1369..67bbc5551f53 100644 --- a/services/notifications/requirements/_base.txt +++ b/services/notifications/requirements/_base.txt @@ -17,6 +17,7 @@ aiodocker==0.24.0 aiofiles==24.1.0 # via # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/notifications-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in aiohappyeyeballs==2.6.1 # via aiohttp @@ -37,8 +38,14 @@ aiohttp==3.12.12 # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -53,8 +60,10 @@ aiormq==6.8.1 # via aio-pika aiosignal==1.3.2 # via aiohttp +aiosmtplib==4.0.1 + # via -r requirements/../../../packages/notifications-library/requirements/_base.in alembic==1.15.1 - # via -r requirements/../../../packages/postgres-database/requirements/_base.in + # via -r requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/_base.in amqp==5.3.1 # via kombu annotated-types==0.7.0 @@ -72,6 +81,7 @@ arrow==1.3.0 # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in asgi-lifespan==2.1.0 @@ -106,8 +116,14 @@ certifi==2025.1.31 # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -206,8 +222,14 @@ httpx==0.28.1 # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -247,8 +269,14 @@ jinja2==3.1.6 # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -258,12 +286,14 @@ jinja2==3.1.6 # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/notifications-library/requirements/_base.in # fastapi jsonschema==4.23.0 # via # -r requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in jsonschema-specifications==2024.10.1 # via jsonschema @@ -286,8 +316,14 @@ mako==1.3.9 # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -356,7 +392,7 @@ opentelemetry-instrumentation-aio-pika==0.52b1 opentelemetry-instrumentation-asgi==0.52b1 # via opentelemetry-instrumentation-fastapi opentelemetry-instrumentation-asyncpg==0.52b1 - # via -r requirements/../../../packages/postgres-database/requirements/_base.in + # via -r requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/_base.in opentelemetry-instrumentation-fastapi==0.52b1 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in opentelemetry-instrumentation-httpx==0.52b1 @@ -417,8 +453,14 @@ orjson==3.10.16 # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -439,7 +481,11 @@ orjson==3.10.16 # -r requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in - # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/notifications-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in @@ -489,8 +535,14 @@ pydantic==2.11.0 # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -514,8 +566,13 @@ pydantic==2.11.0 # -r requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in - # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in - # -r requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/notifications-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in @@ -544,7 +601,11 @@ pydantic-extra-types==2.10.3 # -r requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in - # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/notifications-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in @@ -567,8 +628,14 @@ pydantic-settings==2.7.0 # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -583,6 +650,8 @@ pydantic-settings==2.7.0 # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in @@ -619,8 +688,14 @@ pyyaml==6.0.2 # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -650,8 +725,14 @@ redis==5.2.1 # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -681,8 +762,14 @@ referencing==0.35.1 # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -700,6 +787,7 @@ rich==13.9.4 # via # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # rich-toolkit @@ -735,8 +823,14 @@ sqlalchemy==1.4.54 # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -746,7 +840,7 @@ sqlalchemy==1.4.54 # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt - # -r requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/_base.in # alembic starlette==0.46.1 # via @@ -765,8 +859,14 @@ starlette==0.46.1 # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -797,6 +897,7 @@ typer==0.15.2 # via # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # fastapi-cli @@ -837,8 +938,14 @@ urllib3==2.3.0 # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/notifications-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -876,7 +983,7 @@ wrapt==1.17.2 yarl==1.18.3 # via # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in - # -r requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # aio-pika # aiohttp diff --git a/services/notifications/requirements/ci.txt b/services/notifications/requirements/ci.txt index 0bf441d418f1..910eff4c58d8 100644 --- a/services/notifications/requirements/ci.txt +++ b/services/notifications/requirements/ci.txt @@ -15,7 +15,7 @@ simcore-celery-library @ ../../packages/celery-library/ simcore-common-library @ ../../packages/common-library/ simcore-models-library @ ../../packages/models-library/ -simcore-postgres-database @ ../../packages/postgres-database/ +simcore-notifications-library @ ../../packages/notifications-library/ pytest-simcore @ ../../packages/pytest-simcore/ simcore-service-library[fastapi] @ ../../packages/service-library/ simcore-settings-library @ ../../packages/settings-library/ diff --git a/services/notifications/requirements/dev.txt b/services/notifications/requirements/dev.txt index f34ee2bd24eb..2fae6cc869f2 100644 --- a/services/notifications/requirements/dev.txt +++ b/services/notifications/requirements/dev.txt @@ -15,7 +15,7 @@ --editable ../../packages/celery-library --editable ../../packages/common-library --editable ../../packages/models-library ---editable ../../packages/postgres-database +--editable ../../packages/notifications-library --editable ../../packages/pytest-simcore --editable ../../packages/service-library[fastapi] --editable ../../packages/settings-library diff --git a/services/notifications/requirements/prod.txt b/services/notifications/requirements/prod.txt index 76ada91ee6f4..7fdfae145ffc 100644 --- a/services/notifications/requirements/prod.txt +++ b/services/notifications/requirements/prod.txt @@ -13,7 +13,7 @@ simcore-celery-library @ ../../packages/celery-library/ simcore-common-library @ ../../packages/common-library/ simcore-models-library @ ../../packages/models-library/ -simcore-postgres-database @ ../../packages/postgres-database/ +simcore-notifications-library @ ../../packages/notifications-library/ simcore-service-library[fastapi] @ ../../packages/service-library/ simcore-settings-library @ ../../packages/settings-library/ From ab3de30f21bbf9e748a8b85cee0dceb5f4b3dba3 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 7 Jul 2025 15:54:42 +0200 Subject: [PATCH 36/69] fix: typecheck --- services/storage/src/simcore_service_storage/api/rest/_files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/storage/src/simcore_service_storage/api/rest/_files.py b/services/storage/src/simcore_service_storage/api/rest/_files.py index e7ab6364c794..3f2c229adbb3 100644 --- a/services/storage/src/simcore_service_storage/api/rest/_files.py +++ b/services/storage/src/simcore_service_storage/api/rest/_files.py @@ -349,7 +349,7 @@ async def is_completed_upload_file( # first check if the task is in the app if task_status.is_done: task_result = TypeAdapter(FileMetaData).validate_python( - await task_manager.get_result( + await task_manager.get_task_result( context=async_job_name_data.model_dump(), task_uuid=TaskUUID(future_id), ) From ebd8cba779ec4984af56ee503c72cbaa286f6862 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 7 Jul 2025 15:56:16 +0200 Subject: [PATCH 37/69] fix: dependency --- services/notifications/requirements/_base.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/services/notifications/requirements/_base.txt b/services/notifications/requirements/_base.txt index 9fd6b136c379..51aad4bf64ec 100644 --- a/services/notifications/requirements/_base.txt +++ b/services/notifications/requirements/_base.txt @@ -363,7 +363,6 @@ opentelemetry-api==1.34.1 # opentelemetry-instrumentation-requests # opentelemetry-sdk # opentelemetry-semantic-conventions -opentelemetry-exporter-otlp-proto-common==1.31.1 opentelemetry-exporter-otlp==1.34.1 # via # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in From dacbe7b6a1b0d96ca9a91795cbce8cb55a591639 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 7 Jul 2025 16:00:14 +0200 Subject: [PATCH 38/69] tests: add test client --- services/notifications/tests/unit/conftest.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/services/notifications/tests/unit/conftest.py b/services/notifications/tests/unit/conftest.py index 439dccb53b5a..b7b4468e2994 100644 --- a/services/notifications/tests/unit/conftest.py +++ b/services/notifications/tests/unit/conftest.py @@ -14,6 +14,7 @@ from celery.worker.worker import WorkController from celery_library.signals import on_worker_init, on_worker_shutdown from fastapi import FastAPI +from fastapi.testclient import TestClient from pytest_mock import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict from servicelib.fastapi.celery.app_server import FastAPIAppServer @@ -143,3 +144,8 @@ async def notifications_rabbitmq_rpc_client( rpc_client = await rabbitmq_rpc_client("pytest_notifications_rpc_client") assert rpc_client return rpc_client + + +@pytest.fixture +def test_client(fastapi_app: FastAPI) -> TestClient: + return TestClient(fastapi_app) From dcae6ec56e094b168235326fcc838bcad21a3879 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 7 Jul 2025 16:06:10 +0200 Subject: [PATCH 39/69] tests: fix health --- .../tests/unit/test_api_rest__health.py | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/services/notifications/tests/unit/test_api_rest__health.py b/services/notifications/tests/unit/test_api_rest__health.py index ba418fe7bc3e..003f58975889 100644 --- a/services/notifications/tests/unit/test_api_rest__health.py +++ b/services/notifications/tests/unit/test_api_rest__health.py @@ -7,15 +7,12 @@ from fastapi.testclient import TestClient from models_library.api_schemas__common.health import HealthCheckGet from models_library.errors import ( - POSRGRES_DATABASE_UNHEALTHY_MSG, RABBITMQ_CLIENT_UNHEALTHY_MSG, ) -from models_library.healthchecks import IsNonResponsive from pytest_mock import MockerFixture from simcore_service_notifications.api.rest._health import HealthCheckError pytest_simcore_core_services_selection = [ - "postgres", "rabbit", ] @@ -26,23 +23,6 @@ def test_health_ok(test_client: TestClient): assert HealthCheckGet.model_validate(response.json()) -@pytest.fixture -def mock_postgres_liveness(mocker: MockerFixture, test_client: TestClient) -> None: - mocker.patch.object( - test_client.app.state.postgres_liveness, - "_liveness_result", - new=IsNonResponsive(reason="fake"), - ) - - -def test_health_postgres_unhealthy( - mock_postgres_liveness: None, test_client: TestClient -): - with pytest.raises(HealthCheckError) as exc: - test_client.get("/") - assert POSRGRES_DATABASE_UNHEALTHY_MSG in f"{exc.value}" - - @pytest.fixture def mock_rabbit_healthy(mocker: MockerFixture, test_client: TestClient) -> None: mocker.patch.object( From f9fb7729defffac5c176d8232df4ed6a10f806b6 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 7 Jul 2025 16:09:30 +0200 Subject: [PATCH 40/69] tests: fix service selection --- services/notifications/tests/unit/test_api_rest__health.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/services/notifications/tests/unit/test_api_rest__health.py b/services/notifications/tests/unit/test_api_rest__health.py index 003f58975889..b596b9b0cf0c 100644 --- a/services/notifications/tests/unit/test_api_rest__health.py +++ b/services/notifications/tests/unit/test_api_rest__health.py @@ -6,14 +6,13 @@ from fastapi import status from fastapi.testclient import TestClient from models_library.api_schemas__common.health import HealthCheckGet -from models_library.errors import ( - RABBITMQ_CLIENT_UNHEALTHY_MSG, -) +from models_library.errors import RABBITMQ_CLIENT_UNHEALTHY_MSG from pytest_mock import MockerFixture from simcore_service_notifications.api.rest._health import HealthCheckError pytest_simcore_core_services_selection = [ "rabbit", + "redis", ] From 09d60260a913a8b15d34ecab9f1b553b63c698d8 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 8 Jul 2025 10:42:50 +0200 Subject: [PATCH 41/69] fix: rename --- .../simcore_service_notifications/api/rpc/_notifications.py | 6 ++++-- .../services/notifications_service.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/services/notifications/src/simcore_service_notifications/api/rpc/_notifications.py b/services/notifications/src/simcore_service_notifications/api/rpc/_notifications.py index bead32a8a827..241a7bf1f4ec 100644 --- a/services/notifications/src/simcore_service_notifications/api/rpc/_notifications.py +++ b/services/notifications/src/simcore_service_notifications/api/rpc/_notifications.py @@ -14,6 +14,8 @@ async def send_notification_message( message: NotificationMessage, recipients: list[Recipient], ) -> None: - await notifications_service.send_notification( - task_manager, message=message, recipients=recipients + await notifications_service.send_notification_message( + task_manager, + message=message, + recipients=recipients, ) diff --git a/services/notifications/src/simcore_service_notifications/services/notifications_service.py b/services/notifications/src/simcore_service_notifications/services/notifications_service.py index 12f2358beac2..3db54dfcd058 100644 --- a/services/notifications/src/simcore_service_notifications/services/notifications_service.py +++ b/services/notifications/src/simcore_service_notifications/services/notifications_service.py @@ -9,7 +9,7 @@ _logger = logging.getLogger(__name__) -async def send_notification( +async def send_notification_message( task_manager: TaskManager, *, message: NotificationMessage, @@ -18,7 +18,7 @@ async def send_notification( for recipient in recipients: await task_manager.send_task( name=f"notifications.{recipient.type}", - context=TaskContext(), + context=TaskContext(), # TODO: TaskFilter queue=TaskQueue.DEFAULT, message=message, recipient=recipient, From 6dd7fd733606b5d45b73f16bc918b99e8f5c13ad Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 8 Jul 2025 16:49:24 +0200 Subject: [PATCH 42/69] tests: update models --- .../rpc/notifications/messages.py | 31 ++++++++++--------- .../rpc_interfaces/notifications/messages.py | 4 +-- .../api/rpc/_notifications.py | 4 +-- .../clients/celery.py | 6 ++-- .../modules/celery/_email_tasks.py | 14 +++------ .../modules/celery/tasks.py | 6 ++-- .../services/notifications_service.py | 17 +++++----- .../notifications/tests/unit/test_tasks.py | 6 ++-- 8 files changed, 39 insertions(+), 49 deletions(-) diff --git a/packages/models-library/src/models_library/rpc/notifications/messages.py b/packages/models-library/src/models_library/rpc/notifications/messages.py index 8c51a7a9ba34..2a75e97fd96a 100644 --- a/packages/models-library/src/models_library/rpc/notifications/messages.py +++ b/packages/models-library/src/models_library/rpc/notifications/messages.py @@ -1,28 +1,29 @@ from abc import ABC -from typing import Annotated, Any, TypeAlias +from typing import Any, Literal -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, EmailStr -class BaseRecipient(BaseModel, ABC): +class Channel(BaseModel, ABC): type: str + model_config = ConfigDict( + frozen=True, + ) -class SMSRecipient(BaseRecipient): - type: Annotated[str, Field(frozen=True)] = "sms" - phone_number: str +class EmailChannel(Channel): + type: Literal["email"] = "email" + to: EmailStr + reply_to: EmailStr | None = None -class EmailRecipient(BaseRecipient): - type: Annotated[str, Field(frozen=True)] = "email" - address: str - -Recipient: TypeAlias = Annotated[ - EmailRecipient | SMSRecipient, Field(discriminator="type") -] +class SMSChannel(Channel): + type: Literal["sms"] = "sms" + phone_number: str # Consider using phone number validation library here class NotificationMessage(BaseModel): - event: str - context: dict[str, Any] | None = None + event_type: str # e.g. "account.registered" + channel: Channel + context: dict[str, Any] | None = None # Additional context for the notification diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/messages.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/messages.py index 7434396532d9..39905263a6cf 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/messages.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/messages.py @@ -2,7 +2,7 @@ from models_library.api_schemas_notifications import NOTIFICATIONS_RPC_NAMESPACE from models_library.rabbitmq_basic_types import RPCMethodName -from models_library.rpc.notifications.messages import NotificationMessage, Recipient +from models_library.rpc.notifications.messages import NotificationMessage from pydantic import NonNegativeInt, TypeAdapter from ... import RabbitMQRPCClient @@ -14,12 +14,10 @@ async def send_notification_message( rabbitmq_rpc_client: RabbitMQRPCClient, *, message: NotificationMessage, - recipients: list[Recipient], ) -> None: await rabbitmq_rpc_client.request( NOTIFICATIONS_RPC_NAMESPACE, TypeAdapter(RPCMethodName).validate_python("send_notification_message"), timeout_s=_DEFAULT_TIMEOUT_S, message=message, - recipients=recipients, ) diff --git a/services/notifications/src/simcore_service_notifications/api/rpc/_notifications.py b/services/notifications/src/simcore_service_notifications/api/rpc/_notifications.py index 241a7bf1f4ec..8487d601f885 100644 --- a/services/notifications/src/simcore_service_notifications/api/rpc/_notifications.py +++ b/services/notifications/src/simcore_service_notifications/api/rpc/_notifications.py @@ -1,4 +1,4 @@ -from models_library.rpc.notifications.messages import NotificationMessage, Recipient +from models_library.rpc.notifications.messages import NotificationMessage from servicelib.celery.task_manager import TaskManager from servicelib.rabbitmq import RPCRouter @@ -12,10 +12,8 @@ async def send_notification_message( task_manager: TaskManager, *, message: NotificationMessage, - recipients: list[Recipient], ) -> None: await notifications_service.send_notification_message( task_manager, message=message, - recipients=recipients, ) diff --git a/services/notifications/src/simcore_service_notifications/clients/celery.py b/services/notifications/src/simcore_service_notifications/clients/celery.py index 24ba2a19765c..87074e266317 100644 --- a/services/notifications/src/simcore_service_notifications/clients/celery.py +++ b/services/notifications/src/simcore_service_notifications/clients/celery.py @@ -7,9 +7,9 @@ from fastapi import FastAPI from fastapi_lifespan_manager import State from models_library.rpc.notifications.messages import ( - EmailRecipient, + EmailChannel, NotificationMessage, - SMSRecipient, + SMSChannel, ) from settings_library.celery import CelerySettings @@ -28,7 +28,7 @@ async def celery_lifespan(app: FastAPI) -> AsyncIterator[State]: ) register_celery_types() - register_pydantic_types(NotificationMessage, EmailRecipient, SMSRecipient) + register_pydantic_types(NotificationMessage, EmailChannel, SMSChannel) yield {} diff --git a/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py b/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py index 086e4f9f3bb0..53c1258cf5d1 100644 --- a/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py +++ b/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py @@ -2,11 +2,8 @@ import logging -from celery import Task # type: ignore[import-untyped] -from models_library.rpc.notifications.messages import ( - EmailRecipient, - NotificationMessage, -) +from celery import Task +from models_library.rpc.notifications.messages import EmailChannel, NotificationMessage from servicelib.celery.models import TaskID _logger = logging.getLogger(__name__) @@ -18,8 +15,7 @@ async def send_email( task: Task, task_id: TaskID, message: NotificationMessage, - recipient: EmailRecipient, ) -> None: - # TODO: render email template with message and recipient details - # and send the email using an email service - _logger.info("Sending email notification to %s", recipient.address) + assert isinstance(message.channel, EmailChannel) # nosec + + _logger.info("Sending email notification to %s", message.channel.to) diff --git a/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py b/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py index 3d5a55271e13..c9d42a501946 100644 --- a/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py +++ b/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py @@ -5,9 +5,9 @@ from celery_library.task import register_task from celery_library.types import register_celery_types, register_pydantic_types from models_library.rpc.notifications.messages import ( - EmailRecipient, + EmailChannel, NotificationMessage, - SMSRecipient, + SMSChannel, ) from servicelib.logging_utils import log_context @@ -25,7 +25,7 @@ class TaskQueue(StrEnum): def setup_worker_tasks(app: Celery) -> None: register_celery_types() - register_pydantic_types(NotificationMessage, EmailRecipient, SMSRecipient) + register_pydantic_types(NotificationMessage, EmailChannel, SMSChannel) with log_context(_logger, logging.INFO, msg="worker tasks registration"): register_task( diff --git a/services/notifications/src/simcore_service_notifications/services/notifications_service.py b/services/notifications/src/simcore_service_notifications/services/notifications_service.py index 3db54dfcd058..3187431296b8 100644 --- a/services/notifications/src/simcore_service_notifications/services/notifications_service.py +++ b/services/notifications/src/simcore_service_notifications/services/notifications_service.py @@ -1,6 +1,6 @@ import logging -from models_library.rpc.notifications.messages import NotificationMessage, Recipient +from models_library.rpc.notifications.messages import NotificationMessage from servicelib.celery.models import TaskContext from servicelib.celery.task_manager import TaskManager @@ -13,13 +13,10 @@ async def send_notification_message( task_manager: TaskManager, *, message: NotificationMessage, - recipients: list[Recipient], ) -> None: - for recipient in recipients: - await task_manager.send_task( - name=f"notifications.{recipient.type}", - context=TaskContext(), # TODO: TaskFilter - queue=TaskQueue.DEFAULT, - message=message, - recipient=recipient, - ) + await task_manager.send_task( + name=f"notifications.{message.event_type}", + context=TaskContext(), # TODO: TaskFilter + queue=TaskQueue.DEFAULT, + message=message, + ) diff --git a/services/notifications/tests/unit/test_tasks.py b/services/notifications/tests/unit/test_tasks.py index 4da88f490d28..620068e5d458 100644 --- a/services/notifications/tests/unit/test_tasks.py +++ b/services/notifications/tests/unit/test_tasks.py @@ -4,7 +4,7 @@ from servicelib.rabbitmq.rpc_interfaces.notifications.messages import ( send_notification_message, ) -from simcore_service_notifications.clients.celery import EmailRecipient +from simcore_service_notifications.clients.celery import EmailChannel pytest_simcore_core_services_selection = [ "rabbit", @@ -23,8 +23,8 @@ async def test_send_email( await send_notification_message( notifications_rabbitmq_rpc_client, message=NotificationMessage( - event="test_event", + event_type="on_account_requested", + channel=EmailChannel(to="test@example.com"), context={"key": "value"}, ), - recipients=[EmailRecipient(address="test@example.com")], ) From a657d3e98a9ef5adcd67d48a68f5d4e7994dd571 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 8 Jul 2025 22:14:46 +0200 Subject: [PATCH 43/69] feat: add async_jobs --- .../src/simcore_service_notifications/api/rpc/routes.py | 6 +++++- .../tests/unit/{test_tasks.py => test_send_email_tasks.py} | 0 2 files changed, 5 insertions(+), 1 deletion(-) rename services/notifications/tests/unit/{test_tasks.py => test_send_email_tasks.py} (100%) diff --git a/services/notifications/src/simcore_service_notifications/api/rpc/routes.py b/services/notifications/src/simcore_service_notifications/api/rpc/routes.py index 9968a5a4eff2..08994ec187eb 100644 --- a/services/notifications/src/simcore_service_notifications/api/rpc/routes.py +++ b/services/notifications/src/simcore_service_notifications/api/rpc/routes.py @@ -1,5 +1,6 @@ from collections.abc import AsyncIterator +from celery_library.rpc import _async_jobs from fastapi import FastAPI from fastapi_lifespan_manager import State from models_library.api_schemas_notifications import NOTIFICATIONS_RPC_NAMESPACE @@ -9,7 +10,10 @@ from ...clients.rabbitmq import get_rabbitmq_rpc_server from . import _notifications -ROUTERS: list[RPCRouter] = [_notifications.router] +ROUTERS: list[RPCRouter] = [ + _async_jobs.router, + _notifications.router, +] async def rpc_api_routes_lifespan(app: FastAPI) -> AsyncIterator[State]: diff --git a/services/notifications/tests/unit/test_tasks.py b/services/notifications/tests/unit/test_send_email_tasks.py similarity index 100% rename from services/notifications/tests/unit/test_tasks.py rename to services/notifications/tests/unit/test_send_email_tasks.py From 628042cb36200e5274dbee2a19d56c01d3561617 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 9 Jul 2025 11:32:12 +0200 Subject: [PATCH 44/69] refactor: decouple models --- .../rpc/notifications/account.py | 15 +++++++ .../rpc/notifications/messages.py | 29 ------------- .../rpc/notifications/notifications.py | 32 ++++++++++++++ .../rpc_interfaces/notifications/messages.py | 10 ++--- .../api/rpc/_notifications.py | 10 ++--- .../clients/celery.py | 6 +-- .../modules/celery/_email_tasks.py | 42 ++++++++++++++++--- .../modules/celery/tasks.py | 12 +++--- .../services/notifications_service.py | 11 ++--- .../tests/unit/test_send_email_tasks.py | 26 ++++++++---- 10 files changed, 128 insertions(+), 65 deletions(-) create mode 100644 packages/models-library/src/models_library/rpc/notifications/account.py delete mode 100644 packages/models-library/src/models_library/rpc/notifications/messages.py create mode 100644 packages/models-library/src/models_library/rpc/notifications/notifications.py diff --git a/packages/models-library/src/models_library/rpc/notifications/account.py b/packages/models-library/src/models_library/rpc/notifications/account.py new file mode 100644 index 000000000000..0a0723621291 --- /dev/null +++ b/packages/models-library/src/models_library/rpc/notifications/account.py @@ -0,0 +1,15 @@ +from typing import Literal + +from pydantic import EmailStr + +from .notifications import Event + + +class AccountRequestedEvent(Event): + type: Literal["account.requested"] = "account.requested" + + first_name: str + last_name: str + email: EmailStr + + # TODO: add more fields as needed diff --git a/packages/models-library/src/models_library/rpc/notifications/messages.py b/packages/models-library/src/models_library/rpc/notifications/messages.py deleted file mode 100644 index 2a75e97fd96a..000000000000 --- a/packages/models-library/src/models_library/rpc/notifications/messages.py +++ /dev/null @@ -1,29 +0,0 @@ -from abc import ABC -from typing import Any, Literal - -from pydantic import BaseModel, ConfigDict, EmailStr - - -class Channel(BaseModel, ABC): - type: str - - model_config = ConfigDict( - frozen=True, - ) - - -class EmailChannel(Channel): - type: Literal["email"] = "email" - to: EmailStr - reply_to: EmailStr | None = None - - -class SMSChannel(Channel): - type: Literal["sms"] = "sms" - phone_number: str # Consider using phone number validation library here - - -class NotificationMessage(BaseModel): - event_type: str # e.g. "account.registered" - channel: Channel - context: dict[str, Any] | None = None # Additional context for the notification diff --git a/packages/models-library/src/models_library/rpc/notifications/notifications.py b/packages/models-library/src/models_library/rpc/notifications/notifications.py new file mode 100644 index 000000000000..8fdb399d72ac --- /dev/null +++ b/packages/models-library/src/models_library/rpc/notifications/notifications.py @@ -0,0 +1,32 @@ +from typing import Annotated, Literal, TypeAlias + +from pydantic import BaseModel, ConfigDict, EmailStr, Field + + +class Event(BaseModel): + type: str + + model_config = ConfigDict( + frozen=True, + ) + + +class EmailChannel(BaseModel): + type: Literal["email"] = "email" + + to: EmailStr + reply_to: EmailStr | None = None + + +class SMSChannel(BaseModel): + type: Literal["sms"] = "sms" + + phone_number: str # Consider using phone number validation library here + + +Channel: TypeAlias = EmailChannel | SMSChannel + + +class Notification(BaseModel): + event: Event + channel: Annotated[Channel, Field(discriminator="type")] diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/messages.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/messages.py index 39905263a6cf..516dfc834368 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/messages.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/messages.py @@ -2,7 +2,7 @@ from models_library.api_schemas_notifications import NOTIFICATIONS_RPC_NAMESPACE from models_library.rabbitmq_basic_types import RPCMethodName -from models_library.rpc.notifications.messages import NotificationMessage +from models_library.rpc.notifications.notifications import Notification from pydantic import NonNegativeInt, TypeAdapter from ... import RabbitMQRPCClient @@ -10,14 +10,14 @@ _DEFAULT_TIMEOUT_S: Final[NonNegativeInt] = 30 -async def send_notification_message( +async def send_notification( rabbitmq_rpc_client: RabbitMQRPCClient, *, - message: NotificationMessage, + notification: Notification, ) -> None: await rabbitmq_rpc_client.request( NOTIFICATIONS_RPC_NAMESPACE, - TypeAdapter(RPCMethodName).validate_python("send_notification_message"), + TypeAdapter(RPCMethodName).validate_python("send_notification"), timeout_s=_DEFAULT_TIMEOUT_S, - message=message, + notification=notification, ) diff --git a/services/notifications/src/simcore_service_notifications/api/rpc/_notifications.py b/services/notifications/src/simcore_service_notifications/api/rpc/_notifications.py index 8487d601f885..d2ed14bb265a 100644 --- a/services/notifications/src/simcore_service_notifications/api/rpc/_notifications.py +++ b/services/notifications/src/simcore_service_notifications/api/rpc/_notifications.py @@ -1,4 +1,4 @@ -from models_library.rpc.notifications.messages import NotificationMessage +from models_library.rpc.notifications.notifications import Notification from servicelib.celery.task_manager import TaskManager from servicelib.rabbitmq import RPCRouter @@ -8,12 +8,12 @@ @router.expose(reraise_if_error_type=()) -async def send_notification_message( +async def send_notification( task_manager: TaskManager, *, - message: NotificationMessage, + notification: Notification, ) -> None: - await notifications_service.send_notification_message( + await notifications_service.send_notification( task_manager, - message=message, + notification=notification, ) diff --git a/services/notifications/src/simcore_service_notifications/clients/celery.py b/services/notifications/src/simcore_service_notifications/clients/celery.py index 87074e266317..e777af87eabc 100644 --- a/services/notifications/src/simcore_service_notifications/clients/celery.py +++ b/services/notifications/src/simcore_service_notifications/clients/celery.py @@ -6,9 +6,9 @@ from celery_library.types import register_celery_types, register_pydantic_types from fastapi import FastAPI from fastapi_lifespan_manager import State -from models_library.rpc.notifications.messages import ( +from models_library.rpc.notifications.notifications import ( EmailChannel, - NotificationMessage, + Notification, SMSChannel, ) from settings_library.celery import CelerySettings @@ -28,7 +28,7 @@ async def celery_lifespan(app: FastAPI) -> AsyncIterator[State]: ) register_celery_types() - register_pydantic_types(NotificationMessage, EmailChannel, SMSChannel) + register_pydantic_types(Notification, EmailChannel, SMSChannel) yield {} diff --git a/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py b/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py index 53c1258cf5d1..c0a2bbe1cb34 100644 --- a/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py +++ b/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py @@ -3,7 +3,7 @@ import logging from celery import Task -from models_library.rpc.notifications.messages import EmailChannel, NotificationMessage +from models_library.rpc.notifications.notifications import EmailChannel, Notification from servicelib.celery.models import TaskID _logger = logging.getLogger(__name__) @@ -11,11 +11,43 @@ EMAIL_CHANNEL_NAME = "email" -async def send_email( +async def send_email_notification( task: Task, task_id: TaskID, - message: NotificationMessage, + notification: Notification, ) -> None: - assert isinstance(message.channel, EmailChannel) # nosec + assert isinstance(notification.channel, EmailChannel) # nosec - _logger.info("Sending email notification to %s", message.channel.to) + _logger.info("Sending email notification to %s", notification.channel.to) + + # event_extra_data = event_extra_data | (asdict(sharer_data) if sharer_data else {}) + + # parts = render_email_parts( + # env=create_render_environment_from_notifications_library( + # undefined=StrictUndefined + # ), + # event_name=event_name, + # user=user_data, + # product=product_data, + # # extras + # **event_extra_data, + # ) + + # from_ = get_support_address(product_data) + # to = get_user_address(user_data) + + # assert from_.addr_spec == product_data.support_email + # assert to.addr_spec == user_email + + # msg = compose_email( + # from_, + # to, + # subject=parts.subject, + # content_text=parts.text_content, + # content_html=parts.html_content, + # ) + # if event_attachments: + # add_attachments(msg, event_attachments) + + # async with create_email_session(settings=SMTPSettings.create_from_envs()) as smtp: + # await smtp.send_message(msg) diff --git a/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py b/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py index c9d42a501946..48058c26907d 100644 --- a/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py +++ b/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py @@ -4,14 +4,14 @@ from celery import Celery # type: ignore[import-untyped] from celery_library.task import register_task from celery_library.types import register_celery_types, register_pydantic_types -from models_library.rpc.notifications.messages import ( +from models_library.rpc.notifications.account import AccountRequestedEvent +from models_library.rpc.notifications.notifications import ( EmailChannel, - NotificationMessage, SMSChannel, ) from servicelib.logging_utils import log_context -from ...modules.celery._email_tasks import EMAIL_CHANNEL_NAME, send_email +from ...modules.celery._email_tasks import EMAIL_CHANNEL_NAME, send_email_notification _logger = logging.getLogger(__name__) @@ -25,9 +25,11 @@ class TaskQueue(StrEnum): def setup_worker_tasks(app: Celery) -> None: register_celery_types() - register_pydantic_types(NotificationMessage, EmailChannel, SMSChannel) + register_pydantic_types(AccountRequestedEvent, EmailChannel, SMSChannel) with log_context(_logger, logging.INFO, msg="worker tasks registration"): register_task( - app, send_email, ".".join((_NOTIFICATIONS_PREFIX, EMAIL_CHANNEL_NAME)) + app, + send_email_notification, + ".".join((_NOTIFICATIONS_PREFIX, EMAIL_CHANNEL_NAME)), ) diff --git a/services/notifications/src/simcore_service_notifications/services/notifications_service.py b/services/notifications/src/simcore_service_notifications/services/notifications_service.py index 3187431296b8..78527f4afb6d 100644 --- a/services/notifications/src/simcore_service_notifications/services/notifications_service.py +++ b/services/notifications/src/simcore_service_notifications/services/notifications_service.py @@ -1,6 +1,6 @@ import logging -from models_library.rpc.notifications.messages import NotificationMessage +from models_library.rpc.notifications.notifications import Notification from servicelib.celery.models import TaskContext from servicelib.celery.task_manager import TaskManager @@ -9,14 +9,15 @@ _logger = logging.getLogger(__name__) -async def send_notification_message( +async def send_notification( task_manager: TaskManager, *, - message: NotificationMessage, + notification: Notification, ) -> None: await task_manager.send_task( - name=f"notifications.{message.event_type}", + # send to the specific channel worker + name=f"notifications.{notification.channel.type}", context=TaskContext(), # TODO: TaskFilter queue=TaskQueue.DEFAULT, - message=message, + notification=notification, ) diff --git a/services/notifications/tests/unit/test_send_email_tasks.py b/services/notifications/tests/unit/test_send_email_tasks.py index 620068e5d458..99fd681f7d71 100644 --- a/services/notifications/tests/unit/test_send_email_tasks.py +++ b/services/notifications/tests/unit/test_send_email_tasks.py @@ -1,8 +1,12 @@ import pytest -from models_library.rpc.notifications.messages import NotificationMessage +from faker import Faker +from models_library.rpc.notifications.notifications import ( + AccountRequestedEvent, + Notification, +) from servicelib.rabbitmq import RabbitMQRPCClient from servicelib.rabbitmq.rpc_interfaces.notifications.messages import ( - send_notification_message, + send_notification, ) from simcore_service_notifications.clients.celery import EmailChannel @@ -17,14 +21,20 @@ "mock_celery_worker", "fastapi_app", ) -async def test_send_email( +async def test_account_requested( notifications_rabbitmq_rpc_client: RabbitMQRPCClient, + faker: Faker, ): - await send_notification_message( + email = faker.email() + + await send_notification( notifications_rabbitmq_rpc_client, - message=NotificationMessage( - event_type="on_account_requested", - channel=EmailChannel(to="test@example.com"), - context={"key": "value"}, + notification=Notification( + event=AccountRequestedEvent( + first_name=faker.first_name(), + last_name=faker.last_name(), + email=email, + ), + channel=EmailChannel(to=email), ), ) From 817b2a7f009b09280463cf86a78661203a3fab2d Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 9 Jul 2025 11:40:07 +0200 Subject: [PATCH 45/69] refactor: packages --- .../src/models_library/rpc/notifications/account.py | 2 +- .../rpc/notifications/{notifications.py => schemas.py} | 0 .../rabbitmq/rpc_interfaces/notifications/messages.py | 2 +- .../api/rpc/_notifications.py | 2 +- .../src/simcore_service_notifications/clients/celery.py | 2 +- .../modules/celery/_email_tasks.py | 2 +- .../simcore_service_notifications/modules/celery/tasks.py | 2 +- .../services/notifications_service.py | 2 +- .../notifications/tests/unit/test_send_email_tasks.py | 8 ++++---- 9 files changed, 11 insertions(+), 11 deletions(-) rename packages/models-library/src/models_library/rpc/notifications/{notifications.py => schemas.py} (100%) diff --git a/packages/models-library/src/models_library/rpc/notifications/account.py b/packages/models-library/src/models_library/rpc/notifications/account.py index 0a0723621291..89cc18e8ac7f 100644 --- a/packages/models-library/src/models_library/rpc/notifications/account.py +++ b/packages/models-library/src/models_library/rpc/notifications/account.py @@ -2,7 +2,7 @@ from pydantic import EmailStr -from .notifications import Event +from .schemas import Event class AccountRequestedEvent(Event): diff --git a/packages/models-library/src/models_library/rpc/notifications/notifications.py b/packages/models-library/src/models_library/rpc/notifications/schemas.py similarity index 100% rename from packages/models-library/src/models_library/rpc/notifications/notifications.py rename to packages/models-library/src/models_library/rpc/notifications/schemas.py diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/messages.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/messages.py index 516dfc834368..c252f21de656 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/messages.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/messages.py @@ -2,7 +2,7 @@ from models_library.api_schemas_notifications import NOTIFICATIONS_RPC_NAMESPACE from models_library.rabbitmq_basic_types import RPCMethodName -from models_library.rpc.notifications.notifications import Notification +from models_library.rpc.notifications.schemas import Notification from pydantic import NonNegativeInt, TypeAdapter from ... import RabbitMQRPCClient diff --git a/services/notifications/src/simcore_service_notifications/api/rpc/_notifications.py b/services/notifications/src/simcore_service_notifications/api/rpc/_notifications.py index d2ed14bb265a..a14c8e34dbae 100644 --- a/services/notifications/src/simcore_service_notifications/api/rpc/_notifications.py +++ b/services/notifications/src/simcore_service_notifications/api/rpc/_notifications.py @@ -1,4 +1,4 @@ -from models_library.rpc.notifications.notifications import Notification +from models_library.rpc.notifications.schemas import Notification from servicelib.celery.task_manager import TaskManager from servicelib.rabbitmq import RPCRouter diff --git a/services/notifications/src/simcore_service_notifications/clients/celery.py b/services/notifications/src/simcore_service_notifications/clients/celery.py index e777af87eabc..e58da8c25000 100644 --- a/services/notifications/src/simcore_service_notifications/clients/celery.py +++ b/services/notifications/src/simcore_service_notifications/clients/celery.py @@ -6,7 +6,7 @@ from celery_library.types import register_celery_types, register_pydantic_types from fastapi import FastAPI from fastapi_lifespan_manager import State -from models_library.rpc.notifications.notifications import ( +from models_library.rpc.notifications.schemas import ( EmailChannel, Notification, SMSChannel, diff --git a/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py b/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py index c0a2bbe1cb34..e82303235787 100644 --- a/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py +++ b/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py @@ -3,7 +3,7 @@ import logging from celery import Task -from models_library.rpc.notifications.notifications import EmailChannel, Notification +from models_library.rpc.notifications.schemas import EmailChannel, Notification from servicelib.celery.models import TaskID _logger = logging.getLogger(__name__) diff --git a/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py b/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py index 48058c26907d..746204d2e2f6 100644 --- a/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py +++ b/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py @@ -5,7 +5,7 @@ from celery_library.task import register_task from celery_library.types import register_celery_types, register_pydantic_types from models_library.rpc.notifications.account import AccountRequestedEvent -from models_library.rpc.notifications.notifications import ( +from models_library.rpc.notifications.schemas import ( EmailChannel, SMSChannel, ) diff --git a/services/notifications/src/simcore_service_notifications/services/notifications_service.py b/services/notifications/src/simcore_service_notifications/services/notifications_service.py index 78527f4afb6d..554de172a5f8 100644 --- a/services/notifications/src/simcore_service_notifications/services/notifications_service.py +++ b/services/notifications/src/simcore_service_notifications/services/notifications_service.py @@ -1,6 +1,6 @@ import logging -from models_library.rpc.notifications.notifications import Notification +from models_library.rpc.notifications.schemas import Notification from servicelib.celery.models import TaskContext from servicelib.celery.task_manager import TaskManager diff --git a/services/notifications/tests/unit/test_send_email_tasks.py b/services/notifications/tests/unit/test_send_email_tasks.py index 99fd681f7d71..9d740fc29a6c 100644 --- a/services/notifications/tests/unit/test_send_email_tasks.py +++ b/services/notifications/tests/unit/test_send_email_tasks.py @@ -1,9 +1,7 @@ import pytest from faker import Faker -from models_library.rpc.notifications.notifications import ( - AccountRequestedEvent, - Notification, -) +from models_library.rpc.notifications.account import AccountRequestedEvent +from models_library.rpc.notifications.schemas import Notification from servicelib.rabbitmq import RabbitMQRPCClient from servicelib.rabbitmq.rpc_interfaces.notifications.messages import ( send_notification, @@ -38,3 +36,5 @@ async def test_account_requested( channel=EmailChannel(to=email), ), ) + + # TODO: wait for the email to be sent and check From 6088a9a4a1d72900e9767638936e04e191696fa2 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 9 Jul 2025 11:46:45 +0200 Subject: [PATCH 46/69] fix: validate phone numbers --- .../src/models_library/rpc/notifications/schemas.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/models-library/src/models_library/rpc/notifications/schemas.py b/packages/models-library/src/models_library/rpc/notifications/schemas.py index 8fdb399d72ac..d90db1ee10e9 100644 --- a/packages/models-library/src/models_library/rpc/notifications/schemas.py +++ b/packages/models-library/src/models_library/rpc/notifications/schemas.py @@ -1,6 +1,7 @@ from typing import Annotated, Literal, TypeAlias from pydantic import BaseModel, ConfigDict, EmailStr, Field +from pydantic_extra_types.phone_numbers import PhoneNumber class Event(BaseModel): @@ -21,7 +22,7 @@ class EmailChannel(BaseModel): class SMSChannel(BaseModel): type: Literal["sms"] = "sms" - phone_number: str # Consider using phone number validation library here + phone_number: PhoneNumber Channel: TypeAlias = EmailChannel | SMSChannel From ac0cdff53c2ae7b8564eb93e57a5993065e55b95 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 9 Jul 2025 11:57:55 +0200 Subject: [PATCH 47/69] fix: add phonennumbers dep --- packages/models-library/requirements/_base.in | 1 + packages/models-library/requirements/_base.txt | 2 ++ services/notifications/requirements/_base.txt | 13 +++++++------ services/notifications/requirements/_test.txt | 12 +++++++----- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/packages/models-library/requirements/_base.in b/packages/models-library/requirements/_base.in index b33d20bdd6b0..d0f947ed1105 100644 --- a/packages/models-library/requirements/_base.in +++ b/packages/models-library/requirements/_base.in @@ -7,6 +7,7 @@ arrow jsonschema orjson +phonenumbers pydantic[email] pydantic-settings pydantic-extra-types diff --git a/packages/models-library/requirements/_base.txt b/packages/models-library/requirements/_base.txt index d680cfc7b086..9beb39871848 100644 --- a/packages/models-library/requirements/_base.txt +++ b/packages/models-library/requirements/_base.txt @@ -22,6 +22,8 @@ orjson==3.10.15 # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/common-library/requirements/_base.in # -r requirements/_base.in +phonenumbers==9.0.9 + # via -r requirements/_base.in pydantic==2.10.6 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt diff --git a/services/notifications/requirements/_base.txt b/services/notifications/requirements/_base.txt index 51aad4bf64ec..6adb0af4709c 100644 --- a/services/notifications/requirements/_base.txt +++ b/services/notifications/requirements/_base.txt @@ -153,12 +153,6 @@ click-plugins==1.1.1.2 # via celery click-repl==0.3.0 # via celery -deprecated==1.2.18 - # via - # opentelemetry-api - # opentelemetry-exporter-otlp-proto-grpc - # opentelemetry-exporter-otlp-proto-http - # opentelemetry-semantic-conventions dnspython==2.7.0 # via email-validator email-validator==2.2.0 @@ -498,6 +492,13 @@ packaging==24.2 # opentelemetry-instrumentation pamqp==3.3.0 # via aiormq +phonenumbers==9.0.9 + # via + # -r requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in prometheus-client==0.21.1 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in prompt-toolkit==3.0.51 diff --git a/services/notifications/requirements/_test.txt b/services/notifications/requirements/_test.txt index 4b7853abe2d6..6d9e1ade6984 100644 --- a/services/notifications/requirements/_test.txt +++ b/services/notifications/requirements/_test.txt @@ -93,7 +93,9 @@ packaging==24.2 # kombu # pytest pluggy==1.5.0 - # via pytest + # via + # pytest + # pytest-cov prompt-toolkit==3.0.51 # via # -c requirements/_base.txt @@ -102,6 +104,10 @@ psutil==7.0.0 # via # -c requirements/_base.txt # pytest-celery +pygments==2.19.1 + # via + # -c requirements/_base.txt + # pytest pytest==8.4.1 # via # -r requirements/_test.in @@ -109,10 +115,6 @@ pytest==8.4.1 # pytest-cov # pytest-docker-tools # pytest-mock -pygments==2.19.1 - # via - # -c requirements/_base.txt - # pytest pytest-asyncio==1.0.0 # via -r requirements/_test.in pytest-celery==1.2.0 From f0644999c866003453f5a89cf1be0110444c1132 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 9 Jul 2025 14:16:25 +0200 Subject: [PATCH 48/69] refactor: models --- .../rpc/notifications/__init__.py | 5 ++ .../rpc/notifications/_notifications.py | 11 ++++ .../rpc/notifications/account.py | 15 ----- .../rpc/notifications/channels/__init__.py | 16 +++++ .../notifications/channels/_email_channel.py | 16 +++++ .../notifications/channels/_sms_channel.py | 10 +++ .../rpc/notifications/events/__init__.py | 19 ++++++ .../notifications/events/_account_events.py | 23 +++++++ .../rpc/notifications/schemas.py | 33 ---------- .../rpc_interfaces/notifications/messages.py | 2 +- .../api/rpc/_notifications.py | 2 +- .../clients/celery.py | 4 +- .../modules/celery/_email_tasks.py | 62 +++++++++---------- .../modules/celery/tasks.py | 7 +-- .../services/notifications_service.py | 2 +- .../tests/unit/test_send_email_tasks.py | 15 +++-- 16 files changed, 149 insertions(+), 93 deletions(-) create mode 100644 packages/models-library/src/models_library/rpc/notifications/_notifications.py delete mode 100644 packages/models-library/src/models_library/rpc/notifications/account.py create mode 100644 packages/models-library/src/models_library/rpc/notifications/channels/__init__.py create mode 100644 packages/models-library/src/models_library/rpc/notifications/channels/_email_channel.py create mode 100644 packages/models-library/src/models_library/rpc/notifications/channels/_sms_channel.py create mode 100644 packages/models-library/src/models_library/rpc/notifications/events/__init__.py create mode 100644 packages/models-library/src/models_library/rpc/notifications/events/_account_events.py delete mode 100644 packages/models-library/src/models_library/rpc/notifications/schemas.py diff --git a/packages/models-library/src/models_library/rpc/notifications/__init__.py b/packages/models-library/src/models_library/rpc/notifications/__init__.py index e69de29bb2d1..a26e186033dc 100644 --- a/packages/models-library/src/models_library/rpc/notifications/__init__.py +++ b/packages/models-library/src/models_library/rpc/notifications/__init__.py @@ -0,0 +1,5 @@ +from ._notifications import Notification + +__all__: tuple[str, ...] = ("Notification",) + +# nopycln: file diff --git a/packages/models-library/src/models_library/rpc/notifications/_notifications.py b/packages/models-library/src/models_library/rpc/notifications/_notifications.py new file mode 100644 index 000000000000..65b69f2b042a --- /dev/null +++ b/packages/models-library/src/models_library/rpc/notifications/_notifications.py @@ -0,0 +1,11 @@ +from typing import Annotated + +from pydantic import BaseModel, Field + +from .channels import Channel +from .events import Event + + +class Notification(BaseModel): + event: Annotated[Event, Field(discriminator="type")] + channel: Annotated[Channel, Field(discriminator="type")] diff --git a/packages/models-library/src/models_library/rpc/notifications/account.py b/packages/models-library/src/models_library/rpc/notifications/account.py deleted file mode 100644 index 89cc18e8ac7f..000000000000 --- a/packages/models-library/src/models_library/rpc/notifications/account.py +++ /dev/null @@ -1,15 +0,0 @@ -from typing import Literal - -from pydantic import EmailStr - -from .schemas import Event - - -class AccountRequestedEvent(Event): - type: Literal["account.requested"] = "account.requested" - - first_name: str - last_name: str - email: EmailStr - - # TODO: add more fields as needed diff --git a/packages/models-library/src/models_library/rpc/notifications/channels/__init__.py b/packages/models-library/src/models_library/rpc/notifications/channels/__init__.py new file mode 100644 index 000000000000..708e3003df99 --- /dev/null +++ b/packages/models-library/src/models_library/rpc/notifications/channels/__init__.py @@ -0,0 +1,16 @@ +from typing import TypeAlias + +from ._email_channel import EmailAddress, EmailChannel +from ._sms_channel import SMSChannel + +Channel: TypeAlias = EmailChannel | SMSChannel + + +__all__: tuple[str, ...] = ( + "Channel", + "EmailAddress", + "EmailChannel", + "SMSChannel", +) + +# nopycln: file diff --git a/packages/models-library/src/models_library/rpc/notifications/channels/_email_channel.py b/packages/models-library/src/models_library/rpc/notifications/channels/_email_channel.py new file mode 100644 index 000000000000..4d28fed5fb04 --- /dev/null +++ b/packages/models-library/src/models_library/rpc/notifications/channels/_email_channel.py @@ -0,0 +1,16 @@ +from typing import Literal + +from pydantic import BaseModel + + +class EmailAddress(BaseModel): + display_name: str | None = None + addr_spec: str + + +class EmailChannel(BaseModel): + type: Literal["email"] = "email" + + from_addr: EmailAddress + to_addr: EmailAddress + reply_to_addr: EmailAddress | None = None diff --git a/packages/models-library/src/models_library/rpc/notifications/channels/_sms_channel.py b/packages/models-library/src/models_library/rpc/notifications/channels/_sms_channel.py new file mode 100644 index 000000000000..70ed96c5bdb3 --- /dev/null +++ b/packages/models-library/src/models_library/rpc/notifications/channels/_sms_channel.py @@ -0,0 +1,10 @@ +from typing import Literal + +from pydantic import BaseModel +from pydantic_extra_types.phone_numbers import PhoneNumber + + +class SMSChannel(BaseModel): + type: Literal["sms"] = "sms" + + phone_number: PhoneNumber diff --git a/packages/models-library/src/models_library/rpc/notifications/events/__init__.py b/packages/models-library/src/models_library/rpc/notifications/events/__init__.py new file mode 100644 index 000000000000..b45ed0c018b1 --- /dev/null +++ b/packages/models-library/src/models_library/rpc/notifications/events/__init__.py @@ -0,0 +1,19 @@ +from typing import TypeAlias + +from ._account_events import ( + AccountApprovedEvent, + AccountRejectedEvent, + AccountRequestedEvent, +) + +Event: TypeAlias = AccountRequestedEvent | AccountApprovedEvent | AccountRejectedEvent + + +__all__: tuple[str, ...] = ( + "AccountApprovedEvent", + "AccountRejectedEvent", + "AccountRequestedEvent", + "Event", +) + +# nopycln: file diff --git a/packages/models-library/src/models_library/rpc/notifications/events/_account_events.py b/packages/models-library/src/models_library/rpc/notifications/events/_account_events.py new file mode 100644 index 000000000000..cc29286dc257 --- /dev/null +++ b/packages/models-library/src/models_library/rpc/notifications/events/_account_events.py @@ -0,0 +1,23 @@ +from typing import Literal + +from pydantic import BaseModel, EmailStr + + +class AccountRequestedEvent(BaseModel): + type: Literal["account_requested"] = "account_requested" + + first_name: str + last_name: str + email: EmailStr + + # TODO: add more fields as needed + + +class AccountApprovedEvent(BaseModel): + type: Literal["account_approved"] = "account_approved" + + +class AccountRejectedEvent(BaseModel): + type: Literal["account_rejected"] = "account_rejected" + + reason: str diff --git a/packages/models-library/src/models_library/rpc/notifications/schemas.py b/packages/models-library/src/models_library/rpc/notifications/schemas.py deleted file mode 100644 index d90db1ee10e9..000000000000 --- a/packages/models-library/src/models_library/rpc/notifications/schemas.py +++ /dev/null @@ -1,33 +0,0 @@ -from typing import Annotated, Literal, TypeAlias - -from pydantic import BaseModel, ConfigDict, EmailStr, Field -from pydantic_extra_types.phone_numbers import PhoneNumber - - -class Event(BaseModel): - type: str - - model_config = ConfigDict( - frozen=True, - ) - - -class EmailChannel(BaseModel): - type: Literal["email"] = "email" - - to: EmailStr - reply_to: EmailStr | None = None - - -class SMSChannel(BaseModel): - type: Literal["sms"] = "sms" - - phone_number: PhoneNumber - - -Channel: TypeAlias = EmailChannel | SMSChannel - - -class Notification(BaseModel): - event: Event - channel: Annotated[Channel, Field(discriminator="type")] diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/messages.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/messages.py index c252f21de656..f04d3c8effa9 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/messages.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/messages.py @@ -2,7 +2,7 @@ from models_library.api_schemas_notifications import NOTIFICATIONS_RPC_NAMESPACE from models_library.rabbitmq_basic_types import RPCMethodName -from models_library.rpc.notifications.schemas import Notification +from models_library.rpc.notifications import Notification from pydantic import NonNegativeInt, TypeAdapter from ... import RabbitMQRPCClient diff --git a/services/notifications/src/simcore_service_notifications/api/rpc/_notifications.py b/services/notifications/src/simcore_service_notifications/api/rpc/_notifications.py index a14c8e34dbae..f59c5db57761 100644 --- a/services/notifications/src/simcore_service_notifications/api/rpc/_notifications.py +++ b/services/notifications/src/simcore_service_notifications/api/rpc/_notifications.py @@ -1,4 +1,4 @@ -from models_library.rpc.notifications.schemas import Notification +from models_library.rpc.notifications import Notification from servicelib.celery.task_manager import TaskManager from servicelib.rabbitmq import RPCRouter diff --git a/services/notifications/src/simcore_service_notifications/clients/celery.py b/services/notifications/src/simcore_service_notifications/clients/celery.py index e58da8c25000..5452a20b7e36 100644 --- a/services/notifications/src/simcore_service_notifications/clients/celery.py +++ b/services/notifications/src/simcore_service_notifications/clients/celery.py @@ -6,9 +6,9 @@ from celery_library.types import register_celery_types, register_pydantic_types from fastapi import FastAPI from fastapi_lifespan_manager import State -from models_library.rpc.notifications.schemas import ( +from models_library.rpc.notifications import Notification +from models_library.rpc.notifications.channels import ( EmailChannel, - Notification, SMSChannel, ) from settings_library.celery import CelerySettings diff --git a/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py b/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py index e82303235787..d70b903280c7 100644 --- a/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py +++ b/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py @@ -1,10 +1,19 @@ # pylint: disable=unused-argument import logging +from email.headerregistry import Address from celery import Task -from models_library.rpc.notifications.schemas import EmailChannel, Notification +from jinja2 import StrictUndefined +from models_library.rpc.notifications import Notification +from models_library.rpc.notifications.channels import EmailChannel +from notifications_library._email import compose_email, create_email_session +from notifications_library._email_render import render_email_parts +from notifications_library._render import ( + create_render_environment_from_notifications_library, +) from servicelib.celery.models import TaskID +from settings_library.email import SMTPSettings _logger = logging.getLogger(__name__) @@ -16,38 +25,29 @@ async def send_email_notification( task_id: TaskID, notification: Notification, ) -> None: + _ = task, task_id assert isinstance(notification.channel, EmailChannel) # nosec - _logger.info("Sending email notification to %s", notification.channel.to) - - # event_extra_data = event_extra_data | (asdict(sharer_data) if sharer_data else {}) - - # parts = render_email_parts( - # env=create_render_environment_from_notifications_library( - # undefined=StrictUndefined - # ), - # event_name=event_name, - # user=user_data, - # product=product_data, - # # extras - # **event_extra_data, - # ) - - # from_ = get_support_address(product_data) - # to = get_user_address(user_data) - - # assert from_.addr_spec == product_data.support_email - # assert to.addr_spec == user_email - - # msg = compose_email( - # from_, - # to, - # subject=parts.subject, - # content_text=parts.text_content, - # content_html=parts.html_content, - # ) + _logger.info("Sending email notification to %s", notification.channel.to_addr) + + parts = render_email_parts( + env=create_render_environment_from_notifications_library( + undefined=StrictUndefined + ), + event_name=f"on_{notification.event.type}", + **notification.event.model_dump(), + ) + + msg = compose_email( + Address(**notification.channel.from_addr.model_dump()), + Address(**notification.channel.to_addr.model_dump()), + subject=parts.subject, + content_text=parts.text_content, + content_html=parts.html_content, + ) + # if event_attachments: # add_attachments(msg, event_attachments) - # async with create_email_session(settings=SMTPSettings.create_from_envs()) as smtp: - # await smtp.send_message(msg) + async with create_email_session(settings=SMTPSettings.create_from_envs()) as smtp: + await smtp.send_message(msg) diff --git a/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py b/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py index 746204d2e2f6..1cee7388d6ed 100644 --- a/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py +++ b/services/notifications/src/simcore_service_notifications/modules/celery/tasks.py @@ -4,11 +4,8 @@ from celery import Celery # type: ignore[import-untyped] from celery_library.task import register_task from celery_library.types import register_celery_types, register_pydantic_types -from models_library.rpc.notifications.account import AccountRequestedEvent -from models_library.rpc.notifications.schemas import ( - EmailChannel, - SMSChannel, -) +from models_library.rpc.notifications.channels import EmailChannel, SMSChannel +from models_library.rpc.notifications.events import AccountRequestedEvent from servicelib.logging_utils import log_context from ...modules.celery._email_tasks import EMAIL_CHANNEL_NAME, send_email_notification diff --git a/services/notifications/src/simcore_service_notifications/services/notifications_service.py b/services/notifications/src/simcore_service_notifications/services/notifications_service.py index 554de172a5f8..c66156893dea 100644 --- a/services/notifications/src/simcore_service_notifications/services/notifications_service.py +++ b/services/notifications/src/simcore_service_notifications/services/notifications_service.py @@ -1,6 +1,6 @@ import logging -from models_library.rpc.notifications.schemas import Notification +from models_library.rpc.notifications import Notification from servicelib.celery.models import TaskContext from servicelib.celery.task_manager import TaskManager diff --git a/services/notifications/tests/unit/test_send_email_tasks.py b/services/notifications/tests/unit/test_send_email_tasks.py index 9d740fc29a6c..da7b44eab3c1 100644 --- a/services/notifications/tests/unit/test_send_email_tasks.py +++ b/services/notifications/tests/unit/test_send_email_tasks.py @@ -1,12 +1,14 @@ import pytest from faker import Faker -from models_library.rpc.notifications.account import AccountRequestedEvent -from models_library.rpc.notifications.schemas import Notification +from models_library.rpc.notifications import Notification +from models_library.rpc.notifications.channels import EmailAddress, EmailChannel +from models_library.rpc.notifications.events._account_events import ( + AccountRequestedEvent, +) from servicelib.rabbitmq import RabbitMQRPCClient from servicelib.rabbitmq.rpc_interfaces.notifications.messages import ( send_notification, ) -from simcore_service_notifications.clients.celery import EmailChannel pytest_simcore_core_services_selection = [ "rabbit", @@ -33,7 +35,12 @@ async def test_account_requested( last_name=faker.last_name(), email=email, ), - channel=EmailChannel(to=email), + channel=EmailChannel( + from_addr=EmailAddress(addr_spec=""), + to_addr=EmailAddress( + addr_spec=email, + ), + ), ), ) From e021ae352c6281af24a6c61950c3f41c3b475045 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 9 Jul 2025 15:12:24 +0200 Subject: [PATCH 49/69] fix: events --- .../src/common_library/pydantic_basic_types.py | 11 ++++++++++- .../rpc/notifications/channels/_email_channel.py | 4 ++-- .../rpc/notifications/events/_account_events.py | 3 ++- .../notifications/tests/unit/test_send_email_tasks.py | 2 +- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/packages/common-library/src/common_library/pydantic_basic_types.py b/packages/common-library/src/common_library/pydantic_basic_types.py index 452c118dae95..fc5b10d17cfc 100644 --- a/packages/common-library/src/common_library/pydantic_basic_types.py +++ b/packages/common-library/src/common_library/pydantic_basic_types.py @@ -1,7 +1,7 @@ from re import Pattern from typing import Annotated, Final, TypeAlias -from pydantic import Field +from pydantic import Field, StringConstraints from pydantic_core import core_schema # https://en.wikipedia.org/wiki/List_of_TCP_and_UDP_port_numbers#Registered_ports @@ -77,3 +77,12 @@ class LongTruncatedStr(ConstrainedStr): # Analogous to ShortTruncatedStr strip_whitespace = True curtail_length = 65536 # same as github descripton + + +NotEmptyStr: TypeAlias = Annotated[ + str, + StringConstraints( + min_length=1, + strip_whitespace=True, + ), +] diff --git a/packages/models-library/src/models_library/rpc/notifications/channels/_email_channel.py b/packages/models-library/src/models_library/rpc/notifications/channels/_email_channel.py index 4d28fed5fb04..c9116545a558 100644 --- a/packages/models-library/src/models_library/rpc/notifications/channels/_email_channel.py +++ b/packages/models-library/src/models_library/rpc/notifications/channels/_email_channel.py @@ -1,11 +1,11 @@ from typing import Literal -from pydantic import BaseModel +from pydantic import BaseModel, EmailStr class EmailAddress(BaseModel): display_name: str | None = None - addr_spec: str + addr_spec: EmailStr class EmailChannel(BaseModel): diff --git a/packages/models-library/src/models_library/rpc/notifications/events/_account_events.py b/packages/models-library/src/models_library/rpc/notifications/events/_account_events.py index cc29286dc257..9069f95739bf 100644 --- a/packages/models-library/src/models_library/rpc/notifications/events/_account_events.py +++ b/packages/models-library/src/models_library/rpc/notifications/events/_account_events.py @@ -1,5 +1,6 @@ from typing import Literal +from common_library.pydantic_basic_types import NotEmptyStr from pydantic import BaseModel, EmailStr @@ -20,4 +21,4 @@ class AccountApprovedEvent(BaseModel): class AccountRejectedEvent(BaseModel): type: Literal["account_rejected"] = "account_rejected" - reason: str + reason: NotEmptyStr diff --git a/services/notifications/tests/unit/test_send_email_tasks.py b/services/notifications/tests/unit/test_send_email_tasks.py index da7b44eab3c1..b55f372721ef 100644 --- a/services/notifications/tests/unit/test_send_email_tasks.py +++ b/services/notifications/tests/unit/test_send_email_tasks.py @@ -36,7 +36,7 @@ async def test_account_requested( email=email, ), channel=EmailChannel( - from_addr=EmailAddress(addr_spec=""), + from_addr=EmailAddress(addr_spec=faker.email()), to_addr=EmailAddress( addr_spec=email, ), From f322d02f760f5ae43cb3f1d1e6893143f38bb607 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 9 Jul 2025 15:21:32 +0200 Subject: [PATCH 50/69] feat: revert postgres removal --- .../api/rest/_health.py | 7 ++- .../api/rest/dependencies.py | 9 ++++ .../clients/postgres/__init__.py | 33 ++++++++++++++ .../clients/postgres/_liveness.py | 43 +++++++++++++++++++ .../core/events.py | 6 +++ 5 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 services/notifications/src/simcore_service_notifications/clients/postgres/__init__.py create mode 100644 services/notifications/src/simcore_service_notifications/clients/postgres/_liveness.py diff --git a/services/notifications/src/simcore_service_notifications/api/rest/_health.py b/services/notifications/src/simcore_service_notifications/api/rest/_health.py index 80e4fccf0c39..d8f7e3e600e8 100644 --- a/services/notifications/src/simcore_service_notifications/api/rest/_health.py +++ b/services/notifications/src/simcore_service_notifications/api/rest/_health.py @@ -4,11 +4,13 @@ from fastapi import APIRouter, Depends from models_library.api_schemas__common.health import HealthCheckGet from models_library.errors import ( + POSRGRES_DATABASE_UNHEALTHY_MSG, RABBITMQ_CLIENT_UNHEALTHY_MSG, ) from servicelib.rabbitmq import RabbitMQClient -from .dependencies import get_rabbitmq_client +from ...clients.postgres import PostgresLiveness +from .dependencies import get_postgres_liveness, get_rabbitmq_client router = APIRouter() @@ -20,8 +22,11 @@ class HealthCheckError(RuntimeError): @router.get("/", response_model=HealthCheckGet) async def check_service_health( rabbitmq_client: Annotated[RabbitMQClient, Depends(get_rabbitmq_client)], + postgres_liveness: Annotated[PostgresLiveness, Depends(get_postgres_liveness)], ): if not rabbitmq_client.healthy: raise HealthCheckError(RABBITMQ_CLIENT_UNHEALTHY_MSG) + if not postgres_liveness.is_responsive: + raise HealthCheckError(POSRGRES_DATABASE_UNHEALTHY_MSG) return HealthCheckGet(timestamp=f"{__name__}@{arrow.utcnow().datetime.isoformat()}") diff --git a/services/notifications/src/simcore_service_notifications/api/rest/dependencies.py b/services/notifications/src/simcore_service_notifications/api/rest/dependencies.py index 800efcfe15d8..962154ea9f77 100644 --- a/services/notifications/src/simcore_service_notifications/api/rest/dependencies.py +++ b/services/notifications/src/simcore_service_notifications/api/rest/dependencies.py @@ -5,6 +5,9 @@ from fastapi import Depends, FastAPI, Request from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient +from ...clients.postgres import PostgresLiveness +from ...clients.postgres import get_postgres_liveness as get_postgress_db_liveness + def get_application(request: Request) -> FastAPI: return cast(FastAPI, request.app) @@ -15,3 +18,9 @@ def get_rabbitmq_client( ) -> RabbitMQRPCClient: assert isinstance(app.state.rabbitmq_rpc_server, RabbitMQRPCClient) # nosec return app.state.rabbitmq_rpc_server + + +def get_postgres_liveness( + app: Annotated[FastAPI, Depends(get_application)], +) -> PostgresLiveness: + return get_postgress_db_liveness(app) diff --git a/services/notifications/src/simcore_service_notifications/clients/postgres/__init__.py b/services/notifications/src/simcore_service_notifications/clients/postgres/__init__.py new file mode 100644 index 000000000000..e0883fba1ced --- /dev/null +++ b/services/notifications/src/simcore_service_notifications/clients/postgres/__init__.py @@ -0,0 +1,33 @@ +import logging +from collections.abc import AsyncIterator + +from fastapi import FastAPI +from fastapi_lifespan_manager import State +from servicelib.fastapi.postgres_lifespan import PostgresLifespanState +from servicelib.logging_utils import log_context + +from ._liveness import PostgresLiveness + +_logger = logging.getLogger(__name__) + + +async def postgres_lifespan(app: FastAPI, state: State) -> AsyncIterator[State]: + app.state.engine = state[PostgresLifespanState.POSTGRES_ASYNC_ENGINE] + + app.state.postgres_liveness = PostgresLiveness(app) + + with log_context(_logger, logging.INFO, msg="setup postgres health"): + await app.state.postgres_liveness.setup() + + yield {} + + with log_context(_logger, logging.INFO, msg="teardown postgres health"): + await app.state.postgres_liveness.teardown() + + +def get_postgres_liveness(app: FastAPI) -> PostgresLiveness: + assert isinstance(app.state.postgres_liveness, PostgresLiveness) # nosec + return app.state.postgres_liveness + + +__all__: tuple[str, ...] = ("PostgresLiveness",) diff --git a/services/notifications/src/simcore_service_notifications/clients/postgres/_liveness.py b/services/notifications/src/simcore_service_notifications/clients/postgres/_liveness.py new file mode 100644 index 000000000000..6d26fd83e939 --- /dev/null +++ b/services/notifications/src/simcore_service_notifications/clients/postgres/_liveness.py @@ -0,0 +1,43 @@ +import logging +from asyncio import Task +from datetime import timedelta +from typing import Final + +from common_library.async_tools import cancel_wait_task +from fastapi import FastAPI +from models_library.healthchecks import IsResponsive, LivenessResult +from servicelib.background_task import create_periodic_task +from servicelib.db_asyncpg_utils import check_postgres_liveness +from servicelib.fastapi.db_asyncpg_engine import get_engine +from servicelib.logging_utils import log_catch + +_logger = logging.getLogger(__name__) + +_LVENESS_CHECK_INTERVAL: Final[timedelta] = timedelta(seconds=10) + + +class PostgresLiveness: + def __init__(self, app: FastAPI) -> None: + self.app = app + + self._liveness_result: LivenessResult = IsResponsive(elapsed=timedelta(0)) + self._task: Task | None = None + + async def _check_task(self) -> None: + self._liveness_result = await check_postgres_liveness(get_engine(self.app)) + + @property + def is_responsive(self) -> bool: + return isinstance(self._liveness_result, IsResponsive) + + async def setup(self) -> None: + self._task = create_periodic_task( + self._check_task, + interval=_LVENESS_CHECK_INTERVAL, + task_name="posgress_liveness_check", + ) + + async def teardown(self) -> None: + if self._task is not None: + with log_catch(_logger, reraise=False): + await cancel_wait_task(self._task, max_delay=5) diff --git a/services/notifications/src/simcore_service_notifications/core/events.py b/services/notifications/src/simcore_service_notifications/core/events.py index e1e94b3b7588..11c7ec6dab64 100644 --- a/services/notifications/src/simcore_service_notifications/core/events.py +++ b/services/notifications/src/simcore_service_notifications/core/events.py @@ -8,11 +8,13 @@ ) from servicelib.fastapi.postgres_lifespan import ( create_postgres_database_input_state, + postgres_database_lifespan, ) from .._meta import APP_FINISHED_BANNER_MSG, APP_STARTED_BANNER_MSG from ..api.rpc.routes import rpc_api_routes_lifespan from ..clients.celery import celery_lifespan +from ..clients.postgres import postgres_lifespan from ..clients.rabbitmq import rabbitmq_lifespan from .settings import ApplicationSettings @@ -39,6 +41,10 @@ def create_app_lifespan(settings: ApplicationSettings) -> LifespanManager: app_lifespan = LifespanManager() app_lifespan.add(_settings_lifespan) + # - postgres + app_lifespan.add(postgres_database_lifespan) + app_lifespan.add(postgres_lifespan) + if settings.NOTIFICATIONS_CELERY and not settings.NOTIFICATIONS_WORKER_MODE: # - rabbitmq app_lifespan.add(rabbitmq_lifespan) From 7cecec8e3bf6ad761e9a72a5c9a4194ed9a290cf Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 9 Jul 2025 15:31:44 +0200 Subject: [PATCH 51/69] fix: add postgres deps --- services/notifications/requirements/_base.in | 1 + services/notifications/requirements/_base.txt | 42 ++++++++++++++++++- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/services/notifications/requirements/_base.in b/services/notifications/requirements/_base.in index 7d5ff96d812a..b392120e16f7 100644 --- a/services/notifications/requirements/_base.in +++ b/services/notifications/requirements/_base.in @@ -11,6 +11,7 @@ --requirement ../../../packages/notifications-library/requirements/_base.in --requirement ../../../packages/models-library/requirements/_base.in --requirement ../../../packages/settings-library/requirements/_base.in +--requirement ../../../packages/postgres-database/requirements/_base.in # service-library[fastapi] --requirement ../../../packages/service-library/requirements/_base.in --requirement ../../../packages/service-library/requirements/_fastapi.in diff --git a/services/notifications/requirements/_base.txt b/services/notifications/requirements/_base.txt index 6adb0af4709c..67998f8b7490 100644 --- a/services/notifications/requirements/_base.txt +++ b/services/notifications/requirements/_base.txt @@ -46,6 +46,8 @@ aiohttp==3.12.12 # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/notifications-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -63,7 +65,9 @@ aiosignal==1.3.2 aiosmtplib==4.0.1 # via -r requirements/../../../packages/notifications-library/requirements/_base.in alembic==1.15.1 - # via -r requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/_base.in + # via + # -r requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/_base.in amqp==5.3.1 # via kombu annotated-types==0.7.0 @@ -124,6 +128,8 @@ certifi==2025.1.31 # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/notifications-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -224,6 +230,8 @@ httpx==0.28.1 # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/notifications-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -271,6 +279,8 @@ jinja2==3.1.6 # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/notifications-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -318,6 +328,8 @@ mako==1.3.10 # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/notifications-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -386,7 +398,9 @@ opentelemetry-instrumentation-aio-pika==0.55b1 opentelemetry-instrumentation-asgi==0.55b1 # via opentelemetry-instrumentation-fastapi opentelemetry-instrumentation-asyncpg==0.55b1 - # via -r requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/_base.in + # via + # -r requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/_base.in opentelemetry-instrumentation-fastapi==0.55b1 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in opentelemetry-instrumentation-httpx==0.55b1 @@ -455,6 +469,8 @@ orjson==3.10.16 # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/notifications-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -480,6 +496,7 @@ orjson==3.10.16 # -r requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in @@ -544,6 +561,8 @@ pydantic==2.11.0 # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/notifications-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -574,6 +593,8 @@ pydantic==2.11.0 # -r requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/_base.in # -r requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in @@ -607,6 +628,7 @@ pydantic-extra-types==2.10.3 # -r requirements/../../../packages/notifications-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in @@ -637,6 +659,8 @@ pydantic-settings==2.7.0 # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/notifications-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -697,6 +721,8 @@ pyyaml==6.0.2 # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/notifications-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -734,6 +760,8 @@ redis==5.2.1 # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/notifications-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -771,6 +799,8 @@ referencing==0.35.1 # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/notifications-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -832,6 +862,8 @@ sqlalchemy==1.4.54 # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/notifications-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -842,6 +874,7 @@ sqlalchemy==1.4.54 # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/_base.in # alembic starlette==0.46.1 # via @@ -868,6 +901,8 @@ starlette==0.46.1 # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/notifications-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -951,6 +986,8 @@ urllib3==2.3.0 # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/notifications-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/notifications-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -988,6 +1025,7 @@ yarl==1.18.3 # via # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/notifications-library/requirements/../../../packages/postgres-database/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # aio-pika # aiohttp From 1c87d24ab6de6539774ead07084f7cb6a237da71 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 9 Jul 2025 15:33:41 +0200 Subject: [PATCH 52/69] fix: postgres deps --- services/notifications/requirements/ci.txt | 1 + services/notifications/requirements/dev.txt | 1 + services/notifications/requirements/prod.txt | 1 + 3 files changed, 3 insertions(+) diff --git a/services/notifications/requirements/ci.txt b/services/notifications/requirements/ci.txt index 910eff4c58d8..3a699961edc7 100644 --- a/services/notifications/requirements/ci.txt +++ b/services/notifications/requirements/ci.txt @@ -16,6 +16,7 @@ simcore-celery-library @ ../../packages/celery-library/ simcore-common-library @ ../../packages/common-library/ simcore-models-library @ ../../packages/models-library/ simcore-notifications-library @ ../../packages/notifications-library/ +simcore-postgres-database @ ../../packages/postgres-database/ pytest-simcore @ ../../packages/pytest-simcore/ simcore-service-library[fastapi] @ ../../packages/service-library/ simcore-settings-library @ ../../packages/settings-library/ diff --git a/services/notifications/requirements/dev.txt b/services/notifications/requirements/dev.txt index 2fae6cc869f2..5e21f4fc4abd 100644 --- a/services/notifications/requirements/dev.txt +++ b/services/notifications/requirements/dev.txt @@ -16,6 +16,7 @@ --editable ../../packages/common-library --editable ../../packages/models-library --editable ../../packages/notifications-library +--editable ../../packages/postgres-database --editable ../../packages/pytest-simcore --editable ../../packages/service-library[fastapi] --editable ../../packages/settings-library diff --git a/services/notifications/requirements/prod.txt b/services/notifications/requirements/prod.txt index 7fdfae145ffc..badd27a0079b 100644 --- a/services/notifications/requirements/prod.txt +++ b/services/notifications/requirements/prod.txt @@ -14,6 +14,7 @@ simcore-celery-library @ ../../packages/celery-library/ simcore-common-library @ ../../packages/common-library/ simcore-models-library @ ../../packages/models-library/ simcore-notifications-library @ ../../packages/notifications-library/ +simcore-postgres-database @ ../../packages/postgres-database/ simcore-service-library[fastapi] @ ../../packages/service-library/ simcore-settings-library @ ../../packages/settings-library/ From b3074b48c55cdad3048343a70287970e050ee652 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 9 Jul 2025 15:37:27 +0200 Subject: [PATCH 53/69] tests: fix postgres --- services/notifications/tests/unit/conftest.py | 4 ++++ services/notifications/tests/unit/test_api_rest__health.py | 1 + services/notifications/tests/unit/test_send_email_tasks.py | 1 + 3 files changed, 6 insertions(+) diff --git a/services/notifications/tests/unit/conftest.py b/services/notifications/tests/unit/conftest.py index b7b4468e2994..49c01479c9ba 100644 --- a/services/notifications/tests/unit/conftest.py +++ b/services/notifications/tests/unit/conftest.py @@ -7,6 +7,7 @@ from typing import Any, Final import pytest +import sqlalchemy as sa from asgi_lifespan import LifespanManager from celery import Celery from celery.contrib.testing.worker import start_worker @@ -35,11 +36,14 @@ def app_environment( monkeypatch: pytest.MonkeyPatch, mock_environment: EnvVarsDict, + postgres_db: sa.engine.Engine, # wait for postgres service to start + postgres_env_vars_dict: EnvVarsDict, ) -> EnvVarsDict: return setenvs_from_dict( monkeypatch, { **mock_environment, + **postgres_env_vars_dict, }, ) diff --git a/services/notifications/tests/unit/test_api_rest__health.py b/services/notifications/tests/unit/test_api_rest__health.py index b596b9b0cf0c..6461709b3f89 100644 --- a/services/notifications/tests/unit/test_api_rest__health.py +++ b/services/notifications/tests/unit/test_api_rest__health.py @@ -11,6 +11,7 @@ from simcore_service_notifications.api.rest._health import HealthCheckError pytest_simcore_core_services_selection = [ + "postgres", "rabbit", "redis", ] diff --git a/services/notifications/tests/unit/test_send_email_tasks.py b/services/notifications/tests/unit/test_send_email_tasks.py index b55f372721ef..09d9320f0fdb 100644 --- a/services/notifications/tests/unit/test_send_email_tasks.py +++ b/services/notifications/tests/unit/test_send_email_tasks.py @@ -11,6 +11,7 @@ ) pytest_simcore_core_services_selection = [ + "postgres", "rabbit", "redis", ] From 8b06ed3d7b0a3c2bf78484ebc359359ebe0dace9 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 9 Jul 2025 15:41:39 +0200 Subject: [PATCH 54/69] tests: fix postgres plugin --- services/notifications/tests/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/services/notifications/tests/conftest.py b/services/notifications/tests/conftest.py index 13c51cf1a40b..4f3864524f78 100644 --- a/services/notifications/tests/conftest.py +++ b/services/notifications/tests/conftest.py @@ -12,6 +12,7 @@ "pytest_simcore.docker_compose", "pytest_simcore.docker_swarm", "pytest_simcore.environment_configs", + "pytest_simcore.postgres_service", "pytest_simcore.rabbit_service", "pytest_simcore.redis_service", "pytest_simcore.repository_paths", From 4793012b791763995edc63fa387a1fde73b4c408 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 9 Jul 2025 15:42:21 +0200 Subject: [PATCH 55/69] fix: typecheck --- .../modules/celery/_email_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py b/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py index d70b903280c7..a7fa37872051 100644 --- a/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py +++ b/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py @@ -3,7 +3,7 @@ import logging from email.headerregistry import Address -from celery import Task +from celery import Task # type: ignore[import-untyped] from jinja2 import StrictUndefined from models_library.rpc.notifications import Notification from models_library.rpc.notifications.channels import EmailChannel From 158f7800bc4029524d121bc00c525b69027e3763 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 9 Jul 2025 15:56:33 +0200 Subject: [PATCH 56/69] fix: test --- services/notifications/tests/unit/test_send_email_tasks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/notifications/tests/unit/test_send_email_tasks.py b/services/notifications/tests/unit/test_send_email_tasks.py index 09d9320f0fdb..88ff2beb49e0 100644 --- a/services/notifications/tests/unit/test_send_email_tasks.py +++ b/services/notifications/tests/unit/test_send_email_tasks.py @@ -26,7 +26,7 @@ async def test_account_requested( notifications_rabbitmq_rpc_client: RabbitMQRPCClient, faker: Faker, ): - email = faker.email() + user_email = faker.email() await send_notification( notifications_rabbitmq_rpc_client, @@ -34,12 +34,12 @@ async def test_account_requested( event=AccountRequestedEvent( first_name=faker.first_name(), last_name=faker.last_name(), - email=email, + email=user_email, ), channel=EmailChannel( from_addr=EmailAddress(addr_spec=faker.email()), to_addr=EmailAddress( - addr_spec=email, + addr_spec=user_email, ), ), ), From 40729d81176b1586a25cf510265463002d241cf0 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 10 Jul 2025 20:47:20 +0200 Subject: [PATCH 57/69] fix: task filter import --- .../services/notifications_service.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/notifications/src/simcore_service_notifications/services/notifications_service.py b/services/notifications/src/simcore_service_notifications/services/notifications_service.py index e9edf0d66d42..02ba7571145f 100644 --- a/services/notifications/src/simcore_service_notifications/services/notifications_service.py +++ b/services/notifications/src/simcore_service_notifications/services/notifications_service.py @@ -1,7 +1,7 @@ import logging from models_library.rpc.notifications import Notification -from servicelib.celery.models import TaskContext +from servicelib.celery.models import TaskFilter from servicelib.celery.task_manager import TaskManager from ..modules.celery.tasks import TaskQueue @@ -16,8 +16,8 @@ async def send_notification( ) -> None: await task_manager.send_task( # send to the specific channel worker - name=f"notifications.{notification.channel.type}", - context=TaskContext(), # TODO: TaskFilter + task_name=f"notifications.{notification.channel.type}", + task_filter=TaskFilter(), # TODO: TaskFilter task_queue=TaskQueue.DEFAULT, notification=notification, ) From c318e9021ff2db2dcf8f67f36717fda83f5cdb68 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 11 Jul 2025 10:52:44 +0200 Subject: [PATCH 58/69] fix: imports --- .../rabbitmq/rpc_interfaces/notifications/__init__.py | 3 +++ .../notifications/{messages.py => _notifications.py} | 0 services/notifications/tests/unit/test_send_email_tasks.py | 4 ++-- 3 files changed, 5 insertions(+), 2 deletions(-) rename packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/{messages.py => _notifications.py} (100%) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/__init__.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/__init__.py index e69de29bb2d1..11795609ea92 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/__init__.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/__init__.py @@ -0,0 +1,3 @@ +from ._notifications import send_notification + +__all__: tuple[str, ...] = ("send_notification",) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/messages.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/_notifications.py similarity index 100% rename from packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/messages.py rename to packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/_notifications.py diff --git a/services/notifications/tests/unit/test_send_email_tasks.py b/services/notifications/tests/unit/test_send_email_tasks.py index 88ff2beb49e0..1bad4fbfde32 100644 --- a/services/notifications/tests/unit/test_send_email_tasks.py +++ b/services/notifications/tests/unit/test_send_email_tasks.py @@ -2,11 +2,11 @@ from faker import Faker from models_library.rpc.notifications import Notification from models_library.rpc.notifications.channels import EmailAddress, EmailChannel -from models_library.rpc.notifications.events._account_events import ( +from models_library.rpc.notifications.events import ( AccountRequestedEvent, ) from servicelib.rabbitmq import RabbitMQRPCClient -from servicelib.rabbitmq.rpc_interfaces.notifications.messages import ( +from servicelib.rabbitmq.rpc_interfaces.notifications import ( send_notification, ) From 24c11544f99ce78203ae6e7c6f3918a8e9eb42d1 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 11 Jul 2025 16:09:41 +0200 Subject: [PATCH 59/69] fix: models --- .../rpc/notifications/events/__init__.py | 6 ++++ .../notifications/events/_account_events.py | 33 +++++++++++++++-- .../core/events.py | 2 +- services/notifications/tests/unit/conftest.py | 36 +++++++++---------- .../tests/unit/test_send_email_tasks.py | 28 ++++++++++++--- 5 files changed, 77 insertions(+), 28 deletions(-) diff --git a/packages/models-library/src/models_library/rpc/notifications/events/__init__.py b/packages/models-library/src/models_library/rpc/notifications/events/__init__.py index b45ed0c018b1..56d07abf0da4 100644 --- a/packages/models-library/src/models_library/rpc/notifications/events/__init__.py +++ b/packages/models-library/src/models_library/rpc/notifications/events/__init__.py @@ -4,6 +4,9 @@ AccountApprovedEvent, AccountRejectedEvent, AccountRequestedEvent, + ProductData, + ProductUIData, + UserData, ) Event: TypeAlias = AccountRequestedEvent | AccountApprovedEvent | AccountRejectedEvent @@ -14,6 +17,9 @@ "AccountRejectedEvent", "AccountRequestedEvent", "Event", + "ProductData", + "ProductUIData", + "UserData", ) # nopycln: file diff --git a/packages/models-library/src/models_library/rpc/notifications/events/_account_events.py b/packages/models-library/src/models_library/rpc/notifications/events/_account_events.py index 9069f95739bf..c7df26e04c09 100644 --- a/packages/models-library/src/models_library/rpc/notifications/events/_account_events.py +++ b/packages/models-library/src/models_library/rpc/notifications/events/_account_events.py @@ -1,12 +1,12 @@ from typing import Literal from common_library.pydantic_basic_types import NotEmptyStr +from models_library.products import ProductName from pydantic import BaseModel, EmailStr -class AccountRequestedEvent(BaseModel): - type: Literal["account_requested"] = "account_requested" - +class UserData(BaseModel): + username: str first_name: str last_name: str email: EmailStr @@ -14,6 +14,33 @@ class AccountRequestedEvent(BaseModel): # TODO: add more fields as needed +class ProductUIData(BaseModel): + project_alias: str + logo_url: str | None = ( + None # default_logo = "https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/refs/heads/master/services/static-webserver/client/source/resource/osparc/osparc-white.svg" in base.html + ) + strong_color: str | None = ( + None # default_strong_color = "rgb(131, 0, 191)" in base.html + ) + + +class ProductData(BaseModel): + product_name: ProductName + display_name: str + vendor_display_inline: str + support_email: str + homepage_url: str | None # default_homepage = "https://osparc.io/" in base.html + ui: ProductUIData + + +class AccountRequestedEvent(BaseModel): + type: Literal["account_requested"] = "account_requested" + + user: UserData + product: ProductData + host: str + + class AccountApprovedEvent(BaseModel): type: Literal["account_approved"] = "account_approved" diff --git a/services/notifications/src/simcore_service_notifications/core/events.py b/services/notifications/src/simcore_service_notifications/core/events.py index 11c7ec6dab64..f7059ccc7288 100644 --- a/services/notifications/src/simcore_service_notifications/core/events.py +++ b/services/notifications/src/simcore_service_notifications/core/events.py @@ -45,7 +45,7 @@ def create_app_lifespan(settings: ApplicationSettings) -> LifespanManager: app_lifespan.add(postgres_database_lifespan) app_lifespan.add(postgres_lifespan) - if settings.NOTIFICATIONS_CELERY and not settings.NOTIFICATIONS_WORKER_MODE: + if not settings.NOTIFICATIONS_WORKER_MODE: # - rabbitmq app_lifespan.add(rabbitmq_lifespan) diff --git a/services/notifications/tests/unit/conftest.py b/services/notifications/tests/unit/conftest.py index 49c01479c9ba..c0d104369503 100644 --- a/services/notifications/tests/unit/conftest.py +++ b/services/notifications/tests/unit/conftest.py @@ -38,43 +38,42 @@ def app_environment( mock_environment: EnvVarsDict, postgres_db: sa.engine.Engine, # wait for postgres service to start postgres_env_vars_dict: EnvVarsDict, + rabbit_service: RabbitSettings, + redis_service: RedisSettings, ) -> EnvVarsDict: return setenvs_from_dict( monkeypatch, { **mock_environment, **postgres_env_vars_dict, + "RABBIT_HOST": rabbit_service.RABBIT_HOST, + "RABBIT_PASSWORD": rabbit_service.RABBIT_PASSWORD.get_secret_value(), + "RABBIT_PORT": f"{rabbit_service.RABBIT_PORT}", + "RABBIT_SECURE": f"{rabbit_service.RABBIT_SECURE}", + "RABBIT_USER": rabbit_service.RABBIT_USER, + "REDIS_SECURE": redis_service.REDIS_SECURE, + "REDIS_HOST": redis_service.REDIS_HOST, + "REDIS_PORT": f"{redis_service.REDIS_PORT}", + "REDIS_PASSWORD": redis_service.REDIS_PASSWORD.get_secret_value(), }, ) -@pytest.fixture -def enabled_rabbitmq( - app_environment: EnvVarsDict, rabbit_service: RabbitSettings -) -> RabbitSettings: - return rabbit_service - - -@pytest.fixture -def enabled_redis( - app_environment: EnvVarsDict, redis_service: RedisSettings -) -> RedisSettings: - return redis_service - - @pytest.fixture def app_settings( app_environment: EnvVarsDict, - enabled_rabbitmq: RabbitSettings, - enabled_redis: RedisSettings, + monkeypatch: pytest.MonkeyPatch, ) -> ApplicationSettings: + monkeypatch.setenv("NOTIFICATIONS_WORKER_MODE", "false") settings = ApplicationSettings.create_from_envs() print(f"{settings.model_dump_json(indent=2)=}") return settings @pytest.fixture -async def fastapi_app(app_settings: ApplicationSettings) -> AsyncIterator[FastAPI]: +async def mock_fastapi_app( + mock_celery_app: None, app_settings: ApplicationSettings +) -> AsyncIterator[FastAPI]: app: FastAPI = create_app(app_settings) async with LifespanManager(app, startup_timeout=30, shutdown_timeout=30): @@ -111,7 +110,6 @@ def mock_celery_app(mocker: MockerFixture, celery_config: dict[str, Any]) -> Cel async def mock_celery_worker( app_environment: EnvVarsDict, celery_app: Celery, - fastapi_app: FastAPI, monkeypatch: pytest.MonkeyPatch, ) -> AsyncIterator[Any]: monkeypatch.setenv("NOTIFICATIONS_WORKER_MODE", "true") @@ -121,7 +119,7 @@ def _on_worker_init_wrapper(sender: WorkController, **_kwargs): assert app_settings.NOTIFICATIONS_CELERY # nosec return partial( on_worker_init, - FastAPIAppServer(app=fastapi_app), + FastAPIAppServer(app=create_app(app_settings)), app_settings.NOTIFICATIONS_CELERY, )(sender, **_kwargs) diff --git a/services/notifications/tests/unit/test_send_email_tasks.py b/services/notifications/tests/unit/test_send_email_tasks.py index 1bad4fbfde32..2803f7d5f6cd 100644 --- a/services/notifications/tests/unit/test_send_email_tasks.py +++ b/services/notifications/tests/unit/test_send_email_tasks.py @@ -4,6 +4,9 @@ from models_library.rpc.notifications.channels import EmailAddress, EmailChannel from models_library.rpc.notifications.events import ( AccountRequestedEvent, + ProductData, + ProductUIData, + UserData, ) from servicelib.rabbitmq import RabbitMQRPCClient from servicelib.rabbitmq.rpc_interfaces.notifications import ( @@ -18,9 +21,8 @@ @pytest.mark.usefixtures( - "mock_celery_app", "mock_celery_worker", - "fastapi_app", + "mock_fastapi_app", ) async def test_account_requested( notifications_rabbitmq_rpc_client: RabbitMQRPCClient, @@ -32,9 +34,25 @@ async def test_account_requested( notifications_rabbitmq_rpc_client, notification=Notification( event=AccountRequestedEvent( - first_name=faker.first_name(), - last_name=faker.last_name(), - email=user_email, + user=UserData( + username=faker.user_name(), + first_name=faker.first_name(), + last_name=faker.last_name(), + email=user_email, + ), + product=ProductData( + product_name=faker.company(), + display_name=faker.company(), + vendor_display_inline=faker.company_suffix(), + support_email=faker.email(), + homepage_url=faker.url(), + ui=ProductUIData( + project_alias=faker.word(), + logo_url=faker.image_url(), + strong_color=faker.color_name(), + ), + ), + host=faker.url(), ), channel=EmailChannel( from_addr=EmailAddress(addr_spec=faker.email()), From 8d416f7a8da858a7b92af5b908af08a8ed1f755a Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 14 Jul 2025 09:29:00 +0200 Subject: [PATCH 60/69] tests: disable tracing --- services/notifications/tests/unit/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/services/notifications/tests/unit/conftest.py b/services/notifications/tests/unit/conftest.py index c0d104369503..1a4df419d8b9 100644 --- a/services/notifications/tests/unit/conftest.py +++ b/services/notifications/tests/unit/conftest.py @@ -46,6 +46,7 @@ def app_environment( { **mock_environment, **postgres_env_vars_dict, + "NOTIFICATIONS_TRACING": "null", "RABBIT_HOST": rabbit_service.RABBIT_HOST, "RABBIT_PASSWORD": rabbit_service.RABBIT_PASSWORD.get_secret_value(), "RABBIT_PORT": f"{rabbit_service.RABBIT_PORT}", From 999f337e987e99f8080565a88470c072521a5c6b Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 14 Jul 2025 16:05:08 +0200 Subject: [PATCH 61/69] feat: improve models --- .../notifications/events/_account_events.py | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/packages/models-library/src/models_library/rpc/notifications/events/_account_events.py b/packages/models-library/src/models_library/rpc/notifications/events/_account_events.py index c7df26e04c09..804bb311e925 100644 --- a/packages/models-library/src/models_library/rpc/notifications/events/_account_events.py +++ b/packages/models-library/src/models_library/rpc/notifications/events/_account_events.py @@ -1,8 +1,8 @@ -from typing import Literal +from typing import Any, Literal from common_library.pydantic_basic_types import NotEmptyStr from models_library.products import ProductName -from pydantic import BaseModel, EmailStr +from pydantic import BaseModel, EmailStr, HttpUrl class UserData(BaseModel): @@ -33,19 +33,26 @@ class ProductData(BaseModel): ui: ProductUIData -class AccountRequestedEvent(BaseModel): - type: Literal["account_requested"] = "account_requested" - +class BaseAccountEvent(BaseModel): user: UserData product: ProductData - host: str -class AccountApprovedEvent(BaseModel): +class AccountRequestedEvent(BaseAccountEvent): + type: Literal["account_requested"] = "account_requested" + + host: HttpUrl + product_info: dict[str, Any] = {} + request_form: dict[str, Any] = {} + + +class AccountApprovedEvent(BaseAccountEvent): type: Literal["account_approved"] = "account_approved" + link: HttpUrl + -class AccountRejectedEvent(BaseModel): +class AccountRejectedEvent(BaseAccountEvent): type: Literal["account_rejected"] = "account_rejected" reason: NotEmptyStr From 5d9fc5cf75f728b71b98fc42627450c6ce7415ac Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 14 Jul 2025 16:05:33 +0200 Subject: [PATCH 62/69] feat: add dumps util --- .../src/notifications_library/_render.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/packages/notifications-library/src/notifications_library/_render.py b/packages/notifications-library/src/notifications_library/_render.py index 4be7ba3225c1..f6d40e3a9cfe 100644 --- a/packages/notifications-library/src/notifications_library/_render.py +++ b/packages/notifications-library/src/notifications_library/_render.py @@ -1,26 +1,39 @@ +import functools +import json import logging from pathlib import Path +from typing import Any import notifications_library +from common_library.json_serialization import pydantic_encoder from jinja2 import Environment, FileSystemLoader, PackageLoader, select_autoescape +from models_library.utils._original_fastapi_encoders import jsonable_encoder _logger = logging.getLogger(__name__) +def _safe_json_dumps(obj: Any, **kwargs): + return json.dumps(jsonable_encoder(obj), default=pydantic_encoder, **kwargs) + + def create_render_environment_from_notifications_library(**kwargs) -> Environment: - return Environment( + env = Environment( loader=PackageLoader(notifications_library.__name__, "templates"), autoescape=select_autoescape(["html", "xml"]), **kwargs ) + env.globals["dumps"] = functools.partial(_safe_json_dumps, indent=1) + return env def create_render_environment_from_folder(top_dir: Path) -> Environment: assert top_dir.exists() # nosec assert top_dir.is_dir() # nosec - return Environment( + env = Environment( loader=FileSystemLoader(top_dir), autoescape=select_autoescape( ["html", "xml"], ), ) + env.globals["dumps"] = functools.partial(_safe_json_dumps, indent=1) + return env From 906c67fe54e7c68027d3e6527067c2ee5651bc78 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 14 Jul 2025 16:09:01 +0200 Subject: [PATCH 63/69] tests: add new test --- .../tests/unit/test_send_email_tasks.py | 50 ++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/services/notifications/tests/unit/test_send_email_tasks.py b/services/notifications/tests/unit/test_send_email_tasks.py index 2803f7d5f6cd..ea0f4d38f12f 100644 --- a/services/notifications/tests/unit/test_send_email_tasks.py +++ b/services/notifications/tests/unit/test_send_email_tasks.py @@ -3,11 +3,13 @@ from models_library.rpc.notifications import Notification from models_library.rpc.notifications.channels import EmailAddress, EmailChannel from models_library.rpc.notifications.events import ( + AccountApprovedEvent, AccountRequestedEvent, ProductData, ProductUIData, UserData, ) +from pydantic import HttpUrl from servicelib.rabbitmq import RabbitMQRPCClient from servicelib.rabbitmq.rpc_interfaces.notifications import ( send_notification, @@ -52,7 +54,53 @@ async def test_account_requested( strong_color=faker.color_name(), ), ), - host=faker.url(), + host=HttpUrl(faker.url()), + ), + channel=EmailChannel( + from_addr=EmailAddress(addr_spec=faker.email()), + to_addr=EmailAddress( + addr_spec=user_email, + ), + ), + ), + ) + + # TODO: wait for the email to be sent and check + + +@pytest.mark.usefixtures( + "mock_celery_worker", + "mock_fastapi_app", +) +async def test_account_approved( + notifications_rabbitmq_rpc_client: RabbitMQRPCClient, + faker: Faker, +): + user_email = faker.email() + + await send_notification( + notifications_rabbitmq_rpc_client, + notification=Notification( + event=AccountApprovedEvent( + user=UserData( + username=faker.user_name(), + first_name=faker.first_name(), + last_name=faker.last_name(), + email=user_email, + ), + product=ProductData( + product_name=faker.company(), + display_name=faker.company(), + vendor_display_inline=faker.company_suffix(), + support_email=faker.email(), + homepage_url=faker.url(), + ui=ProductUIData( + project_alias=faker.word(), + logo_url=faker.image_url(), + strong_color=faker.color(), + ), + ), + link=HttpUrl(faker.url()), ), channel=EmailChannel( from_addr=EmailAddress(addr_spec=faker.email()), From eebbeb293f9dad7cd57e7e3c50b413264251d627 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 14 Jul 2025 16:17:25 +0200 Subject: [PATCH 64/69] fix: add missing ipinfo --- .../models_library/rpc/notifications/events/_account_events.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/models-library/src/models_library/rpc/notifications/events/_account_events.py b/packages/models-library/src/models_library/rpc/notifications/events/_account_events.py index 804bb311e925..e509e906719d 100644 --- a/packages/models-library/src/models_library/rpc/notifications/events/_account_events.py +++ b/packages/models-library/src/models_library/rpc/notifications/events/_account_events.py @@ -42,8 +42,11 @@ class AccountRequestedEvent(BaseAccountEvent): type: Literal["account_requested"] = "account_requested" host: HttpUrl + + # NOTE: following are kept for backward compatibility product_info: dict[str, Any] = {} request_form: dict[str, Any] = {} + ipinfo: dict[str, Any] = {} class AccountApprovedEvent(BaseAccountEvent): From 1a3eca33e9af9e7ffb1382b399802796fe31f646 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 14 Jul 2025 16:43:48 +0200 Subject: [PATCH 65/69] fix: address display name default --- .../rpc/notifications/channels/_email_channel.py | 2 +- services/notifications/tests/unit/conftest.py | 10 ++++++++++ .../notifications/tests/unit/test_send_email_tasks.py | 9 ++++++++- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/packages/models-library/src/models_library/rpc/notifications/channels/_email_channel.py b/packages/models-library/src/models_library/rpc/notifications/channels/_email_channel.py index c9116545a558..e97b9e14ceba 100644 --- a/packages/models-library/src/models_library/rpc/notifications/channels/_email_channel.py +++ b/packages/models-library/src/models_library/rpc/notifications/channels/_email_channel.py @@ -4,7 +4,7 @@ class EmailAddress(BaseModel): - display_name: str | None = None + display_name: str = "" addr_spec: EmailStr diff --git a/services/notifications/tests/unit/conftest.py b/services/notifications/tests/unit/conftest.py index 1a4df419d8b9..45d08d6235c4 100644 --- a/services/notifications/tests/unit/conftest.py +++ b/services/notifications/tests/unit/conftest.py @@ -14,6 +14,7 @@ from celery.signals import worker_init, worker_shutdown from celery.worker.worker import WorkController from celery_library.signals import on_worker_init, on_worker_shutdown +from faker import Faker from fastapi import FastAPI from fastapi.testclient import TestClient from pytest_mock import MockerFixture @@ -152,3 +153,12 @@ async def notifications_rabbitmq_rpc_client( @pytest.fixture def test_client(fastapi_app: FastAPI) -> TestClient: return TestClient(fastapi_app) + + +@pytest.fixture +def fake_ipinfo(faker: Faker) -> dict[str, Any]: + return { + "x-real-ip": faker.ipv4(), + "x-forwarded-for": faker.ipv4(), + "peername": faker.ipv4(), + } diff --git a/services/notifications/tests/unit/test_send_email_tasks.py b/services/notifications/tests/unit/test_send_email_tasks.py index ea0f4d38f12f..538be29c2c78 100644 --- a/services/notifications/tests/unit/test_send_email_tasks.py +++ b/services/notifications/tests/unit/test_send_email_tasks.py @@ -1,3 +1,5 @@ +from typing import Any + import pytest from faker import Faker from models_library.rpc.notifications import Notification @@ -28,6 +30,7 @@ ) async def test_account_requested( notifications_rabbitmq_rpc_client: RabbitMQRPCClient, + fake_ipinfo: dict[str, Any], faker: Faker, ): user_email = faker.email() @@ -55,6 +58,7 @@ async def test_account_requested( ), ), host=HttpUrl(faker.url()), + ipinfo=fake_ipinfo, ), channel=EmailChannel( from_addr=EmailAddress(addr_spec=faker.email()), @@ -103,8 +107,11 @@ async def test_account_approved( link=HttpUrl(faker.url()), ), channel=EmailChannel( - from_addr=EmailAddress(addr_spec=faker.email()), + from_addr=EmailAddress( + display_name=faker.name(), addr_spec=faker.email() + ), to_addr=EmailAddress( + display_name=faker.name(), addr_spec=user_email, ), ), From 5f03a50773940afa23833c40ec7bc450da866170 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 15 Jul 2025 11:19:09 +0200 Subject: [PATCH 66/69] fix: authors --- services/notifications/setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/services/notifications/setup.py b/services/notifications/setup.py index c50365f3e259..23398f671226 100755 --- a/services/notifications/setup.py +++ b/services/notifications/setup.py @@ -23,7 +23,10 @@ def read_reqs(reqs_path: Path) -> set[str]: NAME = "simcore-service-notifications" VERSION = (CURRENT_DIR / "VERSION").read_text().strip() -AUTHORS = ("Andrei Neagu (GitHK)",) +AUTHORS = ( + "Giancarlo Romeo (giancarloromeo)", + "Andrei Neagu (GitHK)", +) DESCRIPTION = "Service used for sending notifications to users via different channels" PROD_REQUIREMENTS = tuple( From ef872a3c5f61c9af52f06b3423086b85ebf53032 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 15 Jul 2025 16:06:43 +0200 Subject: [PATCH 67/69] fix: typecheck --- .../modules/celery/worker_main.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/services/notifications/src/simcore_service_notifications/modules/celery/worker_main.py b/services/notifications/src/simcore_service_notifications/modules/celery/worker_main.py index 904313a94af2..467b08c53c09 100644 --- a/services/notifications/src/simcore_service_notifications/modules/celery/worker_main.py +++ b/services/notifications/src/simcore_service_notifications/modules/celery/worker_main.py @@ -1,4 +1,3 @@ -import logging from functools import partial from celery.signals import worker_init, worker_shutdown # type: ignore[import-untyped] @@ -8,7 +7,7 @@ on_worker_shutdown, ) from servicelib.fastapi.celery.app_server import FastAPIAppServer -from servicelib.logging_utils import config_all_loggers +from servicelib.logging_utils import setup_loggers from ...core.application import create_app from ...core.settings import ApplicationSettings @@ -16,12 +15,12 @@ _settings = ApplicationSettings.create_from_envs() -logging.basicConfig(level=_settings.log_level) # NOSONAR -logging.root.setLevel(_settings.log_level) -config_all_loggers( +setup_loggers( log_format_local_dev_enabled=_settings.NOTIFICATIONS_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=_settings.NOTIFICATIONS_LOG_FILTER_MAPPING, tracing_settings=_settings.NOTIFICATIONS_TRACING, + log_base_level=_settings.log_level, + noisy_loggers=None, ) From 77197a8fe0c59c836d6e3945b24840b879b0d187 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 25 Aug 2025 15:17:43 +0200 Subject: [PATCH 68/69] fix: upgrade click --- services/notifications/requirements/_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/notifications/requirements/_test.txt b/services/notifications/requirements/_test.txt index 407077bfacaf..6c6b4da278ef 100644 --- a/services/notifications/requirements/_test.txt +++ b/services/notifications/requirements/_test.txt @@ -29,7 +29,7 @@ charset-normalizer==3.4.1 # via # -c requirements/_base.txt # requests -click==8.1.8 +click==8.2.1 # via # -c requirements/_base.txt # celery From 919dfeea9891104c91d0b7e36ebd3e8bc7f04c98 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 26 Aug 2025 16:41:38 +0200 Subject: [PATCH 69/69] mionr --- .../rabbitmq/rpc_interfaces/notifications/_notifications.py | 2 +- .../src/simcore_service_notifications/core/events.py | 1 + .../modules/celery/_email_tasks.py | 5 +++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/_notifications.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/_notifications.py index f04d3c8effa9..a95df0977138 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/_notifications.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/notifications/_notifications.py @@ -17,7 +17,7 @@ async def send_notification( ) -> None: await rabbitmq_rpc_client.request( NOTIFICATIONS_RPC_NAMESPACE, - TypeAdapter(RPCMethodName).validate_python("send_notification"), + TypeAdapter(RPCMethodName).validate_python(send_notification.__name__), timeout_s=_DEFAULT_TIMEOUT_S, notification=notification, ) diff --git a/services/notifications/src/simcore_service_notifications/core/events.py b/services/notifications/src/simcore_service_notifications/core/events.py index 0d25e5ddb5d7..ac73d999af98 100644 --- a/services/notifications/src/simcore_service_notifications/core/events.py +++ b/services/notifications/src/simcore_service_notifications/core/events.py @@ -47,6 +47,7 @@ def create_app_lifespan( app_lifespan.add(_settings_lifespan) # - postgres + # NOTE: for now we will remove app_lifespan.add(postgres_database_lifespan) app_lifespan.add(postgres_lifespan) diff --git a/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py b/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py index a7fa37872051..25aebfe5b289 100644 --- a/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py +++ b/services/notifications/src/simcore_service_notifications/modules/celery/_email_tasks.py @@ -26,6 +26,11 @@ async def send_email_notification( notification: Notification, ) -> None: _ = task, task_id + + # + # NOTE: task can be used to provide progress + # + assert isinstance(notification.channel, EmailChannel) # nosec _logger.info("Sending email notification to %s", notification.channel.to_addr)