From e8c4c8c1da76fc177e227aa35cb4819b52866727 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 26 Mar 2025 14:45:15 +0100 Subject: [PATCH 01/58] unit tests --- .github/workflows/ci-testing-deploy.yml | 46 +++++++++++++++++++++++ ci/github/unit-testing/notifications.bash | 43 +++++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100755 ci/github/unit-testing/notifications.bash diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml index ad6eacf353b9..d74466c4ba20 100644 --- a/.github/workflows/ci-testing-deploy.yml +++ b/.github/workflows/ci-testing-deploy.yml @@ -602,6 +602,51 @@ jobs: with: flags: unittests #optional + unit-test-notifications: + needs: changes + if: ${{ needs.changes.outputs.notifications == 'true' || github.event_name == 'push' }} + timeout-minutes: 18 # if this timeout gets too small, then split the tests + name: "[unit] notifications" + runs-on: ${{ matrix.os }} + strategy: + matrix: + python: ["3.11"] + os: [ubuntu-24.04] + fail-fast: false + steps: + - uses: actions/checkout@v4 + - name: setup docker buildx + id: buildx + uses: docker/setup-buildx-action@v3 + with: + driver: docker-container + - name: setup python environment + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: install rclone + run: sudo ./ci/github/helpers/install_rclone.bash + - name: install uv + uses: astral-sh/setup-uv@v5 + with: + version: "0.5.x" + enable-cache: false + cache-dependency-glob: "**/notifications/requirements/ci.txt" + - name: show system version + run: ./ci/helpers/show_system_versions.bash + - name: install + run: ./ci/github/unit-testing/notifications.bash install + - name: typecheck + run: ./ci/github/unit-testing/notifications.bash typecheck + - name: test + if: ${{ !cancelled() }} + run: ./ci/github/unit-testing/notifications.bash test + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + flags: unittests #optional unit-test-api: needs: changes @@ -1839,6 +1884,7 @@ jobs: unit-test-common-library, unit-test-notifications-library, unit-test-payments, + unit-test-notifications, unit-test-dynamic-scheduler, unit-test-postgres-database, unit-test-python-linting, diff --git a/ci/github/unit-testing/notifications.bash b/ci/github/unit-testing/notifications.bash new file mode 100755 index 000000000000..4f78013c19e6 --- /dev/null +++ b/ci/github/unit-testing/notifications.bash @@ -0,0 +1,43 @@ +#!/bin/bash +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes +IFS=$'\n\t' + +install() { + make devenv + # shellcheck source=/dev/null + source .venv/bin/activate + pushd services/notifications + make install-ci + popd + uv pip list +} + +test() { + # shellcheck source=/dev/null + source .venv/bin/activate + pushd services/notifications + make test-ci-unit + popd +} + +typecheck() { + # shellcheck source=/dev/null + source .venv/bin/activate + uv pip install mypy + pushd services/notifications + make mypy + popd +} + +# Check if the function exists (bash specific) +if declare -f "$1" >/dev/null; then + # call arguments verbatim + "$@" +else + # Show a helpful error + echo "'$1' is not a known function name" >&2 + exit 1 +fi From 5033ace078788ab39096e836797ff678863916b5 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 26 Mar 2025 14:45:30 +0100 Subject: [PATCH 02/58] rpc namespace --- .../models_library/api_schemas_notifications/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 packages/models-library/src/models_library/api_schemas_notifications/__init__.py diff --git a/packages/models-library/src/models_library/api_schemas_notifications/__init__.py b/packages/models-library/src/models_library/api_schemas_notifications/__init__.py new file mode 100644 index 000000000000..dfa868dc5227 --- /dev/null +++ b/packages/models-library/src/models_library/api_schemas_notifications/__init__.py @@ -0,0 +1,9 @@ +from typing import Final + +from pydantic import TypeAdapter + +from ..rabbitmq_basic_types import RPCNamespace + +NOTIFICATIONS_RPC_NAMESPACE: Final[RPCNamespace] = TypeAdapter( + RPCNamespace +).validate_python("notifications") From b08d0a7d43fbdaaffa60adeea91e370c9f2a93fe Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 26 Mar 2025 14:45:46 +0100 Subject: [PATCH 03/58] added new service --- services/notifications/Dockerfile | 184 +++++++ services/notifications/Makefile | 5 + services/notifications/README.md | 9 + services/notifications/VERSION | 1 + services/notifications/docker/boot.sh | 65 +++ services/notifications/docker/entrypoint.sh | 71 +++ services/notifications/docker/healthcheck.py | 40 ++ services/notifications/requirements/Makefile | 10 + services/notifications/requirements/_base.in | 20 + services/notifications/requirements/_base.txt | 471 ++++++++++++++++++ services/notifications/requirements/_test.in | 24 + services/notifications/requirements/_test.txt | 93 ++++ services/notifications/requirements/_tools.in | 5 + .../notifications/requirements/_tools.txt | 82 +++ services/notifications/requirements/ci.txt | 22 + .../requirements/constraints.txt | 0 services/notifications/requirements/dev.txt | 22 + services/notifications/requirements/prod.txt | 19 + services/notifications/setup.cfg | 15 + services/notifications/setup.py | 67 +++ .../simcore_service_notifications/__init__.py | 1 + .../simcore_service_notifications/_meta.py | 36 ++ .../api/__init__.py | 0 .../api/rest/__init__.py | 0 .../api/rest/_dependencies.py | 26 + .../api/rest/_health.py | 25 + .../api/rest/routes.py | 14 + .../api/rpc/__init__.py | 0 .../api/rpc/routes.py | 19 + .../src/simcore_service_notifications/cli.py | 14 + .../core/__init__.py | 0 .../core/application.py | 71 +++ .../core/settings.py | 71 +++ .../src/simcore_service_notifications/main.py | 3 + .../models/__init__.py | 0 .../services/__init__.py | 0 .../services/rabbitmq.py | 29 ++ services/notifications/tests/conftest.py | 29 ++ .../notifications/tests/integration/.gitkeep | 0 services/notifications/tests/unit/conftest.py | 44 ++ .../tests/unit/test_api_rest__health.py | 17 + services/notifications/tests/unit/test_cli.py | 35 ++ 42 files changed, 1659 insertions(+) create mode 100644 services/notifications/Dockerfile create mode 100644 services/notifications/Makefile create mode 100644 services/notifications/README.md create mode 100644 services/notifications/VERSION create mode 100755 services/notifications/docker/boot.sh create mode 100755 services/notifications/docker/entrypoint.sh create mode 100755 services/notifications/docker/healthcheck.py create mode 100644 services/notifications/requirements/Makefile create mode 100644 services/notifications/requirements/_base.in create mode 100644 services/notifications/requirements/_base.txt create mode 100644 services/notifications/requirements/_test.in create mode 100644 services/notifications/requirements/_test.txt create mode 100644 services/notifications/requirements/_tools.in create mode 100644 services/notifications/requirements/_tools.txt create mode 100644 services/notifications/requirements/ci.txt create mode 100644 services/notifications/requirements/constraints.txt create mode 100644 services/notifications/requirements/dev.txt create mode 100644 services/notifications/requirements/prod.txt create mode 100644 services/notifications/setup.cfg create mode 100755 services/notifications/setup.py create mode 100644 services/notifications/src/simcore_service_notifications/__init__.py create mode 100644 services/notifications/src/simcore_service_notifications/_meta.py create mode 100644 services/notifications/src/simcore_service_notifications/api/__init__.py create mode 100644 services/notifications/src/simcore_service_notifications/api/rest/__init__.py create mode 100644 services/notifications/src/simcore_service_notifications/api/rest/_dependencies.py create mode 100644 services/notifications/src/simcore_service_notifications/api/rest/_health.py create mode 100644 services/notifications/src/simcore_service_notifications/api/rest/routes.py create mode 100644 services/notifications/src/simcore_service_notifications/api/rpc/__init__.py create mode 100644 services/notifications/src/simcore_service_notifications/api/rpc/routes.py create mode 100644 services/notifications/src/simcore_service_notifications/cli.py create mode 100644 services/notifications/src/simcore_service_notifications/core/__init__.py create mode 100644 services/notifications/src/simcore_service_notifications/core/application.py create mode 100644 services/notifications/src/simcore_service_notifications/core/settings.py create mode 100644 services/notifications/src/simcore_service_notifications/main.py create mode 100644 services/notifications/src/simcore_service_notifications/models/__init__.py create mode 100644 services/notifications/src/simcore_service_notifications/services/__init__.py create mode 100644 services/notifications/src/simcore_service_notifications/services/rabbitmq.py create mode 100644 services/notifications/tests/conftest.py create mode 100644 services/notifications/tests/integration/.gitkeep create mode 100644 services/notifications/tests/unit/conftest.py create mode 100644 services/notifications/tests/unit/test_api_rest__health.py create mode 100644 services/notifications/tests/unit/test_cli.py diff --git a/services/notifications/Dockerfile b/services/notifications/Dockerfile new file mode 100644 index 000000000000..0a9ecd6d35db --- /dev/null +++ b/services/notifications/Dockerfile @@ -0,0 +1,184 @@ +# syntax=docker/dockerfile:1 + +# Define arguments in the global scope +ARG PYTHON_VERSION="3.11.9" +ARG UV_VERSION="0.5" + +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build +# we docker image is built based on debian +FROM python:${PYTHON_VERSION}-slim-bookworm AS base + + +# +# USAGE: +# cd sercices/notifications +# docker build -f Dockerfile -t notifications:prod --target production ../../ +# docker run notifications:prod +# +# REQUIRED: context expected at ``osparc-simcore/`` folder because we need access to osparc-simcore/packages + +LABEL maintainer=GitHK + +# for docker apt caching to work this needs to be added: [https://vsupalov.com/buildkit-cache-mount-dockerfile/] +RUN rm -f /etc/apt/apt.conf.d/docker-clean && \ + echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ + set -eux && \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + gosu \ + curl \ + && apt-get clean -y \ + # verify that the binary works + && gosu nobody true + +# simcore-user uid=8004(scu) gid=8004(scu) groups=8004(scu) +ENV SC_USER_ID=8004 \ + SC_USER_NAME=scu \ + SC_BUILD_TARGET=base \ + SC_BOOT_MODE=default + +RUN adduser \ + --uid ${SC_USER_ID} \ + --disabled-password \ + --gecos "" \ + --shell /bin/sh \ + --home /home/${SC_USER_NAME} \ + ${SC_USER_NAME} + + +# Sets utf-8 encoding for Python et al +ENV LANG=C.UTF-8 + +# Turns off writing .pyc files; superfluous on an ephemeral container. +ENV PYTHONDONTWRITEBYTECODE=1 \ + VIRTUAL_ENV=/home/scu/.venv + +# Ensures that the python and pip executables used in the image will be +# those from our virtualenv. +ENV PATH="${VIRTUAL_ENV}/bin:$PATH" + +# rclone installation +ARG TARGETARCH +ENV TARGETARCH=${TARGETARCH} +RUN \ + --mount=type=bind,source=scripts/install_rclone.bash,target=/tmp/install_rclone.bash \ + ./tmp/install_rclone.bash + +# -------------------------- Build stage ------------------- +# Installs build/package management tools and third party dependencies +# +# + /build WORKDIR +# +FROM base AS build + +ENV SC_BUILD_TARGET=build + +RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ + set -eux \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential + +# install UV https://docs.astral.sh/uv/guides/integration/docker/#installing-uv +COPY --from=uv_build /uv /uvx /bin/ + +# NOTE: python virtualenv is used here such that installed +# packages may be moved to production image easily by copying the venv +RUN uv venv "${VIRTUAL_ENV}" + +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --upgrade \ + wheel \ + setuptools + +WORKDIR /build + +# install base 3rd party dependencies + + + +# --------------------------Prod-depends-only stage ------------------- +# This stage is for production only dependencies that get partially wiped out afterwards (final docker image concerns) +# +# + /build +# + services/notifications [scu:scu] WORKDIR +# +FROM build AS prod-only-deps + +ENV SC_BUILD_TARGET prod-only-deps + +WORKDIR /build/services/notifications + +RUN \ + --mount=type=bind,source=packages,target=/build/packages,rw \ + --mount=type=bind,source=services/notifications,target=/build/services/notifications,rw \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip sync \ + requirements/prod.txt \ + && uv pip list + + +# --------------------------Production stage ------------------- +# Final cleanup up to reduce image size and startup setup +# Runs as scu (non-root user) +# +# + /home/scu $HOME = WORKDIR +# + services/notifications [scu:scu] +# +FROM base AS production + +ENV SC_BUILD_TARGET=production \ + SC_BOOT_MODE=production + +ENV PYTHONOPTIMIZE=TRUE +# https://docs.astral.sh/uv/guides/integration/docker/#compiling-bytecode +ENV UV_COMPILE_BYTECODE=1 + +WORKDIR /home/scu + +# ensure home folder is read/writable for user scu +RUN chown -R scu /home/scu + +# Starting from clean base image, copies pre-installed virtualenv from prod-only-deps +COPY --chown=scu:scu --from=prod-only-deps ${VIRTUAL_ENV} ${VIRTUAL_ENV} + +# Copies booting scripts +COPY --chown=scu:scu services/notifications/docker services/notifications/docker +RUN chmod +x services/notifications/docker/*.sh + + +HEALTHCHECK --interval=30s \ + --timeout=20s \ + --start-period=30s \ + --retries=3 \ + CMD ["python3", "services/notifications/docker/healthcheck.py", "http://localhost:8000/health"] + +EXPOSE 8000 + +ENTRYPOINT [ "/bin/sh", "services/notifications/docker/entrypoint.sh" ] +CMD ["/bin/sh", "services/notifications/docker/boot.sh"] + + +# --------------------------Development stage ------------------- +# Source code accessible in host but runs in container +# Runs as myu with same gid/uid as host +# Placed at the end to speed-up the build if images targeting production +# +# + /devel WORKDIR +# + services (mounted volume) +# +FROM build AS development + +ENV SC_BUILD_TARGET=development \ + SC_DEVEL_MOUNT=/devel/services/notifications + +WORKDIR /devel + +RUN chown -R scu:scu "${VIRTUAL_ENV}" + +EXPOSE 8000 +EXPOSE 3000 + +ENTRYPOINT ["/bin/sh", "services/notifications/docker/entrypoint.sh"] +CMD ["/bin/sh", "services/notifications/docker/boot.sh"] diff --git a/services/notifications/Makefile b/services/notifications/Makefile new file mode 100644 index 000000000000..8c208ed2a314 --- /dev/null +++ b/services/notifications/Makefile @@ -0,0 +1,5 @@ +# +# DEVELOPMENT recipes for notifications +# +include ../../scripts/common.Makefile +include ../../scripts/common-service.Makefile diff --git a/services/notifications/README.md b/services/notifications/README.md new file mode 100644 index 000000000000..b678722e9bf0 --- /dev/null +++ b/services/notifications/README.md @@ -0,0 +1,9 @@ +# notifications + + +To develop this project, just + +```cmd +make help + +``` diff --git a/services/notifications/VERSION b/services/notifications/VERSION new file mode 100644 index 000000000000..8acdd82b765e --- /dev/null +++ b/services/notifications/VERSION @@ -0,0 +1 @@ +0.0.1 diff --git a/services/notifications/docker/boot.sh b/services/notifications/docker/boot.sh new file mode 100755 index 000000000000..bd14ca192210 --- /dev/null +++ b/services/notifications/docker/boot.sh @@ -0,0 +1,65 @@ +#!/bin/sh +set -o errexit +set -o nounset + +IFS=$(printf '\n\t') + +INFO="INFO: [$(basename "$0")] " + +echo "$INFO" "Booting in ${SC_BOOT_MODE} mode ..." +echo "$INFO" "User :$(id "$(whoami)")" +echo "$INFO" "Workdir : $(pwd)" + +# +# DEVELOPMENT MODE +# +# - prints environ info +# - installs requirements in mounted volume +# +if [ "${SC_BUILD_TARGET}" = "development" ]; then + echo "$INFO" "Environment :" + printenv | sed 's/=/: /' | sed 's/^/ /' | sort + echo "$INFO" "Python :" + python --version | sed 's/^/ /' + command -v python | sed 's/^/ /' + + cd services/notifications + uv pip --quiet sync requirements/dev.txt + cd - + echo "$INFO" "PIP :" + uv pip list +fi + +if [ "${SC_BOOT_MODE}" = "debug" ]; then + # NOTE: production does NOT pre-installs debugpy + uv pip install debugpy +fi + +# +# RUNNING application +# +APP_LOG_LEVEL=${LOGLEVEL:-${LOG_LEVEL:-${LOGLEVEL:-INFO}}} +NOTIFICATIONS_SERVER_REMOTE_DEBUG_PORT=3000 +SERVER_LOG_LEVEL=$(echo "${APP_LOG_LEVEL}" | tr '[:upper:]' '[:lower:]') +echo "$INFO" "Log-level app/server: $APP_LOG_LEVEL/$SERVER_LOG_LEVEL" + +if [ "${SC_BOOT_MODE}" = "debug" ]; then + reload_dir_packages=$(find /devel/packages -maxdepth 3 -type d -path "*/src/*" ! -path "*.*" -exec echo '--reload-dir {} \' \;) + + exec sh -c " + cd services/notifications/src/simcore_service_notifications && \ + python -m debugpy --listen 0.0.0.0:${NOTIFICATIONS_SERVER_REMOTE_DEBUG_PORT} -m uvicorn main:the_app \ + --host 0.0.0.0 \ + --port 8000 \ + --reload \ + $reload_dir_packages + --reload-dir . \ + --log-level \"${SERVER_LOG_LEVEL}\" + " +else + exec uvicorn simcore_service_notifications.main:the_app \ + --host 0.0.0.0 \ + --port 8000 \ + --log-level "${SERVER_LOG_LEVEL}" \ + --no-access-log +fi diff --git a/services/notifications/docker/entrypoint.sh b/services/notifications/docker/entrypoint.sh new file mode 100755 index 000000000000..1568d6affdce --- /dev/null +++ b/services/notifications/docker/entrypoint.sh @@ -0,0 +1,71 @@ +#!/bin/sh +# +# - Executes *inside* of the container upon start as --user [default root] +# - Notice that the container *starts* as --user [default root] but +# *runs* as non-root user [scu] +# +set -o errexit +set -o nounset + +IFS=$(printf '\n\t') + +INFO="INFO: [$(basename "$0")] " +WARNING="WARNING: [$(basename "$0")] " +ERROR="ERROR: [$(basename "$0")] " + +echo "$INFO" "Entrypoint for stage ${SC_BUILD_TARGET} ..." +echo "$INFO" "User :$(id "$(whoami)")" +echo "$INFO" "Workdir : $(pwd)" +echo "$INFO" "User : $(id scu)" +echo "$INFO" "python : $(command -v python)" +echo "$INFO" "pip : $(command -v pip)" + +# +# DEVELOPMENT MODE +# - expects docker run ... -v $(pwd):$SC_DEVEL_MOUNT +# - mounts source folders +# - deduces host's uid/gip and assigns to user within docker +# +if [ "${SC_BUILD_TARGET}" = "development" ]; then + echo "$INFO" "development mode detected..." + stat "${SC_DEVEL_MOUNT}" >/dev/null 2>&1 || + (echo "$ERROR" "You must mount '$SC_DEVEL_MOUNT' to deduce user and group ids" && exit 1) + + echo "$INFO" "setting correct user id/group id..." + HOST_USERID=$(stat --format=%u "${SC_DEVEL_MOUNT}") + HOST_GROUPID=$(stat --format=%g "${SC_DEVEL_MOUNT}") + CONT_GROUPNAME=$(getent group "${HOST_GROUPID}" | cut --delimiter=: --fields=1) + if [ "$HOST_USERID" -eq 0 ]; then + echo "$WARNING" "Folder mounted owned by root user... adding $SC_USER_NAME to root..." + adduser "$SC_USER_NAME" root + else + echo "$INFO" "Folder mounted owned by user $HOST_USERID:$HOST_GROUPID-'$CONT_GROUPNAME'..." + # take host's credentials in $SC_USER_NAME + if [ -z "$CONT_GROUPNAME" ]; then + echo "$WARNING" "Creating new group grp$SC_USER_NAME" + CONT_GROUPNAME=grp$SC_USER_NAME + addgroup --gid "$HOST_GROUPID" "$CONT_GROUPNAME" + else + echo "$INFO" "group already exists" + fi + echo "$INFO" "Adding $SC_USER_NAME to group $CONT_GROUPNAME..." + adduser "$SC_USER_NAME" "$CONT_GROUPNAME" + + echo "$WARNING" "Changing ownership [this could take some time]" + echo "$INFO" "Changing $SC_USER_NAME:$SC_USER_NAME ($SC_USER_ID:$SC_USER_ID) to $SC_USER_NAME:$CONT_GROUPNAME ($HOST_USERID:$HOST_GROUPID)" + usermod --uid "$HOST_USERID" --gid "$HOST_GROUPID" "$SC_USER_NAME" + + echo "$INFO" "Changing group properties of files around from $SC_USER_ID to group $CONT_GROUPNAME" + find / -path /proc -prune -o -group "$SC_USER_ID" -exec chgrp --no-dereference "$CONT_GROUPNAME" {} \; + # change user property of files already around + echo "$INFO" "Changing ownership properties of files around from $SC_USER_ID to group $CONT_GROUPNAME" + find / -path /proc -prune -o -user "$SC_USER_ID" -exec chown --no-dereference "$SC_USER_NAME" {} \; + fi +fi + + +echo "$INFO Starting $* ..." +echo " $SC_USER_NAME rights : $(id "$SC_USER_NAME")" +echo " local dir : $(ls -al)" + +exec gosu "$SC_USER_NAME" "$@" diff --git a/services/notifications/docker/healthcheck.py b/services/notifications/docker/healthcheck.py new file mode 100755 index 000000000000..9e3f3274a292 --- /dev/null +++ b/services/notifications/docker/healthcheck.py @@ -0,0 +1,40 @@ +#!/bin/python +""" Healthcheck script to run inside docker + +Example of usage in a Dockerfile +``` + COPY --chown=scu:scu docker/healthcheck.py docker/healthcheck.py + HEALTHCHECK --interval=30s \ + --timeout=30s \ + --start-period=1s \ + --retries=3 \ + CMD python3 docker/healthcheck.py http://localhost:8000/ +``` + +Q&A: + 1. why not to use curl instead of a python script? + - SEE https://blog.sixeyed.com/docker-healthchecks-why-not-to-use-curl-or-iwr/ +""" +import os +import sys +from urllib.request import urlopen + +SUCCESS, UNHEALTHY = 0, 1 + +# Disabled if boots with debugger (e.g. debug, pdb-debug, debug-ptvsd, debugpy, etc) +ok = "debug" in os.environ.get("SC_BOOT_MODE", "").lower() + +# Queries host +# pylint: disable=consider-using-with +ok = ( + ok + or urlopen( + "{host}{baseurl}".format( + host=sys.argv[1], baseurl=os.environ.get("SIMCORE_NODE_BASEPATH", "") + ) # adds a base-path if defined in environ + ).getcode() + == 200 +) + + +sys.exit(SUCCESS if ok else UNHEALTHY) diff --git a/services/notifications/requirements/Makefile b/services/notifications/requirements/Makefile new file mode 100644 index 000000000000..e1319af9d7f2 --- /dev/null +++ b/services/notifications/requirements/Makefile @@ -0,0 +1,10 @@ +# +# Targets to pip-compile requirements +# +include ../../../requirements/base.Makefile + +# Add here any extra explicit dependency: e.g. _migration.txt: _base.txt + +_base.in: constraints.txt +_test.in: constraints.txt +_tools.in: constraints.txt diff --git a/services/notifications/requirements/_base.in b/services/notifications/requirements/_base.in new file mode 100644 index 000000000000..91752463674b --- /dev/null +++ b/services/notifications/requirements/_base.in @@ -0,0 +1,20 @@ +# +# Specifies third-party dependencies for 'services/notifications/src' +# +# NOTE: ALL version constraints MUST be commented +--constraint ../../../requirements/constraints.txt +--constraint ./constraints.txt + +# intra-repo required dependencies +--requirement ../../../packages/common-library/requirements/_base.in +--requirement ../../../packages/models-library/requirements/_base.in +--requirement ../../../packages/settings-library/requirements/_base.in +# service-library[fastapi] +--requirement ../../../packages/service-library/requirements/_base.in +--requirement ../../../packages/service-library/requirements/_fastapi.in + + +fastapi +packaging +pydantic +uvicorn diff --git a/services/notifications/requirements/_base.txt b/services/notifications/requirements/_base.txt new file mode 100644 index 000000000000..8f34bfb846fe --- /dev/null +++ b/services/notifications/requirements/_base.txt @@ -0,0 +1,471 @@ +aio-pika==9.5.5 + # via -r requirements/../../../packages/service-library/requirements/_base.in +aiocache==0.12.3 + # via -r requirements/../../../packages/service-library/requirements/_base.in +aiodebug==2.3.0 + # via -r requirements/../../../packages/service-library/requirements/_base.in +aiodocker==0.24.0 + # via -r requirements/../../../packages/service-library/requirements/_base.in +aiofiles==24.1.0 + # via -r requirements/../../../packages/service-library/requirements/_base.in +aiohappyeyeballs==2.6.1 + # via aiohttp +aiohttp==3.11.14 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # aiodocker +aiormq==6.8.1 + # via aio-pika +aiosignal==1.3.2 + # via aiohttp +annotated-types==0.7.0 + # via pydantic +anyio==4.9.0 + # via + # fast-depends + # faststream + # httpx + # starlette +arrow==1.3.0 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in +asgiref==3.8.1 + # via opentelemetry-instrumentation-asgi +attrs==25.3.0 + # via + # aiohttp + # jsonschema + # referencing +certifi==2025.1.31 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # httpcore + # httpx + # requests +charset-normalizer==3.4.1 + # via requests +click==8.1.8 + # via + # typer + # uvicorn +deprecated==1.2.18 + # via + # opentelemetry-api + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http + # opentelemetry-semantic-conventions +dnspython==2.7.0 + # via email-validator +email-validator==2.2.0 + # via pydantic +exceptiongroup==1.2.2 + # via aio-pika +fast-depends==2.4.12 + # via faststream +fastapi==0.115.12 + # via + # -r requirements/../../../packages/service-library/requirements/_fastapi.in + # -r requirements/_base.in + # fastapi-lifespan-manager +fastapi-lifespan-manager==0.1.4 + # via -r requirements/../../../packages/service-library/requirements/_fastapi.in +faststream==0.5.37 + # via -r requirements/../../../packages/service-library/requirements/_base.in +frozenlist==1.5.0 + # via + # aiohttp + # aiosignal +googleapis-common-protos==1.69.2 + # via + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +grpcio==1.71.0 + # via opentelemetry-exporter-otlp-proto-grpc +h11==0.14.0 + # via + # httpcore + # uvicorn +httpcore==1.0.7 + # via httpx +httpx==0.28.1 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/service-library/requirements/_fastapi.in +idna==3.10 + # via + # anyio + # email-validator + # httpx + # requests + # yarl +importlib-metadata==8.6.1 + # via opentelemetry-api +jsonschema==4.23.0 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in +jsonschema-specifications==2024.10.1 + # via jsonschema +markdown-it-py==3.0.0 + # via rich +mdurl==0.1.2 + # via markdown-it-py +multidict==6.2.0 + # via + # aiohttp + # yarl +opentelemetry-api==1.31.1 + # via + # -r requirements/../../../packages/service-library/requirements/_base.in + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http + # opentelemetry-instrumentation + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-httpx + # opentelemetry-instrumentation-logging + # opentelemetry-instrumentation-redis + # opentelemetry-instrumentation-requests + # opentelemetry-sdk + # opentelemetry-semantic-conventions +opentelemetry-exporter-otlp==1.31.1 + # via -r requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-exporter-otlp-proto-common==1.31.1 + # via + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +opentelemetry-exporter-otlp-proto-grpc==1.31.1 + # via opentelemetry-exporter-otlp +opentelemetry-exporter-otlp-proto-http==1.31.1 + # via opentelemetry-exporter-otlp +opentelemetry-instrumentation==0.52b1 + # via + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-httpx + # opentelemetry-instrumentation-logging + # opentelemetry-instrumentation-redis + # opentelemetry-instrumentation-requests +opentelemetry-instrumentation-asgi==0.52b1 + # via opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-fastapi==0.52b1 + # via -r requirements/../../../packages/service-library/requirements/_fastapi.in +opentelemetry-instrumentation-httpx==0.52b1 + # via -r requirements/../../../packages/service-library/requirements/_fastapi.in +opentelemetry-instrumentation-logging==0.52b1 + # via -r requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-instrumentation-redis==0.52b1 + # via -r requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-instrumentation-requests==0.52b1 + # via -r requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-proto==1.31.1 + # via + # opentelemetry-exporter-otlp-proto-common + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +opentelemetry-sdk==1.31.1 + # via + # -r requirements/../../../packages/service-library/requirements/_base.in + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +opentelemetry-semantic-conventions==0.52b1 + # via + # opentelemetry-instrumentation + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-httpx + # opentelemetry-instrumentation-redis + # opentelemetry-instrumentation-requests + # opentelemetry-sdk +opentelemetry-util-http==0.52b1 + # via + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-httpx + # opentelemetry-instrumentation-requests +orjson==3.10.16 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in +packaging==24.2 + # via + # -r requirements/_base.in + # opentelemetry-instrumentation +pamqp==3.3.0 + # via aiormq +prometheus-client==0.21.1 + # via + # -r requirements/../../../packages/service-library/requirements/_fastapi.in + # prometheus-fastapi-instrumentator +prometheus-fastapi-instrumentator==7.1.0 + # via -r requirements/../../../packages/service-library/requirements/_fastapi.in +propcache==0.3.1 + # via + # aiohttp + # yarl +protobuf==5.29.4 + # via + # googleapis-common-protos + # opentelemetry-proto +psutil==7.0.0 + # via -r requirements/../../../packages/service-library/requirements/_base.in +pycryptodome==3.22.0 + # via stream-zip +pydantic==2.10.6 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/_base.in + # fast-depends + # fastapi + # pydantic-extra-types + # pydantic-settings +pydantic-core==2.27.2 + # via pydantic +pydantic-extra-types==2.10.3 + # via + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in +pydantic-settings==2.7.0 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in +pygments==2.19.1 + # via rich +pyinstrument==5.0.1 + # via -r requirements/../../../packages/service-library/requirements/_base.in +python-dateutil==2.9.0.post0 + # via arrow +python-dotenv==1.1.0 + # via pydantic-settings +pyyaml==6.0.2 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/service-library/requirements/_base.in +redis==5.2.1 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/service-library/requirements/_base.in +referencing==0.35.1 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # jsonschema + # jsonschema-specifications +requests==2.32.3 + # via opentelemetry-exporter-otlp-proto-http +rich==13.9.4 + # via + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in + # typer +rpds-py==0.23.1 + # via + # jsonschema + # referencing +shellingham==1.5.4 + # via typer +six==1.17.0 + # via python-dateutil +sniffio==1.3.1 + # via anyio +starlette==0.46.1 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # fastapi + # prometheus-fastapi-instrumentator +stream-zip==0.0.83 + # via -r requirements/../../../packages/service-library/requirements/_base.in +tenacity==9.0.0 + # via -r requirements/../../../packages/service-library/requirements/_base.in +toolz==1.0.0 + # via -r requirements/../../../packages/service-library/requirements/_base.in +tqdm==4.67.1 + # via -r requirements/../../../packages/service-library/requirements/_base.in +typer==0.15.2 + # via + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in +types-python-dateutil==2.9.0.20241206 + # via arrow +typing-extensions==4.13.0 + # via + # aiodebug + # anyio + # fastapi + # faststream + # opentelemetry-sdk + # pydantic + # pydantic-core + # pydantic-extra-types + # typer +urllib3==2.3.0 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # requests +uvicorn==0.34.0 + # via + # -r requirements/../../../packages/service-library/requirements/_fastapi.in + # -r requirements/_base.in +wrapt==1.17.2 + # via + # deprecated + # opentelemetry-instrumentation + # opentelemetry-instrumentation-httpx + # opentelemetry-instrumentation-redis +yarl==1.18.3 + # via + # -r requirements/../../../packages/service-library/requirements/_base.in + # aio-pika + # aiohttp + # aiormq +zipp==3.21.0 + # via importlib-metadata diff --git a/services/notifications/requirements/_test.in b/services/notifications/requirements/_test.in new file mode 100644 index 000000000000..0269b0db4202 --- /dev/null +++ b/services/notifications/requirements/_test.in @@ -0,0 +1,24 @@ +# +# Specifies dependencies required to run 'services/notifications/test' +# both for unit and integration tests!! +# +--constraint ../../../requirements/constraints.txt +--constraint ./constraints.txt + +# Adds base AS CONSTRAINT specs, not requirement. +# - Resulting _text.txt is a frozen list of EXTRA packages for testing, besides _base.txt +# +--constraint _base.txt + + +asgi-lifespan +coverage +docker +faker +httpx +pytest +pytest-asyncio +pytest-cov +pytest-mock +pytest-runner +python-dotenv diff --git a/services/notifications/requirements/_test.txt b/services/notifications/requirements/_test.txt new file mode 100644 index 000000000000..39c817aa9271 --- /dev/null +++ b/services/notifications/requirements/_test.txt @@ -0,0 +1,93 @@ +anyio==4.9.0 + # via + # -c requirements/_base.txt + # httpx +asgi-lifespan==2.1.0 + # via -r requirements/_test.in +certifi==2025.1.31 + # via + # -c requirements/../../../requirements/constraints.txt + # -c requirements/_base.txt + # httpcore + # httpx + # requests +charset-normalizer==3.4.1 + # via + # -c requirements/_base.txt + # requests +coverage==7.7.1 + # via + # -r requirements/_test.in + # pytest-cov +docker==7.1.0 + # via -r requirements/_test.in +faker==37.1.0 + # via -r requirements/_test.in +h11==0.14.0 + # via + # -c requirements/_base.txt + # httpcore +httpcore==1.0.7 + # via + # -c requirements/_base.txt + # httpx +httpx==0.28.1 + # via + # -c requirements/../../../requirements/constraints.txt + # -c requirements/_base.txt + # -r requirements/_test.in +idna==3.10 + # via + # -c requirements/_base.txt + # anyio + # httpx + # requests +iniconfig==2.1.0 + # via pytest +packaging==24.2 + # via + # -c requirements/_base.txt + # pytest +pluggy==1.5.0 + # via pytest +pytest==8.3.5 + # via + # -r requirements/_test.in + # pytest-asyncio + # pytest-cov + # pytest-mock +pytest-asyncio==0.23.8 + # via + # -c requirements/../../../requirements/constraints.txt + # -r requirements/_test.in +pytest-cov==6.0.0 + # via -r requirements/_test.in +pytest-mock==3.14.0 + # via -r requirements/_test.in +pytest-runner==6.0.1 + # via -r requirements/_test.in +python-dotenv==1.1.0 + # via + # -c requirements/_base.txt + # -r requirements/_test.in +requests==2.32.3 + # via + # -c requirements/_base.txt + # docker +sniffio==1.3.1 + # via + # -c requirements/_base.txt + # anyio + # asgi-lifespan +typing-extensions==4.13.0 + # via + # -c requirements/_base.txt + # anyio +tzdata==2025.2 + # via faker +urllib3==2.3.0 + # via + # -c requirements/../../../requirements/constraints.txt + # -c requirements/_base.txt + # docker + # requests diff --git a/services/notifications/requirements/_tools.in b/services/notifications/requirements/_tools.in new file mode 100644 index 000000000000..1def82c12a30 --- /dev/null +++ b/services/notifications/requirements/_tools.in @@ -0,0 +1,5 @@ +--constraint ../../../requirements/constraints.txt +--constraint _base.txt +--constraint _test.txt + +--requirement ../../../requirements/devenv.txt diff --git a/services/notifications/requirements/_tools.txt b/services/notifications/requirements/_tools.txt new file mode 100644 index 000000000000..4deff3bbf27d --- /dev/null +++ b/services/notifications/requirements/_tools.txt @@ -0,0 +1,82 @@ +astroid==3.3.9 + # via pylint +black==25.1.0 + # via -r requirements/../../../requirements/devenv.txt +build==1.2.2.post1 + # via pip-tools +bump2version==1.0.1 + # via -r requirements/../../../requirements/devenv.txt +cfgv==3.4.0 + # via pre-commit +click==8.1.8 + # via + # -c requirements/_base.txt + # black + # pip-tools +dill==0.3.9 + # via pylint +distlib==0.3.9 + # via virtualenv +filelock==3.18.0 + # via virtualenv +identify==2.6.9 + # via pre-commit +isort==6.0.1 + # via + # -r requirements/../../../requirements/devenv.txt + # pylint +mccabe==0.7.0 + # via pylint +mypy==1.15.0 + # via -r requirements/../../../requirements/devenv.txt +mypy-extensions==1.0.0 + # via + # black + # mypy +nodeenv==1.9.1 + # via pre-commit +packaging==24.2 + # via + # -c requirements/_base.txt + # -c requirements/_test.txt + # black + # build +pathspec==0.12.1 + # via black +pip==25.0.1 + # via pip-tools +pip-tools==7.4.1 + # via -r requirements/../../../requirements/devenv.txt +platformdirs==4.3.7 + # via + # black + # pylint + # virtualenv +pre-commit==4.2.0 + # via -r requirements/../../../requirements/devenv.txt +pylint==3.3.6 + # via -r requirements/../../../requirements/devenv.txt +pyproject-hooks==1.2.0 + # via + # build + # pip-tools +pyyaml==6.0.2 + # via + # -c requirements/../../../requirements/constraints.txt + # -c requirements/_base.txt + # pre-commit +ruff==0.11.2 + # via -r requirements/../../../requirements/devenv.txt +setuptools==78.1.0 + # via pip-tools +tomlkit==0.13.2 + # via pylint +typing-extensions==4.13.0 + # via + # -c requirements/_base.txt + # -c requirements/_test.txt + # mypy +virtualenv==20.29.3 + # via pre-commit +wheel==0.45.1 + # via pip-tools diff --git a/services/notifications/requirements/ci.txt b/services/notifications/requirements/ci.txt new file mode 100644 index 000000000000..431489bb1343 --- /dev/null +++ b/services/notifications/requirements/ci.txt @@ -0,0 +1,22 @@ +# Shortcut to install all packages for the contigous integration (CI) of 'services/notifications' +# +# - As ci.txt but w/ tests +# +# Usage: +# pip install -r requirements/ci.txt +# + +# installs base + tests requirements +--requirement _base.txt +--requirement _test.txt +--requirement _tools.txt + +# installs this repo's packages +simcore-common-library @ ../../packages/common-library +simcore-models-library @ ../../packages/models-library +pytest-simcore @ ../../packages/pytest-simcore +simcore-service-library[fastapi] @ ../../packages/service-library +simcore-settings-library @ ../../packages/settings-library + +# installs current package +simcore-service-notifications @ . diff --git a/services/notifications/requirements/constraints.txt b/services/notifications/requirements/constraints.txt new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/services/notifications/requirements/dev.txt b/services/notifications/requirements/dev.txt new file mode 100644 index 000000000000..46a7ed46c894 --- /dev/null +++ b/services/notifications/requirements/dev.txt @@ -0,0 +1,22 @@ +# Shortcut to install all packages needed to develop 'services/notifications' +# +# - As ci.txt but with current and repo packages in develop (edit) mode +# +# Usage: +# pip install -r requirements/dev.txt +# + +# installs base + tests + tools requirements +--requirement _base.txt +--requirement _test.txt +--requirement _tools.txt + +# installs this repo's packages +--editable ../../packages/common-library +--editable ../../packages/models-library +--editable ../../packages/pytest-simcore +--editable ../../packages/service-library[fastapi] +--editable ../../packages/settings-library + +# installs current package +--editable . diff --git a/services/notifications/requirements/prod.txt b/services/notifications/requirements/prod.txt new file mode 100644 index 000000000000..80c1b291e379 --- /dev/null +++ b/services/notifications/requirements/prod.txt @@ -0,0 +1,19 @@ +# Shortcut to install 'services/notifications' for production +# +# - As ci.txt but w/o tests +# +# Usage: +# pip install -r requirements/prod.txt +# + +# installs base requirements +--requirement _base.txt + +# installs this repo's packages +simcore-models-library @ ../../packages/models-library +simcore-common-library @ ../../packages/common-library/ +simcore-service-library[fastapi] @ ../../packages/service-library +simcore-settings-library @ ../../packages/settings-library + +# installs current package +simcore-service-notifications @ . diff --git a/services/notifications/setup.cfg b/services/notifications/setup.cfg new file mode 100644 index 000000000000..e420fe7efd21 --- /dev/null +++ b/services/notifications/setup.cfg @@ -0,0 +1,15 @@ +[bumpversion] +current_version = 0.0.1 +commit = True +message = services/notifications version: {current_version} → {new_version} +tag = False +commit_args = --no-verify + +[bumpversion:file:VERSION] + +[tool:pytest] +asyncio_mode = auto + +[mypy] +plugins = + pydantic.mypy diff --git a/services/notifications/setup.py b/services/notifications/setup.py new file mode 100755 index 000000000000..f383da6097ba --- /dev/null +++ b/services/notifications/setup.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 + +import re +import sys +from pathlib import Path + +from setuptools import find_packages, setup + + +def read_reqs(reqs_path: Path) -> set[str]: + return { + r + for r in re.findall( + r"(^[^#\n-][\w\[,\]]+[-~>=<.\w]*)", + reqs_path.read_text(), + re.MULTILINE, + ) + if isinstance(r, str) + } + + +CURRENT_DIR = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent + +NAME = "simcore-service-notifications" +VERSION = (CURRENT_DIR / "VERSION").read_text().strip() +AUTHORS = ("Andrei Neagu (GitHK)",) +DESCRIPTION = "Service for executing commands on docker nodes" +README = (CURRENT_DIR / "README.md").read_text() + +PROD_REQUIREMENTS = tuple( + read_reqs(CURRENT_DIR / "requirements" / "_base.txt") + | { + "simcore-models-library", + "simcore-service-library", + "simcore-settings-library", + } +) + +TEST_REQUIREMENTS = tuple(read_reqs(CURRENT_DIR / "requirements" / "_test.txt")) + +SETUP = { + "name": NAME, + "version": VERSION, + "author": AUTHORS, + "description": DESCRIPTION, + "long_description": README, + "license": "MIT license", + "python_requires": "~=3.11", + "packages": find_packages(where="src"), + "package_dir": { + "": "src", + }, + "include_package_data": True, + "install_requires": PROD_REQUIREMENTS, + "test_suite": "tests", + "tests_require": TEST_REQUIREMENTS, + "extras_require": {"test": TEST_REQUIREMENTS}, + "entry_points": { + "console_scripts": [ + "simcore-service-notifications = simcore_service_notifications.cli:main", + "simcore-service = simcore_service_notifications.cli:main", + ], + }, +} + +if __name__ == "__main__": + setup(**SETUP) diff --git a/services/notifications/src/simcore_service_notifications/__init__.py b/services/notifications/src/simcore_service_notifications/__init__.py new file mode 100644 index 000000000000..94fc632e7af1 --- /dev/null +++ b/services/notifications/src/simcore_service_notifications/__init__.py @@ -0,0 +1 @@ +from ._meta import __version__ diff --git a/services/notifications/src/simcore_service_notifications/_meta.py b/services/notifications/src/simcore_service_notifications/_meta.py new file mode 100644 index 000000000000..1f054b45d73c --- /dev/null +++ b/services/notifications/src/simcore_service_notifications/_meta.py @@ -0,0 +1,36 @@ +"""Application's metadata""" + +from importlib.metadata import distribution, version +from typing import Final + +from packaging.version import Version + +_current_distribution = distribution("simcore-service-notifications") +__version__: str = version("simcore-service-notifications") + + +APP_NAME: Final[str] = _current_distribution.metadata["Name"] +VERSION: Final[Version] = Version(__version__) +API_VTAG: str = f"v{VERSION.major}" + + +def get_summary() -> str: + return _current_distribution.metadata.get_all("Summary", [""])[-1] + + +SUMMARY: Final[str] = get_summary() + + +APP_STARTED_BANNER_MSG = rf""" + _______ _ ___ _ _ +(_______) _ (_) / __(_) _ (_) + _ _ ___ _| |_ _ _| |__ _ ____ _____ _| |_ _ ___ ____ ___ +| | | |/ _ (_ _| (_ __| |/ ___(____ (_ _| |/ _ \| _ \ /___) +| | | | |_| || |_| | | | | ( (___/ ___ | | |_| | |_| | | | |___ | +|_| |_|\___/ \__|_| |_| |_|\____\_____| \__|_|\___/|_| |_(___/ + {API_VTAG}""" + + +APP_FINISHED_BANNER_MSG = "{:=^100}".format( + f"🎉 App {APP_NAME}=={VERSION} shutdown completed 🎉" +) diff --git a/services/notifications/src/simcore_service_notifications/api/__init__.py b/services/notifications/src/simcore_service_notifications/api/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/services/notifications/src/simcore_service_notifications/api/rest/__init__.py b/services/notifications/src/simcore_service_notifications/api/rest/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/services/notifications/src/simcore_service_notifications/api/rest/_dependencies.py b/services/notifications/src/simcore_service_notifications/api/rest/_dependencies.py new file mode 100644 index 000000000000..8c335c90c92a --- /dev/null +++ b/services/notifications/src/simcore_service_notifications/api/rest/_dependencies.py @@ -0,0 +1,26 @@ +"""Free functions to inject dependencies in routes handlers""" + +from typing import Annotated, cast + +from fastapi import Depends, FastAPI, Request +from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient + +from ...core.settings import ApplicationSettings + + +def get_application(request: Request) -> FastAPI: + return cast(FastAPI, request.app) + + +def get_settings( + app: Annotated[FastAPI, Depends(get_application)], +) -> ApplicationSettings: + assert isinstance(app.state.settings, ApplicationSettings) # nosec + return app.state.settings + + +def get_rabbitmq_client( + app: Annotated[FastAPI, Depends(get_application)], +) -> RabbitMQRPCClient: + assert isinstance(app.state.rabbitmq_rpc_server, RabbitMQRPCClient) # nosec + return app.state.rabbitmq_rpc_server diff --git a/services/notifications/src/simcore_service_notifications/api/rest/_health.py b/services/notifications/src/simcore_service_notifications/api/rest/_health.py new file mode 100644 index 000000000000..c6924ea53d39 --- /dev/null +++ b/services/notifications/src/simcore_service_notifications/api/rest/_health.py @@ -0,0 +1,25 @@ +from typing import Annotated + +import arrow +from fastapi import APIRouter, Depends +from models_library.api_schemas__common.health import HealthCheckGet +from models_library.errors import RABBITMQ_CLIENT_UNHEALTHY_MSG +from servicelib.rabbitmq import RabbitMQClient + +from ._dependencies import get_rabbitmq_client + +router = APIRouter() + + +class HealthCheckError(RuntimeError): + """Failed a health check""" + + +@router.get("/health", response_model=HealthCheckGet) +async def check_service_health( + rabbitmq_client: Annotated[RabbitMQClient, Depends(get_rabbitmq_client)], +): + if not rabbitmq_client.healthy: + raise HealthCheckError(RABBITMQ_CLIENT_UNHEALTHY_MSG) + + return HealthCheckGet(timestamp=f"{__name__}@{arrow.utcnow().datetime.isoformat()}") diff --git a/services/notifications/src/simcore_service_notifications/api/rest/routes.py b/services/notifications/src/simcore_service_notifications/api/rest/routes.py new file mode 100644 index 000000000000..18688cf2f4d7 --- /dev/null +++ b/services/notifications/src/simcore_service_notifications/api/rest/routes.py @@ -0,0 +1,14 @@ +from fastapi import FastAPI, HTTPException +from servicelib.fastapi.exceptions_utils import ( + handle_errors_as_500, + http_exception_as_json_response, +) + +from . import _health + + +def setup_rest_api(app: FastAPI): + app.include_router(_health.router) + + app.add_exception_handler(Exception, handle_errors_as_500) + app.add_exception_handler(HTTPException, http_exception_as_json_response) diff --git a/services/notifications/src/simcore_service_notifications/api/rpc/__init__.py b/services/notifications/src/simcore_service_notifications/api/rpc/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/services/notifications/src/simcore_service_notifications/api/rpc/routes.py b/services/notifications/src/simcore_service_notifications/api/rpc/routes.py new file mode 100644 index 000000000000..bf0507457103 --- /dev/null +++ b/services/notifications/src/simcore_service_notifications/api/rpc/routes.py @@ -0,0 +1,19 @@ +from fastapi import FastAPI +from models_library.api_schemas_notifications import NOTIFICATIONS_RPC_NAMESPACE +from servicelib.rabbitmq import RPCRouter + +from ...services.rabbitmq import get_rabbitmq_rpc_server + +ROUTERS: list[RPCRouter] = [ + # `_internal_module.router`` +] + + +def setup_rpc_api_routes(app: FastAPI) -> None: + async def startup() -> None: + rpc_server = get_rabbitmq_rpc_server(app) + + for router in ROUTERS: + await rpc_server.register_router(router, NOTIFICATIONS_RPC_NAMESPACE, app) + + app.add_event_handler("startup", startup) diff --git a/services/notifications/src/simcore_service_notifications/cli.py b/services/notifications/src/simcore_service_notifications/cli.py new file mode 100644 index 000000000000..2bddceb17179 --- /dev/null +++ b/services/notifications/src/simcore_service_notifications/cli.py @@ -0,0 +1,14 @@ +import logging + +import typer +from settings_library.utils_cli import create_settings_command, create_version_callback + +from ._meta import APP_NAME, __version__ +from .core.settings import ApplicationSettings + +log = logging.getLogger(__name__) + +main = typer.Typer(name=APP_NAME) + +main.command()(create_settings_command(settings_cls=ApplicationSettings, logger=log)) +main.callback()(create_version_callback(__version__)) diff --git a/services/notifications/src/simcore_service_notifications/core/__init__.py b/services/notifications/src/simcore_service_notifications/core/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/services/notifications/src/simcore_service_notifications/core/application.py b/services/notifications/src/simcore_service_notifications/core/application.py new file mode 100644 index 000000000000..db048efc714e --- /dev/null +++ b/services/notifications/src/simcore_service_notifications/core/application.py @@ -0,0 +1,71 @@ +import logging + +from fastapi import FastAPI +from servicelib.fastapi.openapi import ( + get_common_oas_options, + override_fastapi_openapi_method, +) +from servicelib.fastapi.tracing import initialize_tracing +from servicelib.logging_utils import config_all_loggers + +from .._meta import ( + API_VTAG, + APP_FINISHED_BANNER_MSG, + APP_NAME, + APP_STARTED_BANNER_MSG, + SUMMARY, + VERSION, +) +from ..api.rest.routes import setup_rest_api +from ..api.rpc.routes import setup_rpc_api_routes +from ..services.rabbitmq import setup_rabbitmq +from .settings import ApplicationSettings + +logger = logging.getLogger(__name__) + + +def _setup_logger(settings: ApplicationSettings): + # SEE https://github.com/ITISFoundation/osparc-simcore/issues/3148 + logging.basicConfig(level=settings.LOG_LEVEL.value) # NOSONAR + logging.root.setLevel(settings.LOG_LEVEL.value) + config_all_loggers( + log_format_local_dev_enabled=settings.NOTIFICATIONS_VOLUMES_LOG_FORMAT_LOCAL_DEV_ENABLED, + logger_filter_mapping=settings.NOTIFICATIONS_VOLUMES_LOG_FILTER_MAPPING, + tracing_settings=settings.NOTIFICATIONS_TRACING, + ) + + +def create_app() -> FastAPI: + settings = ApplicationSettings.create_from_envs() + _setup_logger(settings) + logger.debug(settings.model_dump_json(indent=2)) + + assert settings.SC_BOOT_MODE # nosec + app = FastAPI( + debug=settings.SC_BOOT_MODE.is_devel_mode(), + title=APP_NAME, + description=SUMMARY, + version=f"{VERSION}", + openapi_url=f"/api/{API_VTAG}/openapi.json", + **get_common_oas_options(is_devel_mode=settings.SC_BOOT_MODE.is_devel_mode()), + ) + override_fastapi_openapi_method(app) + app.state.settings = settings + + setup_rabbitmq(app) + setup_rest_api(app) + setup_rpc_api_routes(app) + + if settings.NOTIFICATIONS_TRACING: + initialize_tracing(app, settings.NOTIFICATIONS_TRACING, APP_NAME) + + async def _on_startup() -> None: + print(APP_STARTED_BANNER_MSG, flush=True) # noqa: T201 + + async def _on_shutdown() -> None: + print(APP_FINISHED_BANNER_MSG, flush=True) # noqa: T201 + + app.add_event_handler("startup", _on_startup) + app.add_event_handler("shutdown", _on_shutdown) + + return app diff --git a/services/notifications/src/simcore_service_notifications/core/settings.py b/services/notifications/src/simcore_service_notifications/core/settings.py new file mode 100644 index 000000000000..014838ce2955 --- /dev/null +++ b/services/notifications/src/simcore_service_notifications/core/settings.py @@ -0,0 +1,71 @@ +from typing import Annotated + +from common_library.basic_types import DEFAULT_FACTORY +from models_library.basic_types import BootModeEnum, LogLevel +from pydantic import AliasChoices, Field, field_validator +from servicelib.logging_utils_filtering import LoggerName, MessageSubstring +from settings_library.base import BaseCustomSettings +from settings_library.rabbit import RabbitSettings +from settings_library.tracing import TracingSettings +from settings_library.utils_logging import MixinLoggingSettings + + +class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): + LOG_LEVEL: Annotated[ + LogLevel, + Field( + validation_alias=AliasChoices( + "NOTIFICATIONS_LOGLEVEL", + "LOG_LEVEL", + "LOGLEVEL", + ), + ), + ] = LogLevel.WARNING + + SC_BOOT_MODE: BootModeEnum | None + + NOTIFICATIONS_VOLUMES_LOG_FORMAT_LOCAL_DEV_ENABLED: Annotated[ + bool, + Field( + validation_alias=AliasChoices( + "NOTIFICATIONS_VOLUMES_LOG_FORMAT_LOCAL_DEV_ENABLED", + "LOG_FORMAT_LOCAL_DEV_ENABLED", + ), + description=( + "Enables local development log format. WARNING: make sure it is " + "disabled if you want to have structured logs!" + ), + ), + ] = False + + NOTIFICATIONS_VOLUMES_LOG_FILTER_MAPPING: Annotated[ + dict[LoggerName, list[MessageSubstring]], + Field( + default_factory=dict, + validation_alias=AliasChoices( + "NOTIFICATIONS_VOLUMES_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING" + ), + description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", + ), + ] = DEFAULT_FACTORY + + NOTIFICATIONS_RABBITMQ: Annotated[ + RabbitSettings, + Field( + description="settings for service/rabbitmq", + json_schema_extra={"auto_default_from_env": True}, + ), + ] + + NOTIFICATIONS_TRACING: Annotated[ + TracingSettings | None, + Field( + description="settings for opentelemetry tracing", + json_schema_extra={"auto_default_from_env": True}, + ), + ] + + @field_validator("LOG_LEVEL") + @classmethod + def valid_log_level(cls, value) -> LogLevel: + return LogLevel(cls.validate_log_level(value)) diff --git a/services/notifications/src/simcore_service_notifications/main.py b/services/notifications/src/simcore_service_notifications/main.py new file mode 100644 index 000000000000..8b2e0ed31966 --- /dev/null +++ b/services/notifications/src/simcore_service_notifications/main.py @@ -0,0 +1,3 @@ +from simcore_service_notifications.core.application import create_app + +the_app = create_app() diff --git a/services/notifications/src/simcore_service_notifications/models/__init__.py b/services/notifications/src/simcore_service_notifications/models/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/services/notifications/src/simcore_service_notifications/services/__init__.py b/services/notifications/src/simcore_service_notifications/services/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/services/notifications/src/simcore_service_notifications/services/rabbitmq.py b/services/notifications/src/simcore_service_notifications/services/rabbitmq.py new file mode 100644 index 000000000000..8d3f1b9be24c --- /dev/null +++ b/services/notifications/src/simcore_service_notifications/services/rabbitmq.py @@ -0,0 +1,29 @@ +from typing import cast + +from fastapi import FastAPI +from servicelib.rabbitmq import RabbitMQRPCClient, wait_till_rabbitmq_responsive +from settings_library.rabbit import RabbitSettings + + +def setup_rabbitmq(app: FastAPI) -> None: + settings: RabbitSettings = app.state.settings.NOTIFICATIONS_RABBITMQ + app.state.rabbitmq_rpc_server = None + + async def _on_startup() -> None: + await wait_till_rabbitmq_responsive(settings.dsn) + + app.state.rabbitmq_rpc_server = await RabbitMQRPCClient.create( + client_name="dynamic_scheduler_rpc_server", settings=settings + ) + + async def _on_shutdown() -> None: + if app.state.rabbitmq_rpc_server: + await app.state.rabbitmq_rpc_server.close() + + app.add_event_handler("startup", _on_startup) + app.add_event_handler("shutdown", _on_shutdown) + + +def get_rabbitmq_rpc_server(app: FastAPI) -> RabbitMQRPCClient: + assert app.state.rabbitmq_rpc_server # nosec + return cast(RabbitMQRPCClient, app.state.rabbitmq_rpc_server) diff --git a/services/notifications/tests/conftest.py b/services/notifications/tests/conftest.py new file mode 100644 index 000000000000..a848f56562d5 --- /dev/null +++ b/services/notifications/tests/conftest.py @@ -0,0 +1,29 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument + + +import pytest +from models_library.basic_types import BootModeEnum +from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict + +pytest_plugins = [ + "pytest_simcore.docker_compose", + "pytest_simcore.docker_swarm", + "pytest_simcore.rabbit_service", + "pytest_simcore.repository_paths", +] + + +@pytest.fixture +def mock_environment(monkeypatch: pytest.MonkeyPatch) -> EnvVarsDict: + return setenvs_from_dict( + monkeypatch, + { + "LOGLEVEL": "DEBUG", + "SC_BOOT_MODE": BootModeEnum.DEBUG, + "RABBIT_HOST": "test", + "RABBIT_PASSWORD": "test", + "RABBIT_SECURE": "false", + "RABBIT_USER": "test", + }, + ) diff --git a/services/notifications/tests/integration/.gitkeep b/services/notifications/tests/integration/.gitkeep new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/services/notifications/tests/unit/conftest.py b/services/notifications/tests/unit/conftest.py new file mode 100644 index 000000000000..56e46e5a2e6d --- /dev/null +++ b/services/notifications/tests/unit/conftest.py @@ -0,0 +1,44 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument + +from collections.abc import AsyncIterator + +import pytest +from asgi_lifespan import LifespanManager +from fastapi import FastAPI +from fastapi.testclient import TestClient +from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict +from settings_library.rabbit import RabbitSettings +from simcore_service_notifications.core.application import create_app + + +@pytest.fixture +def service_env( + monkeypatch: pytest.MonkeyPatch, + mock_environment: EnvVarsDict, + rabbit_service: RabbitSettings, +) -> EnvVarsDict: + return setenvs_from_dict( + monkeypatch, + { + **mock_environment, + "RABBIT_HOST": rabbit_service.RABBIT_HOST, + "RABBIT_PASSWORD": rabbit_service.RABBIT_PASSWORD.get_secret_value(), + "RABBIT_PORT": f"{rabbit_service.RABBIT_PORT}", + "RABBIT_SECURE": f"{rabbit_service.RABBIT_SECURE}", + "RABBIT_USER": rabbit_service.RABBIT_USER, + }, + ) + + +@pytest.fixture +async def initialized_app(service_env: EnvVarsDict) -> AsyncIterator[FastAPI]: + app: FastAPI = create_app() + + async with LifespanManager(app): + yield app + + +@pytest.fixture +def test_client(initialized_app: FastAPI) -> TestClient: + return TestClient(initialized_app) diff --git a/services/notifications/tests/unit/test_api_rest__health.py b/services/notifications/tests/unit/test_api_rest__health.py new file mode 100644 index 000000000000..9f0904c182ea --- /dev/null +++ b/services/notifications/tests/unit/test_api_rest__health.py @@ -0,0 +1,17 @@ +# pylint: disable=protected-access +# pylint: disable=redefined-outer-name + + +from fastapi import status +from fastapi.testclient import TestClient +from models_library.api_schemas__common.health import HealthCheckGet + +pytest_simcore_core_services_selection = [ + "rabbit", +] + + +def test_health_ok(test_client: TestClient): + response = test_client.get("/health") + assert response.status_code == status.HTTP_200_OK + assert HealthCheckGet.model_validate(response.json()) diff --git a/services/notifications/tests/unit/test_cli.py b/services/notifications/tests/unit/test_cli.py new file mode 100644 index 000000000000..d3833a6a3d00 --- /dev/null +++ b/services/notifications/tests/unit/test_cli.py @@ -0,0 +1,35 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument + +import traceback + +import pytest +from click.testing import Result +from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict +from simcore_service_notifications.cli import main +from typer.testing import CliRunner + + +@pytest.fixture +def cli_runner(mock_environment: EnvVarsDict) -> CliRunner: + return CliRunner() + + +def _format_cli_error(result: Result) -> str: + assert result.exception + tb_message = "\n".join(traceback.format_tb(result.exception.__traceback__)) + return f"Below exception was raised by the cli:\n{tb_message}" + + +def test_process_cli_options(cli_runner: CliRunner): + result = cli_runner.invoke(main, ["--help"]) + print(result.stdout) + assert result.exit_code == 0, _format_cli_error(result) + + result = cli_runner.invoke(main, ["settings"]) + print(result.stdout) + assert result.exit_code == 0, _format_cli_error(result) + + result = cli_runner.invoke(main, ["--version"]) + print(result.stdout) + assert result.exit_code == 0, _format_cli_error(result) From d192df5fc84c74629493906562b5b89b5fbf37bc Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 26 Mar 2025 14:46:07 +0100 Subject: [PATCH 04/58] added service in compose files --- .env-devel | 3 +++ Makefile | 1 + services/docker-compose-build.yml | 17 +++++++++++++++++ services/docker-compose-deploy.yml | 2 ++ services/docker-compose.devel.yml | 10 ++++++++++ services/docker-compose.local.yml | 8 ++++++++ services/docker-compose.yml | 26 ++++++++++++++++++++++++++ 7 files changed, 67 insertions(+) diff --git a/.env-devel b/.env-devel index d5df4deab06b..0199c5c1ed52 100644 --- a/.env-devel +++ b/.env-devel @@ -164,6 +164,9 @@ INVITATIONS_TRACING=null LOG_FORMAT_LOCAL_DEV_ENABLED=1 LOG_FILTER_MAPPING='{}' +NOTIFICATIONS_LOGLEVEL=INFO +NOTIFICATIONS_TRACING=null + PAYMENTS_ACCESS_TOKEN_EXPIRE_MINUTES=30 PAYMENTS_ACCESS_TOKEN_SECRET_KEY=2c0411810565e063309be1457009fb39ce023946f6a354e6935107b57676 PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT=10000 diff --git a/Makefile b/Makefile index e1e51612e5d9..4b28fb44e747 100644 --- a/Makefile +++ b/Makefile @@ -44,6 +44,7 @@ SERVICES_NAMES_TO_BUILD := \ efs-guardian \ invitations \ migration \ + notifications \ payments \ resource-usage-tracker \ dynamic-scheduler \ diff --git a/services/docker-compose-build.yml b/services/docker-compose-build.yml index 7fe0baf70245..34fc98dc2643 100644 --- a/services/docker-compose-build.yml +++ b/services/docker-compose-build.yml @@ -169,6 +169,23 @@ services: org.opencontainers.image.source: "${VCS_URL}" org.opencontainers.image.revision: "${VCS_REF}" + notifications: + image: local/notifications:${BUILD_TARGET:?build_target_required} + build: + context: ../ + dockerfile: services/notifications/Dockerfile + cache_from: + - local/notifications:${BUILD_TARGET:?build_target_required} + - ${DOCKER_REGISTRY:-itisfoundation}/notifications:master-github-latest + - ${DOCKER_REGISTRY:-itisfoundation}/notifications:staging-github-latest + - ${DOCKER_REGISTRY:-itisfoundation}/notifications:release-github-latest + target: production + labels: + org.label-schema.schema-version: "1.0" + org.opencontainers.image.created: "${BUILD_DATE}" + org.opencontainers.image.source: "${VCS_URL}" + org.opencontainers.image.revision: "${VCS_REF}" + resource-usage-tracker: image: local/resource-usage-tracker:${BUILD_TARGET:?build_target_required} build: diff --git a/services/docker-compose-deploy.yml b/services/docker-compose-deploy.yml index f3997a0b11d2..e6c21da36dbf 100644 --- a/services/docker-compose-deploy.yml +++ b/services/docker-compose-deploy.yml @@ -25,6 +25,8 @@ services: image: ${DOCKER_REGISTRY:-itisfoundation}/invitations:${DOCKER_IMAGE_TAG:-latest} migration: image: ${DOCKER_REGISTRY:-itisfoundation}/migration:${DOCKER_IMAGE_TAG:-latest} + notifications: + image: ${DOCKER_REGISTRY:-itisfoundation}/notifications:${DOCKER_IMAGE_TAG:-latest} payments: image: ${DOCKER_REGISTRY:-itisfoundation}/payments:${DOCKER_IMAGE_TAG:-latest} dynamic-scheduler: diff --git a/services/docker-compose.devel.yml b/services/docker-compose.devel.yml index a9928bc4cdaa..0fe3894401dc 100644 --- a/services/docker-compose.devel.yml +++ b/services/docker-compose.devel.yml @@ -72,6 +72,16 @@ services: - ../packages:/devel/packages - ${HOST_UV_CACHE_DIR}:/home/scu/.cache/uv + notifications: + environment: + <<: *common-environment + NOTIFICATIONS_PROFILING : ${NOTIFICATIONS_PROFILING} + NOTIFICATIONS_LOGLEVEL: DEBUG + volumes: + - ./notifications:/devel/services/notifications + - ../packages:/devel/packages + - ${HOST_UV_CACHE_DIR}:/home/scu/.cache/uv + clusters-keeper: environment: <<: *common-environment diff --git a/services/docker-compose.local.yml b/services/docker-compose.local.yml index 12a2abd44c6b..07ab2ef096f8 100644 --- a/services/docker-compose.local.yml +++ b/services/docker-compose.local.yml @@ -88,6 +88,14 @@ services: - "8008:8000" - "3017:3000" + notifications: + environment: + <<: *common_environment + INVITATIONS_REMOTE_DEBUGGING_PORT : 3000 + ports: + - "8015:8000" + - "3022:3000" + payments: environment: <<: *common_environment diff --git a/services/docker-compose.yml b/services/docker-compose.yml index be90de5707cb..9b445f29e9dd 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -1095,6 +1095,32 @@ services: TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT: ${TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT} TRACING_OPENTELEMETRY_COLLECTOR_PORT: ${TRACING_OPENTELEMETRY_COLLECTOR_PORT} + notifications: + image: ${DOCKER_REGISTRY:-itisfoundation}/notifications:${DOCKER_IMAGE_TAG:-latest} + init: true + hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" + deploy: + mode: global + resources: + limits: + cpus: "1.0" + memory: 1024M + + environment: + NOTIFICATIONS_LOGLEVEL: ${NOTIFICATIONS_LOGLEVEL} + LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + LOG_FILTER_MAPPING : ${LOG_FILTER_MAPPING} + RABBIT_HOST: ${RABBIT_HOST} + RABBIT_PASSWORD: ${RABBIT_PASSWORD} + RABBIT_PORT: ${RABBIT_PORT} + RABBIT_USER: ${RABBIT_USER} + RABBIT_SECURE: ${RABBIT_SECURE} + + NOTIFICATIONS_TRACING: ${NOTIFICATIONS_TRACING} + TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT: ${TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT} + TRACING_OPENTELEMETRY_COLLECTOR_PORT: ${TRACING_OPENTELEMETRY_COLLECTOR_PORT} + + dask-sidecar: image: ${DOCKER_REGISTRY:-itisfoundation}/dask-sidecar:${DOCKER_IMAGE_TAG:-latest} init: true From 4251212193a35e6ce48d3512be0a07d766f4c5f7 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 26 Mar 2025 15:10:21 +0100 Subject: [PATCH 05/58] make CI trigger --- .github/workflows/ci-testing-deploy.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml index d74466c4ba20..bc13e56f6239 100644 --- a/.github/workflows/ci-testing-deploy.yml +++ b/.github/workflows/ci-testing-deploy.yml @@ -150,6 +150,12 @@ jobs: - 'services/docker-compose*' - 'scripts/mypy/*' - 'mypy.ini' + notifications: + - 'packages/**' + - 'services/notifications/**' + - 'services/docker-compose*' + - 'scripts/mypy/*' + - 'mypy.ini' api: - 'api/**' api-server: From d045e2ac561e204f511205f4ac668711a589e36d Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 26 Mar 2025 15:11:42 +0100 Subject: [PATCH 06/58] migrate to lifespan from setup --- .../api/rest/routes.py | 2 +- .../api/rpc/routes.py | 14 +++--- .../core/application.py | 44 +++++++++++-------- .../services/rabbitmq.py | 20 ++++----- 4 files changed, 43 insertions(+), 37 deletions(-) diff --git a/services/notifications/src/simcore_service_notifications/api/rest/routes.py b/services/notifications/src/simcore_service_notifications/api/rest/routes.py index 18688cf2f4d7..3ea9f2859155 100644 --- a/services/notifications/src/simcore_service_notifications/api/rest/routes.py +++ b/services/notifications/src/simcore_service_notifications/api/rest/routes.py @@ -7,7 +7,7 @@ from . import _health -def setup_rest_api(app: FastAPI): +def initialize_rest_api(app: FastAPI) -> None: app.include_router(_health.router) app.add_exception_handler(Exception, handle_errors_as_500) diff --git a/services/notifications/src/simcore_service_notifications/api/rpc/routes.py b/services/notifications/src/simcore_service_notifications/api/rpc/routes.py index bf0507457103..ad9c4469415d 100644 --- a/services/notifications/src/simcore_service_notifications/api/rpc/routes.py +++ b/services/notifications/src/simcore_service_notifications/api/rpc/routes.py @@ -1,4 +1,7 @@ +from collections.abc import AsyncIterator + from fastapi import FastAPI +from fastapi_lifespan_manager import State from models_library.api_schemas_notifications import NOTIFICATIONS_RPC_NAMESPACE from servicelib.rabbitmq import RPCRouter @@ -9,11 +12,10 @@ ] -def setup_rpc_api_routes(app: FastAPI) -> None: - async def startup() -> None: - rpc_server = get_rabbitmq_rpc_server(app) +async def lifespan_rpc_api_routes(app: FastAPI) -> AsyncIterator[State]: + rpc_server = get_rabbitmq_rpc_server(app) - for router in ROUTERS: - await rpc_server.register_router(router, NOTIFICATIONS_RPC_NAMESPACE, app) + for router in ROUTERS: + await rpc_server.register_router(router, NOTIFICATIONS_RPC_NAMESPACE, app) - app.add_event_handler("startup", startup) + yield {} diff --git a/services/notifications/src/simcore_service_notifications/core/application.py b/services/notifications/src/simcore_service_notifications/core/application.py index db048efc714e..c75f2b24072c 100644 --- a/services/notifications/src/simcore_service_notifications/core/application.py +++ b/services/notifications/src/simcore_service_notifications/core/application.py @@ -1,6 +1,9 @@ import logging +from collections.abc import AsyncIterator from fastapi import FastAPI +from fastapi_lifespan_manager import State +from servicelib.fastapi.lifespan_utils import LifespanGenerator, combine_lifespans from servicelib.fastapi.openapi import ( get_common_oas_options, override_fastapi_openapi_method, @@ -16,15 +19,15 @@ SUMMARY, VERSION, ) -from ..api.rest.routes import setup_rest_api -from ..api.rpc.routes import setup_rpc_api_routes -from ..services.rabbitmq import setup_rabbitmq +from ..api.rest.routes import initialize_rest_api +from ..api.rpc.routes import lifespan_rpc_api_routes +from ..services.rabbitmq import lifespan_rabbitmq from .settings import ApplicationSettings -logger = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) -def _setup_logger(settings: ApplicationSettings): +def _initialise_logger(settings: ApplicationSettings): # SEE https://github.com/ITISFoundation/osparc-simcore/issues/3148 logging.basicConfig(level=settings.LOG_LEVEL.value) # NOSONAR logging.root.setLevel(settings.LOG_LEVEL.value) @@ -35,10 +38,23 @@ def _setup_logger(settings: ApplicationSettings): ) +async def _lifespan_banner(app: FastAPI) -> AsyncIterator[State]: + _ = app + print(APP_STARTED_BANNER_MSG, flush=True) # noqa: T201 + yield {} + print(APP_FINISHED_BANNER_MSG, flush=True) # noqa: T201 + + def create_app() -> FastAPI: settings = ApplicationSettings.create_from_envs() - _setup_logger(settings) - logger.debug(settings.model_dump_json(indent=2)) + _logger.debug(settings.model_dump_json(indent=2)) + + _initialise_logger(settings) + + lifespans: list[LifespanGenerator] = [ + lifespan_rabbitmq, + lifespan_rpc_api_routes, + ] assert settings.SC_BOOT_MODE # nosec app = FastAPI( @@ -47,25 +63,15 @@ def create_app() -> FastAPI: description=SUMMARY, version=f"{VERSION}", openapi_url=f"/api/{API_VTAG}/openapi.json", + lifespan=combine_lifespans(*lifespans, _lifespan_banner), **get_common_oas_options(is_devel_mode=settings.SC_BOOT_MODE.is_devel_mode()), ) override_fastapi_openapi_method(app) app.state.settings = settings - setup_rabbitmq(app) - setup_rest_api(app) - setup_rpc_api_routes(app) + initialize_rest_api(app) if settings.NOTIFICATIONS_TRACING: initialize_tracing(app, settings.NOTIFICATIONS_TRACING, APP_NAME) - async def _on_startup() -> None: - print(APP_STARTED_BANNER_MSG, flush=True) # noqa: T201 - - async def _on_shutdown() -> None: - print(APP_FINISHED_BANNER_MSG, flush=True) # noqa: T201 - - app.add_event_handler("startup", _on_startup) - app.add_event_handler("shutdown", _on_shutdown) - return app diff --git a/services/notifications/src/simcore_service_notifications/services/rabbitmq.py b/services/notifications/src/simcore_service_notifications/services/rabbitmq.py index 8d3f1b9be24c..6185d4e64c31 100644 --- a/services/notifications/src/simcore_service_notifications/services/rabbitmq.py +++ b/services/notifications/src/simcore_service_notifications/services/rabbitmq.py @@ -1,27 +1,25 @@ +from collections.abc import AsyncIterator from typing import cast from fastapi import FastAPI +from fastapi_lifespan_manager import State from servicelib.rabbitmq import RabbitMQRPCClient, wait_till_rabbitmq_responsive from settings_library.rabbit import RabbitSettings -def setup_rabbitmq(app: FastAPI) -> None: +async def lifespan_rabbitmq(app: FastAPI) -> AsyncIterator[State]: settings: RabbitSettings = app.state.settings.NOTIFICATIONS_RABBITMQ app.state.rabbitmq_rpc_server = None - async def _on_startup() -> None: - await wait_till_rabbitmq_responsive(settings.dsn) + await wait_till_rabbitmq_responsive(settings.dsn) - app.state.rabbitmq_rpc_server = await RabbitMQRPCClient.create( - client_name="dynamic_scheduler_rpc_server", settings=settings - ) + app.state.rabbitmq_rpc_server = await RabbitMQRPCClient.create( + client_name="dynamic_scheduler_rpc_server", settings=settings + ) - async def _on_shutdown() -> None: - if app.state.rabbitmq_rpc_server: - await app.state.rabbitmq_rpc_server.close() + yield {} - app.add_event_handler("startup", _on_startup) - app.add_event_handler("shutdown", _on_shutdown) + await app.state.rabbitmq_rpc_server.close() def get_rabbitmq_rpc_server(app: FastAPI) -> RabbitMQRPCClient: From 45f0fa674a7f9f42fa3e8261ec120f3eabb9e039 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 26 Mar 2025 15:28:33 +0100 Subject: [PATCH 07/58] trigger ci --- .github/workflows/ci-testing-deploy.yml | 1 + .../src/simcore_service_notifications/models/__init__.py | 0 2 files changed, 1 insertion(+) delete mode 100644 services/notifications/src/simcore_service_notifications/models/__init__.py diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml index bc13e56f6239..a7d28d300e61 100644 --- a/.github/workflows/ci-testing-deploy.yml +++ b/.github/workflows/ci-testing-deploy.yml @@ -62,6 +62,7 @@ jobs: settings-library: ${{ steps.filter.outputs.settings-library }} simcore-sdk: ${{ steps.filter.outputs.simcore-sdk }} agent: ${{ steps.filter.outputs.agent }} + notifications: ${{ steps.filter.outputs.notifications }} api: ${{ steps.filter.outputs.api }} api-server: ${{ steps.filter.outputs.api-server }} autoscaling: ${{ steps.filter.outputs.autoscaling }} diff --git a/services/notifications/src/simcore_service_notifications/models/__init__.py b/services/notifications/src/simcore_service_notifications/models/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 From 885aadf4dc66a4fff9afd7165985f43b1cc54121 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 26 Mar 2025 15:40:14 +0100 Subject: [PATCH 08/58] remove unused --- .../api/rest/_dependencies.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/services/notifications/src/simcore_service_notifications/api/rest/_dependencies.py b/services/notifications/src/simcore_service_notifications/api/rest/_dependencies.py index 8c335c90c92a..800efcfe15d8 100644 --- a/services/notifications/src/simcore_service_notifications/api/rest/_dependencies.py +++ b/services/notifications/src/simcore_service_notifications/api/rest/_dependencies.py @@ -5,20 +5,11 @@ from fastapi import Depends, FastAPI, Request from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient -from ...core.settings import ApplicationSettings - def get_application(request: Request) -> FastAPI: return cast(FastAPI, request.app) -def get_settings( - app: Annotated[FastAPI, Depends(get_application)], -) -> ApplicationSettings: - assert isinstance(app.state.settings, ApplicationSettings) # nosec - return app.state.settings - - def get_rabbitmq_client( app: Annotated[FastAPI, Depends(get_application)], ) -> RabbitMQRPCClient: From e37453c234858b2aeabf5e2e5c478eb44c3cffb7 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 26 Mar 2025 16:03:12 +0100 Subject: [PATCH 09/58] using CSR pattern --- .../src/simcore_service_notifications/core/application.py | 6 +++--- .../{api/rest/_dependencies.py => core/dependencies.py} | 0 .../{api/rest/routes.py => core/routing_rest.py} | 4 ++-- .../{api/rpc/routes.py => core/routing_rpc.py} | 4 ++-- .../{api => domains}/__init__.py | 0 .../{api/rest => domains/health}/__init__.py | 0 .../{api/rest/_health.py => domains/health/controller.py} | 2 +- .../{api/rpc => domains/rabbitmq}/__init__.py | 0 .../{services/rabbitmq.py => domains/rabbitmq/service.py} | 0 .../src/simcore_service_notifications/services/__init__.py | 0 .../{test_api_rest__health.py => test_domain_health.py} | 0 11 files changed, 8 insertions(+), 8 deletions(-) rename services/notifications/src/simcore_service_notifications/{api/rest/_dependencies.py => core/dependencies.py} (100%) rename services/notifications/src/simcore_service_notifications/{api/rest/routes.py => core/routing_rest.py} (77%) rename services/notifications/src/simcore_service_notifications/{api/rpc/routes.py => core/routing_rpc.py} (82%) rename services/notifications/src/simcore_service_notifications/{api => domains}/__init__.py (100%) rename services/notifications/src/simcore_service_notifications/{api/rest => domains/health}/__init__.py (100%) rename services/notifications/src/simcore_service_notifications/{api/rest/_health.py => domains/health/controller.py} (93%) rename services/notifications/src/simcore_service_notifications/{api/rpc => domains/rabbitmq}/__init__.py (100%) rename services/notifications/src/simcore_service_notifications/{services/rabbitmq.py => domains/rabbitmq/service.py} (100%) delete mode 100644 services/notifications/src/simcore_service_notifications/services/__init__.py rename services/notifications/tests/unit/{test_api_rest__health.py => test_domain_health.py} (100%) diff --git a/services/notifications/src/simcore_service_notifications/core/application.py b/services/notifications/src/simcore_service_notifications/core/application.py index c75f2b24072c..504945739b73 100644 --- a/services/notifications/src/simcore_service_notifications/core/application.py +++ b/services/notifications/src/simcore_service_notifications/core/application.py @@ -19,9 +19,9 @@ SUMMARY, VERSION, ) -from ..api.rest.routes import initialize_rest_api -from ..api.rpc.routes import lifespan_rpc_api_routes -from ..services.rabbitmq import lifespan_rabbitmq +from ..domains.rabbitmq.service import lifespan_rabbitmq +from .routing_rest import initialize_rest_api +from .routing_rpc import lifespan_rpc_api_routes from .settings import ApplicationSettings _logger = logging.getLogger(__name__) diff --git a/services/notifications/src/simcore_service_notifications/api/rest/_dependencies.py b/services/notifications/src/simcore_service_notifications/core/dependencies.py similarity index 100% rename from services/notifications/src/simcore_service_notifications/api/rest/_dependencies.py rename to services/notifications/src/simcore_service_notifications/core/dependencies.py diff --git a/services/notifications/src/simcore_service_notifications/api/rest/routes.py b/services/notifications/src/simcore_service_notifications/core/routing_rest.py similarity index 77% rename from services/notifications/src/simcore_service_notifications/api/rest/routes.py rename to services/notifications/src/simcore_service_notifications/core/routing_rest.py index 3ea9f2859155..993c299c4aa7 100644 --- a/services/notifications/src/simcore_service_notifications/api/rest/routes.py +++ b/services/notifications/src/simcore_service_notifications/core/routing_rest.py @@ -4,11 +4,11 @@ http_exception_as_json_response, ) -from . import _health +from ..domains.health.controller import router as health_router def initialize_rest_api(app: FastAPI) -> None: - app.include_router(_health.router) + app.include_router(health_router) app.add_exception_handler(Exception, handle_errors_as_500) app.add_exception_handler(HTTPException, http_exception_as_json_response) diff --git a/services/notifications/src/simcore_service_notifications/api/rpc/routes.py b/services/notifications/src/simcore_service_notifications/core/routing_rpc.py similarity index 82% rename from services/notifications/src/simcore_service_notifications/api/rpc/routes.py rename to services/notifications/src/simcore_service_notifications/core/routing_rpc.py index ad9c4469415d..9a4789bb9706 100644 --- a/services/notifications/src/simcore_service_notifications/api/rpc/routes.py +++ b/services/notifications/src/simcore_service_notifications/core/routing_rpc.py @@ -5,10 +5,10 @@ from models_library.api_schemas_notifications import NOTIFICATIONS_RPC_NAMESPACE from servicelib.rabbitmq import RPCRouter -from ...services.rabbitmq import get_rabbitmq_rpc_server +from ..domains.rabbitmq.service import get_rabbitmq_rpc_server ROUTERS: list[RPCRouter] = [ - # `_internal_module.router`` + # import form various domains and attach here ] diff --git a/services/notifications/src/simcore_service_notifications/api/__init__.py b/services/notifications/src/simcore_service_notifications/domains/__init__.py similarity index 100% rename from services/notifications/src/simcore_service_notifications/api/__init__.py rename to services/notifications/src/simcore_service_notifications/domains/__init__.py diff --git a/services/notifications/src/simcore_service_notifications/api/rest/__init__.py b/services/notifications/src/simcore_service_notifications/domains/health/__init__.py similarity index 100% rename from services/notifications/src/simcore_service_notifications/api/rest/__init__.py rename to services/notifications/src/simcore_service_notifications/domains/health/__init__.py diff --git a/services/notifications/src/simcore_service_notifications/api/rest/_health.py b/services/notifications/src/simcore_service_notifications/domains/health/controller.py similarity index 93% rename from services/notifications/src/simcore_service_notifications/api/rest/_health.py rename to services/notifications/src/simcore_service_notifications/domains/health/controller.py index c6924ea53d39..b5c9deae8312 100644 --- a/services/notifications/src/simcore_service_notifications/api/rest/_health.py +++ b/services/notifications/src/simcore_service_notifications/domains/health/controller.py @@ -6,7 +6,7 @@ from models_library.errors import RABBITMQ_CLIENT_UNHEALTHY_MSG from servicelib.rabbitmq import RabbitMQClient -from ._dependencies import get_rabbitmq_client +from ...core.dependencies import get_rabbitmq_client router = APIRouter() diff --git a/services/notifications/src/simcore_service_notifications/api/rpc/__init__.py b/services/notifications/src/simcore_service_notifications/domains/rabbitmq/__init__.py similarity index 100% rename from services/notifications/src/simcore_service_notifications/api/rpc/__init__.py rename to services/notifications/src/simcore_service_notifications/domains/rabbitmq/__init__.py diff --git a/services/notifications/src/simcore_service_notifications/services/rabbitmq.py b/services/notifications/src/simcore_service_notifications/domains/rabbitmq/service.py similarity index 100% rename from services/notifications/src/simcore_service_notifications/services/rabbitmq.py rename to services/notifications/src/simcore_service_notifications/domains/rabbitmq/service.py diff --git a/services/notifications/src/simcore_service_notifications/services/__init__.py b/services/notifications/src/simcore_service_notifications/services/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/services/notifications/tests/unit/test_api_rest__health.py b/services/notifications/tests/unit/test_domain_health.py similarity index 100% rename from services/notifications/tests/unit/test_api_rest__health.py rename to services/notifications/tests/unit/test_domain_health.py From 04cd03b4ec935eca063df3b5d7ca6e612e1442b8 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 26 Mar 2025 16:20:12 +0100 Subject: [PATCH 10/58] fixed routing --- .../src/simcore_service_notifications/core/routing_rest.py | 2 +- .../domains/health/{controller.py => controllers/rest.py} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename services/notifications/src/simcore_service_notifications/domains/health/{controller.py => controllers/rest.py} (92%) diff --git a/services/notifications/src/simcore_service_notifications/core/routing_rest.py b/services/notifications/src/simcore_service_notifications/core/routing_rest.py index 993c299c4aa7..44ccf078968b 100644 --- a/services/notifications/src/simcore_service_notifications/core/routing_rest.py +++ b/services/notifications/src/simcore_service_notifications/core/routing_rest.py @@ -4,7 +4,7 @@ http_exception_as_json_response, ) -from ..domains.health.controller import router as health_router +from ..domains.health.controllers.rest import router as health_router def initialize_rest_api(app: FastAPI) -> None: diff --git a/services/notifications/src/simcore_service_notifications/domains/health/controller.py b/services/notifications/src/simcore_service_notifications/domains/health/controllers/rest.py similarity index 92% rename from services/notifications/src/simcore_service_notifications/domains/health/controller.py rename to services/notifications/src/simcore_service_notifications/domains/health/controllers/rest.py index b5c9deae8312..2adc9a86e68e 100644 --- a/services/notifications/src/simcore_service_notifications/domains/health/controller.py +++ b/services/notifications/src/simcore_service_notifications/domains/health/controllers/rest.py @@ -6,7 +6,7 @@ from models_library.errors import RABBITMQ_CLIENT_UNHEALTHY_MSG from servicelib.rabbitmq import RabbitMQClient -from ...core.dependencies import get_rabbitmq_client +from ....core.dependencies import get_rabbitmq_client router = APIRouter() From 2bd819e19a472d79ffc7d0f73a91ef6b1cc6a627 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 26 Mar 2025 16:21:05 +0100 Subject: [PATCH 11/58] added missing env vars --- services/docker-compose.yml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 9b445f29e9dd..f492ea585e20 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -1107,16 +1107,25 @@ services: memory: 1024M environment: - NOTIFICATIONS_LOGLEVEL: ${NOTIFICATIONS_LOGLEVEL} - LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} LOG_FILTER_MAPPING : ${LOG_FILTER_MAPPING} + LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + + NOTIFICATIONS_LOGLEVEL: ${NOTIFICATIONS_LOGLEVEL} + NOTIFICATIONS_TRACING: ${NOTIFICATIONS_TRACING} + + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_ENDPOINT: ${POSTGRES_ENDPOINT} + POSTGRES_HOST: ${POSTGRES_HOST} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_PORT: ${POSTGRES_PORT} + POSTGRES_USER: ${POSTGRES_USER} + RABBIT_HOST: ${RABBIT_HOST} RABBIT_PASSWORD: ${RABBIT_PASSWORD} RABBIT_PORT: ${RABBIT_PORT} - RABBIT_USER: ${RABBIT_USER} RABBIT_SECURE: ${RABBIT_SECURE} + RABBIT_USER: ${RABBIT_USER} - NOTIFICATIONS_TRACING: ${NOTIFICATIONS_TRACING} TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT: ${TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT} TRACING_OPENTELEMETRY_COLLECTOR_PORT: ${TRACING_OPENTELEMETRY_COLLECTOR_PORT} From 5a89d0b96fe8d48132abcb026dbd9ba347892730 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 28 Mar 2025 05:21:00 +0100 Subject: [PATCH 12/58] added posrtgres to requirements --- services/notifications/requirements/_base.in | 1 + services/notifications/requirements/_base.txt | 80 ++++++++++++++++++- services/notifications/requirements/ci.txt | 1 + services/notifications/requirements/dev.txt | 1 + services/notifications/requirements/prod.txt | 1 + 5 files changed, 83 insertions(+), 1 deletion(-) diff --git a/services/notifications/requirements/_base.in b/services/notifications/requirements/_base.in index 91752463674b..d820f83a1534 100644 --- a/services/notifications/requirements/_base.in +++ b/services/notifications/requirements/_base.in @@ -9,6 +9,7 @@ --requirement ../../../packages/common-library/requirements/_base.in --requirement ../../../packages/models-library/requirements/_base.in --requirement ../../../packages/settings-library/requirements/_base.in +--requirement ../../../packages/postgres-database/requirements/_base.in # service-library[fastapi] --requirement ../../../packages/service-library/requirements/_base.in --requirement ../../../packages/service-library/requirements/_fastapi.in diff --git a/services/notifications/requirements/_base.txt b/services/notifications/requirements/_base.txt index 8f34bfb846fe..4bfe2f0f4a86 100644 --- a/services/notifications/requirements/_base.txt +++ b/services/notifications/requirements/_base.txt @@ -15,6 +15,8 @@ aiohttp==3.11.14 # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -29,6 +31,8 @@ aiormq==6.8.1 # via aio-pika aiosignal==1.3.2 # via aiohttp +alembic==1.15.1 + # via -r requirements/../../../packages/postgres-database/requirements/_base.in annotated-types==0.7.0 # via pydantic anyio==4.9.0 @@ -44,6 +48,8 @@ arrow==1.3.0 # -r requirements/../../../packages/service-library/requirements/_base.in asgiref==3.8.1 # via opentelemetry-instrumentation-asgi +asyncpg==0.30.0 + # via sqlalchemy attrs==25.3.0 # via # aiohttp @@ -54,6 +60,8 @@ certifi==2025.1.31 # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -103,6 +111,8 @@ googleapis-common-protos==1.69.2 # via # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http +greenlet==3.1.1 + # via sqlalchemy grpcio==1.71.0 # via opentelemetry-exporter-otlp-proto-grpc h11==0.14.0 @@ -116,6 +126,8 @@ httpx==0.28.1 # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -141,8 +153,27 @@ jsonschema==4.23.0 # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in jsonschema-specifications==2024.10.1 # via jsonschema +mako==1.3.9 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # alembic markdown-it-py==3.0.0 # via rich +markupsafe==3.0.2 + # via mako mdurl==0.1.2 # via markdown-it-py multidict==6.2.0 @@ -156,6 +187,7 @@ opentelemetry-api==1.31.1 # opentelemetry-exporter-otlp-proto-http # opentelemetry-instrumentation # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx # opentelemetry-instrumentation-logging @@ -176,6 +208,7 @@ opentelemetry-exporter-otlp-proto-http==1.31.1 opentelemetry-instrumentation==0.52b1 # via # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx # opentelemetry-instrumentation-logging @@ -183,6 +216,8 @@ opentelemetry-instrumentation==0.52b1 # opentelemetry-instrumentation-requests opentelemetry-instrumentation-asgi==0.52b1 # via opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-asyncpg==0.52b1 + # via -r requirements/../../../packages/postgres-database/requirements/_base.in opentelemetry-instrumentation-fastapi==0.52b1 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in opentelemetry-instrumentation-httpx==0.52b1 @@ -207,6 +242,7 @@ opentelemetry-semantic-conventions==0.52b1 # via # opentelemetry-instrumentation # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-asyncpg # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-httpx # opentelemetry-instrumentation-redis @@ -223,6 +259,8 @@ orjson==3.10.16 # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -235,6 +273,7 @@ orjson==3.10.16 # -r requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in @@ -262,6 +301,8 @@ protobuf==5.29.4 # opentelemetry-proto psutil==7.0.0 # via -r requirements/../../../packages/service-library/requirements/_base.in +psycopg2-binary==2.9.10 + # via sqlalchemy pycryptodome==3.22.0 # via stream-zip pydantic==2.10.6 @@ -269,6 +310,8 @@ pydantic==2.10.6 # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -281,6 +324,8 @@ pydantic==2.10.6 # -r requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in @@ -301,6 +346,7 @@ pydantic-extra-types==2.10.3 # -r requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in @@ -311,6 +357,8 @@ pydantic-settings==2.7.0 # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -337,6 +385,8 @@ pyyaml==6.0.2 # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -352,6 +402,8 @@ redis==5.2.1 # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -367,6 +419,8 @@ referencing==0.35.1 # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -385,7 +439,7 @@ rich==13.9.4 # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # typer -rpds-py==0.23.1 +rpds-py==0.24.0 # via # jsonschema # referencing @@ -395,11 +449,31 @@ six==1.17.0 # via python-dateutil sniffio==1.3.1 # via anyio +sqlalchemy==1.4.54 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/postgres-database/requirements/_base.in + # alembic starlette==0.46.1 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -428,6 +502,7 @@ types-python-dateutil==2.9.0.20241206 typing-extensions==4.13.0 # via # aiodebug + # alembic # anyio # fastapi # faststream @@ -441,6 +516,8 @@ urllib3==2.3.0 # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -463,6 +540,7 @@ wrapt==1.17.2 # opentelemetry-instrumentation-redis yarl==1.18.3 # via + # -r requirements/../../../packages/postgres-database/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # aio-pika # aiohttp diff --git a/services/notifications/requirements/ci.txt b/services/notifications/requirements/ci.txt index 431489bb1343..4248619bbd46 100644 --- a/services/notifications/requirements/ci.txt +++ b/services/notifications/requirements/ci.txt @@ -14,6 +14,7 @@ # installs this repo's packages simcore-common-library @ ../../packages/common-library simcore-models-library @ ../../packages/models-library +simcore-postgres-datase @ ../../packages/postgres-datase pytest-simcore @ ../../packages/pytest-simcore simcore-service-library[fastapi] @ ../../packages/service-library simcore-settings-library @ ../../packages/settings-library diff --git a/services/notifications/requirements/dev.txt b/services/notifications/requirements/dev.txt index 46a7ed46c894..4e73fc7a83a3 100644 --- a/services/notifications/requirements/dev.txt +++ b/services/notifications/requirements/dev.txt @@ -14,6 +14,7 @@ # installs this repo's packages --editable ../../packages/common-library --editable ../../packages/models-library +--editable ../../packages/postgres-database --editable ../../packages/pytest-simcore --editable ../../packages/service-library[fastapi] --editable ../../packages/settings-library diff --git a/services/notifications/requirements/prod.txt b/services/notifications/requirements/prod.txt index 80c1b291e379..a392aca13a6a 100644 --- a/services/notifications/requirements/prod.txt +++ b/services/notifications/requirements/prod.txt @@ -12,6 +12,7 @@ # installs this repo's packages simcore-models-library @ ../../packages/models-library simcore-common-library @ ../../packages/common-library/ +simcore-postgres-database @ ../../packages/postgres-database/ simcore-service-library[fastapi] @ ../../packages/service-library simcore-settings-library @ ../../packages/settings-library From 9e26f0bf7af7776589b9a790dbba99acaf24cb0e Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 28 Mar 2025 05:24:29 +0100 Subject: [PATCH 13/58] added rest of service --- .../src/models_library/errors.py | 1 + .../core/application.py | 2 + .../core/dependencies.py | 11 +++++ .../core/settings.py | 8 ++++ .../domains/database/__init__.py | 0 .../domains/database/_health.py | 40 +++++++++++++++++++ .../domains/database/service.py | 39 ++++++++++++++++++ .../domains/health/controllers/rest.py | 12 +++++- .../domains/rabbitmq/service.py | 9 +++-- services/notifications/tests/conftest.py | 1 + services/notifications/tests/unit/conftest.py | 2 + services/notifications/tests/unit/test_cli.py | 7 +++- .../tests/unit/test_domain_health.py | 1 + 13 files changed, 127 insertions(+), 6 deletions(-) create mode 100644 services/notifications/src/simcore_service_notifications/domains/database/__init__.py create mode 100644 services/notifications/src/simcore_service_notifications/domains/database/_health.py create mode 100644 services/notifications/src/simcore_service_notifications/domains/database/service.py diff --git a/packages/models-library/src/models_library/errors.py b/packages/models-library/src/models_library/errors.py index cec882e12b74..88cad99df10e 100644 --- a/packages/models-library/src/models_library/errors.py +++ b/packages/models-library/src/models_library/errors.py @@ -35,6 +35,7 @@ class ErrorDict(_ErrorDictRequired, total=False): RABBITMQ_CLIENT_UNHEALTHY_MSG = "RabbitMQ client is in a bad state!" +POSRGRES_DATABASE_UNHEALTHY_MSG = "PosrgrePostgres db is in a bad state!" REDIS_CLIENT_UNHEALTHY_MSG = "Redis cannot be reached!" DOCKER_API_PROXY_UNHEALTHY_MSG = "docker-api-proxy service is not reachable!" diff --git a/services/notifications/src/simcore_service_notifications/core/application.py b/services/notifications/src/simcore_service_notifications/core/application.py index 504945739b73..d926e89b3371 100644 --- a/services/notifications/src/simcore_service_notifications/core/application.py +++ b/services/notifications/src/simcore_service_notifications/core/application.py @@ -19,6 +19,7 @@ SUMMARY, VERSION, ) +from ..domains.database.service import lifespan_postgres from ..domains.rabbitmq.service import lifespan_rabbitmq from .routing_rest import initialize_rest_api from .routing_rpc import lifespan_rpc_api_routes @@ -53,6 +54,7 @@ def create_app() -> FastAPI: lifespans: list[LifespanGenerator] = [ lifespan_rabbitmq, + lifespan_postgres, lifespan_rpc_api_routes, ] diff --git a/services/notifications/src/simcore_service_notifications/core/dependencies.py b/services/notifications/src/simcore_service_notifications/core/dependencies.py index 800efcfe15d8..d89b4d231e9e 100644 --- a/services/notifications/src/simcore_service_notifications/core/dependencies.py +++ b/services/notifications/src/simcore_service_notifications/core/dependencies.py @@ -5,6 +5,11 @@ from fastapi import Depends, FastAPI, Request from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient +from ..domains.database.service import ( + PostgresHealth, +) +from ..domains.database.service import get_postgress_health as get_postgress_db_health + def get_application(request: Request) -> FastAPI: return cast(FastAPI, request.app) @@ -15,3 +20,9 @@ def get_rabbitmq_client( ) -> RabbitMQRPCClient: assert isinstance(app.state.rabbitmq_rpc_server, RabbitMQRPCClient) # nosec return app.state.rabbitmq_rpc_server + + +def get_postgress_health( + app: Annotated[FastAPI, Depends(get_application)], +) -> PostgresHealth: + return get_postgress_db_health(app) diff --git a/services/notifications/src/simcore_service_notifications/core/settings.py b/services/notifications/src/simcore_service_notifications/core/settings.py index 014838ce2955..beaa3dde2c7b 100644 --- a/services/notifications/src/simcore_service_notifications/core/settings.py +++ b/services/notifications/src/simcore_service_notifications/core/settings.py @@ -5,6 +5,7 @@ from pydantic import AliasChoices, Field, field_validator from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.base import BaseCustomSettings +from settings_library.postgres import PostgresSettings from settings_library.rabbit import RabbitSettings from settings_library.tracing import TracingSettings from settings_library.utils_logging import MixinLoggingSettings @@ -57,6 +58,13 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): ), ] + NOTIFICATIONS_POSTGRES: Annotated[ + PostgresSettings, + Field( + json_schema_extra={"auto_default_from_env": True}, + ), + ] + NOTIFICATIONS_TRACING: Annotated[ TracingSettings | None, Field( diff --git a/services/notifications/src/simcore_service_notifications/domains/database/__init__.py b/services/notifications/src/simcore_service_notifications/domains/database/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/services/notifications/src/simcore_service_notifications/domains/database/_health.py b/services/notifications/src/simcore_service_notifications/domains/database/_health.py new file mode 100644 index 000000000000..439ec02b2aca --- /dev/null +++ b/services/notifications/src/simcore_service_notifications/domains/database/_health.py @@ -0,0 +1,40 @@ +import logging +from asyncio import Task +from datetime import timedelta + +from fastapi import FastAPI +from models_library.healthchecks import IsResponsive, LivenessResult +from servicelib.async_utils import cancel_wait_task +from servicelib.background_task import create_periodic_task +from servicelib.db_asyncpg_utils import check_postgres_liveness +from servicelib.fastapi.db_asyncpg_engine import get_engine +from servicelib.logging_utils import log_catch + +_logger = logging.getLogger(__name__) + + +class PostgresHealth: + def __init__(self, app: FastAPI) -> None: + self.app = app + + self._liveness_result: LivenessResult = IsResponsive(elapsed=timedelta(0)) + self._task: Task | None = None + + async def _check_task(self) -> None: + self._liveness_result = await check_postgres_liveness(get_engine(self.app)) + + @property + def is_responsive(self) -> bool: + return isinstance(self._liveness_result, IsResponsive) + + async def setup(self) -> None: + self._task = create_periodic_task( + self._check_task, + interval=timedelta(seconds=10), + task_name="posgress_liveness_check", + ) + + async def teardown(self) -> None: + if self._task is not None: + with log_catch(_logger, reraise=False): + await cancel_wait_task(self._task, max_delay=5) diff --git a/services/notifications/src/simcore_service_notifications/domains/database/service.py b/services/notifications/src/simcore_service_notifications/domains/database/service.py new file mode 100644 index 000000000000..0e60e49713f3 --- /dev/null +++ b/services/notifications/src/simcore_service_notifications/domains/database/service.py @@ -0,0 +1,39 @@ +import logging +from collections.abc import AsyncIterator + +from fastapi import FastAPI +from fastapi_lifespan_manager import State +from servicelib.fastapi.db_asyncpg_engine import ( + close_db_connection, + connect_to_db, +) +from servicelib.logging_utils import log_context + +from ...core.settings import ApplicationSettings +from ._health import PostgresHealth + +_logger = logging.getLogger(__name__) + + +async def lifespan_postgres(app: FastAPI) -> AsyncIterator[State]: + settings: ApplicationSettings = app.state.settings + + app.state.postgress_liveness = PostgresHealth(app) + + with log_context(_logger, logging.INFO, msg="connecting to postgres"): + await connect_to_db(app, settings.NOTIFICATIONS_POSTGRES) + await app.state.postgress_liveness.setup() + + yield {} + + with log_context(_logger, logging.INFO, msg="disconnecting from postgres"): + await app.state.postgress_liveness.teardown() + await close_db_connection(app) + + +def get_postgress_health(app: FastAPI) -> PostgresHealth: + assert isinstance(app.state.postgress_liveness, PostgresHealth) # nosec + return app.state.postgress_liveness + + +__all__: tuple[str, ...] = ("PostgresHealth",) diff --git a/services/notifications/src/simcore_service_notifications/domains/health/controllers/rest.py b/services/notifications/src/simcore_service_notifications/domains/health/controllers/rest.py index 2adc9a86e68e..ce51f92fc2a3 100644 --- a/services/notifications/src/simcore_service_notifications/domains/health/controllers/rest.py +++ b/services/notifications/src/simcore_service_notifications/domains/health/controllers/rest.py @@ -3,10 +3,14 @@ import arrow from fastapi import APIRouter, Depends from models_library.api_schemas__common.health import HealthCheckGet -from models_library.errors import RABBITMQ_CLIENT_UNHEALTHY_MSG +from models_library.errors import ( + POSRGRES_DATABASE_UNHEALTHY_MSG, + RABBITMQ_CLIENT_UNHEALTHY_MSG, +) from servicelib.rabbitmq import RabbitMQClient -from ....core.dependencies import get_rabbitmq_client +from ....core.dependencies import get_postgress_health, get_rabbitmq_client +from ...database.service import PostgresHealth router = APIRouter() @@ -18,8 +22,12 @@ class HealthCheckError(RuntimeError): @router.get("/health", response_model=HealthCheckGet) async def check_service_health( rabbitmq_client: Annotated[RabbitMQClient, Depends(get_rabbitmq_client)], + posrgress_health: Annotated[PostgresHealth, Depends(get_postgress_health)], ): if not rabbitmq_client.healthy: raise HealthCheckError(RABBITMQ_CLIENT_UNHEALTHY_MSG) + if not posrgress_health.is_responsive: + raise HealthCheckError(POSRGRES_DATABASE_UNHEALTHY_MSG) + return HealthCheckGet(timestamp=f"{__name__}@{arrow.utcnow().datetime.isoformat()}") diff --git a/services/notifications/src/simcore_service_notifications/domains/rabbitmq/service.py b/services/notifications/src/simcore_service_notifications/domains/rabbitmq/service.py index 6185d4e64c31..2a656540316f 100644 --- a/services/notifications/src/simcore_service_notifications/domains/rabbitmq/service.py +++ b/services/notifications/src/simcore_service_notifications/domains/rabbitmq/service.py @@ -6,15 +6,18 @@ from servicelib.rabbitmq import RabbitMQRPCClient, wait_till_rabbitmq_responsive from settings_library.rabbit import RabbitSettings +from ...core.settings import ApplicationSettings + async def lifespan_rabbitmq(app: FastAPI) -> AsyncIterator[State]: - settings: RabbitSettings = app.state.settings.NOTIFICATIONS_RABBITMQ + settings: ApplicationSettings = app.state.settings + rabbit_settings: RabbitSettings = settings.NOTIFICATIONS_RABBITMQ app.state.rabbitmq_rpc_server = None - await wait_till_rabbitmq_responsive(settings.dsn) + await wait_till_rabbitmq_responsive(rabbit_settings.dsn) app.state.rabbitmq_rpc_server = await RabbitMQRPCClient.create( - client_name="dynamic_scheduler_rpc_server", settings=settings + client_name="dynamic_scheduler_rpc_server", settings=rabbit_settings ) yield {} diff --git a/services/notifications/tests/conftest.py b/services/notifications/tests/conftest.py index a848f56562d5..65c289240d7f 100644 --- a/services/notifications/tests/conftest.py +++ b/services/notifications/tests/conftest.py @@ -9,6 +9,7 @@ pytest_plugins = [ "pytest_simcore.docker_compose", "pytest_simcore.docker_swarm", + "pytest_simcore.postgres_service", "pytest_simcore.rabbit_service", "pytest_simcore.repository_paths", ] diff --git a/services/notifications/tests/unit/conftest.py b/services/notifications/tests/unit/conftest.py index 56e46e5a2e6d..7d581eddc3e4 100644 --- a/services/notifications/tests/unit/conftest.py +++ b/services/notifications/tests/unit/conftest.py @@ -17,6 +17,7 @@ def service_env( monkeypatch: pytest.MonkeyPatch, mock_environment: EnvVarsDict, rabbit_service: RabbitSettings, + postgres_env_vars_dict: EnvVarsDict, ) -> EnvVarsDict: return setenvs_from_dict( monkeypatch, @@ -27,6 +28,7 @@ def service_env( "RABBIT_PORT": f"{rabbit_service.RABBIT_PORT}", "RABBIT_SECURE": f"{rabbit_service.RABBIT_SECURE}", "RABBIT_USER": rabbit_service.RABBIT_USER, + **postgres_env_vars_dict, }, ) diff --git a/services/notifications/tests/unit/test_cli.py b/services/notifications/tests/unit/test_cli.py index d3833a6a3d00..71f124b1f65d 100644 --- a/services/notifications/tests/unit/test_cli.py +++ b/services/notifications/tests/unit/test_cli.py @@ -9,9 +9,14 @@ from simcore_service_notifications.cli import main from typer.testing import CliRunner +pytest_simcore_core_services_selection = [ + "postgres", + "rabbit", +] + @pytest.fixture -def cli_runner(mock_environment: EnvVarsDict) -> CliRunner: +def cli_runner(service_env: EnvVarsDict) -> CliRunner: return CliRunner() diff --git a/services/notifications/tests/unit/test_domain_health.py b/services/notifications/tests/unit/test_domain_health.py index 9f0904c182ea..13497658172f 100644 --- a/services/notifications/tests/unit/test_domain_health.py +++ b/services/notifications/tests/unit/test_domain_health.py @@ -7,6 +7,7 @@ from models_library.api_schemas__common.health import HealthCheckGet pytest_simcore_core_services_selection = [ + "postgres", "rabbit", ] From d0f3e0299475a41f71f8c63cb3ede3a073060581 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 28 Mar 2025 05:27:23 +0100 Subject: [PATCH 14/58] removed unused --- services/notifications/tests/conftest.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/services/notifications/tests/conftest.py b/services/notifications/tests/conftest.py index 65c289240d7f..6b5a947232d5 100644 --- a/services/notifications/tests/conftest.py +++ b/services/notifications/tests/conftest.py @@ -2,10 +2,6 @@ # pylint: disable=unused-argument -import pytest -from models_library.basic_types import BootModeEnum -from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict - pytest_plugins = [ "pytest_simcore.docker_compose", "pytest_simcore.docker_swarm", @@ -13,18 +9,3 @@ "pytest_simcore.rabbit_service", "pytest_simcore.repository_paths", ] - - -@pytest.fixture -def mock_environment(monkeypatch: pytest.MonkeyPatch) -> EnvVarsDict: - return setenvs_from_dict( - monkeypatch, - { - "LOGLEVEL": "DEBUG", - "SC_BOOT_MODE": BootModeEnum.DEBUG, - "RABBIT_HOST": "test", - "RABBIT_PASSWORD": "test", - "RABBIT_SECURE": "false", - "RABBIT_USER": "test", - }, - ) From 441706e8cf3be1d18660c5e362645a254fa19b8c Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 28 Mar 2025 05:45:28 +0100 Subject: [PATCH 15/58] revert --- services/notifications/tests/conftest.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/services/notifications/tests/conftest.py b/services/notifications/tests/conftest.py index 6b5a947232d5..a310c11b5d51 100644 --- a/services/notifications/tests/conftest.py +++ b/services/notifications/tests/conftest.py @@ -2,6 +2,10 @@ # pylint: disable=unused-argument +import pytest +from models_library.basic_types import BootModeEnum +from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict + pytest_plugins = [ "pytest_simcore.docker_compose", "pytest_simcore.docker_swarm", @@ -9,3 +13,14 @@ "pytest_simcore.rabbit_service", "pytest_simcore.repository_paths", ] + + +@pytest.fixture +def mock_environment(monkeypatch: pytest.MonkeyPatch) -> EnvVarsDict: + return setenvs_from_dict( + monkeypatch, + { + "LOGLEVEL": "DEBUG", + "SC_BOOT_MODE": BootModeEnum.DEBUG, + }, + ) From d9659dd9cbcbef441fe0e199c66657429e214c4d Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 28 Mar 2025 05:51:28 +0100 Subject: [PATCH 16/58] added spec generation --- services/notifications/.env-devel | 14 ++++++ services/notifications/Makefile | 13 ++++++ services/notifications/openapi.json | 66 +++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+) create mode 100644 services/notifications/.env-devel create mode 100644 services/notifications/openapi.json diff --git a/services/notifications/.env-devel b/services/notifications/.env-devel new file mode 100644 index 000000000000..323bf6862ddd --- /dev/null +++ b/services/notifications/.env-devel @@ -0,0 +1,14 @@ +SC_BOOT_MODE=debug + +RABBIT_HOST=rabbit +RABBIT_PASSWORD=adminadmin +RABBIT_PORT=5672 +RABBIT_SECURE=false +RABBIT_USER=admin + +POSTGRES_DB=simcoredb +POSTGRES_ENDPOINT=postgres:5432 +POSTGRES_HOST=postgres +POSTGRES_PASSWORD=adminadmin +POSTGRES_PORT=5432 +POSTGRES_USER=scu diff --git a/services/notifications/Makefile b/services/notifications/Makefile index 8c208ed2a314..e85180bc847e 100644 --- a/services/notifications/Makefile +++ b/services/notifications/Makefile @@ -3,3 +3,16 @@ # include ../../scripts/common.Makefile include ../../scripts/common-service.Makefile + + +.PHONY: openapi.json +openapi-specs: openapi.json +openapi.json: .env + # generating openapi specs file (need to have the environment set for this) + @set -o allexport; \ + source $<; \ + set +o allexport; \ + python3 -c "import json; from $(APP_PACKAGE_NAME).main import *; print( json.dumps(the_app.openapi(), indent=2) )" > $@ + + # validates OAS file: $@ + $(call validate_openapi_specs,$@) diff --git a/services/notifications/openapi.json b/services/notifications/openapi.json new file mode 100644 index 000000000000..feb2fff4626e --- /dev/null +++ b/services/notifications/openapi.json @@ -0,0 +1,66 @@ +{ + "openapi": "3.1.0", + "info": { + "title": "simcore-service-notifications", + "description": "Service for executing commands on docker nodes", + "version": "0.0.1" + }, + "servers": [ + { + "url": "/", + "description": "Default server: requests directed to serving url" + }, + { + "url": "http://{host}:{port}", + "description": "Development server: can configure any base url", + "variables": { + "host": { + "default": "127.0.0.1" + }, + "port": { + "default": "8000" + } + } + } + ], + "paths": { + "/health": { + "get": { + "summary": "Check Service Health", + "operationId": "check_service_health_health_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HealthCheckGet" + } + } + } + } + } + } + } + }, + "components": { + "schemas": { + "HealthCheckGet": { + "properties": { + "timestamp": { + "type": "string", + "title": "Timestamp" + } + }, + "type": "object", + "required": [ + "timestamp" + ], + "title": "HealthCheckGet", + "example": { + "timestamp": "simcore_service_directorv2.api.routes.health@2023-07-03T12:59:12.024551+00:00" + } + } + } + } +} From e5ea1eaf499ccc216cb8cdb710fe30aef1c4605e Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 28 Mar 2025 05:55:16 +0100 Subject: [PATCH 17/58] fixed install --- services/notifications/requirements/ci.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/notifications/requirements/ci.txt b/services/notifications/requirements/ci.txt index 4248619bbd46..b738654d3be5 100644 --- a/services/notifications/requirements/ci.txt +++ b/services/notifications/requirements/ci.txt @@ -14,7 +14,7 @@ # installs this repo's packages simcore-common-library @ ../../packages/common-library simcore-models-library @ ../../packages/models-library -simcore-postgres-datase @ ../../packages/postgres-datase +simcore-postgres-database @ ../../packages/postgres-database pytest-simcore @ ../../packages/pytest-simcore simcore-service-library[fastapi] @ ../../packages/service-library simcore-settings-library @ ../../packages/settings-library From 44560dbc964da8c317857b6b88c8eeef1ca389f0 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 28 Mar 2025 05:57:02 +0100 Subject: [PATCH 18/58] module can now be imported --- .../domains/health/controllers/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 services/notifications/src/simcore_service_notifications/domains/health/controllers/__init__.py diff --git a/services/notifications/src/simcore_service_notifications/domains/health/controllers/__init__.py b/services/notifications/src/simcore_service_notifications/domains/health/controllers/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 From 878dd0a2ec477c378c4a4f678eda3d6400a6993d Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 28 Mar 2025 06:08:01 +0100 Subject: [PATCH 19/58] bumped timeouts --- services/notifications/tests/unit/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/notifications/tests/unit/conftest.py b/services/notifications/tests/unit/conftest.py index 7d581eddc3e4..c09ed434b9a6 100644 --- a/services/notifications/tests/unit/conftest.py +++ b/services/notifications/tests/unit/conftest.py @@ -37,7 +37,7 @@ def service_env( async def initialized_app(service_env: EnvVarsDict) -> AsyncIterator[FastAPI]: app: FastAPI = create_app() - async with LifespanManager(app): + async with LifespanManager(app, startup_timeout=30, shutdown_timeout=30): yield app From 7647fdab77d9d02d24c92f55f02f301b8dffe589 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 28 Mar 2025 06:23:27 +0100 Subject: [PATCH 20/58] constrained aiopika --- services/notifications/requirements/_base.txt | 13 +++++++++---- services/notifications/requirements/constraints.txt | 1 + 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/services/notifications/requirements/_base.txt b/services/notifications/requirements/_base.txt index 4bfe2f0f4a86..d02cec468ed1 100644 --- a/services/notifications/requirements/_base.txt +++ b/services/notifications/requirements/_base.txt @@ -1,5 +1,7 @@ -aio-pika==9.5.5 - # via -r requirements/../../../packages/service-library/requirements/_base.in +aio-pika==9.5.3 + # via + # -c requirements/./constraints.txt + # -r requirements/../../../packages/service-library/requirements/_base.in aiocache==0.12.3 # via -r requirements/../../../packages/service-library/requirements/_base.in aiodebug==2.3.0 @@ -305,7 +307,7 @@ psycopg2-binary==2.9.10 # via sqlalchemy pycryptodome==3.22.0 # via stream-zip -pydantic==2.10.6 +pydantic==2.11.0 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -339,7 +341,7 @@ pydantic==2.10.6 # fastapi # pydantic-extra-types # pydantic-settings -pydantic-core==2.27.2 +pydantic-core==2.33.0 # via pydantic pydantic-extra-types==2.10.3 # via @@ -511,6 +513,9 @@ typing-extensions==4.13.0 # pydantic-core # pydantic-extra-types # typer + # typing-inspection +typing-inspection==0.4.0 + # via pydantic urllib3==2.3.0 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt diff --git a/services/notifications/requirements/constraints.txt b/services/notifications/requirements/constraints.txt index e69de29bb2d1..5b162c9ebd09 100644 --- a/services/notifications/requirements/constraints.txt +++ b/services/notifications/requirements/constraints.txt @@ -0,0 +1 @@ +aio-pika<=9.5.3 From c5a5ffd2874a01075be0627b064240e40d441c9a Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 28 Mar 2025 06:28:23 +0100 Subject: [PATCH 21/58] revert --- services/notifications/requirements/_base.txt | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/services/notifications/requirements/_base.txt b/services/notifications/requirements/_base.txt index d02cec468ed1..3e5d8ce47dd2 100644 --- a/services/notifications/requirements/_base.txt +++ b/services/notifications/requirements/_base.txt @@ -1,7 +1,5 @@ -aio-pika==9.5.3 - # via - # -c requirements/./constraints.txt - # -r requirements/../../../packages/service-library/requirements/_base.in +aio-pika==9.5.5 + # via -r requirements/../../../packages/service-library/requirements/_base.in aiocache==0.12.3 # via -r requirements/../../../packages/service-library/requirements/_base.in aiodebug==2.3.0 From 0e4076384202b96a2e102ea301ab60349082ea6b Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 28 Mar 2025 06:38:33 +0100 Subject: [PATCH 22/58] alligned files --- services/notifications/requirements/ci.txt | 11 +++++------ services/notifications/requirements/prod.txt | 6 +++--- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/services/notifications/requirements/ci.txt b/services/notifications/requirements/ci.txt index b738654d3be5..e082494f2928 100644 --- a/services/notifications/requirements/ci.txt +++ b/services/notifications/requirements/ci.txt @@ -12,12 +12,11 @@ --requirement _tools.txt # installs this repo's packages -simcore-common-library @ ../../packages/common-library -simcore-models-library @ ../../packages/models-library -simcore-postgres-database @ ../../packages/postgres-database -pytest-simcore @ ../../packages/pytest-simcore -simcore-service-library[fastapi] @ ../../packages/service-library -simcore-settings-library @ ../../packages/settings-library +simcore-common-library @ ../../packages/common-library/ +simcore-models-library @ ../../packages/models-library/ +simcore-postgres-database @ ../../packages/postgres-database/ +simcore-service-library[fastapi] @ ../../packages/service-library/ +simcore-settings-library @ ../../packages/settings-library/ # installs current package simcore-service-notifications @ . diff --git a/services/notifications/requirements/prod.txt b/services/notifications/requirements/prod.txt index a392aca13a6a..f203156b59cd 100644 --- a/services/notifications/requirements/prod.txt +++ b/services/notifications/requirements/prod.txt @@ -10,11 +10,11 @@ --requirement _base.txt # installs this repo's packages -simcore-models-library @ ../../packages/models-library simcore-common-library @ ../../packages/common-library/ +simcore-models-library @ ../../packages/models-library/ simcore-postgres-database @ ../../packages/postgres-database/ -simcore-service-library[fastapi] @ ../../packages/service-library -simcore-settings-library @ ../../packages/settings-library +simcore-service-library[fastapi] @ ../../packages/service-library/ +simcore-settings-library @ ../../packages/settings-library/ # installs current package simcore-service-notifications @ . From b9fac906b1f673e5df3951cb2a31bf8a684273be Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 28 Mar 2025 06:44:19 +0100 Subject: [PATCH 23/58] added missing --- services/notifications/requirements/ci.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/services/notifications/requirements/ci.txt b/services/notifications/requirements/ci.txt index e082494f2928..21975753559d 100644 --- a/services/notifications/requirements/ci.txt +++ b/services/notifications/requirements/ci.txt @@ -15,6 +15,7 @@ simcore-common-library @ ../../packages/common-library/ simcore-models-library @ ../../packages/models-library/ simcore-postgres-database @ ../../packages/postgres-database/ +pytest-simcore @ ../../packages/pytest-simcore/ simcore-service-library[fastapi] @ ../../packages/service-library/ simcore-settings-library @ ../../packages/settings-library/ From dfe5af5d0879d950b37e8df996fe1c711b8d9e88 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 28 Mar 2025 06:45:05 +0100 Subject: [PATCH 24/58] removed constraints --- services/notifications/requirements/constraints.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/services/notifications/requirements/constraints.txt b/services/notifications/requirements/constraints.txt index 5b162c9ebd09..e69de29bb2d1 100644 --- a/services/notifications/requirements/constraints.txt +++ b/services/notifications/requirements/constraints.txt @@ -1 +0,0 @@ -aio-pika<=9.5.3 From 7ba0b4610a819b7de20ac2ddf82be08b9ce52671 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 28 Mar 2025 08:03:49 +0100 Subject: [PATCH 25/58] making test async to try in ci --- services/notifications/tests/unit/test_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/notifications/tests/unit/test_cli.py b/services/notifications/tests/unit/test_cli.py index 71f124b1f65d..68a23976b96e 100644 --- a/services/notifications/tests/unit/test_cli.py +++ b/services/notifications/tests/unit/test_cli.py @@ -26,7 +26,7 @@ def _format_cli_error(result: Result) -> str: return f"Below exception was raised by the cli:\n{tb_message}" -def test_process_cli_options(cli_runner: CliRunner): +async def test_process_cli_options(cli_runner: CliRunner): result = cli_runner.invoke(main, ["--help"]) print(result.stdout) assert result.exit_code == 0, _format_cli_error(result) From 1913266b33393e6f35853e46f55468fe1e9cc444 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 28 Mar 2025 08:47:53 +0100 Subject: [PATCH 26/58] rename --- .../src/simcore_service_notifications/core/routing_rest.py | 2 +- .../domains/health/{controllers => cnt}/__init__.py | 0 .../domains/health/{controllers => cnt}/rest.py | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename services/notifications/src/simcore_service_notifications/domains/health/{controllers => cnt}/__init__.py (100%) rename services/notifications/src/simcore_service_notifications/domains/health/{controllers => cnt}/rest.py (100%) diff --git a/services/notifications/src/simcore_service_notifications/core/routing_rest.py b/services/notifications/src/simcore_service_notifications/core/routing_rest.py index 44ccf078968b..e23db8427283 100644 --- a/services/notifications/src/simcore_service_notifications/core/routing_rest.py +++ b/services/notifications/src/simcore_service_notifications/core/routing_rest.py @@ -4,7 +4,7 @@ http_exception_as_json_response, ) -from ..domains.health.controllers.rest import router as health_router +from ..domains.health.cnt.rest import router as health_router def initialize_rest_api(app: FastAPI) -> None: diff --git a/services/notifications/src/simcore_service_notifications/domains/health/controllers/__init__.py b/services/notifications/src/simcore_service_notifications/domains/health/cnt/__init__.py similarity index 100% rename from services/notifications/src/simcore_service_notifications/domains/health/controllers/__init__.py rename to services/notifications/src/simcore_service_notifications/domains/health/cnt/__init__.py diff --git a/services/notifications/src/simcore_service_notifications/domains/health/controllers/rest.py b/services/notifications/src/simcore_service_notifications/domains/health/cnt/rest.py similarity index 100% rename from services/notifications/src/simcore_service_notifications/domains/health/controllers/rest.py rename to services/notifications/src/simcore_service_notifications/domains/health/cnt/rest.py From 82f64189c74e49752e5b9f5b1709afb956c5cf33 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 28 Mar 2025 08:47:58 +0100 Subject: [PATCH 27/58] =?UTF-8?q?services/notifications=20version:=200.0.1?= =?UTF-8?q?=20=E2=86=92=200.0.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/notifications/VERSION | 2 +- services/notifications/setup.cfg | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/services/notifications/VERSION b/services/notifications/VERSION index 8acdd82b765e..4e379d2bfeab 100644 --- a/services/notifications/VERSION +++ b/services/notifications/VERSION @@ -1 +1 @@ -0.0.1 +0.0.2 diff --git a/services/notifications/setup.cfg b/services/notifications/setup.cfg index e420fe7efd21..a626845e2baf 100644 --- a/services/notifications/setup.cfg +++ b/services/notifications/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.0.1 +current_version = 0.0.2 commit = True message = services/notifications version: {current_version} → {new_version} tag = False @@ -11,5 +11,5 @@ commit_args = --no-verify asyncio_mode = auto [mypy] -plugins = +plugins = pydantic.mypy From 22a6c69fb0d1832736401c01edee1fb9966966b1 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 28 Mar 2025 08:50:20 +0100 Subject: [PATCH 28/58] revert changes --- .../src/simcore_service_notifications/core/routing_rest.py | 2 +- .../domains/health/{cnt => controllers}/__init__.py | 0 .../domains/health/{cnt => controllers}/rest.py | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename services/notifications/src/simcore_service_notifications/domains/health/{cnt => controllers}/__init__.py (100%) rename services/notifications/src/simcore_service_notifications/domains/health/{cnt => controllers}/rest.py (100%) diff --git a/services/notifications/src/simcore_service_notifications/core/routing_rest.py b/services/notifications/src/simcore_service_notifications/core/routing_rest.py index e23db8427283..44ccf078968b 100644 --- a/services/notifications/src/simcore_service_notifications/core/routing_rest.py +++ b/services/notifications/src/simcore_service_notifications/core/routing_rest.py @@ -4,7 +4,7 @@ http_exception_as_json_response, ) -from ..domains.health.cnt.rest import router as health_router +from ..domains.health.controllers.rest import router as health_router def initialize_rest_api(app: FastAPI) -> None: diff --git a/services/notifications/src/simcore_service_notifications/domains/health/cnt/__init__.py b/services/notifications/src/simcore_service_notifications/domains/health/controllers/__init__.py similarity index 100% rename from services/notifications/src/simcore_service_notifications/domains/health/cnt/__init__.py rename to services/notifications/src/simcore_service_notifications/domains/health/controllers/__init__.py diff --git a/services/notifications/src/simcore_service_notifications/domains/health/cnt/rest.py b/services/notifications/src/simcore_service_notifications/domains/health/controllers/rest.py similarity index 100% rename from services/notifications/src/simcore_service_notifications/domains/health/cnt/rest.py rename to services/notifications/src/simcore_service_notifications/domains/health/controllers/rest.py From 852ae92942a6ea0bb94288f5ebaeac82bbc39e0d Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 28 Mar 2025 08:50:39 +0100 Subject: [PATCH 29/58] =?UTF-8?q?services/notifications=20version:=200.0.2?= =?UTF-8?q?=20=E2=86=92=200.0.3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/notifications/VERSION | 2 +- services/notifications/setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/notifications/VERSION b/services/notifications/VERSION index 4e379d2bfeab..bcab45af15a0 100644 --- a/services/notifications/VERSION +++ b/services/notifications/VERSION @@ -1 +1 @@ -0.0.2 +0.0.3 diff --git a/services/notifications/setup.cfg b/services/notifications/setup.cfg index a626845e2baf..6f9323e263bb 100644 --- a/services/notifications/setup.cfg +++ b/services/notifications/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.0.2 +current_version = 0.0.3 commit = True message = services/notifications version: {current_version} → {new_version} tag = False From 2347d89871b066148f64dec4ed1f0fa0e17daa0e Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 9 Apr 2025 11:22:23 +0200 Subject: [PATCH 30/58] rename --- .../{domains => api}/__init__.py | 0 .../{domains/database => api/rest}/__init__.py | 0 .../health/controllers/rest.py => api/rest/_health.py} | 4 ++-- .../{core/routing_rest.py => api/rest/routing.py} | 2 +- .../{domains/health => api/rpc}/__init__.py | 0 .../{core/routing_rpc.py => api/rpc/routing.py} | 2 +- .../src/simcore_service_notifications/core/application.py | 8 ++++---- .../simcore_service_notifications/core/dependencies.py | 6 ++---- .../domains/rabbitmq/__init__.py | 0 .../{domains/health/controllers => services}/__init__.py | 0 .../database/service.py => services/postgres/__init__.py} | 0 .../{domains/database => services/postgres}/_health.py | 0 .../{domains/rabbitmq/service.py => services/rabbitmq.py} | 2 +- .../{test_domain_health.py => test_api_rest__health.py} | 0 14 files changed, 11 insertions(+), 13 deletions(-) rename services/notifications/src/simcore_service_notifications/{domains => api}/__init__.py (100%) rename services/notifications/src/simcore_service_notifications/{domains/database => api/rest}/__init__.py (100%) rename services/notifications/src/simcore_service_notifications/{domains/health/controllers/rest.py => api/rest/_health.py} (88%) rename services/notifications/src/simcore_service_notifications/{core/routing_rest.py => api/rest/routing.py} (84%) rename services/notifications/src/simcore_service_notifications/{domains/health => api/rpc}/__init__.py (100%) rename services/notifications/src/simcore_service_notifications/{core/routing_rpc.py => api/rpc/routing.py} (89%) delete mode 100644 services/notifications/src/simcore_service_notifications/domains/rabbitmq/__init__.py rename services/notifications/src/simcore_service_notifications/{domains/health/controllers => services}/__init__.py (100%) rename services/notifications/src/simcore_service_notifications/{domains/database/service.py => services/postgres/__init__.py} (100%) rename services/notifications/src/simcore_service_notifications/{domains/database => services/postgres}/_health.py (100%) rename services/notifications/src/simcore_service_notifications/{domains/rabbitmq/service.py => services/rabbitmq.py} (95%) rename services/notifications/tests/unit/{test_domain_health.py => test_api_rest__health.py} (100%) diff --git a/services/notifications/src/simcore_service_notifications/domains/__init__.py b/services/notifications/src/simcore_service_notifications/api/__init__.py similarity index 100% rename from services/notifications/src/simcore_service_notifications/domains/__init__.py rename to services/notifications/src/simcore_service_notifications/api/__init__.py diff --git a/services/notifications/src/simcore_service_notifications/domains/database/__init__.py b/services/notifications/src/simcore_service_notifications/api/rest/__init__.py similarity index 100% rename from services/notifications/src/simcore_service_notifications/domains/database/__init__.py rename to services/notifications/src/simcore_service_notifications/api/rest/__init__.py diff --git a/services/notifications/src/simcore_service_notifications/domains/health/controllers/rest.py b/services/notifications/src/simcore_service_notifications/api/rest/_health.py similarity index 88% rename from services/notifications/src/simcore_service_notifications/domains/health/controllers/rest.py rename to services/notifications/src/simcore_service_notifications/api/rest/_health.py index ce51f92fc2a3..aba07f3ce4f2 100644 --- a/services/notifications/src/simcore_service_notifications/domains/health/controllers/rest.py +++ b/services/notifications/src/simcore_service_notifications/api/rest/_health.py @@ -9,8 +9,8 @@ ) from servicelib.rabbitmq import RabbitMQClient -from ....core.dependencies import get_postgress_health, get_rabbitmq_client -from ...database.service import PostgresHealth +from ...core.dependencies import get_postgress_health, get_rabbitmq_client +from ...services.postgres import PostgresHealth router = APIRouter() diff --git a/services/notifications/src/simcore_service_notifications/core/routing_rest.py b/services/notifications/src/simcore_service_notifications/api/rest/routing.py similarity index 84% rename from services/notifications/src/simcore_service_notifications/core/routing_rest.py rename to services/notifications/src/simcore_service_notifications/api/rest/routing.py index 44ccf078968b..5fae78c42ce0 100644 --- a/services/notifications/src/simcore_service_notifications/core/routing_rest.py +++ b/services/notifications/src/simcore_service_notifications/api/rest/routing.py @@ -4,7 +4,7 @@ http_exception_as_json_response, ) -from ..domains.health.controllers.rest import router as health_router +from ._health import router as health_router def initialize_rest_api(app: FastAPI) -> None: diff --git a/services/notifications/src/simcore_service_notifications/domains/health/__init__.py b/services/notifications/src/simcore_service_notifications/api/rpc/__init__.py similarity index 100% rename from services/notifications/src/simcore_service_notifications/domains/health/__init__.py rename to services/notifications/src/simcore_service_notifications/api/rpc/__init__.py diff --git a/services/notifications/src/simcore_service_notifications/core/routing_rpc.py b/services/notifications/src/simcore_service_notifications/api/rpc/routing.py similarity index 89% rename from services/notifications/src/simcore_service_notifications/core/routing_rpc.py rename to services/notifications/src/simcore_service_notifications/api/rpc/routing.py index 9a4789bb9706..92c322a7a80b 100644 --- a/services/notifications/src/simcore_service_notifications/core/routing_rpc.py +++ b/services/notifications/src/simcore_service_notifications/api/rpc/routing.py @@ -5,7 +5,7 @@ from models_library.api_schemas_notifications import NOTIFICATIONS_RPC_NAMESPACE from servicelib.rabbitmq import RPCRouter -from ..domains.rabbitmq.service import get_rabbitmq_rpc_server +from ...services.rabbitmq import get_rabbitmq_rpc_server ROUTERS: list[RPCRouter] = [ # import form various domains and attach here diff --git a/services/notifications/src/simcore_service_notifications/core/application.py b/services/notifications/src/simcore_service_notifications/core/application.py index d926e89b3371..d759c432b01b 100644 --- a/services/notifications/src/simcore_service_notifications/core/application.py +++ b/services/notifications/src/simcore_service_notifications/core/application.py @@ -19,10 +19,10 @@ SUMMARY, VERSION, ) -from ..domains.database.service import lifespan_postgres -from ..domains.rabbitmq.service import lifespan_rabbitmq -from .routing_rest import initialize_rest_api -from .routing_rpc import lifespan_rpc_api_routes +from ..api.rest.routing import initialize_rest_api +from ..api.rpc.routing import lifespan_rpc_api_routes +from ..services.postgres.service import lifespan_postgres +from ..services.rabbitmq import lifespan_rabbitmq from .settings import ApplicationSettings _logger = logging.getLogger(__name__) diff --git a/services/notifications/src/simcore_service_notifications/core/dependencies.py b/services/notifications/src/simcore_service_notifications/core/dependencies.py index d89b4d231e9e..f0405ae4a7bb 100644 --- a/services/notifications/src/simcore_service_notifications/core/dependencies.py +++ b/services/notifications/src/simcore_service_notifications/core/dependencies.py @@ -5,10 +5,8 @@ from fastapi import Depends, FastAPI, Request from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient -from ..domains.database.service import ( - PostgresHealth, -) -from ..domains.database.service import get_postgress_health as get_postgress_db_health +from ..services.postgres import PostgresHealth +from ..services.postgres import get_postgress_health as get_postgress_db_health def get_application(request: Request) -> FastAPI: diff --git a/services/notifications/src/simcore_service_notifications/domains/rabbitmq/__init__.py b/services/notifications/src/simcore_service_notifications/domains/rabbitmq/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/services/notifications/src/simcore_service_notifications/domains/health/controllers/__init__.py b/services/notifications/src/simcore_service_notifications/services/__init__.py similarity index 100% rename from services/notifications/src/simcore_service_notifications/domains/health/controllers/__init__.py rename to services/notifications/src/simcore_service_notifications/services/__init__.py diff --git a/services/notifications/src/simcore_service_notifications/domains/database/service.py b/services/notifications/src/simcore_service_notifications/services/postgres/__init__.py similarity index 100% rename from services/notifications/src/simcore_service_notifications/domains/database/service.py rename to services/notifications/src/simcore_service_notifications/services/postgres/__init__.py diff --git a/services/notifications/src/simcore_service_notifications/domains/database/_health.py b/services/notifications/src/simcore_service_notifications/services/postgres/_health.py similarity index 100% rename from services/notifications/src/simcore_service_notifications/domains/database/_health.py rename to services/notifications/src/simcore_service_notifications/services/postgres/_health.py diff --git a/services/notifications/src/simcore_service_notifications/domains/rabbitmq/service.py b/services/notifications/src/simcore_service_notifications/services/rabbitmq.py similarity index 95% rename from services/notifications/src/simcore_service_notifications/domains/rabbitmq/service.py rename to services/notifications/src/simcore_service_notifications/services/rabbitmq.py index 2a656540316f..34095d7f38c3 100644 --- a/services/notifications/src/simcore_service_notifications/domains/rabbitmq/service.py +++ b/services/notifications/src/simcore_service_notifications/services/rabbitmq.py @@ -6,7 +6,7 @@ from servicelib.rabbitmq import RabbitMQRPCClient, wait_till_rabbitmq_responsive from settings_library.rabbit import RabbitSettings -from ...core.settings import ApplicationSettings +from ..core.settings import ApplicationSettings async def lifespan_rabbitmq(app: FastAPI) -> AsyncIterator[State]: diff --git a/services/notifications/tests/unit/test_domain_health.py b/services/notifications/tests/unit/test_api_rest__health.py similarity index 100% rename from services/notifications/tests/unit/test_domain_health.py rename to services/notifications/tests/unit/test_api_rest__health.py From 2af93ea5cf1f4bfdbb1b66c5a2475cffbc47aebe Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 9 Apr 2025 13:02:00 +0200 Subject: [PATCH 31/58] fixed broken tests --- .../api/rpc/routing.py | 2 +- .../core/application.py | 37 +++-------- .../core/events.py | 63 +++++++++++++++++++ .../core/settings.py | 2 + .../services/postgres/__init__.py | 5 +- .../services/rabbitmq.py | 2 +- services/notifications/tests/unit/conftest.py | 2 + 7 files changed, 82 insertions(+), 31 deletions(-) create mode 100644 services/notifications/src/simcore_service_notifications/core/events.py diff --git a/services/notifications/src/simcore_service_notifications/api/rpc/routing.py b/services/notifications/src/simcore_service_notifications/api/rpc/routing.py index 92c322a7a80b..79ac6afcf0ae 100644 --- a/services/notifications/src/simcore_service_notifications/api/rpc/routing.py +++ b/services/notifications/src/simcore_service_notifications/api/rpc/routing.py @@ -12,7 +12,7 @@ ] -async def lifespan_rpc_api_routes(app: FastAPI) -> AsyncIterator[State]: +async def rpc_api_routes_lifespan(app: FastAPI) -> AsyncIterator[State]: rpc_server = get_rabbitmq_rpc_server(app) for router in ROUTERS: diff --git a/services/notifications/src/simcore_service_notifications/core/application.py b/services/notifications/src/simcore_service_notifications/core/application.py index d759c432b01b..04d17dab8b2f 100644 --- a/services/notifications/src/simcore_service_notifications/core/application.py +++ b/services/notifications/src/simcore_service_notifications/core/application.py @@ -1,28 +1,19 @@ import logging -from collections.abc import AsyncIterator from fastapi import FastAPI -from fastapi_lifespan_manager import State -from servicelib.fastapi.lifespan_utils import LifespanGenerator, combine_lifespans from servicelib.fastapi.openapi import ( get_common_oas_options, override_fastapi_openapi_method, ) +from servicelib.fastapi.prometheus_instrumentation import ( + initialize_prometheus_instrumentation, +) from servicelib.fastapi.tracing import initialize_tracing from servicelib.logging_utils import config_all_loggers -from .._meta import ( - API_VTAG, - APP_FINISHED_BANNER_MSG, - APP_NAME, - APP_STARTED_BANNER_MSG, - SUMMARY, - VERSION, -) +from .._meta import API_VTAG, APP_NAME, SUMMARY, VERSION from ..api.rest.routing import initialize_rest_api -from ..api.rpc.routing import lifespan_rpc_api_routes -from ..services.postgres.service import lifespan_postgres -from ..services.rabbitmq import lifespan_rabbitmq +from . import events from .settings import ApplicationSettings _logger = logging.getLogger(__name__) @@ -39,25 +30,12 @@ def _initialise_logger(settings: ApplicationSettings): ) -async def _lifespan_banner(app: FastAPI) -> AsyncIterator[State]: - _ = app - print(APP_STARTED_BANNER_MSG, flush=True) # noqa: T201 - yield {} - print(APP_FINISHED_BANNER_MSG, flush=True) # noqa: T201 - - def create_app() -> FastAPI: settings = ApplicationSettings.create_from_envs() _logger.debug(settings.model_dump_json(indent=2)) _initialise_logger(settings) - lifespans: list[LifespanGenerator] = [ - lifespan_rabbitmq, - lifespan_postgres, - lifespan_rpc_api_routes, - ] - assert settings.SC_BOOT_MODE # nosec app = FastAPI( debug=settings.SC_BOOT_MODE.is_devel_mode(), @@ -65,7 +43,7 @@ def create_app() -> FastAPI: description=SUMMARY, version=f"{VERSION}", openapi_url=f"/api/{API_VTAG}/openapi.json", - lifespan=combine_lifespans(*lifespans, _lifespan_banner), + lifespan=events.create_app_lifespan(), **get_common_oas_options(is_devel_mode=settings.SC_BOOT_MODE.is_devel_mode()), ) override_fastapi_openapi_method(app) @@ -73,6 +51,9 @@ def create_app() -> FastAPI: initialize_rest_api(app) + if settings.NOTIFICATIONS_PROMETHEUS_INSTRUMENTATION_ENABLED: + initialize_prometheus_instrumentation(app) + if settings.NOTIFICATIONS_TRACING: initialize_tracing(app, settings.NOTIFICATIONS_TRACING, APP_NAME) diff --git a/services/notifications/src/simcore_service_notifications/core/events.py b/services/notifications/src/simcore_service_notifications/core/events.py new file mode 100644 index 000000000000..b32ea900c4b3 --- /dev/null +++ b/services/notifications/src/simcore_service_notifications/core/events.py @@ -0,0 +1,63 @@ +from collections.abc import AsyncIterator + +from fastapi import FastAPI +from fastapi_lifespan_manager import LifespanManager, State +from servicelib.fastapi.postgres_lifespan import ( + PostgresLifespanState, + postgres_database_lifespan, +) +from servicelib.fastapi.prometheus_instrumentation import ( + lifespan_prometheus_instrumentation, +) + +from .._meta import APP_FINISHED_BANNER_MSG, APP_STARTED_BANNER_MSG +from ..api.rpc.routing import rpc_api_routes_lifespan +from ..services.postgres import postgres_lifespan +from ..services.rabbitmq import rabbitmq_lifespan +from .settings import ApplicationSettings + + +async def _banner_lifespan(app: FastAPI) -> AsyncIterator[State]: + _ = app + print(APP_STARTED_BANNER_MSG, flush=True) # noqa: T201 + yield {} + print(APP_FINISHED_BANNER_MSG, flush=True) # noqa: T201 + + +async def _main_lifespan(app: FastAPI) -> AsyncIterator[State]: + settings: ApplicationSettings = app.state.settings + yield { + PostgresLifespanState.POSTGRES_SETTINGS: settings.NOTIFICATIONS_POSTGRES, + "prometheus_instrumentation_enabled": settings.NOTIFICATIONS_PROMETHEUS_INSTRUMENTATION_ENABLED, + } + + +async def _prometheus_instrumentation_lifespan( # TODO: put this into one single setup like we did for the DB + app: FastAPI, state: State +) -> AsyncIterator[State]: + if state.get("prometheus_instrumentation_enabled", False): + async for prometheus_state in lifespan_prometheus_instrumentation(app): + yield prometheus_state + + +def create_app_lifespan(): + # WARNING: order matters + app_lifespan = LifespanManager() + app_lifespan.add(_main_lifespan) + + # - postgres + app_lifespan.add(postgres_database_lifespan) + app_lifespan.add(postgres_lifespan) + + # - rabbitmq + app_lifespan.add(rabbitmq_lifespan) + + # - rpc api routes + app_lifespan.add(rpc_api_routes_lifespan) + + # - prometheus instrumentation + app_lifespan.add(_prometheus_instrumentation_lifespan) + + app_lifespan.add(_banner_lifespan) + + return app_lifespan diff --git a/services/notifications/src/simcore_service_notifications/core/settings.py b/services/notifications/src/simcore_service_notifications/core/settings.py index beaa3dde2c7b..6f7e13a546e3 100644 --- a/services/notifications/src/simcore_service_notifications/core/settings.py +++ b/services/notifications/src/simcore_service_notifications/core/settings.py @@ -73,6 +73,8 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): ), ] + NOTIFICATIONS_PROMETHEUS_INSTRUMENTATION_ENABLED: bool = True + @field_validator("LOG_LEVEL") @classmethod def valid_log_level(cls, value) -> LogLevel: diff --git a/services/notifications/src/simcore_service_notifications/services/postgres/__init__.py b/services/notifications/src/simcore_service_notifications/services/postgres/__init__.py index 0e60e49713f3..413c2e038d98 100644 --- a/services/notifications/src/simcore_service_notifications/services/postgres/__init__.py +++ b/services/notifications/src/simcore_service_notifications/services/postgres/__init__.py @@ -7,6 +7,7 @@ close_db_connection, connect_to_db, ) +from servicelib.fastapi.postgres_lifespan import PostgresLifespanState from servicelib.logging_utils import log_context from ...core.settings import ApplicationSettings @@ -15,7 +16,9 @@ _logger = logging.getLogger(__name__) -async def lifespan_postgres(app: FastAPI) -> AsyncIterator[State]: +async def postgres_lifespan(app: FastAPI, state: State) -> AsyncIterator[State]: + app.state.engine = state[PostgresLifespanState.POSTGRES_ASYNC_ENGINE] + settings: ApplicationSettings = app.state.settings app.state.postgress_liveness = PostgresHealth(app) diff --git a/services/notifications/src/simcore_service_notifications/services/rabbitmq.py b/services/notifications/src/simcore_service_notifications/services/rabbitmq.py index 34095d7f38c3..3c205c401621 100644 --- a/services/notifications/src/simcore_service_notifications/services/rabbitmq.py +++ b/services/notifications/src/simcore_service_notifications/services/rabbitmq.py @@ -9,7 +9,7 @@ from ..core.settings import ApplicationSettings -async def lifespan_rabbitmq(app: FastAPI) -> AsyncIterator[State]: +async def rabbitmq_lifespan(app: FastAPI) -> AsyncIterator[State]: settings: ApplicationSettings = app.state.settings rabbit_settings: RabbitSettings = settings.NOTIFICATIONS_RABBITMQ app.state.rabbitmq_rpc_server = None diff --git a/services/notifications/tests/unit/conftest.py b/services/notifications/tests/unit/conftest.py index c09ed434b9a6..e1f57c7c5c76 100644 --- a/services/notifications/tests/unit/conftest.py +++ b/services/notifications/tests/unit/conftest.py @@ -4,6 +4,7 @@ from collections.abc import AsyncIterator import pytest +import sqlalchemy as sa from asgi_lifespan import LifespanManager from fastapi import FastAPI from fastapi.testclient import TestClient @@ -17,6 +18,7 @@ def service_env( monkeypatch: pytest.MonkeyPatch, mock_environment: EnvVarsDict, rabbit_service: RabbitSettings, + postgres_db: sa.engine.Engine, postgres_env_vars_dict: EnvVarsDict, ) -> EnvVarsDict: return setenvs_from_dict( From e769ee3b756ded359b009e45ae10e6414a64f012 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 10 Apr 2025 15:43:38 +0200 Subject: [PATCH 32/58] renamed and repalce --- .../src/simcore_service_notifications/core/events.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/services/notifications/src/simcore_service_notifications/core/events.py b/services/notifications/src/simcore_service_notifications/core/events.py index b32ea900c4b3..a9382441e944 100644 --- a/services/notifications/src/simcore_service_notifications/core/events.py +++ b/services/notifications/src/simcore_service_notifications/core/events.py @@ -3,7 +3,7 @@ from fastapi import FastAPI from fastapi_lifespan_manager import LifespanManager, State from servicelib.fastapi.postgres_lifespan import ( - PostgresLifespanState, + create_input_state, postgres_database_lifespan, ) from servicelib.fastapi.prometheus_instrumentation import ( @@ -24,15 +24,16 @@ async def _banner_lifespan(app: FastAPI) -> AsyncIterator[State]: print(APP_FINISHED_BANNER_MSG, flush=True) # noqa: T201 -async def _main_lifespan(app: FastAPI) -> AsyncIterator[State]: +async def _settings_lifespan(app: FastAPI) -> AsyncIterator[State]: settings: ApplicationSettings = app.state.settings yield { - PostgresLifespanState.POSTGRES_SETTINGS: settings.NOTIFICATIONS_POSTGRES, + **create_input_state(settings.NOTIFICATIONS_POSTGRES), + # TODO this shoudl also be replaced ensire thing should be repalced "prometheus_instrumentation_enabled": settings.NOTIFICATIONS_PROMETHEUS_INSTRUMENTATION_ENABLED, } -async def _prometheus_instrumentation_lifespan( # TODO: put this into one single setup like we did for the DB +async def _prometheus_instrumentation_lifespan( app: FastAPI, state: State ) -> AsyncIterator[State]: if state.get("prometheus_instrumentation_enabled", False): @@ -43,7 +44,7 @@ async def _prometheus_instrumentation_lifespan( # TODO: put this into one singl def create_app_lifespan(): # WARNING: order matters app_lifespan = LifespanManager() - app_lifespan.add(_main_lifespan) + app_lifespan.add(_settings_lifespan) # - postgres app_lifespan.add(postgres_database_lifespan) From 1b15400f29decdef952f75b71445193d2fbe481a Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 11 Apr 2025 07:14:20 +0200 Subject: [PATCH 33/58] using new style lifespans --- .../core/events.py | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/services/notifications/src/simcore_service_notifications/core/events.py b/services/notifications/src/simcore_service_notifications/core/events.py index a9382441e944..1eca188ac621 100644 --- a/services/notifications/src/simcore_service_notifications/core/events.py +++ b/services/notifications/src/simcore_service_notifications/core/events.py @@ -3,11 +3,12 @@ from fastapi import FastAPI from fastapi_lifespan_manager import LifespanManager, State from servicelib.fastapi.postgres_lifespan import ( - create_input_state, + create_postgres_database_input_state, postgres_database_lifespan, ) from servicelib.fastapi.prometheus_instrumentation import ( - lifespan_prometheus_instrumentation, + create_prometheus_instrumentationmain_input_state, + prometheus_instrumentation_lifespan, ) from .._meta import APP_FINISHED_BANNER_MSG, APP_STARTED_BANNER_MSG @@ -27,20 +28,13 @@ async def _banner_lifespan(app: FastAPI) -> AsyncIterator[State]: async def _settings_lifespan(app: FastAPI) -> AsyncIterator[State]: settings: ApplicationSettings = app.state.settings yield { - **create_input_state(settings.NOTIFICATIONS_POSTGRES), - # TODO this shoudl also be replaced ensire thing should be repalced - "prometheus_instrumentation_enabled": settings.NOTIFICATIONS_PROMETHEUS_INSTRUMENTATION_ENABLED, + **create_postgres_database_input_state(settings.NOTIFICATIONS_POSTGRES), + **create_prometheus_instrumentationmain_input_state( + enabled=settings.NOTIFICATIONS_PROMETHEUS_INSTRUMENTATION_ENABLED + ), } -async def _prometheus_instrumentation_lifespan( - app: FastAPI, state: State -) -> AsyncIterator[State]: - if state.get("prometheus_instrumentation_enabled", False): - async for prometheus_state in lifespan_prometheus_instrumentation(app): - yield prometheus_state - - def create_app_lifespan(): # WARNING: order matters app_lifespan = LifespanManager() @@ -57,7 +51,7 @@ def create_app_lifespan(): app_lifespan.add(rpc_api_routes_lifespan) # - prometheus instrumentation - app_lifespan.add(_prometheus_instrumentation_lifespan) + app_lifespan.add(prometheus_instrumentation_lifespan) app_lifespan.add(_banner_lifespan) From 26865a8a75c2411185e1efe7667cfec6360fb37b Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 11 Apr 2025 07:23:05 +0200 Subject: [PATCH 34/58] remove unused --- services/docker-compose.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 0e51faac9871..3a42ee23de74 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -1099,12 +1099,6 @@ services: image: ${DOCKER_REGISTRY:-itisfoundation}/notifications:${DOCKER_IMAGE_TAG:-latest} init: true hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" - deploy: - mode: global - resources: - limits: - cpus: "1.0" - memory: 1024M environment: LOG_FILTER_MAPPING : ${LOG_FILTER_MAPPING} From f9f9155d95f00cdc0da75b1dcd94661a52881fa4 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 11 Apr 2025 07:50:30 +0200 Subject: [PATCH 35/58] updated specs --- services/notifications/openapi.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/notifications/openapi.json b/services/notifications/openapi.json index feb2fff4626e..f2f44875d4e8 100644 --- a/services/notifications/openapi.json +++ b/services/notifications/openapi.json @@ -3,7 +3,7 @@ "info": { "title": "simcore-service-notifications", "description": "Service for executing commands on docker nodes", - "version": "0.0.1" + "version": "0.0.3" }, "servers": [ { From 38bfa823964b9f4311c58246a5b8d4a85f400e79 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 11 Apr 2025 07:51:24 +0200 Subject: [PATCH 36/58] using free port to deploy --- services/docker-compose.local.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/docker-compose.local.yml b/services/docker-compose.local.yml index a215bfc454b7..f07fa42453a9 100644 --- a/services/docker-compose.local.yml +++ b/services/docker-compose.local.yml @@ -94,7 +94,7 @@ services: INVITATIONS_REMOTE_DEBUGGING_PORT : 3000 ports: - "8015:8000" - - "3022:3000" + - "3023:3000" payments: environment: From fccd907de0bea3282c57bcdbd78d587f7b6dc0d8 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 11 Apr 2025 08:16:43 +0200 Subject: [PATCH 37/58] added health endpoint --- packages/pytest-simcore/src/pytest_simcore/simcore_services.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/pytest-simcore/src/pytest_simcore/simcore_services.py b/packages/pytest-simcore/src/pytest_simcore/simcore_services.py index d231fa253078..802adbe735db 100644 --- a/packages/pytest-simcore/src/pytest_simcore/simcore_services.py +++ b/packages/pytest-simcore/src/pytest_simcore/simcore_services.py @@ -47,6 +47,7 @@ "autoscaling": "/", "clusters-keeper": "/", "dask-scheduler": "/health", + "notifications": "/health", "datcore-adapter": "/v0/live", "director-v2": "/", "dynamic-schdlr": "/", From 2bf1f343846e30f592cbbbd90141d933d92f2d43 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 11 Apr 2025 09:51:50 +0200 Subject: [PATCH 38/58] removing rclone --- .github/workflows/ci-testing-deploy.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml index a7d28d300e61..ef3241d0562c 100644 --- a/.github/workflows/ci-testing-deploy.yml +++ b/.github/workflows/ci-testing-deploy.yml @@ -631,8 +631,6 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - - name: install rclone - run: sudo ./ci/github/helpers/install_rclone.bash - name: install uv uses: astral-sh/setup-uv@v5 with: From 6a5ab9b0e6065ad3f2616f7484056831ef8856ed Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 11 Apr 2025 09:52:19 +0200 Subject: [PATCH 39/58] change log level --- .env-devel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env-devel b/.env-devel index 0199c5c1ed52..3169f8fa259a 100644 --- a/.env-devel +++ b/.env-devel @@ -132,7 +132,7 @@ DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT=01:00:00 DIRECTOR_V2_TRACING=null # DYNAMIC_SCHEDULER ---- -DYNAMIC_SCHEDULER_LOGLEVEL=DEBUG +DYNAMIC_SCHEDULER_LOGLEVEL=INFO DYNAMIC_SCHEDULER_PROFILING=1 DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER=0 DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT=01:00:00 From c549b1750448f889654a528d2786ebf2f51081fe Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 11 Apr 2025 09:55:13 +0200 Subject: [PATCH 40/58] rename with better naming --- packages/models-library/src/models_library/errors.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/models-library/src/models_library/errors.py b/packages/models-library/src/models_library/errors.py index 88cad99df10e..d1498f63474f 100644 --- a/packages/models-library/src/models_library/errors.py +++ b/packages/models-library/src/models_library/errors.py @@ -34,10 +34,10 @@ class ErrorDict(_ErrorDictRequired, total=False): ctx: dict[str, Any] -RABBITMQ_CLIENT_UNHEALTHY_MSG = "RabbitMQ client is in a bad state!" -POSRGRES_DATABASE_UNHEALTHY_MSG = "PosrgrePostgres db is in a bad state!" +RABBITMQ_CLIENT_UNHEALTHY_MSG = "RabbitMQ cannot be reached!" +POSRGRES_DATABASE_UNHEALTHY_MSG = "Postgres cannot be reached!" REDIS_CLIENT_UNHEALTHY_MSG = "Redis cannot be reached!" -DOCKER_API_PROXY_UNHEALTHY_MSG = "docker-api-proxy service is not reachable!" +DOCKER_API_PROXY_UNHEALTHY_MSG = "docker-api-proxy cannot be reached!" # NOTE: Here we do not just import as 'from pydantic.error_wrappers import ErrorDict' From a07de4e5876937cacc2d375ac456e46e879cc72b Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 11 Apr 2025 11:28:37 +0200 Subject: [PATCH 41/58] move /health endpoint to / --- packages/pytest-simcore/src/pytest_simcore/simcore_services.py | 2 +- services/notifications/Dockerfile | 2 +- .../src/simcore_service_notifications/api/rest/_health.py | 2 +- services/notifications/tests/unit/test_api_rest__health.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/simcore_services.py b/packages/pytest-simcore/src/pytest_simcore/simcore_services.py index 802adbe735db..54a0b8393952 100644 --- a/packages/pytest-simcore/src/pytest_simcore/simcore_services.py +++ b/packages/pytest-simcore/src/pytest_simcore/simcore_services.py @@ -47,7 +47,7 @@ "autoscaling": "/", "clusters-keeper": "/", "dask-scheduler": "/health", - "notifications": "/health", + "notifications": "/", "datcore-adapter": "/v0/live", "director-v2": "/", "dynamic-schdlr": "/", diff --git a/services/notifications/Dockerfile b/services/notifications/Dockerfile index 0a9ecd6d35db..833f27e2f0e3 100644 --- a/services/notifications/Dockerfile +++ b/services/notifications/Dockerfile @@ -152,7 +152,7 @@ HEALTHCHECK --interval=30s \ --timeout=20s \ --start-period=30s \ --retries=3 \ - CMD ["python3", "services/notifications/docker/healthcheck.py", "http://localhost:8000/health"] + CMD ["python3", "services/notifications/docker/healthcheck.py", "http://localhost:8000/"] EXPOSE 8000 diff --git a/services/notifications/src/simcore_service_notifications/api/rest/_health.py b/services/notifications/src/simcore_service_notifications/api/rest/_health.py index aba07f3ce4f2..7a9f549225e8 100644 --- a/services/notifications/src/simcore_service_notifications/api/rest/_health.py +++ b/services/notifications/src/simcore_service_notifications/api/rest/_health.py @@ -19,7 +19,7 @@ class HealthCheckError(RuntimeError): """Failed a health check""" -@router.get("/health", response_model=HealthCheckGet) +@router.get("/", response_model=HealthCheckGet) async def check_service_health( rabbitmq_client: Annotated[RabbitMQClient, Depends(get_rabbitmq_client)], posrgress_health: Annotated[PostgresHealth, Depends(get_postgress_health)], diff --git a/services/notifications/tests/unit/test_api_rest__health.py b/services/notifications/tests/unit/test_api_rest__health.py index 13497658172f..59054b1286c4 100644 --- a/services/notifications/tests/unit/test_api_rest__health.py +++ b/services/notifications/tests/unit/test_api_rest__health.py @@ -13,6 +13,6 @@ def test_health_ok(test_client: TestClient): - response = test_client.get("/health") + response = test_client.get("/") assert response.status_code == status.HTTP_200_OK assert HealthCheckGet.model_validate(response.json()) From bc3f051e9838919adde6c3fc4ac9c428618fdb65 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 11 Apr 2025 11:31:49 +0200 Subject: [PATCH 42/58] remove unused --- services/notifications/README.md | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 services/notifications/README.md diff --git a/services/notifications/README.md b/services/notifications/README.md deleted file mode 100644 index b678722e9bf0..000000000000 --- a/services/notifications/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# notifications - - -To develop this project, just - -```cmd -make help - -``` From 2d6ec3e8a3b435902c0ef7c67d2963d6c4f09a84 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 11 Apr 2025 12:47:14 +0200 Subject: [PATCH 43/58] removed unused --- services/notifications/.env-devel | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 services/notifications/.env-devel diff --git a/services/notifications/.env-devel b/services/notifications/.env-devel deleted file mode 100644 index 323bf6862ddd..000000000000 --- a/services/notifications/.env-devel +++ /dev/null @@ -1,14 +0,0 @@ -SC_BOOT_MODE=debug - -RABBIT_HOST=rabbit -RABBIT_PASSWORD=adminadmin -RABBIT_PORT=5672 -RABBIT_SECURE=false -RABBIT_USER=admin - -POSTGRES_DB=simcoredb -POSTGRES_ENDPOINT=postgres:5432 -POSTGRES_HOST=postgres -POSTGRES_PASSWORD=adminadmin -POSTGRES_PORT=5432 -POSTGRES_USER=scu From fc65e448aa01a57fedc2333bafc765f4e6fc1a99 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 22 Apr 2025 13:00:14 +0200 Subject: [PATCH 44/58] added owners --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8705bf17e2e9..7a263044f3e9 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -35,6 +35,7 @@ Makefile @pcrespov @sanderegg /services/efs-guardian/ @matusdrobuliak66 /services/invitations/ @pcrespov /services/migration/ @pcrespov +/services/notifications/ @GitHK /services/payments/ @pcrespov @matusdrobuliak66 /services/resource-usage-tracker/ @matusdrobuliak66 /services/static-webserver/ @GitHK From 574934bf5f7199ca1731d4f8383082db8020f9b7 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 22 Apr 2025 13:01:54 +0200 Subject: [PATCH 45/58] health updates --- services/notifications/setup.cfg | 2 +- .../api/rest/_health.py | 6 ++--- .../{core => api/rest}/dependencies.py | 6 ++--- .../api/rpc/routing.py | 4 +++- .../core/application.py | 4 +++- .../services/postgres/__init__.py | 24 +++++++------------ 6 files changed, 21 insertions(+), 25 deletions(-) rename services/notifications/src/simcore_service_notifications/{core => api/rest}/dependencies.py (82%) diff --git a/services/notifications/setup.cfg b/services/notifications/setup.cfg index 6f9323e263bb..bb340a8b81fa 100644 --- a/services/notifications/setup.cfg +++ b/services/notifications/setup.cfg @@ -11,5 +11,5 @@ commit_args = --no-verify asyncio_mode = auto [mypy] -plugins = +plugins = pydantic.mypy diff --git a/services/notifications/src/simcore_service_notifications/api/rest/_health.py b/services/notifications/src/simcore_service_notifications/api/rest/_health.py index 7a9f549225e8..e14b7cbfca24 100644 --- a/services/notifications/src/simcore_service_notifications/api/rest/_health.py +++ b/services/notifications/src/simcore_service_notifications/api/rest/_health.py @@ -9,8 +9,8 @@ ) from servicelib.rabbitmq import RabbitMQClient -from ...core.dependencies import get_postgress_health, get_rabbitmq_client from ...services.postgres import PostgresHealth +from .dependencies import get_postgres_health, get_rabbitmq_client router = APIRouter() @@ -22,12 +22,12 @@ class HealthCheckError(RuntimeError): @router.get("/", response_model=HealthCheckGet) async def check_service_health( rabbitmq_client: Annotated[RabbitMQClient, Depends(get_rabbitmq_client)], - posrgress_health: Annotated[PostgresHealth, Depends(get_postgress_health)], + postgres_health: Annotated[PostgresHealth, Depends(get_postgres_health)], ): if not rabbitmq_client.healthy: raise HealthCheckError(RABBITMQ_CLIENT_UNHEALTHY_MSG) - if not posrgress_health.is_responsive: + if not postgres_health.is_responsive: raise HealthCheckError(POSRGRES_DATABASE_UNHEALTHY_MSG) return HealthCheckGet(timestamp=f"{__name__}@{arrow.utcnow().datetime.isoformat()}") diff --git a/services/notifications/src/simcore_service_notifications/core/dependencies.py b/services/notifications/src/simcore_service_notifications/api/rest/dependencies.py similarity index 82% rename from services/notifications/src/simcore_service_notifications/core/dependencies.py rename to services/notifications/src/simcore_service_notifications/api/rest/dependencies.py index f0405ae4a7bb..8193fcda1081 100644 --- a/services/notifications/src/simcore_service_notifications/core/dependencies.py +++ b/services/notifications/src/simcore_service_notifications/api/rest/dependencies.py @@ -5,8 +5,8 @@ from fastapi import Depends, FastAPI, Request from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient -from ..services.postgres import PostgresHealth -from ..services.postgres import get_postgress_health as get_postgress_db_health +from ...services.postgres import PostgresHealth +from ...services.postgres import get_postgres_health as get_postgress_db_health def get_application(request: Request) -> FastAPI: @@ -20,7 +20,7 @@ def get_rabbitmq_client( return app.state.rabbitmq_rpc_server -def get_postgress_health( +def get_postgres_health( app: Annotated[FastAPI, Depends(get_application)], ) -> PostgresHealth: return get_postgress_db_health(app) diff --git a/services/notifications/src/simcore_service_notifications/api/rpc/routing.py b/services/notifications/src/simcore_service_notifications/api/rpc/routing.py index 79ac6afcf0ae..515dd198bec0 100644 --- a/services/notifications/src/simcore_service_notifications/api/rpc/routing.py +++ b/services/notifications/src/simcore_service_notifications/api/rpc/routing.py @@ -16,6 +16,8 @@ async def rpc_api_routes_lifespan(app: FastAPI) -> AsyncIterator[State]: rpc_server = get_rabbitmq_rpc_server(app) for router in ROUTERS: - await rpc_server.register_router(router, NOTIFICATIONS_RPC_NAMESPACE, app) + await rpc_server.register_router( + router, NOTIFICATIONS_RPC_NAMESPACE, app + ) # pragma: no cover yield {} diff --git a/services/notifications/src/simcore_service_notifications/core/application.py b/services/notifications/src/simcore_service_notifications/core/application.py index 04d17dab8b2f..3b7cfe92c26a 100644 --- a/services/notifications/src/simcore_service_notifications/core/application.py +++ b/services/notifications/src/simcore_service_notifications/core/application.py @@ -55,6 +55,8 @@ def create_app() -> FastAPI: initialize_prometheus_instrumentation(app) if settings.NOTIFICATIONS_TRACING: - initialize_tracing(app, settings.NOTIFICATIONS_TRACING, APP_NAME) + initialize_tracing( + app, settings.NOTIFICATIONS_TRACING, APP_NAME + ) # pragma: no cover return app diff --git a/services/notifications/src/simcore_service_notifications/services/postgres/__init__.py b/services/notifications/src/simcore_service_notifications/services/postgres/__init__.py index 413c2e038d98..aae3b4088ea0 100644 --- a/services/notifications/src/simcore_service_notifications/services/postgres/__init__.py +++ b/services/notifications/src/simcore_service_notifications/services/postgres/__init__.py @@ -3,10 +3,6 @@ from fastapi import FastAPI from fastapi_lifespan_manager import State -from servicelib.fastapi.db_asyncpg_engine import ( - close_db_connection, - connect_to_db, -) from servicelib.fastapi.postgres_lifespan import PostgresLifespanState from servicelib.logging_utils import log_context @@ -19,24 +15,20 @@ async def postgres_lifespan(app: FastAPI, state: State) -> AsyncIterator[State]: app.state.engine = state[PostgresLifespanState.POSTGRES_ASYNC_ENGINE] - settings: ApplicationSettings = app.state.settings + app.state.postgres_health = PostgresHealth(app) - app.state.postgress_liveness = PostgresHealth(app) - - with log_context(_logger, logging.INFO, msg="connecting to postgres"): - await connect_to_db(app, settings.NOTIFICATIONS_POSTGRES) - await app.state.postgress_liveness.setup() + with log_context(_logger, logging.INFO, msg="setup postgres health"): + await app.state.postgres_health.setup() yield {} - with log_context(_logger, logging.INFO, msg="disconnecting from postgres"): - await app.state.postgress_liveness.teardown() - await close_db_connection(app) + with log_context(_logger, logging.INFO, msg="teardown postgres health"): + await app.state.postgres_health.teardown() -def get_postgress_health(app: FastAPI) -> PostgresHealth: - assert isinstance(app.state.postgress_liveness, PostgresHealth) # nosec - return app.state.postgress_liveness +def get_postgres_health(app: FastAPI) -> PostgresHealth: + assert isinstance(app.state.postgres_health, PostgresHealth) # nosec + return app.state.postgres_health __all__: tuple[str, ...] = ("PostgresHealth",) From f11f14a1b68010ff1cc35f9fc4e493cd3b001965 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 22 Apr 2025 13:09:07 +0200 Subject: [PATCH 46/58] refactor --- .../simcore_service_notifications/api/rest/_health.py | 8 ++++---- .../api/rest/dependencies.py | 10 +++++----- .../services/postgres/__init__.py | 11 +++++------ .../services/postgres/{_health.py => _liveness.py} | 2 +- 4 files changed, 15 insertions(+), 16 deletions(-) rename services/notifications/src/simcore_service_notifications/services/postgres/{_health.py => _liveness.py} (98%) diff --git a/services/notifications/src/simcore_service_notifications/api/rest/_health.py b/services/notifications/src/simcore_service_notifications/api/rest/_health.py index e14b7cbfca24..e8d2f6a82b7e 100644 --- a/services/notifications/src/simcore_service_notifications/api/rest/_health.py +++ b/services/notifications/src/simcore_service_notifications/api/rest/_health.py @@ -9,8 +9,8 @@ ) from servicelib.rabbitmq import RabbitMQClient -from ...services.postgres import PostgresHealth -from .dependencies import get_postgres_health, get_rabbitmq_client +from ...services.postgres import PostgresLiveness +from .dependencies import get_postgres_liveness, get_rabbitmq_client router = APIRouter() @@ -22,12 +22,12 @@ class HealthCheckError(RuntimeError): @router.get("/", response_model=HealthCheckGet) async def check_service_health( rabbitmq_client: Annotated[RabbitMQClient, Depends(get_rabbitmq_client)], - postgres_health: Annotated[PostgresHealth, Depends(get_postgres_health)], + postgres_liveness: Annotated[PostgresLiveness, Depends(get_postgres_liveness)], ): if not rabbitmq_client.healthy: raise HealthCheckError(RABBITMQ_CLIENT_UNHEALTHY_MSG) - if not postgres_health.is_responsive: + if not postgres_liveness.is_responsive: raise HealthCheckError(POSRGRES_DATABASE_UNHEALTHY_MSG) return HealthCheckGet(timestamp=f"{__name__}@{arrow.utcnow().datetime.isoformat()}") diff --git a/services/notifications/src/simcore_service_notifications/api/rest/dependencies.py b/services/notifications/src/simcore_service_notifications/api/rest/dependencies.py index 8193fcda1081..7844eeb2ec04 100644 --- a/services/notifications/src/simcore_service_notifications/api/rest/dependencies.py +++ b/services/notifications/src/simcore_service_notifications/api/rest/dependencies.py @@ -5,8 +5,8 @@ from fastapi import Depends, FastAPI, Request from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient -from ...services.postgres import PostgresHealth -from ...services.postgres import get_postgres_health as get_postgress_db_health +from ...services.postgres import PostgresLiveness +from ...services.postgres import get_postgres_liveness as get_postgress_db_liveness def get_application(request: Request) -> FastAPI: @@ -20,7 +20,7 @@ def get_rabbitmq_client( return app.state.rabbitmq_rpc_server -def get_postgres_health( +def get_postgres_liveness( app: Annotated[FastAPI, Depends(get_application)], -) -> PostgresHealth: - return get_postgress_db_health(app) +) -> PostgresLiveness: + return get_postgress_db_liveness(app) diff --git a/services/notifications/src/simcore_service_notifications/services/postgres/__init__.py b/services/notifications/src/simcore_service_notifications/services/postgres/__init__.py index aae3b4088ea0..b34fdec8e793 100644 --- a/services/notifications/src/simcore_service_notifications/services/postgres/__init__.py +++ b/services/notifications/src/simcore_service_notifications/services/postgres/__init__.py @@ -6,8 +6,7 @@ from servicelib.fastapi.postgres_lifespan import PostgresLifespanState from servicelib.logging_utils import log_context -from ...core.settings import ApplicationSettings -from ._health import PostgresHealth +from ._liveness import PostgresLiveness _logger = logging.getLogger(__name__) @@ -15,7 +14,7 @@ async def postgres_lifespan(app: FastAPI, state: State) -> AsyncIterator[State]: app.state.engine = state[PostgresLifespanState.POSTGRES_ASYNC_ENGINE] - app.state.postgres_health = PostgresHealth(app) + app.state.postgres_health = PostgresLiveness(app) with log_context(_logger, logging.INFO, msg="setup postgres health"): await app.state.postgres_health.setup() @@ -26,9 +25,9 @@ async def postgres_lifespan(app: FastAPI, state: State) -> AsyncIterator[State]: await app.state.postgres_health.teardown() -def get_postgres_health(app: FastAPI) -> PostgresHealth: - assert isinstance(app.state.postgres_health, PostgresHealth) # nosec +def get_postgres_liveness(app: FastAPI) -> PostgresLiveness: + assert isinstance(app.state.postgres_health, PostgresLiveness) # nosec return app.state.postgres_health -__all__: tuple[str, ...] = ("PostgresHealth",) +__all__: tuple[str, ...] = ("PostgresLiveness",) diff --git a/services/notifications/src/simcore_service_notifications/services/postgres/_health.py b/services/notifications/src/simcore_service_notifications/services/postgres/_liveness.py similarity index 98% rename from services/notifications/src/simcore_service_notifications/services/postgres/_health.py rename to services/notifications/src/simcore_service_notifications/services/postgres/_liveness.py index 439ec02b2aca..d6587adf6092 100644 --- a/services/notifications/src/simcore_service_notifications/services/postgres/_health.py +++ b/services/notifications/src/simcore_service_notifications/services/postgres/_liveness.py @@ -13,7 +13,7 @@ _logger = logging.getLogger(__name__) -class PostgresHealth: +class PostgresLiveness: def __init__(self, app: FastAPI) -> None: self.app = app From ab870e92a59fd7f2e0d8366079fb7edbb37b52a4 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 22 Apr 2025 13:09:18 +0200 Subject: [PATCH 47/58] remove integrations folder --- services/notifications/tests/integration/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 services/notifications/tests/integration/.gitkeep diff --git a/services/notifications/tests/integration/.gitkeep b/services/notifications/tests/integration/.gitkeep deleted file mode 100644 index e69de29bb2d1..000000000000 From 8f3bb3395d4c92774b0de4e0e764ddbf3283ce69 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 22 Apr 2025 13:28:56 +0200 Subject: [PATCH 48/58] added tests for health --- .../services/postgres/__init__.py | 10 ++--- .../tests/unit/test_api_rest__health.py | 40 ++++++++++++++++++- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/services/notifications/src/simcore_service_notifications/services/postgres/__init__.py b/services/notifications/src/simcore_service_notifications/services/postgres/__init__.py index b34fdec8e793..e0883fba1ced 100644 --- a/services/notifications/src/simcore_service_notifications/services/postgres/__init__.py +++ b/services/notifications/src/simcore_service_notifications/services/postgres/__init__.py @@ -14,20 +14,20 @@ async def postgres_lifespan(app: FastAPI, state: State) -> AsyncIterator[State]: app.state.engine = state[PostgresLifespanState.POSTGRES_ASYNC_ENGINE] - app.state.postgres_health = PostgresLiveness(app) + app.state.postgres_liveness = PostgresLiveness(app) with log_context(_logger, logging.INFO, msg="setup postgres health"): - await app.state.postgres_health.setup() + await app.state.postgres_liveness.setup() yield {} with log_context(_logger, logging.INFO, msg="teardown postgres health"): - await app.state.postgres_health.teardown() + await app.state.postgres_liveness.teardown() def get_postgres_liveness(app: FastAPI) -> PostgresLiveness: - assert isinstance(app.state.postgres_health, PostgresLiveness) # nosec - return app.state.postgres_health + assert isinstance(app.state.postgres_liveness, PostgresLiveness) # nosec + return app.state.postgres_liveness __all__: tuple[str, ...] = ("PostgresLiveness",) diff --git a/services/notifications/tests/unit/test_api_rest__health.py b/services/notifications/tests/unit/test_api_rest__health.py index 59054b1286c4..ba418fe7bc3e 100644 --- a/services/notifications/tests/unit/test_api_rest__health.py +++ b/services/notifications/tests/unit/test_api_rest__health.py @@ -1,10 +1,18 @@ -# pylint: disable=protected-access # pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +import pytest from fastapi import status from fastapi.testclient import TestClient from models_library.api_schemas__common.health import HealthCheckGet +from models_library.errors import ( + POSRGRES_DATABASE_UNHEALTHY_MSG, + RABBITMQ_CLIENT_UNHEALTHY_MSG, +) +from models_library.healthchecks import IsNonResponsive +from pytest_mock import MockerFixture +from simcore_service_notifications.api.rest._health import HealthCheckError pytest_simcore_core_services_selection = [ "postgres", @@ -16,3 +24,33 @@ def test_health_ok(test_client: TestClient): response = test_client.get("/") assert response.status_code == status.HTTP_200_OK assert HealthCheckGet.model_validate(response.json()) + + +@pytest.fixture +def mock_postgres_liveness(mocker: MockerFixture, test_client: TestClient) -> None: + mocker.patch.object( + test_client.app.state.postgres_liveness, + "_liveness_result", + new=IsNonResponsive(reason="fake"), + ) + + +def test_health_postgres_unhealthy( + mock_postgres_liveness: None, test_client: TestClient +): + with pytest.raises(HealthCheckError) as exc: + test_client.get("/") + assert POSRGRES_DATABASE_UNHEALTHY_MSG in f"{exc.value}" + + +@pytest.fixture +def mock_rabbit_healthy(mocker: MockerFixture, test_client: TestClient) -> None: + mocker.patch.object( + test_client.app.state.rabbitmq_rpc_server, "_healthy_state", new=False + ) + + +def test_health_rabbit_unhealthy(mock_rabbit_healthy: None, test_client: TestClient): + with pytest.raises(HealthCheckError) as exc: + test_client.get("/") + assert RABBITMQ_CLIENT_UNHEALTHY_MSG in f"{exc.value}" From e36bd2ba89f485f4b32f54d6d852ed8824ea66d7 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 22 Apr 2025 13:44:09 +0200 Subject: [PATCH 49/58] fixed openapi specs script --- services/notifications/Makefile | 8 +-- services/notifications/openapi.json | 16 +---- .../src/simcore_service_notifications/cli.py | 72 ++++++++++++++++++- 3 files changed, 76 insertions(+), 20 deletions(-) diff --git a/services/notifications/Makefile b/services/notifications/Makefile index e85180bc847e..bc14e6354c18 100644 --- a/services/notifications/Makefile +++ b/services/notifications/Makefile @@ -5,14 +5,14 @@ include ../../scripts/common.Makefile include ../../scripts/common-service.Makefile +.env-ignore: + $(APP_CLI_NAME) echo-dotenv > $@ + .PHONY: openapi.json openapi-specs: openapi.json -openapi.json: .env +openapi.json: .env-ignore ## produces openapi.json # generating openapi specs file (need to have the environment set for this) @set -o allexport; \ source $<; \ set +o allexport; \ python3 -c "import json; from $(APP_PACKAGE_NAME).main import *; print( json.dumps(the_app.openapi(), indent=2) )" > $@ - - # validates OAS file: $@ - $(call validate_openapi_specs,$@) diff --git a/services/notifications/openapi.json b/services/notifications/openapi.json index f2f44875d4e8..e5862ecae45c 100644 --- a/services/notifications/openapi.json +++ b/services/notifications/openapi.json @@ -9,25 +9,13 @@ { "url": "/", "description": "Default server: requests directed to serving url" - }, - { - "url": "http://{host}:{port}", - "description": "Development server: can configure any base url", - "variables": { - "host": { - "default": "127.0.0.1" - }, - "port": { - "default": "8000" - } - } } ], "paths": { - "/health": { + "/": { "get": { "summary": "Check Service Health", - "operationId": "check_service_health_health_get", + "operationId": "check_service_health__get", "responses": { "200": { "description": "Successful Response", diff --git a/services/notifications/src/simcore_service_notifications/cli.py b/services/notifications/src/simcore_service_notifications/cli.py index 2bddceb17179..13bee086290f 100644 --- a/services/notifications/src/simcore_service_notifications/cli.py +++ b/services/notifications/src/simcore_service_notifications/cli.py @@ -1,14 +1,82 @@ import logging +import os import typer -from settings_library.utils_cli import create_settings_command, create_version_callback +from settings_library.postgres import PostgresSettings +from settings_library.rabbit import RabbitSettings +from settings_library.utils_cli import ( + create_settings_command, + create_version_callback, + print_as_envfile, +) from ._meta import APP_NAME, __version__ from .core.settings import ApplicationSettings log = logging.getLogger(__name__) -main = typer.Typer(name=APP_NAME) +main = typer.Typer( + name=APP_NAME, + pretty_exceptions_enable=False, + pretty_exceptions_show_locals=False, +) main.command()(create_settings_command(settings_cls=ApplicationSettings, logger=log)) main.callback()(create_version_callback(__version__)) + + +@main.command() +def echo_dotenv(ctx: typer.Context, *, minimal: bool = True) -> None: + """Generates and displays a valid environment variables file (also known as dot-envfile) + + Usage: + $ simcore-service echo-dotenv > .env + $ cat .env + $ set -o allexport; source .env; set +o allexport + """ + assert ctx # nosec + + # NOTE: we normally DO NOT USE `os.environ` to capture env vars but this is a special case + # The idea here is to have a command that can generate a **valid** `.env` file that can be used + # to initialized the app. For that reason we fill required fields of the `ApplicationSettings` with + # "fake" but valid values (e.g. generating a password or adding tags as `replace-with-api-key). + # Nonetheless, if the caller of this CLI has already some **valid** env vars in the environment we want to use them ... + # and that is why we use `os.environ`. + + settings = ApplicationSettings.create_from_envs( + SC_BOOT_MODE="default", + NOTIFICATIONS_POSTGRES=os.environ.get( + "NOTIFICATIONS_POSTGRES", + PostgresSettings.create_from_envs( + POSTGRES_HOST=os.environ.get( + "POSTGRES_HOST", "replace-with-postgres-host" + ), + POSTGRES_USER=os.environ.get( + "POSTGRES_USER", "replace-with-postgres-user" + ), + POSTGRES_DB=os.environ.get("POSTGRES_DB", "replace-with-postgres-db"), + POSTGRES_PASSWORD=os.environ.get( + "POSTGRES_PASSWORD", "replace-with-postgres-password" + ), + ), + ), + NOTIFICATIONS_RABBITMQ=os.environ.get( + "NOTIFICATIONS_RABBITMQ", + RabbitSettings.create_from_envs( + RABBIT_HOST=os.environ.get("RABBIT_HOST", "replace-with-rabbit-host"), + RABBIT_SECURE=os.environ.get("RABBIT_SECURE", "True"), + RABBIT_USER=os.environ.get("RABBIT_USER", "replace-with-rabbit-user"), + RABBIT_PASSWORD=os.environ.get( + "RABBIT_PASSWORD", "replace-with-rabbit-password" + ), + ), + ), + ) + + print_as_envfile( + settings, + compact=False, + verbose=True, + show_secrets=True, + exclude_unset=minimal, + ) From bc7651d960a44c140ddea3d8bc3eb926d1fd0724 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 22 Apr 2025 13:49:11 +0200 Subject: [PATCH 50/58] refactor services to clients --- .../src/simcore_service_notifications/api/rest/_health.py | 2 +- .../simcore_service_notifications/api/rest/dependencies.py | 4 ++-- .../src/simcore_service_notifications/api/rpc/routing.py | 2 +- .../{services => clients}/__init__.py | 0 .../{services => clients}/postgres/__init__.py | 0 .../{services => clients}/postgres/_liveness.py | 0 .../{services => clients}/rabbitmq.py | 0 .../src/simcore_service_notifications/core/events.py | 4 ++-- 8 files changed, 6 insertions(+), 6 deletions(-) rename services/notifications/src/simcore_service_notifications/{services => clients}/__init__.py (100%) rename services/notifications/src/simcore_service_notifications/{services => clients}/postgres/__init__.py (100%) rename services/notifications/src/simcore_service_notifications/{services => clients}/postgres/_liveness.py (100%) rename services/notifications/src/simcore_service_notifications/{services => clients}/rabbitmq.py (100%) diff --git a/services/notifications/src/simcore_service_notifications/api/rest/_health.py b/services/notifications/src/simcore_service_notifications/api/rest/_health.py index e8d2f6a82b7e..5f38f21d5e03 100644 --- a/services/notifications/src/simcore_service_notifications/api/rest/_health.py +++ b/services/notifications/src/simcore_service_notifications/api/rest/_health.py @@ -9,7 +9,7 @@ ) from servicelib.rabbitmq import RabbitMQClient -from ...services.postgres import PostgresLiveness +from ...clients.postgres import PostgresLiveness from .dependencies import get_postgres_liveness, get_rabbitmq_client router = APIRouter() diff --git a/services/notifications/src/simcore_service_notifications/api/rest/dependencies.py b/services/notifications/src/simcore_service_notifications/api/rest/dependencies.py index 7844eeb2ec04..962154ea9f77 100644 --- a/services/notifications/src/simcore_service_notifications/api/rest/dependencies.py +++ b/services/notifications/src/simcore_service_notifications/api/rest/dependencies.py @@ -5,8 +5,8 @@ from fastapi import Depends, FastAPI, Request from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient -from ...services.postgres import PostgresLiveness -from ...services.postgres import get_postgres_liveness as get_postgress_db_liveness +from ...clients.postgres import PostgresLiveness +from ...clients.postgres import get_postgres_liveness as get_postgress_db_liveness def get_application(request: Request) -> FastAPI: diff --git a/services/notifications/src/simcore_service_notifications/api/rpc/routing.py b/services/notifications/src/simcore_service_notifications/api/rpc/routing.py index 515dd198bec0..2209baa43b56 100644 --- a/services/notifications/src/simcore_service_notifications/api/rpc/routing.py +++ b/services/notifications/src/simcore_service_notifications/api/rpc/routing.py @@ -5,7 +5,7 @@ from models_library.api_schemas_notifications import NOTIFICATIONS_RPC_NAMESPACE from servicelib.rabbitmq import RPCRouter -from ...services.rabbitmq import get_rabbitmq_rpc_server +from ...clients.rabbitmq import get_rabbitmq_rpc_server ROUTERS: list[RPCRouter] = [ # import form various domains and attach here diff --git a/services/notifications/src/simcore_service_notifications/services/__init__.py b/services/notifications/src/simcore_service_notifications/clients/__init__.py similarity index 100% rename from services/notifications/src/simcore_service_notifications/services/__init__.py rename to services/notifications/src/simcore_service_notifications/clients/__init__.py diff --git a/services/notifications/src/simcore_service_notifications/services/postgres/__init__.py b/services/notifications/src/simcore_service_notifications/clients/postgres/__init__.py similarity index 100% rename from services/notifications/src/simcore_service_notifications/services/postgres/__init__.py rename to services/notifications/src/simcore_service_notifications/clients/postgres/__init__.py diff --git a/services/notifications/src/simcore_service_notifications/services/postgres/_liveness.py b/services/notifications/src/simcore_service_notifications/clients/postgres/_liveness.py similarity index 100% rename from services/notifications/src/simcore_service_notifications/services/postgres/_liveness.py rename to services/notifications/src/simcore_service_notifications/clients/postgres/_liveness.py diff --git a/services/notifications/src/simcore_service_notifications/services/rabbitmq.py b/services/notifications/src/simcore_service_notifications/clients/rabbitmq.py similarity index 100% rename from services/notifications/src/simcore_service_notifications/services/rabbitmq.py rename to services/notifications/src/simcore_service_notifications/clients/rabbitmq.py diff --git a/services/notifications/src/simcore_service_notifications/core/events.py b/services/notifications/src/simcore_service_notifications/core/events.py index 1eca188ac621..65a828744e5d 100644 --- a/services/notifications/src/simcore_service_notifications/core/events.py +++ b/services/notifications/src/simcore_service_notifications/core/events.py @@ -13,8 +13,8 @@ from .._meta import APP_FINISHED_BANNER_MSG, APP_STARTED_BANNER_MSG from ..api.rpc.routing import rpc_api_routes_lifespan -from ..services.postgres import postgres_lifespan -from ..services.rabbitmq import rabbitmq_lifespan +from ..clients.postgres import postgres_lifespan +from ..clients.rabbitmq import rabbitmq_lifespan from .settings import ApplicationSettings From 22f69c91d078b8a81822fc0e7e896fa8e62ea589 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 22 Apr 2025 13:49:39 +0200 Subject: [PATCH 51/58] added services as empty --- .../src/simcore_service_notifications/services/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 services/notifications/src/simcore_service_notifications/services/__init__.py diff --git a/services/notifications/src/simcore_service_notifications/services/__init__.py b/services/notifications/src/simcore_service_notifications/services/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 From b6fd7d865d2be0e4dd9e3c19c20d8b55c2dddd98 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 22 Apr 2025 13:57:43 +0200 Subject: [PATCH 52/58] fixed service building --- services/notifications/setup.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/services/notifications/setup.py b/services/notifications/setup.py index f383da6097ba..b515b3b5ca93 100755 --- a/services/notifications/setup.py +++ b/services/notifications/setup.py @@ -25,7 +25,6 @@ def read_reqs(reqs_path: Path) -> set[str]: VERSION = (CURRENT_DIR / "VERSION").read_text().strip() AUTHORS = ("Andrei Neagu (GitHK)",) DESCRIPTION = "Service for executing commands on docker nodes" -README = (CURRENT_DIR / "README.md").read_text() PROD_REQUIREMENTS = tuple( read_reqs(CURRENT_DIR / "requirements" / "_base.txt") @@ -43,7 +42,6 @@ def read_reqs(reqs_path: Path) -> set[str]: "version": VERSION, "author": AUTHORS, "description": DESCRIPTION, - "long_description": README, "license": "MIT license", "python_requires": "~=3.11", "packages": find_packages(where="src"), From 44ea04c5dc505fbf58da1541f6562ef923d9fa35 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 23 Apr 2025 11:32:58 +0200 Subject: [PATCH 53/58] bumped test coverage --- .../dynamic-scheduler/tests/unit/test_cli.py | 2 +- services/notifications/tests/unit/test_cli.py | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/services/dynamic-scheduler/tests/unit/test_cli.py b/services/dynamic-scheduler/tests/unit/test_cli.py index 6bdaa62d1e6d..6c89f5f71dc2 100644 --- a/services/dynamic-scheduler/tests/unit/test_cli.py +++ b/services/dynamic-scheduler/tests/unit/test_cli.py @@ -38,7 +38,7 @@ def test_echo_dotenv(cli_runner: CliRunner, monkeypatch: pytest.MonkeyPatch): with monkeypatch.context() as patch: setenvs_from_dict(patch, environs) - ApplicationSettings.create_from_envs() + assert ApplicationSettings.create_from_envs() def test_list_settings( diff --git a/services/notifications/tests/unit/test_cli.py b/services/notifications/tests/unit/test_cli.py index 68a23976b96e..bcfc7925b61c 100644 --- a/services/notifications/tests/unit/test_cli.py +++ b/services/notifications/tests/unit/test_cli.py @@ -5,8 +5,13 @@ import pytest from click.testing import Result -from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict +from pytest_simcore.helpers.monkeypatch_envs import ( + EnvVarsDict, + load_dotenv, + setenvs_from_dict, +) from simcore_service_notifications.cli import main +from simcore_service_notifications.core.settings import ApplicationSettings from typer.testing import CliRunner pytest_simcore_core_services_selection = [ @@ -38,3 +43,15 @@ async def test_process_cli_options(cli_runner: CliRunner): result = cli_runner.invoke(main, ["--version"]) print(result.stdout) assert result.exit_code == 0, _format_cli_error(result) + + +async def test_echo_dotenv(cli_runner: CliRunner, monkeypatch: pytest.MonkeyPatch): + result = cli_runner.invoke(main, ["echo-dotenv"]) + print(result.stdout) + assert result.exit_code == 0, _format_cli_error(result) + + environs = load_dotenv(result.stdout) + + with monkeypatch.context() as patch: + setenvs_from_dict(patch, environs) + assert ApplicationSettings.create_from_envs() From b5dce44ebfc28b25f39b68fde9048c2acf2110ad Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 23 Apr 2025 11:44:30 +0200 Subject: [PATCH 54/58] fixed failing test --- services/dynamic-sidecar/tests/unit/test_api_rest_health.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_health.py b/services/dynamic-sidecar/tests/unit/test_api_rest_health.py index b7dca61cf637..71ab384d771d 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_health.py +++ b/services/dynamic-sidecar/tests/unit/test_api_rest_health.py @@ -35,4 +35,4 @@ async def test_is_unhealthy_via_rabbitmq(test_client: TestClient) -> None: test_client.application.state.rabbitmq_client._healthy_state = False # noqa: SLF001 response = await test_client.get("/health") assert response.status_code == status.HTTP_503_SERVICE_UNAVAILABLE, response - assert response.json() == {"detail": "RabbitMQ client is in a bad state!"} + assert response.json() == {"detail": "RabbitMQ cannot be reached!"} From 5fae894bb7d8ce84afb8935f08a9259a6d4cd74e Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 24 Apr 2025 14:05:50 +0200 Subject: [PATCH 55/58] sugestions --- services/notifications/Dockerfile | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/services/notifications/Dockerfile b/services/notifications/Dockerfile index 833f27e2f0e3..0718d6f7aa81 100644 --- a/services/notifications/Dockerfile +++ b/services/notifications/Dockerfile @@ -2,7 +2,7 @@ # Define arguments in the global scope ARG PYTHON_VERSION="3.11.9" -ARG UV_VERSION="0.5" +ARG UV_VERSION="0.6" FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv_build # we docker image is built based on debian @@ -11,7 +11,7 @@ FROM python:${PYTHON_VERSION}-slim-bookworm AS base # # USAGE: -# cd sercices/notifications +# cd services/notifications # docker build -f Dockerfile -t notifications:prod --target production ../../ # docker run notifications:prod # @@ -27,7 +27,6 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=private \ apt-get update && \ apt-get install -y --no-install-recommends \ gosu \ - curl \ && apt-get clean -y \ # verify that the binary works && gosu nobody true @@ -58,13 +57,6 @@ ENV PYTHONDONTWRITEBYTECODE=1 \ # those from our virtualenv. ENV PATH="${VIRTUAL_ENV}/bin:$PATH" -# rclone installation -ARG TARGETARCH -ENV TARGETARCH=${TARGETARCH} -RUN \ - --mount=type=bind,source=scripts/install_rclone.bash,target=/tmp/install_rclone.bash \ - ./tmp/install_rclone.bash - # -------------------------- Build stage ------------------- # Installs build/package management tools and third party dependencies # @@ -106,7 +98,7 @@ WORKDIR /build # FROM build AS prod-only-deps -ENV SC_BUILD_TARGET prod-only-deps +ENV SC_BUILD_TARGET=prod-only-deps WORKDIR /build/services/notifications From d98653c0649c1f03ca7f7340744a41fa610fc6a5 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 24 Apr 2025 14:05:56 +0200 Subject: [PATCH 56/58] verison changes --- services/notifications/VERSION | 2 +- services/notifications/openapi.json | 2 +- services/notifications/setup.cfg | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/services/notifications/VERSION b/services/notifications/VERSION index bcab45af15a0..8acdd82b765e 100644 --- a/services/notifications/VERSION +++ b/services/notifications/VERSION @@ -1 +1 @@ -0.0.3 +0.0.1 diff --git a/services/notifications/openapi.json b/services/notifications/openapi.json index e5862ecae45c..3e63b7e994d8 100644 --- a/services/notifications/openapi.json +++ b/services/notifications/openapi.json @@ -3,7 +3,7 @@ "info": { "title": "simcore-service-notifications", "description": "Service for executing commands on docker nodes", - "version": "0.0.3" + "version": "0.0.1" }, "servers": [ { diff --git a/services/notifications/setup.cfg b/services/notifications/setup.cfg index bb340a8b81fa..c3123ece6cf0 100644 --- a/services/notifications/setup.cfg +++ b/services/notifications/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.0.3 +current_version = 1 commit = True message = services/notifications version: {current_version} → {new_version} tag = False From 0fda9c3fc6815230ae1228adf0021b46e9d7ef2c Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 24 Apr 2025 14:18:37 +0200 Subject: [PATCH 57/58] feedback --- services/notifications/setup.py | 2 +- .../src/simcore_service_notifications/api/rpc/routing.py | 2 +- .../clients/postgres/_liveness.py | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/services/notifications/setup.py b/services/notifications/setup.py index b515b3b5ca93..c50365f3e259 100755 --- a/services/notifications/setup.py +++ b/services/notifications/setup.py @@ -24,7 +24,7 @@ def read_reqs(reqs_path: Path) -> set[str]: NAME = "simcore-service-notifications" VERSION = (CURRENT_DIR / "VERSION").read_text().strip() AUTHORS = ("Andrei Neagu (GitHK)",) -DESCRIPTION = "Service for executing commands on docker nodes" +DESCRIPTION = "Service used for sending notifications to users via different channels" PROD_REQUIREMENTS = tuple( read_reqs(CURRENT_DIR / "requirements" / "_base.txt") diff --git a/services/notifications/src/simcore_service_notifications/api/rpc/routing.py b/services/notifications/src/simcore_service_notifications/api/rpc/routing.py index 2209baa43b56..c43bcdb7c05a 100644 --- a/services/notifications/src/simcore_service_notifications/api/rpc/routing.py +++ b/services/notifications/src/simcore_service_notifications/api/rpc/routing.py @@ -8,7 +8,7 @@ from ...clients.rabbitmq import get_rabbitmq_rpc_server ROUTERS: list[RPCRouter] = [ - # import form various domains and attach here + # import and use all routers here ] diff --git a/services/notifications/src/simcore_service_notifications/clients/postgres/_liveness.py b/services/notifications/src/simcore_service_notifications/clients/postgres/_liveness.py index d6587adf6092..57bc7a400768 100644 --- a/services/notifications/src/simcore_service_notifications/clients/postgres/_liveness.py +++ b/services/notifications/src/simcore_service_notifications/clients/postgres/_liveness.py @@ -1,6 +1,7 @@ import logging from asyncio import Task from datetime import timedelta +from typing import Final from fastapi import FastAPI from models_library.healthchecks import IsResponsive, LivenessResult @@ -12,6 +13,8 @@ _logger = logging.getLogger(__name__) +_LVENESS_CHECK_INTERVAL: Final[timedelta] = timedelta(seconds=10) + class PostgresLiveness: def __init__(self, app: FastAPI) -> None: @@ -30,7 +33,7 @@ def is_responsive(self) -> bool: async def setup(self) -> None: self._task = create_periodic_task( self._check_task, - interval=timedelta(seconds=10), + interval=_LVENESS_CHECK_INTERVAL, task_name="posgress_liveness_check", ) From e11f9bb1de5376e1f0659c2691e1e5578d4d14f6 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 24 Apr 2025 15:43:49 +0200 Subject: [PATCH 58/58] update openapi.json --- services/notifications/openapi.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/notifications/openapi.json b/services/notifications/openapi.json index 3e63b7e994d8..501d6014b463 100644 --- a/services/notifications/openapi.json +++ b/services/notifications/openapi.json @@ -2,7 +2,7 @@ "openapi": "3.1.0", "info": { "title": "simcore-service-notifications", - "description": "Service for executing commands on docker nodes", + "description": "Service used for sending notifications to users via different channels", "version": "0.0.1" }, "servers": [