Merge branch 'master' into 7525-use-catalog-rpc-client-instead-of-http

Author: bisgaard-itis

.env-devel

@@ -132,7 +132,7 @@ DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT=01:00:00
 DIRECTOR_V2_TRACING=null
 
 # DYNAMIC_SCHEDULER ----
-DYNAMIC_SCHEDULER_LOGLEVEL=DEBUG
+DYNAMIC_SCHEDULER_LOGLEVEL=INFO
 DYNAMIC_SCHEDULER_PROFILING=1
 DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER=0
 DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT=01:00:00
@@ -164,6 +164,9 @@ INVITATIONS_TRACING=null
 LOG_FORMAT_LOCAL_DEV_ENABLED=1
 LOG_FILTER_MAPPING='{}'
 
+NOTIFICATIONS_LOGLEVEL=INFO
+NOTIFICATIONS_TRACING=null
+
 PAYMENTS_ACCESS_TOKEN_EXPIRE_MINUTES=30
 PAYMENTS_ACCESS_TOKEN_SECRET_KEY=2c0411810565e063309be1457009fb39ce023946f6a354e6935107b57676
 PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT=10000

.github/CODEOWNERS

@@ -35,11 +35,13 @@ Makefile                                           @pcrespov @sanderegg
 /services/efs-guardian/                            @matusdrobuliak66
 /services/invitations/                             @pcrespov
 /services/migration/                               @pcrespov
+/services/notifications/                           @GitHK
 /services/payments/                                @pcrespov @matusdrobuliak66
 /services/resource-usage-tracker/                  @matusdrobuliak66
 /services/static-webserver/                        @GitHK
 /services/static-webserver/client/                 @odeimaiz
 /services/storage/                                 @sanderegg
+/services/storage/modules/celery                   @giancarloromeo
 /services/web/server/                              @pcrespov @sanderegg @GitHK @matusdrobuliak66
 /tests/e2e-frontend/                               @odeimaiz
 /tests/e2e-playwright/                             @matusdrobuliak66

.github/copilot-instructions.md

@@ -23,6 +23,13 @@ This document provides guidelines and best practices for using GitHub Copilot in
 - ensure we use `fastapi` >0.100 compatible code
 - use f-string formatting
 
+
+### Json serialization
+
+- Generally use `json_dumps`/`json_loads` from `common_library.json_serialization` to built-in `json.dumps` / `json.loads`.
+- Prefer Pydantic model methods (e.g., `model.model_dump_json()`) for serialization.
+
+
 ## Node.js-Specific Instructions
 
 - Use ES6+ syntax and features.

.github/workflows/ci-testing-deploy.yml

@@ -62,6 +62,7 @@ jobs:
       settings-library: ${{ steps.filter.outputs.settings-library }}
       simcore-sdk: ${{ steps.filter.outputs.simcore-sdk }}
       agent: ${{ steps.filter.outputs.agent }}
+      notifications: ${{ steps.filter.outputs.notifications }}
       api: ${{ steps.filter.outputs.api }}
       api-server: ${{ steps.filter.outputs.api-server }}
       autoscaling: ${{ steps.filter.outputs.autoscaling }}
@@ -150,6 +151,12 @@ jobs:
               - 'services/docker-compose*'
               - 'scripts/mypy/*'
               - 'mypy.ini'
+            notifications:
+              - 'packages/**'
+              - 'services/notifications/**'
+              - 'services/docker-compose*'
+              - 'scripts/mypy/*'
+              - 'mypy.ini'
             api:
               - 'api/**'
             api-server:
@@ -602,6 +609,49 @@ jobs:
         with:
           flags: unittests #optional
 
+  unit-test-notifications:
+    needs: changes
+    if: ${{ needs.changes.outputs.notifications == 'true' || github.event_name == 'push' }}
+    timeout-minutes: 18 # if this timeout gets too small, then split the tests
+    name: "[unit] notifications"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python: ["3.11"]
+        os: [ubuntu-24.04]
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v4
+      - name: setup docker buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker-container
+      - name: setup python environment
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+      - name: install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "0.5.x"
+          enable-cache: false
+          cache-dependency-glob: "**/notifications/requirements/ci.txt"
+      - name: show system version
+        run: ./ci/helpers/show_system_versions.bash
+      - name: install
+        run: ./ci/github/unit-testing/notifications.bash install
+      - name: typecheck
+        run: ./ci/github/unit-testing/notifications.bash typecheck
+      - name: test
+        if: ${{ !cancelled() }}
+        run: ./ci/github/unit-testing/notifications.bash test
+      - uses: codecov/codecov-action@v5
+        if: ${{ !cancelled() }}
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+        with:
+          flags: unittests #optional
 
   unit-test-api:
     needs: changes
@@ -1834,6 +1884,7 @@ jobs:
         unit-test-common-library,
         unit-test-notifications-library,
         unit-test-payments,
+        unit-test-notifications,
         unit-test-dynamic-scheduler,
         unit-test-postgres-database,
         unit-test-python-linting,

Makefile

@@ -44,6 +44,7 @@ SERVICES_NAMES_TO_BUILD := \
 	efs-guardian \
 	invitations \
   migration \
+	notifications \
 	payments \
 	resource-usage-tracker \
 	dynamic-scheduler \

api/specs/web-server/_long_running_tasks.py

@@ -58,7 +58,7 @@ def get_async_job_status(
     responses=_export_data_responses,
     status_code=status.HTTP_204_NO_CONTENT,
 )
-def abort_async_job(
+def cancel_async_job(
     _path_params: Annotated[_PathParam, Depends()],
 ): ...
 

ci/github/unit-testing/notifications.bash

@@ -0,0 +1,43 @@
+#!/bin/bash
+# http://redsymbol.net/articles/unofficial-bash-strict-mode/
+set -o errexit  # abort on nonzero exitstatus
+set -o nounset  # abort on unbound variable
+set -o pipefail # don't hide errors within pipes
+IFS=$'\n\t'
+
+install() {
+  make devenv
+  # shellcheck source=/dev/null
+  source .venv/bin/activate
+  pushd services/notifications
+  make install-ci
+  popd
+  uv pip list
+}
+
+test() {
+  # shellcheck source=/dev/null
+  source .venv/bin/activate
+  pushd services/notifications
+  make test-ci-unit
+  popd
+}
+
+typecheck() {
+  # shellcheck source=/dev/null
+  source .venv/bin/activate
+  uv pip install mypy
+  pushd services/notifications
+  make mypy
+  popd
+}
+
+# Check if the function exists (bash specific)
+if declare -f "$1" >/dev/null; then
+  # call arguments verbatim
+  "$@"
+else
+  # Show a helpful error
+  echo "'$1' is not a known function name" >&2
+  exit 1
+fi

ci/helpers/requirements/requirements.txt

@@ -1,4 +1,7 @@
 aiohttp==3.9.5
+    # via
+    #   -c requirements/../../../requirements/constraints.txt
+    #   -r requirements/requirements.in
 aiosignal==1.3.1
     # via aiohttp
 annotated-types==0.7.0
@@ -8,11 +11,15 @@ anyio==4.3.0
 attrs==23.2.0
     # via aiohttp
 certifi==2024.12.14
-    # via requests
+    # via
+    #   -c requirements/../../../requirements/constraints.txt
+    #   requests
 charset-normalizer==3.4.1
     # via requests
 docker==7.1.0
+    # via -r requirements/requirements.in
 fastapi==0.115.0
+    # via -r requirements/requirements.in
 frozenlist==1.4.1
     # via
     #   aiohttp
@@ -27,22 +34,27 @@ multidict==6.0.5
     #   aiohttp
     #   yarl
 pydantic==2.10.5
-    # via fastapi
+    # via
+    #   -c requirements/../../../requirements/constraints.txt
+    #   fastapi
 pydantic-core==2.27.2
     # via pydantic
 requests==2.32.3
     # via docker
 sniffio==1.3.1
     # via anyio
 starlette==0.38.6
-    # via fastapi
+    # via
+    #   -c requirements/../../../requirements/constraints.txt
+    #   fastapi
 typing-extensions==4.12.2
     # via
     #   fastapi
     #   pydantic
     #   pydantic-core
 urllib3==2.3.0
     # via
+    #   -c requirements/../../../requirements/constraints.txt
     #   docker
     #   requests
 yarl==1.9.4

packages/aws-library/requirements/_base.txt

@@ -1,8 +1,8 @@
 aio-pika==9.5.5
     # via -r requirements/../../../packages/service-library/requirements/_base.in
-aioboto3==13.3.0
+aioboto3==14.1.0
     # via -r requirements/_base.in
-aiobotocore==2.16.0
+aiobotocore==2.21.1
     # via aioboto3
 aiocache==0.12.3
     # via
@@ -57,22 +57,9 @@ attrs==25.1.0
     #   aiohttp
     #   jsonschema
     #   referencing
-boto3==1.35.81
-    # via
-    #   -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../requirements/constraints.txt
-    #   aiobotocore
-botocore==1.35.81
+boto3==1.37.1
+    # via aiobotocore
+botocore==1.37.1
     # via
     #   aiobotocore
     #   boto3
@@ -134,6 +121,7 @@ importlib-metadata==8.5.0
     # via opentelemetry-api
 jmespath==1.0.1
     # via
+    #   aiobotocore
     #   boto3
     #   botocore
 jsonschema==4.23.0
@@ -148,6 +136,7 @@ mdurl==0.1.2
     # via markdown-it-py
 multidict==6.1.0
     # via
+    #   aiobotocore
     #   aiohttp
     #   yarl
 opentelemetry-api==1.30.0
@@ -311,6 +300,7 @@ pyinstrument==5.0.1
     # via -r requirements/../../../packages/service-library/requirements/_base.in
 python-dateutil==2.9.0.post0
     # via
+    #   aiobotocore
     #   arrow
     #   botocore
 python-dotenv==1.0.1
@@ -372,7 +362,7 @@ rpds-py==0.23.1
     # via
     #   jsonschema
     #   referencing
-s3transfer==0.10.4
+s3transfer==0.11.3
     # via boto3
 sh==2.2.2
     # via -r requirements/_base.in

packages/aws-library/requirements/_test.txt

@@ -20,13 +20,12 @@ aws-xray-sdk==2.14.0
     # via moto
 blinker==1.9.0
     # via flask
-boto3==1.35.81
+boto3==1.37.1
     # via
-    #   -c requirements/../../../requirements/constraints.txt
     #   -c requirements/_base.txt
     #   aws-sam-translator
     #   moto
-botocore==1.35.81
+botocore==1.37.1
     # via
     #   -c requirements/_base.txt
     #   aws-xray-sdk
@@ -142,7 +141,7 @@ markupsafe==3.0.2
     # via
     #   jinja2
     #   werkzeug
-moto==5.1.1
+moto==5.1.4
     # via -r requirements/_test.in
 mpmath==1.3.0
     # via sympy
@@ -258,7 +257,7 @@ rpds-py==0.23.1
     #   -c requirements/_base.txt
     #   jsonschema
     #   referencing
-s3transfer==0.10.4
+s3transfer==0.11.3
     # via
     #   -c requirements/_base.txt
     #   boto3