diff --git a/.codecov.yml b/.codecov.yml index a3f9e9e6dd0..72735bedb30 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -31,6 +31,9 @@ component_management: - component_id: pkg_aws_library paths: - packages/aws-library/** + - component_id: pkg_celery_library + paths: + - packages/celery-library/** - component_id: pkg_dask_task_models_library paths: - packages/dask-task-models-library/** diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d8684350361..b0fde245e8a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -13,6 +13,7 @@ Makefile @pcrespov @sanderegg /api/ @sanderegg @pcrespov @matusdrobuliak66 /ci/ @sanderegg @pcrespov /docs/ @pcrespov +/packages/celery-library/ @giancarloromeo /packages/common-library/ @giancarloromeo /packages/models-library/ @sanderegg @pcrespov @matusdrobuliak66 @giancarloromeo /packages/postgres-database/ @matusdrobuliak66 diff --git a/.github/codeql/codeql-config.yml b/.github/codeql/codeql-config.yml index adac3b13795..739b7c8a5a6 100644 --- a/.github/codeql/codeql-config.yml +++ b/.github/codeql/codeql-config.yml @@ -4,6 +4,7 @@ disable-default-queries: false paths: - packages/aws-library/src + - packages/celery-library/src - packages/dask-task-models-library/src - packages/models-library/src/models_library - packages/postgres-database/src/simcore_postgres_database diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml index 127398d7dcc..05a060c7709 100644 --- a/.github/workflows/ci-testing-deploy.yml +++ b/.github/workflows/ci-testing-deploy.yml @@ -47,6 +47,7 @@ jobs: # Set job outputs to values from filter step outputs: aws-library: ${{ steps.filter.outputs.aws-library }} + celery-library: ${{ steps.filter.outputs.celery-library }} dask-task-models-library: ${{ steps.filter.outputs.dask-task-models-library }} models-library: ${{ steps.filter.outputs.models-library }} common-library: ${{ steps.filter.outputs.common-library }} @@ -94,6 +95,12 @@ jobs: - 'services/docker-compose*' - 'scripts/mypy/*' - 'mypy.ini' + celery-library: + - 'packages/celery-library/**' + - 'packages/pytest-simcore/**' + - 'services/docker-compose*' + - 'scripts/mypy/*' + - 'mypy.ini' dask-task-models-library: - 'packages/dask-task-models-library/**' - 'packages/pytest-simcore/**' @@ -1035,6 +1042,49 @@ jobs: with: flags: unittests #optional + unit-test-celery-library: + needs: changes + if: ${{ needs.changes.outputs.celery-library == 'true' || github.event_name == 'push' || github.event.inputs.force_all_builds == 'true' }} + timeout-minutes: 18 # if this timeout gets too small, then split the tests + name: "[unit] celery-library" + runs-on: ${{ matrix.os }} + strategy: + matrix: + python: ["3.11"] + os: [ubuntu-24.04] + fail-fast: false + steps: + - uses: actions/checkout@v4 + - name: setup docker buildx + id: buildx + uses: docker/setup-buildx-action@v3 + with: + driver: docker-container + - name: setup python environment + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: install uv + uses: astral-sh/setup-uv@v6 + with: + version: "0.6.x" + enable-cache: false + cache-dependency-glob: "**/celery-library/requirements/ci.txt" + - name: show system version + run: ./ci/helpers/show_system_versions.bash + - name: install + run: ./ci/github/unit-testing/celery-library.bash install + - name: typecheck + run: ./ci/github/unit-testing/celery-library.bash typecheck + - name: test + if: ${{ !cancelled() }} + run: ./ci/github/unit-testing/celery-library.bash test + - uses: codecov/codecov-action@v5 + if: ${{ !cancelled() }} + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + flags: unittests #optional unit-test-dask-task-models-library: needs: changes @@ -1824,6 +1874,7 @@ jobs: unit-test-clusters-keeper, unit-test-dask-sidecar, unit-test-aws-library, + unit-test-celery-library, unit-test-dask-task-models-library, unit-test-datcore-adapter, unit-test-director-v2, diff --git a/.vscode/settings.template.json b/.vscode/settings.template.json index 8112185b430..feec2786703 100644 --- a/.vscode/settings.template.json +++ b/.vscode/settings.template.json @@ -33,6 +33,7 @@ "python.analysis.typeCheckingMode": "basic", "python.analysis.extraPaths": [ "./packages/aws-library/src", + "./packages/celery-library/src", "./packages/common-library/src", "./packages/dask-task-models-library/src", "./packages/models-library/src", diff --git a/ci/github/unit-testing/celery-library.bash b/ci/github/unit-testing/celery-library.bash new file mode 100755 index 00000000000..a05004c677f --- /dev/null +++ b/ci/github/unit-testing/celery-library.bash @@ -0,0 +1,43 @@ +#!/bin/bash +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes +IFS=$'\n\t' + +install() { + make devenv + # shellcheck source=/dev/null + source .venv/bin/activate + pushd packages/celery-library + make install-ci + popd + uv pip list +} + +test() { + # shellcheck source=/dev/null + source .venv/bin/activate + pushd packages/celery-library + make tests-ci + popd +} + +typecheck() { + # shellcheck source=/dev/null + source .venv/bin/activate + uv pip install mypy + pushd packages/celery-library + make mypy + popd +} + +# Check if the function exists (bash specific) +if declare -f "$1" >/dev/null; then + # call arguments verbatim + "$@" +else + # Show a helpful error + echo "'$1' is not a known function name" >&2 + exit 1 +fi diff --git a/packages/celery-library/Makefile b/packages/celery-library/Makefile new file mode 100644 index 00000000000..04596d3d124 --- /dev/null +++ b/packages/celery-library/Makefile @@ -0,0 +1,50 @@ +# +# Targets for DEVELOPMENT of Celery Library +# +include ../../scripts/common.Makefile +include ../../scripts/common-package.Makefile + +.PHONY: requirements +requirements: ## compiles pip requirements (.in -> .txt) + @$(MAKE_C) requirements reqs + + +.PHONY: install-dev install-prod install-ci +install-dev install-prod install-ci: _check_venv_active ## install app in development/production or CI mode + # installing in $(subst install-,,$@) mode + @uv pip sync requirements/$(subst install-,,$@).txt + + +.PHONY: tests tests-ci +tests: ## runs unit tests + # running unit tests + @pytest \ + --asyncio-mode=auto \ + --color=yes \ + --cov-config=../../.coveragerc \ + --cov-report=term-missing \ + --cov=celery_library \ + --durations=10 \ + --exitfirst \ + --failed-first \ + --pdb \ + -vv \ + $(CURDIR)/tests + +tests-ci: ## runs unit tests + # running unit tests + @pytest \ + --asyncio-mode=auto \ + --color=yes \ + --cov-append \ + --cov-config=../../.coveragerc \ + --cov-report=term-missing \ + --cov-report=xml \ + --junitxml=junit.xml -o junit_family=legacy \ + --cov=celery_library \ + --durations=10 \ + --log-date-format="%Y-%m-%d %H:%M:%S" \ + --log-format="%(asctime)s %(levelname)s %(message)s" \ + --verbose \ + -m "not heavy_load" \ + $(CURDIR)/tests diff --git a/packages/celery-library/README.md b/packages/celery-library/README.md new file mode 100644 index 00000000000..b64223cfcc6 --- /dev/null +++ b/packages/celery-library/README.md @@ -0,0 +1,20 @@ +# simcore Celery library + +Provides a wrapper around Celery library [1]. + +## Installation + +```console +make help +make install-dev +``` + +## Test + +```console +make help +make test-dev +``` + + +[1] https://github.com/celery/celery diff --git a/packages/celery-library/VERSION b/packages/celery-library/VERSION new file mode 100644 index 00000000000..6e8bf73aa55 --- /dev/null +++ b/packages/celery-library/VERSION @@ -0,0 +1 @@ +0.1.0 diff --git a/packages/celery-library/requirements/Makefile b/packages/celery-library/requirements/Makefile new file mode 100644 index 00000000000..3f25442b790 --- /dev/null +++ b/packages/celery-library/requirements/Makefile @@ -0,0 +1,6 @@ +# +# Targets to pip-compile requirements +# +include ../../../requirements/base.Makefile + +# Add here any extra explicit dependency: e.g. _migration.txt: _base.txt diff --git a/packages/celery-library/requirements/_base.in b/packages/celery-library/requirements/_base.in new file mode 100644 index 00000000000..3fcf6a9e24c --- /dev/null +++ b/packages/celery-library/requirements/_base.in @@ -0,0 +1,10 @@ +# +# Specifies third-party dependencies for 'celery-library' +# +--constraint ../../../requirements/constraints.txt +--requirement ../../../packages/common-library/requirements/_base.in +--requirement ../../../packages/models-library/requirements/_base.in +--requirement ../../../packages/service-library/requirements/_base.in +--requirement ../../../packages/settings-library/requirements/_base.in + +celery[redis] diff --git a/packages/celery-library/requirements/_base.txt b/packages/celery-library/requirements/_base.txt new file mode 100644 index 00000000000..6954b4092b4 --- /dev/null +++ b/packages/celery-library/requirements/_base.txt @@ -0,0 +1,427 @@ +aio-pika==9.5.5 + # via -r requirements/../../../packages/service-library/requirements/_base.in +aiocache==0.12.3 + # via -r requirements/../../../packages/service-library/requirements/_base.in +aiodebug==2.3.0 + # via -r requirements/../../../packages/service-library/requirements/_base.in +aiodocker==0.24.0 + # via -r requirements/../../../packages/service-library/requirements/_base.in +aiofiles==24.1.0 + # via -r requirements/../../../packages/service-library/requirements/_base.in +aiohappyeyeballs==2.6.1 + # via aiohttp +aiohttp==3.11.18 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # aiodocker +aiormq==6.8.1 + # via aio-pika +aiosignal==1.3.2 + # via aiohttp +amqp==5.3.1 + # via kombu +annotated-types==0.7.0 + # via pydantic +anyio==4.9.0 + # via + # fast-depends + # faststream +arrow==1.3.0 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in +attrs==25.3.0 + # via + # aiohttp + # jsonschema + # referencing +billiard==4.2.1 + # via celery +celery==5.5.2 + # via -r requirements/_base.in +certifi==2025.4.26 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # requests +charset-normalizer==3.4.2 + # via requests +click==8.1.8 + # via + # celery + # click-didyoumean + # click-plugins + # click-repl + # typer +click-didyoumean==0.3.1 + # via celery +click-plugins==1.1.1 + # via celery +click-repl==0.3.0 + # via celery +deprecated==1.2.18 + # via + # opentelemetry-api + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http + # opentelemetry-semantic-conventions +dnspython==2.7.0 + # via email-validator +email-validator==2.2.0 + # via pydantic +exceptiongroup==1.3.0 + # via aio-pika +fast-depends==2.4.12 + # via faststream +faststream==0.5.41 + # via -r requirements/../../../packages/service-library/requirements/_base.in +frozenlist==1.6.0 + # via + # aiohttp + # aiosignal +googleapis-common-protos==1.70.0 + # via + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +grpcio==1.71.0 + # via opentelemetry-exporter-otlp-proto-grpc +idna==3.10 + # via + # anyio + # email-validator + # requests + # yarl +importlib-metadata==8.6.1 + # via opentelemetry-api +jsonschema==4.23.0 + # via + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in +jsonschema-specifications==2025.4.1 + # via jsonschema +kombu==5.5.3 + # via celery +markdown-it-py==3.0.0 + # via rich +mdurl==0.1.2 + # via markdown-it-py +multidict==6.4.3 + # via + # aiohttp + # yarl +opentelemetry-api==1.33.1 + # via + # -r requirements/../../../packages/service-library/requirements/_base.in + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http + # opentelemetry-instrumentation + # opentelemetry-instrumentation-aio-pika + # opentelemetry-instrumentation-logging + # opentelemetry-instrumentation-redis + # opentelemetry-instrumentation-requests + # opentelemetry-sdk + # opentelemetry-semantic-conventions +opentelemetry-exporter-otlp==1.33.1 + # via -r requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-exporter-otlp-proto-common==1.33.1 + # via + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +opentelemetry-exporter-otlp-proto-grpc==1.33.1 + # via opentelemetry-exporter-otlp +opentelemetry-exporter-otlp-proto-http==1.33.1 + # via opentelemetry-exporter-otlp +opentelemetry-instrumentation==0.54b1 + # via + # opentelemetry-instrumentation-aio-pika + # opentelemetry-instrumentation-logging + # opentelemetry-instrumentation-redis + # opentelemetry-instrumentation-requests +opentelemetry-instrumentation-aio-pika==0.54b1 + # via -r requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-instrumentation-logging==0.54b1 + # via -r requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-instrumentation-redis==0.54b1 + # via -r requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-instrumentation-requests==0.54b1 + # via -r requirements/../../../packages/service-library/requirements/_base.in +opentelemetry-proto==1.33.1 + # via + # opentelemetry-exporter-otlp-proto-common + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +opentelemetry-sdk==1.33.1 + # via + # -r requirements/../../../packages/service-library/requirements/_base.in + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +opentelemetry-semantic-conventions==0.54b1 + # via + # opentelemetry-instrumentation + # opentelemetry-instrumentation-redis + # opentelemetry-instrumentation-requests + # opentelemetry-sdk +opentelemetry-util-http==0.54b1 + # via opentelemetry-instrumentation-requests +orjson==3.10.18 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in +packaging==25.0 + # via opentelemetry-instrumentation +pamqp==3.3.0 + # via aiormq +prompt-toolkit==3.0.51 + # via click-repl +propcache==0.3.1 + # via + # aiohttp + # yarl +protobuf==5.29.4 + # via + # googleapis-common-protos + # opentelemetry-proto +psutil==7.0.0 + # via -r requirements/../../../packages/service-library/requirements/_base.in +pycryptodome==3.23.0 + # via stream-zip +pydantic==2.11.4 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in + # fast-depends + # pydantic-extra-types + # pydantic-settings +pydantic-core==2.33.2 + # via pydantic +pydantic-extra-types==2.10.4 + # via + # -r requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in +pydantic-settings==2.7.0 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in +pygments==2.19.1 + # via rich +pyinstrument==5.0.1 + # via -r requirements/../../../packages/service-library/requirements/_base.in +pyjwt==2.9.0 + # via redis +python-dateutil==2.9.0.post0 + # via + # arrow + # celery +python-dotenv==1.1.0 + # via pydantic-settings +pyyaml==6.0.2 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/service-library/requirements/_base.in +redis==5.3.0 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/../../../packages/service-library/requirements/_base.in + # celery +referencing==0.35.1 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # jsonschema + # jsonschema-specifications +requests==2.32.3 + # via opentelemetry-exporter-otlp-proto-http +rich==14.0.0 + # via + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in + # typer +rpds-py==0.25.0 + # via + # jsonschema + # referencing +shellingham==1.5.4 + # via typer +six==1.17.0 + # via python-dateutil +sniffio==1.3.1 + # via anyio +stream-zip==0.0.83 + # via -r requirements/../../../packages/service-library/requirements/_base.in +tenacity==9.1.2 + # via -r requirements/../../../packages/service-library/requirements/_base.in +toolz==1.0.0 + # via -r requirements/../../../packages/service-library/requirements/_base.in +tqdm==4.67.1 + # via -r requirements/../../../packages/service-library/requirements/_base.in +typer==0.15.4 + # via + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in +types-python-dateutil==2.9.0.20250516 + # via arrow +typing-extensions==4.13.2 + # via + # aiodebug + # anyio + # exceptiongroup + # faststream + # opentelemetry-sdk + # pydantic + # pydantic-core + # pydantic-extra-types + # typer + # typing-inspection +typing-inspection==0.4.0 + # via pydantic +tzdata==2025.2 + # via kombu +urllib3==2.4.0 + # via + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # requests +vine==5.1.0 + # via + # amqp + # celery + # kombu +wcwidth==0.2.13 + # via prompt-toolkit +wrapt==1.17.2 + # via + # deprecated + # opentelemetry-instrumentation + # opentelemetry-instrumentation-aio-pika + # opentelemetry-instrumentation-redis +yarl==1.20.0 + # via + # -r requirements/../../../packages/service-library/requirements/_base.in + # aio-pika + # aiohttp + # aiormq +zipp==3.21.0 + # via importlib-metadata diff --git a/packages/celery-library/requirements/_test.in b/packages/celery-library/requirements/_test.in new file mode 100644 index 00000000000..e85e3cb5177 --- /dev/null +++ b/packages/celery-library/requirements/_test.in @@ -0,0 +1,27 @@ +# +# Specifies dependencies required to run 'celery-library' +# +--constraint ../../../requirements/constraints.txt + +# Adds base AS CONSTRAINT specs, not requirement. +# - Resulting _text.txt is a frozen list of EXTRA packages for testing, besides _base.txt +# +--constraint _base.txt + +# testing +coverage +faker +httpx +pint +pytest +pytest-asyncio +pytest-benchmark +pytest-celery +pytest-cov +pytest-icdiff +pytest-instafail +pytest-mock +pytest-runner +pytest-sugar +python-dotenv +pyyaml diff --git a/packages/celery-library/requirements/_test.txt b/packages/celery-library/requirements/_test.txt new file mode 100644 index 00000000000..06df05f2779 --- /dev/null +++ b/packages/celery-library/requirements/_test.txt @@ -0,0 +1,228 @@ +amqp==5.3.1 + # via + # -c requirements/_base.txt + # kombu +annotated-types==0.7.0 + # via + # -c requirements/_base.txt + # pydantic +anyio==4.9.0 + # via + # -c requirements/_base.txt + # httpx + # starlette +billiard==4.2.1 + # via + # -c requirements/_base.txt + # celery +celery==5.5.2 + # via + # -c requirements/_base.txt + # pytest-celery +certifi==2025.4.26 + # via + # -c requirements/../../../requirements/constraints.txt + # -c requirements/_base.txt + # httpcore + # httpx + # requests +charset-normalizer==3.4.2 + # via + # -c requirements/_base.txt + # requests +click==8.1.8 + # via + # -c requirements/_base.txt + # celery + # click-didyoumean + # click-plugins + # click-repl +click-didyoumean==0.3.1 + # via + # -c requirements/_base.txt + # celery +click-plugins==1.1.1 + # via + # -c requirements/_base.txt + # celery +click-repl==0.3.0 + # via + # -c requirements/_base.txt + # celery +coverage==7.8.0 + # via + # -r requirements/_test.in + # pytest-cov +debugpy==1.8.14 + # via pytest-celery +docker==7.1.0 + # via + # pytest-celery + # pytest-docker-tools +faker==37.3.0 + # via -r requirements/_test.in +fastapi==0.115.12 + # via -r requirements/_test.in +flexcache==0.3 + # via pint +flexparser==0.4 + # via pint +h11==0.16.0 + # via httpcore +httpcore==1.0.9 + # via httpx +httpx==0.28.1 + # via + # -c requirements/../../../requirements/constraints.txt + # -r requirements/_test.in +icdiff==2.0.7 + # via pytest-icdiff +idna==3.10 + # via + # -c requirements/_base.txt + # anyio + # httpx + # requests +iniconfig==2.1.0 + # via pytest +kombu==5.5.3 + # via + # -c requirements/_base.txt + # celery + # pytest-celery +packaging==25.0 + # via + # -c requirements/_base.txt + # pytest + # pytest-sugar +pint==0.24.4 + # via -r requirements/_test.in +platformdirs==4.3.8 + # via pint +pluggy==1.6.0 + # via pytest +pprintpp==0.4.0 + # via pytest-icdiff +prompt-toolkit==3.0.51 + # via + # -c requirements/_base.txt + # click-repl +psutil==7.0.0 + # via + # -c requirements/_base.txt + # pytest-celery +py-cpuinfo==9.0.0 + # via pytest-benchmark +pydantic==2.11.4 + # via + # -c requirements/../../../requirements/constraints.txt + # -c requirements/_base.txt + # fastapi +pydantic-core==2.33.2 + # via + # -c requirements/_base.txt + # pydantic +pytest==8.3.5 + # via + # -r requirements/_test.in + # pytest-asyncio + # pytest-benchmark + # pytest-cov + # pytest-docker-tools + # pytest-icdiff + # pytest-instafail + # pytest-mock + # pytest-sugar +pytest-asyncio==0.26.0 + # via -r requirements/_test.in +pytest-benchmark==5.1.0 + # via -r requirements/_test.in +pytest-celery==1.2.0 + # via -r requirements/_test.in +pytest-cov==6.1.1 + # via -r requirements/_test.in +pytest-docker-tools==3.1.9 + # via pytest-celery +pytest-icdiff==0.9 + # via -r requirements/_test.in +pytest-instafail==0.5.0 + # via -r requirements/_test.in +pytest-mock==3.14.0 + # via -r requirements/_test.in +pytest-runner==6.0.1 + # via -r requirements/_test.in +pytest-sugar==1.0.0 + # via -r requirements/_test.in +python-dateutil==2.9.0.post0 + # via + # -c requirements/_base.txt + # celery +python-dotenv==1.1.0 + # via + # -c requirements/_base.txt + # -r requirements/_test.in +pyyaml==6.0.2 + # via + # -c requirements/../../../requirements/constraints.txt + # -c requirements/_base.txt + # -r requirements/_test.in +requests==2.32.3 + # via + # -c requirements/_base.txt + # docker +setuptools==80.9.0 + # via pytest-celery +six==1.17.0 + # via + # -c requirements/_base.txt + # python-dateutil +sniffio==1.3.1 + # via + # -c requirements/_base.txt + # anyio +starlette==0.46.2 + # via + # -c requirements/../../../requirements/constraints.txt + # fastapi +tenacity==9.1.2 + # via + # -c requirements/_base.txt + # pytest-celery +termcolor==3.1.0 + # via pytest-sugar +typing-extensions==4.13.2 + # via + # -c requirements/_base.txt + # anyio + # fastapi + # flexcache + # flexparser + # pint + # pydantic + # pydantic-core + # typing-inspection +typing-inspection==0.4.0 + # via + # -c requirements/_base.txt + # pydantic +tzdata==2025.2 + # via + # -c requirements/_base.txt + # faker + # kombu +urllib3==2.4.0 + # via + # -c requirements/../../../requirements/constraints.txt + # -c requirements/_base.txt + # docker + # requests +vine==5.1.0 + # via + # -c requirements/_base.txt + # amqp + # celery + # kombu +wcwidth==0.2.13 + # via + # -c requirements/_base.txt + # prompt-toolkit diff --git a/packages/celery-library/requirements/_tools.in b/packages/celery-library/requirements/_tools.in new file mode 100644 index 00000000000..1def82c12a3 --- /dev/null +++ b/packages/celery-library/requirements/_tools.in @@ -0,0 +1,5 @@ +--constraint ../../../requirements/constraints.txt +--constraint _base.txt +--constraint _test.txt + +--requirement ../../../requirements/devenv.txt diff --git a/packages/celery-library/requirements/_tools.txt b/packages/celery-library/requirements/_tools.txt new file mode 100644 index 00000000000..f2f7f688aac --- /dev/null +++ b/packages/celery-library/requirements/_tools.txt @@ -0,0 +1,87 @@ +astroid==3.3.10 + # via pylint +black==25.1.0 + # via -r requirements/../../../requirements/devenv.txt +build==1.2.2.post1 + # via pip-tools +bump2version==1.0.1 + # via -r requirements/../../../requirements/devenv.txt +cfgv==3.4.0 + # via pre-commit +click==8.1.8 + # via + # -c requirements/_base.txt + # -c requirements/_test.txt + # black + # pip-tools +dill==0.4.0 + # via pylint +distlib==0.3.9 + # via virtualenv +filelock==3.18.0 + # via virtualenv +identify==2.6.10 + # via pre-commit +isort==6.0.1 + # via + # -r requirements/../../../requirements/devenv.txt + # pylint +mccabe==0.7.0 + # via pylint +mypy==1.15.0 + # via -r requirements/../../../requirements/devenv.txt +mypy-extensions==1.1.0 + # via + # black + # mypy +nodeenv==1.9.1 + # via pre-commit +packaging==25.0 + # via + # -c requirements/_base.txt + # -c requirements/_test.txt + # black + # build +pathspec==0.12.1 + # via black +pip==25.1.1 + # via pip-tools +pip-tools==7.4.1 + # via -r requirements/../../../requirements/devenv.txt +platformdirs==4.3.8 + # via + # -c requirements/_test.txt + # black + # pylint + # virtualenv +pre-commit==4.2.0 + # via -r requirements/../../../requirements/devenv.txt +pylint==3.3.7 + # via -r requirements/../../../requirements/devenv.txt +pyproject-hooks==1.2.0 + # via + # build + # pip-tools +pyyaml==6.0.2 + # via + # -c requirements/../../../requirements/constraints.txt + # -c requirements/_base.txt + # -c requirements/_test.txt + # pre-commit +ruff==0.11.10 + # via -r requirements/../../../requirements/devenv.txt +setuptools==80.9.0 + # via + # -c requirements/_test.txt + # pip-tools +tomlkit==0.13.2 + # via pylint +typing-extensions==4.13.2 + # via + # -c requirements/_base.txt + # -c requirements/_test.txt + # mypy +virtualenv==20.31.2 + # via pre-commit +wheel==0.45.1 + # via pip-tools diff --git a/packages/celery-library/requirements/ci.txt b/packages/celery-library/requirements/ci.txt new file mode 100644 index 00000000000..ccbecf3196f --- /dev/null +++ b/packages/celery-library/requirements/ci.txt @@ -0,0 +1,22 @@ +# Shortcut to install all packages for the contigous integration (CI) of 'celery-library' +# +# - As ci.txt but w/ tests +# +# Usage: +# pip install -r requirements/ci.txt +# + +# installs base + tests requirements +--requirement _base.txt +--requirement _test.txt +--requirement _tools.txt + +# installs this repo's packages +simcore-common-library @ ../common-library +simcore-models-library @ ../models-library/ +pytest-simcore @ ../pytest-simcore +simcore-service-library @ ../service-library/ +simcore-settings-library @ ../settings-library/ + +# current module +simcore-celery-library @ . diff --git a/packages/celery-library/requirements/dev.txt b/packages/celery-library/requirements/dev.txt new file mode 100644 index 00000000000..115e5cfb20d --- /dev/null +++ b/packages/celery-library/requirements/dev.txt @@ -0,0 +1,22 @@ +# Shortcut to install all packages needed to develop 'celery-library' +# +# - As ci.txt but with current and repo packages in develop (edit) mode +# +# Usage: +# pip install -r requirements/dev.txt +# + +# installs base + tests requirements +--requirement _base.txt +--requirement _test.txt +--requirement _tools.txt + +# installs this repo's packages +--editable ../common-library/ +--editable ../models-library/ +--editable ../pytest-simcore/ +--editable ../service-library/ +--editable ../settings-library/ + +# current module +--editable . diff --git a/packages/celery-library/setup.cfg b/packages/celery-library/setup.cfg new file mode 100644 index 00000000000..5b10d2dc187 --- /dev/null +++ b/packages/celery-library/setup.cfg @@ -0,0 +1,21 @@ +[bumpversion] +current_version = 0.1.0 +commit = True +message = packages/celery-library version: {current_version} → {new_version} +tag = False +commit_args = --no-verify + +[bumpversion:file:VERSION] + +[bdist_wheel] +universal = 1 + +[aliases] +test = pytest + +[tool:pytest] +asyncio_mode = auto + +[mypy] +plugins = + pydantic.mypy diff --git a/packages/celery-library/setup.py b/packages/celery-library/setup.py new file mode 100644 index 00000000000..226e43f0bf8 --- /dev/null +++ b/packages/celery-library/setup.py @@ -0,0 +1,59 @@ +import re +import sys +from pathlib import Path + +from setuptools import find_packages, setup + + +def read_reqs(reqs_path: Path) -> set[str]: + return { + r + for r in re.findall( + r"(^[^#\n-][\w\[,\]]+[-~>=<.\w]*)", + reqs_path.read_text(), + re.MULTILINE, + ) + if isinstance(r, str) + } + + +CURRENT_DIR = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent + +INSTALL_REQUIREMENTS = tuple( + read_reqs(CURRENT_DIR / "requirements" / "_base.in") +) # WEAK requirements + +TEST_REQUIREMENTS = tuple( + read_reqs(CURRENT_DIR / "requirements" / "_test.txt") +) # STRICT requirements + + +SETUP = { + "name": "simcore-celery-library", + "version": Path(CURRENT_DIR / "VERSION").read_text().strip(), + "author": "Giancarlo Romeo (giancarloromeo)", + "description": "Core service library for Celery", + "python_requires": "~=3.11", + "classifiers": [ + "Development Status :: 2 - Pre-Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Natural Language :: English", + "Programming Language :: Python :: 3.10", + ], + "long_description": Path(CURRENT_DIR / "README.md").read_text(), + "license": "MIT license", + "install_requires": INSTALL_REQUIREMENTS, + "packages": find_packages(where="src"), + "package_data": {"": ["py.typed"]}, + "package_dir": {"": "src"}, + "include_package_data": True, + "test_suite": "tests", + "tests_require": TEST_REQUIREMENTS, + "extras_require": {"test": TEST_REQUIREMENTS}, + "zip_safe": False, +} + + +if __name__ == "__main__": + setup(**SETUP) diff --git a/services/storage/src/simcore_service_storage/modules/celery/backends/__init__.py b/packages/celery-library/src/celery_library/__init__.py similarity index 100% rename from services/storage/src/simcore_service_storage/modules/celery/backends/__init__.py rename to packages/celery-library/src/celery_library/__init__.py diff --git a/packages/celery-library/src/celery_library/backends/__init__.py b/packages/celery-library/src/celery_library/backends/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/storage/src/simcore_service_storage/modules/celery/backends/_redis.py b/packages/celery-library/src/celery_library/backends/_redis.py similarity index 97% rename from services/storage/src/simcore_service_storage/modules/celery/backends/_redis.py rename to packages/celery-library/src/celery_library/backends/_redis.py index 3fd9984fb2a..37a9a415cd5 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/backends/_redis.py +++ b/packages/celery-library/src/celery_library/backends/_redis.py @@ -5,16 +5,16 @@ from models_library.progress_bar import ProgressReport from pydantic import ValidationError -from servicelib.redis._client import RedisClientSDK - -from ..models import ( +from servicelib.celery.models import ( Task, TaskContext, TaskID, TaskMetadata, TaskUUID, - build_task_id_prefix, ) +from servicelib.redis import RedisClientSDK + +from ..utils import build_task_id_prefix _CELERY_TASK_INFO_PREFIX: Final[str] = "celery-task-info-" _CELERY_TASK_ID_KEY_ENCODING = "utf-8" diff --git a/services/storage/src/simcore_service_storage/modules/celery/_common.py b/packages/celery-library/src/celery_library/common.py similarity index 53% rename from services/storage/src/simcore_service_storage/modules/celery/_common.py rename to packages/celery-library/src/celery_library/common.py index 545bb98f682..3b7c9cd22ab 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/_common.py +++ b/packages/celery-library/src/celery_library/common.py @@ -1,12 +1,13 @@ -import logging import ssl from typing import Any from celery import Celery # type: ignore[import-untyped] +from servicelib.redis import RedisClientSDK from settings_library.celery import CelerySettings from settings_library.redis import RedisDatabase -_logger = logging.getLogger(__name__) +from .backends._redis import RedisTaskInfoStore +from .task_manager import CeleryTaskManager def _celery_configure(celery_settings: CelerySettings) -> dict[str, Any]: @@ -25,13 +26,30 @@ def _celery_configure(celery_settings: CelerySettings) -> dict[str, Any]: return base_config -def create_app(celery_settings: CelerySettings) -> Celery: - assert celery_settings +def create_app(settings: CelerySettings) -> Celery: + assert settings return Celery( - broker=celery_settings.CELERY_RABBIT_BROKER.dsn, - backend=celery_settings.CELERY_REDIS_RESULT_BACKEND.build_redis_dsn( + broker=settings.CELERY_RABBIT_BROKER.dsn, + backend=settings.CELERY_REDIS_RESULT_BACKEND.build_redis_dsn( RedisDatabase.CELERY_TASKS, ), - **_celery_configure(celery_settings), + **_celery_configure(settings), + ) + + +async def create_task_manager( + app: Celery, settings: CelerySettings +) -> CeleryTaskManager: + redis_client_sdk = RedisClientSDK( + settings.CELERY_REDIS_RESULT_BACKEND.build_redis_dsn( + RedisDatabase.CELERY_TASKS + ), + client_name="celery_tasks", + ) + + return CeleryTaskManager( + app, + settings, + RedisTaskInfoStore(redis_client_sdk), ) diff --git a/services/storage/src/simcore_service_storage/modules/celery/errors.py b/packages/celery-library/src/celery_library/errors.py similarity index 93% rename from services/storage/src/simcore_service_storage/modules/celery/errors.py rename to packages/celery-library/src/celery_library/errors.py index 0e340f35e71..37b174189f8 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/errors.py +++ b/packages/celery-library/src/celery_library/errors.py @@ -11,7 +11,7 @@ def __str__(self) -> str: return f"{decode_celery_transferrable_error(self)}" -def encore_celery_transferrable_error(error: Exception) -> TransferrableCeleryError: +def encode_celery_transferrable_error(error: Exception) -> TransferrableCeleryError: # NOTE: Celery modifies exceptions during serialization, which can cause # the original error context to be lost. This mechanism ensures the same # error can be recreated on the caller side exactly as it was raised here. diff --git a/packages/celery-library/src/celery_library/py.typed b/packages/celery-library/src/celery_library/py.typed new file mode 100644 index 00000000000..e69de29bb2d diff --git a/packages/celery-library/src/celery_library/signals.py b/packages/celery-library/src/celery_library/signals.py new file mode 100644 index 00000000000..09a29f3b0dc --- /dev/null +++ b/packages/celery-library/src/celery_library/signals.py @@ -0,0 +1,66 @@ +import asyncio +import logging +import threading + +from celery import Celery # type: ignore[import-untyped] +from celery.worker.worker import WorkController # type: ignore[import-untyped] +from servicelib.celery.app_server import STARTUP_TIMEOUT, BaseAppServer +from servicelib.logging_utils import log_context +from settings_library.celery import CelerySettings + +from .common import create_task_manager +from .utils import get_app_server, set_app_server + +_logger = logging.getLogger(__name__) + + +def on_worker_init( + app_server: BaseAppServer, + celery_settings: CelerySettings, + sender: WorkController, + **_kwargs, +) -> None: + startup_complete_event = threading.Event() + + def _init(startup_complete_event: threading.Event) -> None: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + shutdown_event = asyncio.Event() + + async def _setup_task_manager(): + assert sender.app # nosec + assert isinstance(sender.app, Celery) # nosec + + app_server.task_manager = await create_task_manager( + sender.app, + celery_settings, + ) + + set_app_server(sender.app, app_server) + + app_server.event_loop = loop + + loop.run_until_complete(_setup_task_manager()) + loop.run_until_complete( + app_server.startup(startup_complete_event, shutdown_event) + ) + + thread = threading.Thread( + group=None, + target=_init, + name="app_server_init", + args=(startup_complete_event,), + daemon=True, + ) + thread.start() + + startup_complete_event.wait(STARTUP_TIMEOUT * 1.1) + + +def on_worker_shutdown(sender, **_kwargs) -> None: + with log_context(_logger, logging.INFO, "Worker shutdown"): + assert isinstance(sender.app, Celery) + app_server = get_app_server(sender.app) + + app_server.event_loop.run_until_complete(app_server.shutdown()) diff --git a/services/storage/src/simcore_service_storage/modules/celery/_task.py b/packages/celery-library/src/celery_library/task.py similarity index 90% rename from services/storage/src/simcore_service_storage/modules/celery/_task.py rename to packages/celery-library/src/celery_library/task.py index e367a3a73da..f14771cf207 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/_task.py +++ b/packages/celery-library/src/celery_library/task.py @@ -14,11 +14,10 @@ from celery.exceptions import Ignore # type: ignore[import-untyped] from pydantic import NonNegativeInt from servicelib.async_utils import cancel_wait_task +from servicelib.celery.models import TaskID -from . import get_event_loop -from .errors import encore_celery_transferrable_error -from .models import TaskID, TaskId -from .utils import get_fastapi_app +from .errors import encode_celery_transferrable_error +from .utils import get_app_server _logger = logging.getLogger(__name__) @@ -40,15 +39,15 @@ class TaskAbortedError(Exception): ... def _async_task_wrapper( app: Celery, ) -> Callable[ - [Callable[Concatenate[AbortableTask, TaskId, P], Coroutine[Any, Any, R]]], + [Callable[Concatenate[AbortableTask, P], Coroutine[Any, Any, R]]], Callable[Concatenate[AbortableTask, P], R], ]: def decorator( - coro: Callable[Concatenate[AbortableTask, TaskId, P], Coroutine[Any, Any, R]], + coro: Callable[Concatenate[AbortableTask, P], Coroutine[Any, Any, R]], ) -> Callable[Concatenate[AbortableTask, P], R]: @wraps(coro) def wrapper(task: AbortableTask, *args: P.args, **kwargs: P.kwargs) -> R: - fastapi_app = get_fastapi_app(app) + app_server = get_app_server(app) # NOTE: task.request is a thread local object, so we need to pass the id explicitly assert task.request.id is not None # nosec @@ -56,7 +55,7 @@ async def run_task(task_id: TaskID) -> R: try: async with asyncio.TaskGroup() as tg: main_task = tg.create_task( - coro(task, task_id, *args, **kwargs), + coro(task, *args, **kwargs), ) async def abort_monitor(): @@ -90,7 +89,7 @@ async def abort_monitor(): return asyncio.run_coroutine_threadsafe( run_task(task.request.id), - get_event_loop(fastapi_app), + app_server.event_loop, ).result() return wrapper @@ -120,7 +119,7 @@ def wrapper(task: AbortableTask, *args: P.args, **kwargs: P.kwargs) -> R: if isinstance(exc, dont_autoretry_for): _logger.debug("Not retrying for exception %s", type(exc).__name__) # propagate without retry - raise encore_celery_transferrable_error(exc) from exc + raise encode_celery_transferrable_error(exc) from exc exc_type = type(exc).__name__ exc_message = f"{exc}" @@ -134,7 +133,7 @@ def wrapper(task: AbortableTask, *args: P.args, **kwargs: P.kwargs) -> R: raise task.retry( max_retries=max_retries, countdown=delay_between_retries.total_seconds(), - exc=encore_celery_transferrable_error(exc), + exc=encode_celery_transferrable_error(exc), ) from exc return wrapper @@ -145,7 +144,7 @@ def wrapper(task: AbortableTask, *args: P.args, **kwargs: P.kwargs) -> R: @overload def register_task( app: Celery, - fn: Callable[Concatenate[AbortableTask, TaskId, P], Coroutine[Any, Any, R]], + fn: Callable[Concatenate[AbortableTask, TaskID, P], Coroutine[Any, Any, R]], task_name: str | None = None, timeout: timedelta | None = _DEFAULT_TASK_TIMEOUT, max_retries: NonNegativeInt = _DEFAULT_MAX_RETRIES, @@ -169,7 +168,7 @@ def register_task( def register_task( # type: ignore[misc] app: Celery, fn: ( - Callable[Concatenate[AbortableTask, TaskId, P], Coroutine[Any, Any, R]] + Callable[Concatenate[AbortableTask, TaskID, P], Coroutine[Any, Any, R]] | Callable[Concatenate[AbortableTask, P], R] ), task_name: str | None = None, @@ -205,4 +204,5 @@ def register_task( # type: ignore[misc] bind=True, base=AbortableTask, time_limit=None if timeout is None else timeout.total_seconds(), + pydantic=True, )(wrapped_fn) diff --git a/services/storage/src/simcore_service_storage/modules/celery/client.py b/packages/celery-library/src/celery_library/task_manager.py similarity index 93% rename from services/storage/src/simcore_service_storage/modules/celery/client.py rename to packages/celery-library/src/celery_library/task_manager.py index f68baf558fe..7f14d4ddd34 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/client.py +++ b/packages/celery-library/src/celery_library/task_manager.py @@ -9,10 +9,7 @@ ) from common_library.async_tools import make_async from models_library.progress_bar import ProgressReport -from servicelib.logging_utils import log_context -from settings_library.celery import CelerySettings - -from .models import ( +from servicelib.celery.models import ( Task, TaskContext, TaskID, @@ -21,8 +18,11 @@ TaskState, TaskStatus, TaskUUID, - build_task_id, ) +from servicelib.logging_utils import log_context +from settings_library.celery import CelerySettings + +from .utils import build_task_id _logger = logging.getLogger(__name__) @@ -31,8 +31,8 @@ _MAX_PROGRESS_VALUE = 1.0 -@dataclass -class CeleryTaskClient: +@dataclass(frozen=True) +class CeleryTaskManager: _celery_app: Celery _celery_settings: CelerySettings _task_info_store: TaskInfoStore @@ -54,7 +54,7 @@ async def submit_task( self._celery_app.send_task( task_metadata.name, task_id=task_id, - kwargs=task_params, + kwargs={"task_id": task_id} | task_params, queue=task_metadata.queue.value, ) @@ -155,3 +155,9 @@ async def list_tasks(self, task_context: TaskContext) -> list[Task]: msg=f"Listing tasks: {task_context=}", ): return await self._task_info_store.list_tasks(task_context) + + async def set_task_progress(self, task_id: TaskID, report: ProgressReport) -> None: + await self._task_info_store.set_task_progress( + task_id=task_id, + report=report, + ) diff --git a/services/storage/src/simcore_service_storage/modules/celery/_celery_types.py b/packages/celery-library/src/celery_library/types.py similarity index 77% rename from services/storage/src/simcore_service_storage/modules/celery/_celery_types.py rename to packages/celery-library/src/celery_library/types.py index 4ed62e72775..bbd04eabc56 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/_celery_types.py +++ b/packages/celery-library/src/celery_library/types.py @@ -3,14 +3,8 @@ from typing import Any from kombu.utils.json import register_type # type: ignore[import-untyped] -from models_library.api_schemas_storage.storage_schemas import ( - FileUploadCompletionBody, - FoldersBody, -) from pydantic import BaseModel -from ...models import FileMetaData - def _path_encoder(obj): if isinstance(obj, Path): @@ -37,16 +31,6 @@ def _pydantic_model_decoder(clz: type[BaseModel], data: dict[str, Any]) -> BaseM return clz(**data) -def _register_pydantic_types(*models: type[BaseModel]) -> None: - for model in models: - register_type( - model, - _class_full_name(model), - encoder=_pydantic_model_encoder, - decoder=partial(_pydantic_model_decoder, model), - ) - - def register_celery_types() -> None: register_type( Path, @@ -56,6 +40,12 @@ def register_celery_types() -> None: ) register_type(set, _class_full_name(set), encoder=list, decoder=set) - _register_pydantic_types(FileUploadCompletionBody) - _register_pydantic_types(FileMetaData) - _register_pydantic_types(FoldersBody) + +def register_pydantic_types(*models: type[BaseModel]) -> None: + for model in models: + register_type( + model, + _class_full_name(model), + encoder=_pydantic_model_encoder, + decoder=partial(_pydantic_model_decoder, model), + ) diff --git a/packages/celery-library/src/celery_library/utils.py b/packages/celery-library/src/celery_library/utils.py new file mode 100644 index 00000000000..64da3e0c248 --- /dev/null +++ b/packages/celery-library/src/celery_library/utils.py @@ -0,0 +1,31 @@ +from typing import Final + +from celery import Celery # type: ignore[import-untyped] +from servicelib.celery.app_server import BaseAppServer +from servicelib.celery.models import TaskContext, TaskID, TaskUUID + +_APP_SERVER_KEY = "app_server" + +_TASK_ID_KEY_DELIMITATOR: Final[str] = ":" + + +def build_task_id_prefix(task_context: TaskContext) -> str: + return _TASK_ID_KEY_DELIMITATOR.join( + [f"{task_context[key]}" for key in sorted(task_context)] + ) + + +def build_task_id(task_context: TaskContext, task_uuid: TaskUUID) -> TaskID: + return _TASK_ID_KEY_DELIMITATOR.join( + [build_task_id_prefix(task_context), f"{task_uuid}"] + ) + + +def get_app_server(app: Celery) -> BaseAppServer: + app_server = app.conf[_APP_SERVER_KEY] + assert isinstance(app_server, BaseAppServer) + return app_server + + +def set_app_server(app: Celery, app_server: BaseAppServer) -> None: + app.conf[_APP_SERVER_KEY] = app_server diff --git a/packages/celery-library/tests/conftest.py b/packages/celery-library/tests/conftest.py new file mode 100644 index 00000000000..c513d262d7f --- /dev/null +++ b/packages/celery-library/tests/conftest.py @@ -0,0 +1,130 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument + +import datetime +from collections.abc import AsyncIterator, Callable +from functools import partial +from typing import Any + +import pytest +from celery import Celery # type: ignore[import-untyped] +from celery.contrib.testing.worker import TestWorkController, start_worker +from celery.signals import worker_init, worker_shutdown +from celery.worker.worker import WorkController +from celery_library.common import create_task_manager +from celery_library.signals import on_worker_init, on_worker_shutdown +from celery_library.task_manager import CeleryTaskManager +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +from pytest_simcore.helpers.typing_env import EnvVarsDict +from servicelib.celery.app_server import BaseAppServer +from settings_library.celery import CelerySettings +from settings_library.redis import RedisSettings + +pytest_plugins = [ + "pytest_simcore.docker_compose", + "pytest_simcore.docker_swarm", + "pytest_simcore.environment_configs", + "pytest_simcore.redis_service", + "pytest_simcore.repository_paths", +] + + +class FakeAppServer(BaseAppServer): + async def on_startup(self) -> None: + pass + + async def on_shutdown(self) -> None: + pass + + +@pytest.fixture +def register_celery_tasks() -> Callable[[Celery], None]: + """override if tasks are needed""" + + def _(celery_app: Celery) -> None: ... + + return _ + + +@pytest.fixture +def app_environment( + monkeypatch: pytest.MonkeyPatch, + redis_service: RedisSettings, + env_devel_dict: EnvVarsDict, +) -> EnvVarsDict: + return setenvs_from_dict( + monkeypatch, + { + **env_devel_dict, + "REDIS_SECURE": redis_service.REDIS_SECURE, + "REDIS_HOST": redis_service.REDIS_HOST, + "REDIS_PORT": f"{redis_service.REDIS_PORT}", + "REDIS_PASSWORD": redis_service.REDIS_PASSWORD.get_secret_value(), + }, + ) + + +@pytest.fixture +def celery_settings( + app_environment: EnvVarsDict, +) -> CelerySettings: + return CelerySettings.create_from_envs() + + +@pytest.fixture +def app_server() -> BaseAppServer: + return FakeAppServer(app=None) + + +@pytest.fixture(scope="session") +def celery_config() -> dict[str, Any]: + return { + "broker_connection_retry_on_startup": True, + "broker_url": "memory://localhost//", + "result_backend": "cache+memory://localhost//", + "result_expires": datetime.timedelta(days=7), + "result_extended": True, + "pool": "threads", + "task_default_queue": "default", + "task_send_sent_event": True, + "task_track_started": True, + "worker_send_task_events": True, + } + + +@pytest.fixture +async def with_celery_worker( + celery_app: Celery, + app_server: BaseAppServer, + celery_settings: CelerySettings, + register_celery_tasks: Callable[[Celery], None], +) -> AsyncIterator[TestWorkController]: + def _on_worker_init_wrapper(sender: WorkController, **_kwargs): + return partial(on_worker_init, app_server, celery_settings)(sender, **_kwargs) + + worker_init.connect(_on_worker_init_wrapper) + worker_shutdown.connect(on_worker_shutdown) + + register_celery_tasks(celery_app) + + with start_worker( + celery_app, + pool="threads", + concurrency=1, + loglevel="info", + perform_ping_check=False, + queues="default", + ) as worker: + yield worker + + +@pytest.fixture +async def celery_task_manager( + celery_app: Celery, + celery_settings: CelerySettings, + with_celery_worker: TestWorkController, +) -> CeleryTaskManager: + return await create_task_manager( + celery_app, + celery_settings, + ) diff --git a/services/storage/tests/unit/test_modules_celery_errors.py b/packages/celery-library/tests/unit/test_errors.py similarity index 78% rename from services/storage/tests/unit/test_modules_celery_errors.py rename to packages/celery-library/tests/unit/test_errors.py index 74000f7649e..c18e3bb50d6 100644 --- a/services/storage/tests/unit/test_modules_celery_errors.py +++ b/packages/celery-library/tests/unit/test_errors.py @@ -1,9 +1,9 @@ import pytest -from models_library.api_schemas_storage.export_data_async_jobs import AccessRightError -from simcore_service_storage.modules.celery.errors import ( +from celery_library.errors import ( decode_celery_transferrable_error, - encore_celery_transferrable_error, + encode_celery_transferrable_error, ) +from models_library.api_schemas_storage.export_data_async_jobs import AccessRightError @pytest.mark.parametrize( @@ -13,11 +13,11 @@ AccessRightError(user_id=1, file_id="a/path/to/a/file.txt", location_id=0), ], ) -def test_workflow(original_error: Exception): +def test_error(original_error: Exception): try: raise original_error # noqa: TRY301 except Exception as e: # pylint: disable=broad-exception-caught - result = encore_celery_transferrable_error(e) + result = encode_celery_transferrable_error(e) assert decode_celery_transferrable_error(result).args == original_error.args assert f"{decode_celery_transferrable_error(result)}" == f"{original_error}" diff --git a/services/storage/tests/unit/test_modules_celery.py b/packages/celery-library/tests/unit/test_tasks.py similarity index 62% rename from services/storage/tests/unit/test_modules_celery.py rename to packages/celery-library/tests/unit/test_tasks.py index b1819aabb44..4270efcc065 100644 --- a/services/storage/tests/unit/test_modules_celery.py +++ b/packages/celery-library/tests/unit/test_tasks.py @@ -13,55 +13,36 @@ import pytest from celery import Celery, Task from celery.contrib.abortable import AbortableTask +from celery_library.errors import TransferrableCeleryError +from celery_library.task import register_task +from celery_library.task_manager import CeleryTaskManager +from celery_library.utils import get_app_server from common_library.errors_classes import OsparcErrorMixin -from fastapi import FastAPI from models_library.progress_bar import ProgressReport -from servicelib.logging_utils import log_context -from simcore_service_storage.modules.celery import get_celery_client, get_event_loop -from simcore_service_storage.modules.celery._task import ( - AbortableAsyncResult, - register_task, -) -from simcore_service_storage.modules.celery.client import CeleryTaskClient -from simcore_service_storage.modules.celery.errors import TransferrableCeleryError -from simcore_service_storage.modules.celery.models import ( +from servicelib.celery.models import ( TaskContext, TaskID, TaskMetadata, TaskState, ) -from simcore_service_storage.modules.celery.utils import ( - get_celery_worker, - get_fastapi_app, -) -from simcore_service_storage.modules.celery.worker import CeleryTaskWorker +from servicelib.logging_utils import log_context from tenacity import Retrying, retry_if_exception_type, stop_after_delay, wait_fixed _logger = logging.getLogger(__name__) -pytest_simcore_core_services_selection = ["postgres", "rabbit"] +pytest_simcore_core_services_selection = ["redis"] pytest_simcore_ops_services_selection = [] -@pytest.fixture -def celery_client( - initialized_app: FastAPI, - with_storage_celery_worker: CeleryTaskWorker, -) -> CeleryTaskClient: - return get_celery_client(initialized_app) - - async def _fake_file_processor( celery_app: Celery, task_name: str, task_id: str, files: list[str] ) -> str: - worker = get_celery_worker(celery_app) - def sleep_for(seconds: float) -> None: time.sleep(seconds) for n, file in enumerate(files, start=1): with log_context(_logger, logging.INFO, msg=f"Processing file {file}"): - await worker.set_task_progress( + await get_app_server(celery_app).task_manager.set_task_progress( task_id=task_id, report=ProgressReport(actual_value=n / len(files)), ) @@ -70,12 +51,13 @@ def sleep_for(seconds: float) -> None: return "archive.zip" -def fake_file_processor(task: Task, files: list[str]) -> str: +def fake_file_processor(task: Task, task_id: TaskID, files: list[str]) -> str: + assert task_id assert task.name _logger.info("Calling _fake_file_processor") return asyncio.run_coroutine_threadsafe( _fake_file_processor(task.app, task.name, task.request.id, files), - get_event_loop(get_fastapi_app(task.app)), + get_app_server(task.app).event_loop, ).result() @@ -83,7 +65,8 @@ class MyError(OsparcErrorMixin, Exception): msg_template = "Something strange happened: {msg}" -def failure_task(task: Task): +def failure_task(task: Task, task_id: TaskID) -> None: + assert task_id assert task msg = "BOOM!" raise MyError(msg=msg) @@ -92,11 +75,8 @@ def failure_task(task: Task): async def dreamer_task(task: AbortableTask, task_id: TaskID) -> list[int]: numbers = [] for _ in range(30): - if AbortableAsyncResult(task_id, app=task.app).is_aborted(): - _logger.warning("Alarm clock") - return numbers numbers.append(randint(1, 90)) # noqa: S311 - await asyncio.sleep(0.1) + await asyncio.sleep(0.5) return numbers @@ -111,11 +91,11 @@ def _(celery_app: Celery) -> None: async def test_submitting_task_calling_async_function_results_with_success_state( - celery_client: CeleryTaskClient, + celery_task_manager: CeleryTaskManager, ): task_context = TaskContext(user_id=42) - task_uuid = await celery_client.submit_task( + task_uuid = await celery_task_manager.submit_task( TaskMetadata( name=fake_file_processor.__name__, ), @@ -129,23 +109,23 @@ async def test_submitting_task_calling_async_function_results_with_success_state stop=stop_after_delay(30), ): with attempt: - status = await celery_client.get_task_status(task_context, task_uuid) + status = await celery_task_manager.get_task_status(task_context, task_uuid) assert status.task_state == TaskState.SUCCESS assert ( - await celery_client.get_task_status(task_context, task_uuid) + await celery_task_manager.get_task_status(task_context, task_uuid) ).task_state == TaskState.SUCCESS assert ( - await celery_client.get_task_result(task_context, task_uuid) + await celery_task_manager.get_task_result(task_context, task_uuid) ) == "archive.zip" async def test_submitting_task_with_failure_results_with_error( - celery_client: CeleryTaskClient, + celery_task_manager: CeleryTaskManager, ): task_context = TaskContext(user_id=42) - task_uuid = await celery_client.submit_task( + task_uuid = await celery_task_manager.submit_task( TaskMetadata( name=failure_task.__name__, ), @@ -159,26 +139,30 @@ async def test_submitting_task_with_failure_results_with_error( ): with attempt: - raw_result = await celery_client.get_task_result(task_context, task_uuid) + raw_result = await celery_task_manager.get_task_result( + task_context, task_uuid + ) assert isinstance(raw_result, TransferrableCeleryError) - raw_result = await celery_client.get_task_result(task_context, task_uuid) + raw_result = await celery_task_manager.get_task_result(task_context, task_uuid) assert f"{raw_result}" == "Something strange happened: BOOM!" async def test_cancelling_a_running_task_aborts_and_deletes( - celery_client: CeleryTaskClient, + celery_task_manager: CeleryTaskManager, ): task_context = TaskContext(user_id=42) - task_uuid = await celery_client.submit_task( + task_uuid = await celery_task_manager.submit_task( TaskMetadata( name=dreamer_task.__name__, ), task_context=task_context, ) - await celery_client.cancel_task(task_context, task_uuid) + await asyncio.sleep(3.0) + + await celery_task_manager.cancel_task(task_context, task_uuid) for attempt in Retrying( retry=retry_if_exception_type(AssertionError), @@ -186,22 +170,24 @@ async def test_cancelling_a_running_task_aborts_and_deletes( stop=stop_after_delay(30), ): with attempt: - progress = await celery_client.get_task_status(task_context, task_uuid) + progress = await celery_task_manager.get_task_status( + task_context, task_uuid + ) assert progress.task_state == TaskState.ABORTED assert ( - await celery_client.get_task_status(task_context, task_uuid) + await celery_task_manager.get_task_status(task_context, task_uuid) ).task_state == TaskState.ABORTED - assert task_uuid not in await celery_client.list_tasks(task_context) + assert task_uuid not in await celery_task_manager.list_tasks(task_context) async def test_listing_task_uuids_contains_submitted_task( - celery_client: CeleryTaskClient, + celery_task_manager: CeleryTaskManager, ): task_context = TaskContext(user_id=42) - task_uuid = await celery_client.submit_task( + task_uuid = await celery_task_manager.submit_task( TaskMetadata( name=dreamer_task.__name__, ), @@ -214,10 +200,8 @@ async def test_listing_task_uuids_contains_submitted_task( stop=stop_after_delay(10), ): with attempt: - tasks = await celery_client.list_tasks(task_context) - assert len(tasks) == 1 - assert task_uuid == tasks[0].uuid + tasks = await celery_task_manager.list_tasks(task_context) + assert any(task.uuid == task_uuid for task in tasks) - tasks = await celery_client.list_tasks(task_context) - assert len(tasks) == 1 - assert task_uuid == tasks[0].uuid + tasks = await celery_task_manager.list_tasks(task_context) + assert any(task.uuid == task_uuid for task in tasks) diff --git a/packages/service-library/src/servicelib/celery/__init__.py b/packages/service-library/src/servicelib/celery/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/packages/service-library/src/servicelib/celery/app_server.py b/packages/service-library/src/servicelib/celery/app_server.py new file mode 100644 index 00000000000..0c55c0ed919 --- /dev/null +++ b/packages/service-library/src/servicelib/celery/app_server.py @@ -0,0 +1,60 @@ +import asyncio +import datetime +import threading +from abc import ABC, abstractmethod +from asyncio import AbstractEventLoop +from typing import Final, Generic, TypeVar + +from servicelib.celery.task_manager import TaskManager + +STARTUP_TIMEOUT: Final[float] = datetime.timedelta(minutes=1).total_seconds() + +T = TypeVar("T") + + +class BaseAppServer(ABC, Generic[T]): + def __init__(self, app: T) -> None: + self._app: T = app + self._shutdown_event: asyncio.Event | None = None + + @property + def app(self) -> T: + return self._app + + @property + def event_loop(self) -> AbstractEventLoop: + return self._event_loop + + @event_loop.setter + def event_loop(self, loop: AbstractEventLoop) -> None: + self._event_loop = loop + + @property + def task_manager(self) -> TaskManager: + return self._task_manager + + @task_manager.setter + def task_manager(self, manager: TaskManager) -> None: + self._task_manager = manager + + @abstractmethod + async def on_startup(self) -> None: + raise NotImplementedError + + async def startup( + self, completed_event: threading.Event, shutdown_event: asyncio.Event + ) -> None: + self._shutdown_event = shutdown_event + completed_event.set() + await self.on_startup() + await self._shutdown_event.wait() + + @abstractmethod + async def on_shutdown(self) -> None: + raise NotImplementedError + + async def shutdown(self) -> None: + if self._shutdown_event is not None: + self._shutdown_event.set() + + await self.on_shutdown() diff --git a/services/storage/src/simcore_service_storage/modules/celery/models.py b/packages/service-library/src/servicelib/celery/models.py similarity index 74% rename from services/storage/src/simcore_service_storage/modules/celery/models.py rename to packages/service-library/src/servicelib/celery/models.py index 8b19d124ff1..8bc744fcb3e 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/models.py +++ b/packages/service-library/src/servicelib/celery/models.py @@ -1,6 +1,6 @@ -from datetime import timedelta +import datetime from enum import StrEnum -from typing import Annotated, Any, Final, Protocol, TypeAlias +from typing import Annotated, Any, Protocol, TypeAlias from uuid import UUID from models_library.progress_bar import ProgressReport @@ -13,20 +13,6 @@ ] TaskUUID: TypeAlias = UUID -_CELERY_TASK_ID_KEY_SEPARATOR: Final[str] = ":" - - -def build_task_id_prefix(task_context: TaskContext) -> str: - return _CELERY_TASK_ID_KEY_SEPARATOR.join( - [f"{task_context[key]}" for key in sorted(task_context)] - ) - - -def build_task_id(task_context: TaskContext, task_uuid: TaskUUID) -> TaskID: - return _CELERY_TASK_ID_KEY_SEPARATOR.join( - [build_task_id_prefix(task_context), f"{task_uuid}"] - ) - class TaskState(StrEnum): PENDING = "PENDING" @@ -61,7 +47,7 @@ async def create_task( self, task_id: TaskID, task_metadata: TaskMetadata, - expiry: timedelta, + expiry: datetime.timedelta, ) -> None: ... async def exists_task(self, task_id: TaskID) -> bool: ... @@ -87,6 +73,3 @@ class TaskStatus(BaseModel): @property def is_done(self) -> bool: return self.task_state in _TASK_DONE - - -TaskId: TypeAlias = str diff --git a/packages/service-library/src/servicelib/celery/task_manager.py b/packages/service-library/src/servicelib/celery/task_manager.py new file mode 100644 index 00000000000..f8e178348c0 --- /dev/null +++ b/packages/service-library/src/servicelib/celery/task_manager.py @@ -0,0 +1,36 @@ +from typing import Any, Protocol + +from models_library.progress_bar import ProgressReport + +from ..celery.models import ( + Task, + TaskContext, + TaskID, + TaskMetadata, + TaskStatus, + TaskUUID, +) + + +class TaskManager(Protocol): + async def submit_task( + self, task_metadata: TaskMetadata, *, task_context: TaskContext, **task_param + ) -> TaskUUID: ... + + async def cancel_task( + self, task_context: TaskContext, task_uuid: TaskUUID + ) -> None: ... + + async def get_task_result( + self, task_context: TaskContext, task_uuid: TaskUUID + ) -> Any: ... + + async def get_task_status( + self, task_context: TaskContext, task_uuid: TaskUUID + ) -> TaskStatus: ... + + async def list_tasks(self, task_context: TaskContext) -> list[Task]: ... + + async def set_task_progress( + self, task_id: TaskID, report: ProgressReport + ) -> None: ... diff --git a/packages/service-library/src/servicelib/fastapi/celery/__init__.py b/packages/service-library/src/servicelib/fastapi/celery/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/packages/service-library/src/servicelib/fastapi/celery/app_server.py b/packages/service-library/src/servicelib/fastapi/celery/app_server.py new file mode 100644 index 00000000000..3a5ab7303de --- /dev/null +++ b/packages/service-library/src/servicelib/fastapi/celery/app_server.py @@ -0,0 +1,29 @@ +from datetime import timedelta +from typing import Final + +from asgi_lifespan import LifespanManager +from fastapi import FastAPI + +from ...celery.app_server import BaseAppServer + +_SHUTDOWN_TIMEOUT: Final[float] = timedelta(seconds=10).total_seconds() +_STARTUP_TIMEOUT: Final[float] = timedelta(minutes=1).total_seconds() + + +class FastAPIAppServer(BaseAppServer[FastAPI]): + def __init__(self, app: FastAPI): + super().__init__(app) + self._lifespan_manager: LifespanManager | None = None + + async def on_startup(self) -> None: + self._lifespan_manager = LifespanManager( + self.app, + startup_timeout=_STARTUP_TIMEOUT, + shutdown_timeout=_SHUTDOWN_TIMEOUT, + ) + await self._lifespan_manager.__aenter__() + + async def on_shutdown(self) -> None: + if self._lifespan_manager is None: + return + await self._lifespan_manager.__aexit__(None, None, None) diff --git a/services/storage/requirements/_base.in b/services/storage/requirements/_base.in index e2ea1f7474f..1b3efbf0d11 100644 --- a/services/storage/requirements/_base.in +++ b/services/storage/requirements/_base.in @@ -6,6 +6,7 @@ --requirement ../../../packages/aws-library/requirements/_base.in +--requirement ../../../packages/celery-library/requirements/_base.in --requirement ../../../packages/common-library/requirements/_base.in --requirement ../../../packages/models-library/requirements/_base.in --requirement ../../../packages/postgres-database/requirements/_base.in @@ -19,11 +20,12 @@ aiofiles # i/o asgi_lifespan asyncpg # database celery[redis] +fastapi-pagination +httpx opentelemetry-instrumentation-celery opentelemetry-instrumentation-botocore -packaging -fastapi-pagination orjson +packaging pydantic[dotenv] tenacity typer diff --git a/services/storage/requirements/_base.txt b/services/storage/requirements/_base.txt index ac425c262e7..0ba91078bfb 100644 --- a/services/storage/requirements/_base.txt +++ b/services/storage/requirements/_base.txt @@ -1,6 +1,7 @@ aio-pika==9.5.4 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in aioboto3==14.3.0 # via @@ -12,18 +13,22 @@ aiocache==0.12.3 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in aiodebug==2.3.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in aiodocker==0.24.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in aiofiles==24.1.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/_base.in # aioboto3 @@ -43,6 +48,18 @@ aiohttp==3.12.12 # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -84,6 +101,9 @@ arrow==1.3.0 # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in @@ -111,8 +131,8 @@ botocore==1.37.3 # s3transfer botocore-stubs==1.36.17 # via types-aiobotocore -celery==5.4.0 - # via -r requirements/_base.in +celery==5.5.2 + # via -r requirements/../../../packages/celery-library/requirements/_base.in certifi==2025.1.31 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -127,6 +147,18 @@ certifi==2025.1.31 # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -190,6 +222,7 @@ fastapi-pagination==0.12.34 faststream==0.5.34 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in frozenlist==1.5.0 # via @@ -229,6 +262,18 @@ httpx==0.28.1 # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -270,6 +315,18 @@ jinja2==3.1.5 # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -294,11 +351,13 @@ jsonschema==4.23.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in jsonschema-specifications==2024.10.1 # via jsonschema -kombu==5.4.2 +kombu==5.5.3 # via celery mako==1.3.9 # via @@ -314,6 +373,18 @@ mako==1.3.9 # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -345,6 +416,7 @@ multidict==6.1.0 opentelemetry-api==1.30.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http @@ -365,6 +437,7 @@ opentelemetry-api==1.30.0 opentelemetry-exporter-otlp==1.30.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-exporter-otlp-proto-common==1.30.0 # via @@ -407,14 +480,17 @@ opentelemetry-instrumentation-httpx==0.51b0 opentelemetry-instrumentation-logging==0.51b0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-redis==0.51b0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-instrumentation-requests==0.51b0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in opentelemetry-propagator-aws-xray==1.0.2 # via opentelemetry-instrumentation-botocore @@ -426,6 +502,7 @@ opentelemetry-proto==1.30.0 opentelemetry-sdk==1.30.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http @@ -461,6 +538,18 @@ orjson==3.10.15 # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -483,6 +572,14 @@ orjson==3.10.15 # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in @@ -514,6 +611,7 @@ protobuf==5.29.3 psutil==6.1.1 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in psycopg2-binary==2.9.10 # via sqlalchemy @@ -533,6 +631,18 @@ pydantic==2.10.6 # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -559,6 +669,17 @@ pydantic==2.10.6 # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in @@ -590,6 +711,14 @@ pydantic-extra-types==2.10.2 # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in @@ -613,6 +742,18 @@ pydantic-settings==2.7.0 # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -631,6 +772,10 @@ pydantic-settings==2.7.0 # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in @@ -640,6 +785,7 @@ pygments==2.19.1 pyinstrument==5.0.1 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in python-dateutil==2.9.0.post0 # via @@ -667,6 +813,18 @@ pyyaml==6.0.2 # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -682,6 +840,7 @@ pyyaml==6.0.2 # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # uvicorn redis==5.2.1 @@ -698,6 +857,18 @@ redis==5.2.1 # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -713,6 +884,7 @@ redis==5.2.1 # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # celery referencing==0.35.1 @@ -729,6 +901,18 @@ referencing==0.35.1 # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -751,6 +935,8 @@ rich==13.9.4 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # rich-toolkit @@ -787,6 +973,18 @@ sqlalchemy==1.4.54 # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -817,6 +1015,18 @@ starlette==0.45.3 # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -835,24 +1045,30 @@ starlette==0.45.3 stream-zip==0.0.83 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in tenacity==9.0.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/_base.in toolz==1.0.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in tqdm==4.67.1 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in typer==0.15.1 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/_base.in @@ -889,10 +1105,49 @@ typing-extensions==4.12.2 # types-aiobotocore-ec2 # types-aiobotocore-s3 # types-aiobotocore-ssm -tzdata==2025.1 +tzdata==2025.2 + # via kombu +ujson==5.10.0 # via - # celery - # kombu + # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # fastapi urllib3==2.3.0 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -907,6 +1162,18 @@ urllib3==2.3.0 # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/celery-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -953,6 +1220,7 @@ wrapt==1.17.2 yarl==1.18.3 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/celery-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/postgres-database/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # aio-pika diff --git a/services/storage/requirements/_test.txt b/services/storage/requirements/_test.txt index fa83053433b..d8a2498e108 100644 --- a/services/storage/requirements/_test.txt +++ b/services/storage/requirements/_test.txt @@ -59,7 +59,7 @@ botocore==1.37.3 # boto3 # moto # s3transfer -celery==5.4.0 +celery==5.5.2 # via # -c requirements/_base.txt # pytest-celery @@ -199,7 +199,7 @@ jsonschema-specifications==2024.10.1 # -c requirements/_base.txt # jsonschema # openapi-schema-validator -kombu==5.4.2 +kombu==5.5.3 # via # -c requirements/_base.txt # celery @@ -408,10 +408,9 @@ typing-extensions==4.12.2 # pydantic # pydantic-core # sqlalchemy2-stubs -tzdata==2025.1 +tzdata==2025.2 # via # -c requirements/_base.txt - # celery # faker # kombu # pandas diff --git a/services/storage/requirements/ci.txt b/services/storage/requirements/ci.txt index 31b66afbe90..bb397a78f5e 100644 --- a/services/storage/requirements/ci.txt +++ b/services/storage/requirements/ci.txt @@ -13,6 +13,7 @@ # installs this repo's packages simcore-aws-library @ ../../packages/aws-library/ +simcore-celery-library @ ../../packages/celery-library/ simcore-common-library @ ../../packages/common-library/ simcore-models-library @ ../../packages/models-library/ simcore-postgres-database @ ../../packages/postgres-database/ diff --git a/services/storage/requirements/dev.txt b/services/storage/requirements/dev.txt index 253cec8dbcb..b428181c8c5 100644 --- a/services/storage/requirements/dev.txt +++ b/services/storage/requirements/dev.txt @@ -13,6 +13,7 @@ # installs this repo's packages --editable ../../packages/aws-library/ +--editable ../../packages/celery-library/ --editable ../../packages/common-library --editable ../../packages/models-library --editable ../../packages/postgres-database/ diff --git a/services/storage/requirements/prod.txt b/services/storage/requirements/prod.txt index dc4a2da4805..40e94e3f436 100644 --- a/services/storage/requirements/prod.txt +++ b/services/storage/requirements/prod.txt @@ -11,6 +11,7 @@ # installs this repo's packages simcore-aws-library @ ../../packages/aws-library/ +simcore-celery-library @ ../../packages/celery-library/ simcore-common-library @ ../../packages/common-library/ simcore-models-library @ ../../packages/models-library/ simcore-postgres-database @ ../../packages/postgres-database/ diff --git a/services/storage/src/simcore_service_storage/api/_worker_tasks/_files.py b/services/storage/src/simcore_service_storage/api/_worker_tasks/_files.py index 871e8a7bcbc..56a0343fdae 100644 --- a/services/storage/src/simcore_service_storage/api/_worker_tasks/_files.py +++ b/services/storage/src/simcore_service_storage/api/_worker_tasks/_files.py @@ -1,24 +1,24 @@ import logging from celery import Task # type: ignore[import-untyped] +from celery_library.utils import get_app_server from models_library.api_schemas_storage.storage_schemas import ( FileUploadCompletionBody, ) from models_library.projects_nodes_io import LocationID, StorageFileID from models_library.users import UserID +from servicelib.celery.models import TaskID from servicelib.logging_utils import log_context from ...dsm import get_dsm_provider from ...models import FileMetaData -from ...modules.celery.models import TaskId -from ...modules.celery.utils import get_fastapi_app _logger = logging.getLogger(__name__) async def complete_upload_file( task: Task, - task_id: TaskId, + task_id: TaskID, user_id: UserID, location_id: LocationID, file_id: StorageFileID, @@ -30,7 +30,7 @@ async def complete_upload_file( logging.INFO, msg=f"completing upload of file {user_id=}, {location_id=}, {file_id=}", ): - dsm = get_dsm_provider(get_fastapi_app(task.app)).get(location_id) + dsm = get_dsm_provider(get_app_server(task.app).app).get(location_id) # NOTE: completing a multipart upload on AWS can take up to several minutes # if it returns slow we return a 202 - Accepted, the client will have to check later # for completeness diff --git a/services/storage/src/simcore_service_storage/api/_worker_tasks/_paths.py b/services/storage/src/simcore_service_storage/api/_worker_tasks/_paths.py index 2f3d05da547..142c0f3968b 100644 --- a/services/storage/src/simcore_service_storage/api/_worker_tasks/_paths.py +++ b/services/storage/src/simcore_service_storage/api/_worker_tasks/_paths.py @@ -2,22 +2,22 @@ from pathlib import Path from celery import Task # type: ignore[import-untyped] +from celery_library.utils import get_app_server from models_library.projects_nodes_io import LocationID, StorageFileID from models_library.users import UserID from pydantic import ByteSize, TypeAdapter +from servicelib.celery.models import TaskID from servicelib.logging_utils import log_context from servicelib.utils import limited_gather from ...constants import MAX_CONCURRENT_S3_TASKS from ...dsm import get_dsm_provider -from ...modules.celery.models import TaskId -from ...modules.celery.utils import get_fastapi_app _logger = logging.getLogger(__name__) async def compute_path_size( - task: Task, task_id: TaskId, user_id: UserID, location_id: LocationID, path: Path + task: Task, task_id: TaskID, user_id: UserID, location_id: LocationID, path: Path ) -> ByteSize: assert task_id # nosec with log_context( @@ -25,13 +25,13 @@ async def compute_path_size( logging.INFO, msg=f"computing path size {user_id=}, {location_id=}, {path=}", ): - dsm = get_dsm_provider(get_fastapi_app(task.app)).get(location_id) + dsm = get_dsm_provider(get_app_server(task.app).app).get(location_id) return await dsm.compute_path_size(user_id, path=Path(path)) async def delete_paths( task: Task, - task_id: TaskId, + task_id: TaskID, user_id: UserID, location_id: LocationID, paths: set[Path], @@ -42,7 +42,7 @@ async def delete_paths( logging.INFO, msg=f"delete {paths=} in {location_id=} for {user_id=}", ): - dsm = get_dsm_provider(get_fastapi_app(task.app)).get(location_id) + dsm = get_dsm_provider(get_app_server(task.app).app).get(location_id) files_ids: set[StorageFileID] = { TypeAdapter(StorageFileID).validate_python(f"{path}") for path in paths } diff --git a/services/storage/src/simcore_service_storage/api/_worker_tasks/_simcore_s3.py b/services/storage/src/simcore_service_storage/api/_worker_tasks/_simcore_s3.py index 8a9f0a941cc..83f93f6fc4c 100644 --- a/services/storage/src/simcore_service_storage/api/_worker_tasks/_simcore_s3.py +++ b/services/storage/src/simcore_service_storage/api/_worker_tasks/_simcore_s3.py @@ -4,27 +4,27 @@ from aws_library.s3._models import S3ObjectKey from celery import Task # type: ignore[import-untyped] +from celery_library.utils import get_app_server from models_library.api_schemas_storage.storage_schemas import FoldersBody from models_library.api_schemas_webserver.storage import PathToExport from models_library.progress_bar import ProgressReport from models_library.projects_nodes_io import StorageFileID from models_library.users import UserID from pydantic import TypeAdapter +from servicelib.celery.models import TaskID from servicelib.logging_utils import log_context from servicelib.progress_bar import ProgressBarData from ...dsm import get_dsm_provider -from ...modules.celery.models import TaskID, TaskId -from ...modules.celery.utils import get_celery_worker, get_fastapi_app from ...simcore_s3_dsm import SimcoreS3DataManager _logger = logging.getLogger(__name__) async def _task_progress_cb( - task: Task, task_id: TaskId, report: ProgressReport + task: Task, task_id: TaskID, report: ProgressReport ) -> None: - worker = get_celery_worker(task.app) + worker = get_app_server(task.app).task_manager assert task.name # nosec await worker.set_task_progress( task_id=task_id, @@ -33,14 +33,14 @@ async def _task_progress_cb( async def deep_copy_files_from_project( - task: Task, task_id: TaskId, user_id: UserID, body: FoldersBody + task: Task, task_id: TaskID, user_id: UserID, body: FoldersBody ) -> dict[str, Any]: with log_context( _logger, logging.INFO, msg=f"copying {body.source['uuid']} -> {body.destination['uuid']} with {task.request.id}", ): - dsm = get_dsm_provider(get_fastapi_app(task.app)).get( + dsm = get_dsm_provider(get_app_server(task.app).app).get( SimcoreS3DataManager.get_location_id() ) assert isinstance(dsm, SimcoreS3DataManager) # nosec @@ -75,7 +75,7 @@ async def export_data( logging.INFO, f"'{task_id}' export data (for {user_id=}) fom selection: {paths_to_export}", ): - dsm = get_dsm_provider(get_fastapi_app(task.app)).get( + dsm = get_dsm_provider(get_app_server(task.app).app).get( SimcoreS3DataManager.get_location_id() ) assert isinstance(dsm, SimcoreS3DataManager) # nosec @@ -87,7 +87,9 @@ async def export_data( async def _progress_cb(report: ProgressReport) -> None: assert task.name # nosec - await get_celery_worker(task.app).set_task_progress(task_id, report) + await get_app_server(task.app).task_manager.set_task_progress( + task_id, report + ) _logger.debug("'%s' progress %s", task_id, report.percent_value) async with ProgressBarData( diff --git a/services/storage/src/simcore_service_storage/api/_worker_tasks/tasks.py b/services/storage/src/simcore_service_storage/api/_worker_tasks/tasks.py index 6c74cd9792d..02c6ff60dc8 100644 --- a/services/storage/src/simcore_service_storage/api/_worker_tasks/tasks.py +++ b/services/storage/src/simcore_service_storage/api/_worker_tasks/tasks.py @@ -1,11 +1,16 @@ import logging from celery import Celery # type: ignore[import-untyped] +from celery_library.task import register_task +from celery_library.types import register_celery_types, register_pydantic_types from models_library.api_schemas_storage.export_data_async_jobs import AccessRightError +from models_library.api_schemas_storage.storage_schemas import ( + FileUploadCompletionBody, + FoldersBody, +) from servicelib.logging_utils import log_context -from ...modules.celery._celery_types import register_celery_types -from ...modules.celery._task import register_task +from ...models import FileMetaData from ._files import complete_upload_file from ._paths import compute_path_size, delete_paths from ._simcore_s3 import deep_copy_files_from_project, export_data @@ -15,6 +20,8 @@ def setup_worker_tasks(app: Celery) -> None: register_celery_types() + register_pydantic_types(FileUploadCompletionBody, FileMetaData, FoldersBody) + with log_context(_logger, logging.INFO, msg="worker task registration"): register_task(app, export_data, dont_autoretry_for=(AccessRightError,)) register_task(app, compute_path_size) diff --git a/services/storage/src/simcore_service_storage/api/rest/_files.py b/services/storage/src/simcore_service_storage/api/rest/_files.py index f4781841570..84857370260 100644 --- a/services/storage/src/simcore_service_storage/api/rest/_files.py +++ b/services/storage/src/simcore_service_storage/api/rest/_files.py @@ -20,6 +20,8 @@ from models_library.projects_nodes_io import LocationID, StorageFileID from pydantic import AnyUrl, ByteSize, TypeAdapter from servicelib.aiohttp import status +from servicelib.celery.models import TaskMetadata, TaskUUID +from servicelib.celery.task_manager import TaskManager from yarl import URL from ...dsm import get_dsm_provider @@ -34,11 +36,9 @@ StorageQueryParamsBase, UploadLinks, ) -from ...modules.celery.client import CeleryTaskClient -from ...modules.celery.models import TaskMetadata, TaskUUID from ...simcore_s3_dsm import SimcoreS3DataManager from .._worker_tasks._files import complete_upload_file as remote_complete_upload_file -from .dependencies.celery import get_celery_client +from .dependencies.celery import get_task_manager _logger = logging.getLogger(__name__) @@ -270,7 +270,7 @@ async def abort_upload_file( status_code=status.HTTP_202_ACCEPTED, ) async def complete_upload_file( - celery_client: Annotated[CeleryTaskClient, Depends(get_celery_client)], + task_manager: Annotated[TaskManager, Depends(get_task_manager)], query_params: Annotated[StorageQueryParamsBase, Depends()], location_id: LocationID, file_id: StorageFileID, @@ -284,7 +284,7 @@ async def complete_upload_file( user_id=query_params.user_id, product_name=_UNDEFINED_PRODUCT_NAME_FOR_WORKER_TASKS, # NOTE: I would need to change the API here ) - task_uuid = await celery_client.submit_task( + task_uuid = await task_manager.submit_task( TaskMetadata( name=remote_complete_upload_file.__name__, ), @@ -326,7 +326,7 @@ async def complete_upload_file( response_model=Envelope[FileUploadCompleteFutureResponse], ) async def is_completed_upload_file( - celery_client: Annotated[CeleryTaskClient, Depends(get_celery_client)], + task_manager: Annotated[TaskManager, Depends(get_task_manager)], query_params: Annotated[StorageQueryParamsBase, Depends()], location_id: LocationID, file_id: StorageFileID, @@ -340,20 +340,23 @@ async def is_completed_upload_file( user_id=query_params.user_id, product_name=_UNDEFINED_PRODUCT_NAME_FOR_WORKER_TASKS, # NOTE: I would need to change the API here ) - task_status = await celery_client.get_task_status( + task_status = await task_manager.get_task_status( task_context=async_job_name_data.model_dump(), task_uuid=TaskUUID(future_id) ) # first check if the task is in the app if task_status.is_done: - task_result = await celery_client.get_task_result( - task_context=async_job_name_data.model_dump(), task_uuid=TaskUUID(future_id) + task_result = TypeAdapter(FileMetaData).validate_python( + await task_manager.get_task_result( + task_context=async_job_name_data.model_dump(), + task_uuid=TaskUUID(future_id), + ) ) - assert isinstance(task_result, FileMetaData), f"{task_result=}" # nosec new_fmd = task_result assert new_fmd.location_id == location_id # nosec assert new_fmd.file_id == file_id # nosec response = FileUploadCompleteFutureResponse( - state=FileUploadCompleteState.OK, e_tag=new_fmd.entity_tag + state=FileUploadCompleteState.OK, + e_tag=new_fmd.entity_tag, ) else: # the task is still running diff --git a/services/storage/src/simcore_service_storage/api/rest/dependencies/celery.py b/services/storage/src/simcore_service_storage/api/rest/dependencies/celery.py index 58413bba852..1c6cce503aa 100644 --- a/services/storage/src/simcore_service_storage/api/rest/dependencies/celery.py +++ b/services/storage/src/simcore_service_storage/api/rest/dependencies/celery.py @@ -1,13 +1,13 @@ from typing import Annotated +from celery_library.task_manager import CeleryTaskManager from fastapi import Depends, FastAPI from servicelib.fastapi.dependencies import get_app -from ....modules.celery import get_celery_client as _get_celery_client_from_app -from ....modules.celery.client import CeleryTaskClient +from ....modules.celery import get_task_manager_from_app -def get_celery_client( +def get_task_manager( app: Annotated[FastAPI, Depends(get_app)], -) -> CeleryTaskClient: - return _get_celery_client_from_app(app) +) -> CeleryTaskManager: + return get_task_manager_from_app(app) diff --git a/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py b/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py index 3186237eb7e..8628413e83c 100644 --- a/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py +++ b/services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py @@ -3,6 +3,10 @@ import logging from celery.exceptions import CeleryError # type: ignore[import-untyped] +from celery_library.errors import ( + TransferrableCeleryError, + decode_celery_transferrable_error, +) from fastapi import FastAPI from models_library.api_schemas_rpc_async_jobs.async_jobs import ( AsyncJobGet, @@ -17,15 +21,11 @@ JobNotDoneError, JobSchedulerError, ) +from servicelib.celery.models import TaskState from servicelib.logging_utils import log_catch from servicelib.rabbitmq import RPCRouter -from ...modules.celery import get_celery_client -from ...modules.celery.errors import ( - TransferrableCeleryError, - decode_celery_transferrable_error, -) -from ...modules.celery.models import TaskState +from ...modules.celery import get_task_manager_from_app _logger = logging.getLogger(__name__) router = RPCRouter() @@ -36,7 +36,7 @@ async def cancel(app: FastAPI, job_id: AsyncJobId, job_id_data: AsyncJobNameData assert app # nosec assert job_id_data # nosec try: - await get_celery_client(app).cancel_task( + await get_task_manager_from_app(app).cancel_task( task_context=job_id_data.model_dump(), task_uuid=job_id, ) @@ -52,7 +52,7 @@ async def status( assert job_id_data # nosec try: - task_status = await get_celery_client(app).get_task_status( + task_status = await get_task_manager_from_app(app).get_task_status( task_context=job_id_data.model_dump(), task_uuid=job_id, ) @@ -82,13 +82,13 @@ async def result( assert job_id_data # nosec try: - _status = await get_celery_client(app).get_task_status( + _status = await get_task_manager_from_app(app).get_task_status( task_context=job_id_data.model_dump(), task_uuid=job_id, ) if not _status.is_done: raise JobNotDoneError(job_id=job_id) - _result = await get_celery_client(app).get_task_result( + _result = await get_task_manager_from_app(app).get_task_result( task_context=job_id_data.model_dump(), task_uuid=job_id, ) @@ -127,7 +127,7 @@ async def list_jobs( _ = filter_ assert app # nosec try: - tasks = await get_celery_client(app).list_tasks( + tasks = await get_task_manager_from_app(app).list_tasks( task_context=job_id_data.model_dump(), ) except CeleryError as exc: diff --git a/services/storage/src/simcore_service_storage/api/rpc/_paths.py b/services/storage/src/simcore_service_storage/api/rpc/_paths.py index db0e69af38d..9c704d46820 100644 --- a/services/storage/src/simcore_service_storage/api/rpc/_paths.py +++ b/services/storage/src/simcore_service_storage/api/rpc/_paths.py @@ -7,10 +7,10 @@ AsyncJobNameData, ) from models_library.projects_nodes_io import LocationID +from servicelib.celery.models import TaskMetadata from servicelib.rabbitmq import RPCRouter -from ...modules.celery import get_celery_client -from ...modules.celery.models import TaskMetadata +from ...modules.celery import get_task_manager_from_app from .._worker_tasks._paths import compute_path_size as remote_compute_path_size from .._worker_tasks._paths import delete_paths as remote_delete_paths @@ -26,7 +26,7 @@ async def compute_path_size( path: Path, ) -> AsyncJobGet: task_name = remote_compute_path_size.__name__ - task_uuid = await get_celery_client(app).submit_task( + task_uuid = await get_task_manager_from_app(app).submit_task( task_metadata=TaskMetadata( name=task_name, ), @@ -47,7 +47,7 @@ async def delete_paths( paths: set[Path], ) -> AsyncJobGet: task_name = remote_delete_paths.__name__ - task_uuid = await get_celery_client(app).submit_task( + task_uuid = await get_task_manager_from_app(app).submit_task( task_metadata=TaskMetadata( name=task_name, ), diff --git a/services/storage/src/simcore_service_storage/api/rpc/_simcore_s3.py b/services/storage/src/simcore_service_storage/api/rpc/_simcore_s3.py index ba3830c0329..6b0b27f87a5 100644 --- a/services/storage/src/simcore_service_storage/api/rpc/_simcore_s3.py +++ b/services/storage/src/simcore_service_storage/api/rpc/_simcore_s3.py @@ -5,10 +5,10 @@ ) from models_library.api_schemas_storage.storage_schemas import FoldersBody from models_library.api_schemas_webserver.storage import PathToExport +from servicelib.celery.models import TaskMetadata, TasksQueue from servicelib.rabbitmq import RPCRouter -from ...modules.celery import get_celery_client -from ...modules.celery.models import TaskMetadata, TasksQueue +from ...modules.celery import get_task_manager_from_app from .._worker_tasks._simcore_s3 import deep_copy_files_from_project, export_data router = RPCRouter() @@ -21,7 +21,7 @@ async def copy_folders_from_project( body: FoldersBody, ) -> AsyncJobGet: task_name = deep_copy_files_from_project.__name__ - task_uuid = await get_celery_client(app).submit_task( + task_uuid = await get_task_manager_from_app(app).submit_task( task_metadata=TaskMetadata( name=task_name, ), @@ -38,7 +38,7 @@ async def start_export_data( app: FastAPI, job_id_data: AsyncJobNameData, paths_to_export: list[PathToExport] ) -> AsyncJobGet: task_name = export_data.__name__ - task_uuid = await get_celery_client(app).submit_task( + task_uuid = await get_task_manager_from_app(app).submit_task( task_metadata=TaskMetadata( name=task_name, ephemeral=False, diff --git a/services/storage/src/simcore_service_storage/core/application.py b/services/storage/src/simcore_service_storage/core/application.py index d3d2e0c14ff..987878d32cf 100644 --- a/services/storage/src/simcore_service_storage/core/application.py +++ b/services/storage/src/simcore_service_storage/core/application.py @@ -36,7 +36,7 @@ from ..dsm import setup_dsm from ..dsm_cleaner import setup_dsm_cleaner from ..exceptions.handlers import set_exception_handlers -from ..modules.celery import setup_celery_client +from ..modules.celery import setup_task_manager from ..modules.db import setup_db from ..modules.long_running_tasks import setup_rest_api_long_running_tasks_for_uploads from ..modules.rabbitmq import setup as setup_rabbitmq @@ -93,7 +93,9 @@ def create_app(settings: ApplicationSettings) -> FastAPI: # noqa: C901 if not settings.STORAGE_WORKER_MODE: setup_rabbitmq(app) setup_rpc_api_routes(app) - setup_celery_client(app) + + assert settings.STORAGE_CELERY # nosec + setup_task_manager(app, celery_settings=settings.STORAGE_CELERY) setup_rest_api_long_running_tasks_for_uploads(app) setup_rest_api_routes(app, API_VTAG) set_exception_handlers(app) diff --git a/services/storage/src/simcore_service_storage/modules/celery/__init__.py b/services/storage/src/simcore_service_storage/modules/celery/__init__.py index cbf60fda44e..45795d329a3 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/__init__.py +++ b/services/storage/src/simcore_service_storage/modules/celery/__init__.py @@ -1,56 +1,30 @@ -import logging -from asyncio import AbstractEventLoop - +from celery_library.common import create_app, create_task_manager +from celery_library.task_manager import CeleryTaskManager +from celery_library.types import register_celery_types, register_pydantic_types from fastapi import FastAPI -from servicelib.redis._client import RedisClientSDK -from settings_library.redis import RedisDatabase - -from ..._meta import APP_NAME -from ...core.settings import get_application_settings -from ._celery_types import register_celery_types -from ._common import create_app -from .backends._redis import RedisTaskInfoStore -from .client import CeleryTaskClient +from models_library.api_schemas_storage.storage_schemas import ( + FileUploadCompletionBody, + FoldersBody, +) +from settings_library.celery import CelerySettings -_logger = logging.getLogger(__name__) +from ...models import FileMetaData -def setup_celery_client(app: FastAPI) -> None: +def setup_task_manager(app: FastAPI, celery_settings: CelerySettings) -> None: async def on_startup() -> None: - application_settings = get_application_settings(app) - celery_settings = application_settings.STORAGE_CELERY - assert celery_settings # nosec - celery_app = create_app(celery_settings) - redis_client_sdk = RedisClientSDK( - celery_settings.CELERY_REDIS_RESULT_BACKEND.build_redis_dsn( - RedisDatabase.CELERY_TASKS - ), - client_name=f"{APP_NAME}.celery_tasks", - ) - - app.state.celery_client = CeleryTaskClient( - celery_app, - celery_settings, - RedisTaskInfoStore(redis_client_sdk), + app.state.celery_client = await create_task_manager( + create_app(celery_settings), celery_settings ) register_celery_types() + register_pydantic_types(FileUploadCompletionBody, FileMetaData, FoldersBody) app.add_event_handler("startup", on_startup) -def get_celery_client(app: FastAPI) -> CeleryTaskClient: +def get_task_manager_from_app(app: FastAPI) -> CeleryTaskManager: assert hasattr(app.state, "celery_client") # nosec celery_client = app.state.celery_client - assert isinstance(celery_client, CeleryTaskClient) + assert isinstance(celery_client, CeleryTaskManager) # nosec return celery_client - - -def get_event_loop(app: FastAPI) -> AbstractEventLoop: - event_loop = app.state.event_loop - assert isinstance(event_loop, AbstractEventLoop) - return event_loop - - -def set_event_loop(app: FastAPI, event_loop: AbstractEventLoop) -> None: - app.state.event_loop = event_loop diff --git a/services/storage/src/simcore_service_storage/modules/celery/signals.py b/services/storage/src/simcore_service_storage/modules/celery/signals.py deleted file mode 100644 index 113d26c3566..00000000000 --- a/services/storage/src/simcore_service_storage/modules/celery/signals.py +++ /dev/null @@ -1,99 +0,0 @@ -import asyncio -import datetime -import logging -import threading -from typing import Final - -from asgi_lifespan import LifespanManager -from celery import Celery # type: ignore[import-untyped] -from fastapi import FastAPI -from servicelib.logging_utils import log_context -from servicelib.redis._client import RedisClientSDK -from settings_library.redis import RedisDatabase -from simcore_service_storage._meta import APP_NAME - -from ...core.application import create_app -from ...core.settings import ApplicationSettings -from . import set_event_loop -from .backends._redis import RedisTaskInfoStore -from .utils import ( - get_fastapi_app, - set_celery_worker, - set_fastapi_app, -) -from .worker import CeleryTaskWorker - -_logger = logging.getLogger(__name__) - -_SHUTDOWN_TIMEOUT: Final[float] = datetime.timedelta(seconds=10).total_seconds() -_STARTUP_TIMEOUT: Final[float] = datetime.timedelta(minutes=1).total_seconds() - - -def on_worker_init(sender, **_kwargs) -> None: - startup_complete_event = threading.Event() - - def _init(startup_complete_event: threading.Event) -> None: - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - shutdown_event = asyncio.Event() - - app_settings = ApplicationSettings.create_from_envs() - fastapi_app = create_app(app_settings) - - assert app_settings.STORAGE_CELERY - celery_settings = app_settings.STORAGE_CELERY - - async def setup_task_worker(): - redis_client_sdk = RedisClientSDK( - celery_settings.CELERY_REDIS_RESULT_BACKEND.build_redis_dsn( - RedisDatabase.CELERY_TASKS - ), - client_name=f"{APP_NAME}.celery_tasks", - ) - - set_celery_worker( - sender.app, CeleryTaskWorker(RedisTaskInfoStore(redis_client_sdk)) - ) - - async def fastapi_lifespan( - startup_complete_event: threading.Event, shutdown_event: asyncio.Event - ) -> None: - async with LifespanManager( - fastapi_app, - startup_timeout=_STARTUP_TIMEOUT, - shutdown_timeout=_SHUTDOWN_TIMEOUT, - ): - try: - _logger.info("fastapi APP started!") - startup_complete_event.set() - await shutdown_event.wait() - except asyncio.CancelledError: - _logger.warning("Lifespan task cancelled") - - fastapi_app.state.shutdown_event = shutdown_event - set_event_loop(fastapi_app, loop) - - set_fastapi_app(sender.app, fastapi_app) - loop.run_until_complete(setup_task_worker()) - loop.run_until_complete( - fastapi_lifespan(startup_complete_event, shutdown_event) - ) - - thread = threading.Thread( - group=None, - target=_init, - name="fastapi_app", - args=(startup_complete_event,), - daemon=True, - ) - thread.start() - # ensure the fastapi app is ready before going on - startup_complete_event.wait(_STARTUP_TIMEOUT * 1.1) - - -def on_worker_shutdown(sender, **_kwargs) -> None: - with log_context(_logger, logging.INFO, "Worker Shuts-down"): - assert isinstance(sender.app, Celery) - fastapi_app = get_fastapi_app(sender.app) - assert isinstance(fastapi_app, FastAPI) - fastapi_app.state.shutdown_event.set() diff --git a/services/storage/src/simcore_service_storage/modules/celery/utils.py b/services/storage/src/simcore_service_storage/modules/celery/utils.py deleted file mode 100644 index d09c1a1ce41..00000000000 --- a/services/storage/src/simcore_service_storage/modules/celery/utils.py +++ /dev/null @@ -1,27 +0,0 @@ -from celery import Celery # type: ignore[import-untyped] -from fastapi import FastAPI - -from .worker import CeleryTaskWorker - -_WORKER_KEY = "celery_worker" -_FASTAPI_APP_KEY = "fastapi_app" - - -def set_celery_worker(celery_app: Celery, worker: CeleryTaskWorker) -> None: - celery_app.conf[_WORKER_KEY] = worker - - -def get_celery_worker(celery_app: Celery) -> CeleryTaskWorker: - worker = celery_app.conf[_WORKER_KEY] - assert isinstance(worker, CeleryTaskWorker) - return worker - - -def set_fastapi_app(celery_app: Celery, fastapi_app: FastAPI) -> None: - celery_app.conf[_FASTAPI_APP_KEY] = fastapi_app - - -def get_fastapi_app(celery_app: Celery) -> FastAPI: - fastapi_app = celery_app.conf[_FASTAPI_APP_KEY] - assert isinstance(fastapi_app, FastAPI) - return fastapi_app diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker.py b/services/storage/src/simcore_service_storage/modules/celery/worker.py deleted file mode 100644 index a5e98ac09df..00000000000 --- a/services/storage/src/simcore_service_storage/modules/celery/worker.py +++ /dev/null @@ -1,19 +0,0 @@ -import logging -from dataclasses import dataclass - -from models_library.progress_bar import ProgressReport - -from ..celery.models import TaskID, TaskInfoStore - -_logger = logging.getLogger(__name__) - - -@dataclass -class CeleryTaskWorker: - _task_info_store: TaskInfoStore - - async def set_task_progress(self, task_id: TaskID, report: ProgressReport) -> None: - await self._task_info_store.set_task_progress( - task_id=task_id, - report=report, - ) diff --git a/services/storage/src/simcore_service_storage/modules/celery/worker_main.py b/services/storage/src/simcore_service_storage/modules/celery/worker_main.py index 58febcb61f6..ebd9832b9e1 100644 --- a/services/storage/src/simcore_service_storage/modules/celery/worker_main.py +++ b/services/storage/src/simcore_service_storage/modules/celery/worker_main.py @@ -1,17 +1,20 @@ """Main application to be deployed in for example uvicorn.""" import logging +from functools import partial from celery.signals import worker_init, worker_shutdown # type: ignore[import-untyped] -from servicelib.logging_utils import config_all_loggers -from simcore_service_storage.api._worker_tasks.tasks import setup_worker_tasks - -from ...core.settings import ApplicationSettings -from ._common import create_app as create_celery_app -from .signals import ( +from celery_library.common import create_app as create_celery_app +from celery_library.signals import ( on_worker_init, on_worker_shutdown, ) +from servicelib.fastapi.celery.app_server import FastAPIAppServer +from servicelib.logging_utils import config_all_loggers + +from ...api._worker_tasks.tasks import setup_worker_tasks +from ...core.application import create_app +from ...core.settings import ApplicationSettings _settings = ApplicationSettings.create_from_envs() @@ -24,9 +27,20 @@ ) -assert _settings.STORAGE_CELERY +assert _settings.STORAGE_CELERY # nosec app = create_celery_app(_settings.STORAGE_CELERY) -worker_init.connect(on_worker_init) + +app_server = FastAPIAppServer(app=create_app(_settings)) + + +def worker_init_wrapper(sender, **_kwargs): + assert _settings.STORAGE_CELERY # nosec + return partial(on_worker_init, app_server, _settings.STORAGE_CELERY)( + sender, **_kwargs + ) + + +worker_init.connect(worker_init_wrapper) worker_shutdown.connect(on_worker_shutdown) diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index 6bbffe9cd19..ad37ba752e6 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -12,6 +12,7 @@ import random import sys from collections.abc import AsyncIterator, Awaitable, Callable +from functools import partial from pathlib import Path from typing import Any, Final, cast @@ -24,6 +25,8 @@ from celery import Celery from celery.contrib.testing.worker import TestWorkController, start_worker from celery.signals import worker_init, worker_shutdown +from celery.worker.worker import WorkController +from celery_library.signals import on_worker_init, on_worker_shutdown from faker import Faker from fakeredis.aioredis import FakeRedis from fastapi import FastAPI @@ -60,6 +63,7 @@ ) from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.aiohttp import status +from servicelib.fastapi.celery.app_server import FastAPIAppServer from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient from servicelib.utils import limited_gather from settings_library.rabbit import RabbitSettings @@ -71,12 +75,6 @@ from simcore_service_storage.datcore_dsm import DatCoreDataManager from simcore_service_storage.dsm import get_dsm_provider from simcore_service_storage.models import FileMetaData, FileMetaDataAtDB, S3BucketName -from simcore_service_storage.modules.celery.signals import ( - on_worker_init, - on_worker_shutdown, -) -from simcore_service_storage.modules.celery.utils import get_celery_worker -from simcore_service_storage.modules.celery.worker import CeleryTaskWorker from simcore_service_storage.modules.s3 import get_s3_client from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager from sqlalchemy import literal_column @@ -365,7 +363,7 @@ def upload_file( create_upload_file_link_v2: Callable[..., Awaitable[FileUploadSchema]], create_file_of_size: Callable[[ByteSize, str | None], Path], create_simcore_file_id: Callable[[ProjectID, NodeID, str], SimcoreS3FileID], - with_storage_celery_worker: CeleryTaskWorker, + with_storage_celery_worker: TestWorkController, ) -> Callable[ [ByteSize, str, SimcoreS3FileID | None], Awaitable[tuple[Path, SimcoreS3FileID]] ]: @@ -480,7 +478,7 @@ async def create_empty_directory( create_simcore_file_id: Callable[[ProjectID, NodeID, str], SimcoreS3FileID], create_upload_file_link_v2: Callable[..., Awaitable[FileUploadSchema]], client: httpx.AsyncClient, - with_storage_celery_worker: CeleryTaskWorker, + with_storage_celery_worker: TestWorkController, ) -> Callable[[str, ProjectID, NodeID], Awaitable[SimcoreS3FileID]]: async def _directory_creator( dir_name: str, project_id: ProjectID, node_id: NodeID @@ -977,10 +975,7 @@ def celery_config() -> dict[str, Any]: def mock_celery_app(mocker: MockerFixture, celery_config: dict[str, Any]) -> Celery: celery_app = Celery(**celery_config) - for module in ( - "simcore_service_storage.modules.celery._common.create_app", - "simcore_service_storage.modules.celery.create_app", - ): + for module in ("simcore_service_storage.modules.celery.create_app",): mocker.patch(module, return_value=celery_app) return celery_app @@ -996,20 +991,30 @@ def _(celery_app: Celery) -> None: ... @pytest.fixture -async def with_storage_celery_worker_controller( +async def with_storage_celery_worker( app_environment: EnvVarsDict, celery_app: Celery, monkeypatch: pytest.MonkeyPatch, register_celery_tasks: Callable[[Celery], None], ) -> AsyncIterator[TestWorkController]: # Signals must be explicitily connected - worker_init.connect(on_worker_init) + monkeypatch.setenv("STORAGE_WORKER_MODE", "true") + app_settings = ApplicationSettings.create_from_envs() + + app_server = FastAPIAppServer(app=create_app(app_settings)) + + def _on_worker_init_wrapper(sender: WorkController, **_kwargs): + assert app_settings.STORAGE_CELERY # nosec + return partial(on_worker_init, app_server, app_settings.STORAGE_CELERY)( + sender, **_kwargs + ) + + worker_init.connect(_on_worker_init_wrapper) worker_shutdown.connect(on_worker_shutdown) setup_worker_tasks(celery_app) register_celery_tasks(celery_app) - monkeypatch.setenv("STORAGE_WORKER_MODE", "true") with start_worker( celery_app, pool="threads", @@ -1021,14 +1026,6 @@ async def with_storage_celery_worker_controller( yield worker -@pytest.fixture -def with_storage_celery_worker( - with_storage_celery_worker_controller: TestWorkController, -) -> CeleryTaskWorker: - assert isinstance(with_storage_celery_worker_controller.app, Celery) - return get_celery_worker(with_storage_celery_worker_controller.app) - - @pytest.fixture async def storage_rabbitmq_rpc_client( rabbitmq_rpc_client: Callable[[str], Awaitable[RabbitMQRPCClient]], diff --git a/services/storage/tests/unit/test_async_jobs.py b/services/storage/tests/unit/test_async_jobs.py index 36f29a15bd8..26140eb037c 100644 --- a/services/storage/tests/unit/test_async_jobs.py +++ b/services/storage/tests/unit/test_async_jobs.py @@ -10,6 +10,8 @@ import pytest from celery import Celery, Task +from celery.contrib.testing.worker import TestWorkController +from celery_library.task import register_task from fastapi import FastAPI from models_library.api_schemas_rpc_async_jobs.async_jobs import ( AsyncJobGet, @@ -22,16 +24,12 @@ from models_library.api_schemas_storage import STORAGE_RPC_NAMESPACE from models_library.api_schemas_storage.export_data_async_jobs import AccessRightError from models_library.products import ProductName -from models_library.rabbitmq_basic_types import RPCMethodName from models_library.users import UserID +from servicelib.celery.models import TaskID, TaskMetadata from servicelib.rabbitmq import RabbitMQRPCClient, RPCRouter from servicelib.rabbitmq.rpc_interfaces.async_jobs import async_jobs from simcore_service_storage.api.rpc.routes import get_rabbitmq_rpc_server -from simcore_service_storage.modules.celery import get_celery_client -from simcore_service_storage.modules.celery._task import register_task -from simcore_service_storage.modules.celery.client import TaskMetadata -from simcore_service_storage.modules.celery.models import TaskID -from simcore_service_storage.modules.celery.worker import CeleryTaskWorker +from simcore_service_storage.modules.celery import get_task_manager_from_app from tenacity import ( AsyncRetrying, retry_if_exception_type, @@ -54,7 +52,7 @@ async def rpc_sync_job( app: FastAPI, *, job_id_data: AsyncJobNameData, **kwargs: Any ) -> AsyncJobGet: task_name = sync_job.__name__ - task_uuid = await get_celery_client(app).submit_task( + task_uuid = await get_task_manager_from_app(app).submit_task( TaskMetadata(name=task_name), task_context=job_id_data.model_dump(), **kwargs ) @@ -66,7 +64,7 @@ async def rpc_async_job( app: FastAPI, *, job_id_data: AsyncJobNameData, **kwargs: Any ) -> AsyncJobGet: task_name = async_job.__name__ - task_uuid = await get_celery_client(app).submit_task( + task_uuid = await get_task_manager_from_app(app).submit_task( TaskMetadata(name=task_name), task_context=job_id_data.model_dump(), **kwargs ) @@ -94,8 +92,9 @@ async def _process_action(action: str, payload: Any) -> Any: return None -def sync_job(task: Task, action: Action, payload: Any) -> Any: +def sync_job(task: Task, task_id: TaskID, action: Action, payload: Any) -> Any: _ = task + _ = task_id return asyncio.run(_process_action(action, payload)) @@ -126,7 +125,7 @@ async def _start_task_via_rpc( async_job_get = await async_jobs.submit( rabbitmq_rpc_client=client, rpc_namespace=STORAGE_RPC_NAMESPACE, - method_name=RPCMethodName(rpc_task_name), + method_name=rpc_task_name, job_id_data=job_id_data, **kwargs, ) @@ -155,7 +154,7 @@ def _(celery_app: Celery) -> None: async def _wait_for_job( - storage_rabbitmq_rpc_client: RabbitMQRPCClient, + rpc_client: RabbitMQRPCClient, *, async_job_get: AsyncJobGet, job_id_data: AsyncJobNameData, @@ -170,7 +169,7 @@ async def _wait_for_job( ): with attempt: result = await async_jobs.status( - storage_rabbitmq_rpc_client, + rpc_client, rpc_namespace=STORAGE_RPC_NAMESPACE, job_id=async_job_get.job_id, job_id_data=job_id_data, @@ -202,7 +201,7 @@ async def test_async_jobs_workflow( initialized_app: FastAPI, register_rpc_routes: None, storage_rabbitmq_rpc_client: RabbitMQRPCClient, - with_storage_celery_worker: CeleryTaskWorker, + with_storage_celery_worker: TestWorkController, user_id: UserID, product_name: ProductName, exposed_rpc_start: str, @@ -250,7 +249,7 @@ async def test_async_jobs_cancel( initialized_app: FastAPI, register_rpc_routes: None, storage_rabbitmq_rpc_client: RabbitMQRPCClient, - with_storage_celery_worker: CeleryTaskWorker, + with_storage_celery_worker: TestWorkController, user_id: UserID, product_name: ProductName, exposed_rpc_start: str, @@ -315,7 +314,7 @@ async def test_async_jobs_raises( initialized_app: FastAPI, register_rpc_routes: None, storage_rabbitmq_rpc_client: RabbitMQRPCClient, - with_storage_celery_worker: CeleryTaskWorker, + with_storage_celery_worker: TestWorkController, user_id: UserID, product_name: ProductName, exposed_rpc_start: str, diff --git a/services/storage/tests/unit/test_handlers_files.py b/services/storage/tests/unit/test_handlers_files.py index db66eab9878..f07b63cdbe9 100644 --- a/services/storage/tests/unit/test_handlers_files.py +++ b/services/storage/tests/unit/test_handlers_files.py @@ -23,6 +23,7 @@ from aiohttp import ClientSession from aws_library.s3 import S3KeyNotFoundError, S3ObjectKey, SimcoreS3API from aws_library.s3._constants import MULTIPART_UPLOADS_MIN_TOTAL_SIZE +from celery.contrib.testing.worker import TestWorkController from faker import Faker from fastapi import FastAPI from models_library.api_schemas_storage.storage_schemas import ( @@ -53,7 +54,6 @@ from servicelib.aiohttp import status from simcore_service_storage.constants import S3_UNDEFINED_OR_EXTERNAL_MULTIPART_ID from simcore_service_storage.models import FileDownloadResponse, S3BucketName, UploadID -from simcore_service_storage.modules.celery.worker import CeleryTaskWorker from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager from sqlalchemy.ext.asyncio import AsyncEngine from tenacity.asyncio import AsyncRetrying @@ -683,7 +683,7 @@ async def test_upload_real_file_with_s3_client( node_id: NodeID, faker: Faker, s3_client: S3Client, - with_storage_celery_worker: CeleryTaskWorker, + with_storage_celery_worker: TestWorkController, ): file_size = TypeAdapter(ByteSize).validate_python("500Mib") file_name = faker.file_name() diff --git a/services/storage/tests/unit/test_rpc_handlers_paths.py b/services/storage/tests/unit/test_rpc_handlers_paths.py index 98cb75e8cba..cd5db414037 100644 --- a/services/storage/tests/unit/test_rpc_handlers_paths.py +++ b/services/storage/tests/unit/test_rpc_handlers_paths.py @@ -13,6 +13,7 @@ from typing import Any, TypeAlias import pytest +from celery.contrib.testing.worker import TestWorkController from faker import Faker from fastapi import FastAPI from models_library.api_schemas_rpc_async_jobs.async_jobs import ( @@ -34,7 +35,6 @@ compute_path_size, delete_paths, ) -from simcore_service_storage.modules.celery.worker import CeleryTaskWorker from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager pytest_simcore_core_services_selection = ["postgres", "rabbit"] @@ -265,7 +265,7 @@ async def test_path_compute_size_inexistent_path( mock_celery_app: None, initialized_app: FastAPI, storage_rabbitmq_rpc_client: RabbitMQRPCClient, - with_storage_celery_worker: CeleryTaskWorker, + with_storage_celery_worker: TestWorkController, location_id: LocationID, user_id: UserID, faker: Faker, @@ -294,7 +294,7 @@ async def test_delete_paths_empty_set( user_id: UserID, location_id: LocationID, product_name: ProductName, - with_storage_celery_worker: CeleryTaskWorker, + with_storage_celery_worker: TestWorkController, ): await _assert_delete_paths( storage_rabbitmq_rpc_client, @@ -333,7 +333,7 @@ async def test_delete_paths( ], project_params: ProjectWithFilesParams, product_name: ProductName, - with_storage_celery_worker: CeleryTaskWorker, + with_storage_celery_worker: TestWorkController, ): assert ( len(project_params.allowed_file_sizes) == 1 diff --git a/services/storage/tests/unit/test_rpc_handlers_simcore_s3.py b/services/storage/tests/unit/test_rpc_handlers_simcore_s3.py index 751cdae4f2f..7f0d87667f6 100644 --- a/services/storage/tests/unit/test_rpc_handlers_simcore_s3.py +++ b/services/storage/tests/unit/test_rpc_handlers_simcore_s3.py @@ -20,6 +20,7 @@ import pytest import sqlalchemy as sa from celery.contrib.testing.worker import TestWorkController +from celery_library.task_manager import CeleryTaskManager from faker import Faker from fastapi import FastAPI from fastapi.encoders import jsonable_encoder @@ -57,7 +58,6 @@ start_export_data, ) from simcore_postgres_database.storage_models import file_meta_data -from simcore_service_storage.modules.celery.worker import CeleryTaskWorker from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager from sqlalchemy.ext.asyncio import AsyncEngine from yarl import URL @@ -113,7 +113,7 @@ async def test_copy_folders_from_non_existing_project( product_name: ProductName, create_project: Callable[..., Awaitable[dict[str, Any]]], faker: Faker, - with_storage_celery_worker: CeleryTaskWorker, + with_storage_celery_worker: TestWorkController, ): src_project = await create_project() incorrect_src_project = deepcopy(src_project) @@ -154,7 +154,7 @@ async def test_copy_folders_from_empty_project( product_name: ProductName, create_project: Callable[[], Awaitable[dict[str, Any]]], sqlalchemy_async_engine: AsyncEngine, - with_storage_celery_worker: CeleryTaskWorker, + with_storage_celery_worker: TestWorkController, ): # we will copy from src to dst src_project = await create_project() @@ -547,7 +547,7 @@ async def _request_start_export_data( @pytest.fixture def task_progress_spy(mocker: MockerFixture) -> Mock: - return mocker.spy(CeleryTaskWorker, "set_task_progress") + return mocker.spy(CeleryTaskManager, "set_task_progress") @pytest.mark.parametrize( @@ -575,7 +575,7 @@ def task_progress_spy(mocker: MockerFixture) -> Mock: async def test_start_export_data( initialized_app: FastAPI, short_dsm_cleaner_interval: int, - with_storage_celery_worker_controller: TestWorkController, + with_storage_celery_worker: TestWorkController, storage_rabbitmq_rpc_client: RabbitMQRPCClient, user_id: UserID, product_name: ProductName, @@ -621,7 +621,7 @@ async def test_start_export_data( async def test_start_export_data_access_error( initialized_app: FastAPI, short_dsm_cleaner_interval: int, - with_storage_celery_worker_controller: TestWorkController, + with_storage_celery_worker: TestWorkController, storage_rabbitmq_rpc_client: RabbitMQRPCClient, user_id: UserID, product_name: ProductName,