♻️ Extract Celery code to a new simcore library (#7698)

Co-authored-by: Mads Bisgaard <[email protected]>

Author: giancarloromeo

.codecov.yml

@@ -31,6 +31,9 @@ component_management:
     - component_id: pkg_aws_library
       paths:
         - packages/aws-library/**
+    - component_id: pkg_celery_library
+      paths:
+        - packages/celery-library/**
     - component_id: pkg_dask_task_models_library
       paths:
         - packages/dask-task-models-library/**

.github/CODEOWNERS

@@ -13,6 +13,7 @@ Makefile                                           @pcrespov @sanderegg
 /api/                                              @sanderegg @pcrespov @matusdrobuliak66
 /ci/                                               @sanderegg @pcrespov
 /docs/                                             @pcrespov
+/packages/celery-library/                          @giancarloromeo
 /packages/common-library/                          @giancarloromeo
 /packages/models-library/                          @sanderegg @pcrespov @matusdrobuliak66 @giancarloromeo
 /packages/postgres-database/                       @matusdrobuliak66

.github/codeql/codeql-config.yml

@@ -4,6 +4,7 @@ disable-default-queries: false
 
 paths:
   - packages/aws-library/src
+  - packages/celery-library/src
   - packages/dask-task-models-library/src
   - packages/models-library/src/models_library
   - packages/postgres-database/src/simcore_postgres_database

.github/workflows/ci-testing-deploy.yml

@@ -47,6 +47,7 @@ jobs:
     # Set job outputs to values from filter step
     outputs:
       aws-library: ${{ steps.filter.outputs.aws-library }}
+      celery-library: ${{ steps.filter.outputs.celery-library }}
       dask-task-models-library: ${{ steps.filter.outputs.dask-task-models-library }}
       models-library: ${{ steps.filter.outputs.models-library }}
       common-library: ${{ steps.filter.outputs.common-library }}
@@ -94,6 +95,12 @@ jobs:
               - 'services/docker-compose*'
               - 'scripts/mypy/*'
               - 'mypy.ini'
+            celery-library:
+              - 'packages/celery-library/**'
+              - 'packages/pytest-simcore/**'
+              - 'services/docker-compose*'
+              - 'scripts/mypy/*'
+              - 'mypy.ini'
             dask-task-models-library:
               - 'packages/dask-task-models-library/**'
               - 'packages/pytest-simcore/**'
@@ -1035,6 +1042,49 @@ jobs:
         with:
           flags: unittests #optional
 
+  unit-test-celery-library:
+      needs: changes
+      if: ${{ needs.changes.outputs.celery-library == 'true' || github.event_name == 'push' || github.event.inputs.force_all_builds == 'true' }}
+      timeout-minutes: 18 # if this timeout gets too small, then split the tests
+      name: "[unit] celery-library"
+      runs-on: ${{ matrix.os }}
+      strategy:
+        matrix:
+          python: ["3.11"]
+          os: [ubuntu-24.04]
+        fail-fast: false
+      steps:
+        - uses: actions/checkout@v4
+        - name: setup docker buildx
+          id: buildx
+          uses: docker/setup-buildx-action@v3
+          with:
+            driver: docker-container
+        - name: setup python environment
+          uses: actions/setup-python@v5
+          with:
+            python-version: ${{ matrix.python }}
+        - name: install uv
+          uses: astral-sh/setup-uv@v6
+          with:
+            version: "0.6.x"
+            enable-cache: false
+            cache-dependency-glob: "**/celery-library/requirements/ci.txt"
+        - name: show system version
+          run: ./ci/helpers/show_system_versions.bash
+        - name: install
+          run: ./ci/github/unit-testing/celery-library.bash install
+        - name: typecheck
+          run: ./ci/github/unit-testing/celery-library.bash typecheck
+        - name: test
+          if: ${{ !cancelled() }}
+          run: ./ci/github/unit-testing/celery-library.bash test
+        - uses: codecov/codecov-action@v5
+          if: ${{ !cancelled() }}
+          env:
+            CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+          with:
+            flags: unittests #optional
 
   unit-test-dask-task-models-library:
     needs: changes
@@ -1824,6 +1874,7 @@ jobs:
         unit-test-clusters-keeper,
         unit-test-dask-sidecar,
         unit-test-aws-library,
+        unit-test-celery-library,
         unit-test-dask-task-models-library,
         unit-test-datcore-adapter,
         unit-test-director-v2,

.vscode/settings.template.json

@@ -33,6 +33,7 @@
   "python.analysis.typeCheckingMode": "basic",
   "python.analysis.extraPaths": [
     "./packages/aws-library/src",
+    "./packages/celery-library/src",
     "./packages/common-library/src",
     "./packages/dask-task-models-library/src",
     "./packages/models-library/src",

ci/github/unit-testing/celery-library.bash

@@ -0,0 +1,43 @@
+#!/bin/bash
+# http://redsymbol.net/articles/unofficial-bash-strict-mode/
+set -o errexit  # abort on nonzero exitstatus
+set -o nounset  # abort on unbound variable
+set -o pipefail # don't hide errors within pipes
+IFS=$'\n\t'
+
+install() {
+  make devenv
+  # shellcheck source=/dev/null
+  source .venv/bin/activate
+  pushd packages/celery-library
+  make install-ci
+  popd
+  uv pip list
+}
+
+test() {
+  # shellcheck source=/dev/null
+  source .venv/bin/activate
+  pushd packages/celery-library
+  make tests-ci
+  popd
+}
+
+typecheck() {
+  # shellcheck source=/dev/null
+  source .venv/bin/activate
+  uv pip install mypy
+  pushd packages/celery-library
+  make mypy
+  popd
+}
+
+# Check if the function exists (bash specific)
+if declare -f "$1" >/dev/null; then
+  # call arguments verbatim
+  "$@"
+else
+  # Show a helpful error
+  echo "'$1' is not a known function name" >&2
+  exit 1
+fi

packages/celery-library/Makefile

@@ -0,0 +1,50 @@
+#
+# Targets for DEVELOPMENT of Celery Library
+#
+include ../../scripts/common.Makefile
+include ../../scripts/common-package.Makefile
+
+.PHONY: requirements
+requirements: ## compiles pip requirements (.in -> .txt)
+	@$(MAKE_C) requirements reqs
+
+
+.PHONY: install-dev install-prod install-ci
+install-dev install-prod install-ci: _check_venv_active ## install app in development/production or CI mode
+	# installing in $(subst install-,,$@) mode
+	@uv pip sync requirements/$(subst install-,,$@).txt
+
+
+.PHONY: tests tests-ci
+tests: ## runs unit tests
+	# running unit tests
+	@pytest \
+		--asyncio-mode=auto \
+		--color=yes \
+		--cov-config=../../.coveragerc \
+		--cov-report=term-missing \
+		--cov=celery_library \
+		--durations=10 \
+		--exitfirst \
+		--failed-first \
+		--pdb \
+		-vv \
+		$(CURDIR)/tests
+
+tests-ci: ## runs unit tests
+	# running unit tests
+	@pytest \
+		--asyncio-mode=auto \
+		--color=yes \
+		--cov-append \
+		--cov-config=../../.coveragerc \
+		--cov-report=term-missing \
+		--cov-report=xml \
+		--junitxml=junit.xml -o junit_family=legacy \
+		--cov=celery_library \
+		--durations=10 \
+		--log-date-format="%Y-%m-%d %H:%M:%S" \
+    --log-format="%(asctime)s %(levelname)s %(message)s" \
+    --verbose \
+    -m "not heavy_load" \
+		$(CURDIR)/tests

packages/celery-library/README.md

@@ -0,0 +1,20 @@
+# simcore Celery library
+
+Provides a wrapper around Celery library [1].
+
+## Installation
+
+```console
+make help
+make install-dev
+```
+
+## Test
+
+```console
+make help
+make test-dev
+```
+
+
+[1] https://github.com/celery/celery

packages/celery-library/VERSION

@@ -0,0 +1 @@
+0.1.0

packages/celery-library/requirements/Makefile

@@ -0,0 +1,6 @@
+#
+# Targets to pip-compile requirements
+#
+include ../../../requirements/base.Makefile
+
+# Add here any extra explicit dependency: e.g. _migration.txt: _base.txt