make sure celery tasks actually run

Author: bisgaard-itis

packages/celery-library/requirements/_test.in

@@ -11,13 +11,15 @@
 # testing
 coverage
 faker
+fakeredis
 fastapi
 docker
 httpx
 pint
 pytest
 pytest-asyncio
 pytest-benchmark
+pytest-celery
 pytest-cov
 pytest-icdiff
 pytest-instafail

packages/celery-library/requirements/_test.txt

@@ -1,3 +1,7 @@
+amqp==5.3.1
+    # via
+    #   -c requirements/_base.txt
+    #   kombu
 annotated-types==0.7.0
     # via
     #   -c requirements/_base.txt
@@ -7,6 +11,14 @@ anyio==4.9.0
     #   -c requirements/_base.txt
     #   httpx
     #   starlette
+billiard==4.2.1
+    # via
+    #   -c requirements/_base.txt
+    #   celery
+celery==5.5.2
+    # via
+    #   -c requirements/_base.txt
+    #   pytest-celery
 certifi==2025.4.26
     # via
     #   -c requirements/../../../requirements/constraints.txt
@@ -18,14 +30,40 @@ charset-normalizer==3.4.2
     # via
     #   -c requirements/_base.txt
     #   requests
+click==8.2.1
+    # via
+    #   -c requirements/_base.txt
+    #   celery
+    #   click-didyoumean
+    #   click-plugins
+    #   click-repl
+click-didyoumean==0.3.1
+    # via
+    #   -c requirements/_base.txt
+    #   celery
+click-plugins==1.1.1
+    # via
+    #   -c requirements/_base.txt
+    #   celery
+click-repl==0.3.0
+    # via
+    #   -c requirements/_base.txt
+    #   celery
 coverage==7.8.2
     # via
     #   -r requirements/_test.in
     #   pytest-cov
+debugpy==1.8.14
+    # via pytest-celery
 docker==7.1.0
-    # via -r requirements/_test.in
+    # via
+    #   -r requirements/_test.in
+    #   pytest-celery
+    #   pytest-docker-tools
 faker==37.3.0
     # via -r requirements/_test.in
+fakeredis==2.29.0
+    # via -r requirements/_test.in
 fastapi==0.115.12
     # via -r requirements/_test.in
 flexcache==0.3
@@ -50,6 +88,11 @@ idna==3.10
     #   requests
 iniconfig==2.1.0
     # via pytest
+kombu==5.5.3
+    # via
+    #   -c requirements/_base.txt
+    #   celery
+    #   pytest-celery
 packaging==25.0
     # via
     #   -c requirements/_base.txt
@@ -63,6 +106,14 @@ pluggy==1.6.0
     # via pytest
 pprintpp==0.4.0
     # via pytest-icdiff
+prompt-toolkit==3.0.51
+    # via
+    #   -c requirements/_base.txt
+    #   click-repl
+psutil==7.0.0
+    # via
+    #   -c requirements/_base.txt
+    #   pytest-celery
 py-cpuinfo==9.0.0
     # via pytest-benchmark
 pydantic==2.11.5
@@ -74,12 +125,17 @@ pydantic-core==2.33.2
     # via
     #   -c requirements/_base.txt
     #   pydantic
+pyjwt==2.9.0
+    # via
+    #   -c requirements/_base.txt
+    #   redis
 pytest==8.3.5
     # via
     #   -r requirements/_test.in
     #   pytest-asyncio
     #   pytest-benchmark
     #   pytest-cov
+    #   pytest-docker-tools
     #   pytest-icdiff
     #   pytest-instafail
     #   pytest-mock
@@ -88,8 +144,12 @@ pytest-asyncio==1.0.0
     # via -r requirements/_test.in
 pytest-benchmark==5.1.0
     # via -r requirements/_test.in
+pytest-celery==1.2.0
+    # via -r requirements/_test.in
 pytest-cov==6.1.1
     # via -r requirements/_test.in
+pytest-docker-tools==3.1.9
+    # via pytest-celery
 pytest-icdiff==0.9
     # via -r requirements/_test.in
 pytest-instafail==0.5.0
@@ -100,6 +160,10 @@ pytest-runner==6.0.1
     # via -r requirements/_test.in
 pytest-sugar==1.0.0
     # via -r requirements/_test.in
+python-dateutil==2.9.0.post0
+    # via
+    #   -c requirements/_base.txt
+    #   celery
 python-dotenv==1.1.0
     # via
     #   -c requirements/_base.txt
@@ -109,18 +173,35 @@ pyyaml==6.0.2
     #   -c requirements/../../../requirements/constraints.txt
     #   -c requirements/_base.txt
     #   -r requirements/_test.in
+redis==5.3.0
+    # via
+    #   -c requirements/../../../requirements/constraints.txt
+    #   -c requirements/_base.txt
+    #   fakeredis
 requests==2.32.3
     # via
     #   -c requirements/_base.txt
     #   docker
+setuptools==80.9.0
+    # via pytest-celery
+six==1.17.0
+    # via
+    #   -c requirements/_base.txt
+    #   python-dateutil
 sniffio==1.3.1
     # via
     #   -c requirements/_base.txt
     #   anyio
+sortedcontainers==2.4.0
+    # via fakeredis
 starlette==0.46.2
     # via
     #   -c requirements/../../../requirements/constraints.txt
     #   fastapi
+tenacity==9.1.2
+    # via
+    #   -c requirements/_base.txt
+    #   pytest-celery
 termcolor==3.1.0
     # via pytest-sugar
 typing-extensions==4.13.2
@@ -142,9 +223,20 @@ tzdata==2025.2
     # via
     #   -c requirements/_base.txt
     #   faker
+    #   kombu
 urllib3==2.4.0
     # via
     #   -c requirements/../../../requirements/constraints.txt
     #   -c requirements/_base.txt
     #   docker
     #   requests
+vine==5.1.0
+    # via
+    #   -c requirements/_base.txt
+    #   amqp
+    #   celery
+    #   kombu
+wcwidth==0.2.13
+    # via
+    #   -c requirements/_base.txt
+    #   prompt-toolkit

packages/celery-library/requirements/_tools.txt

@@ -11,6 +11,7 @@ cfgv==3.4.0
 click==8.2.1
     # via
     #   -c requirements/_base.txt
+    #   -c requirements/_test.txt
     #   black
     #   pip-tools
 dill==0.4.0
@@ -70,7 +71,9 @@ pyyaml==6.0.2
 ruff==0.11.11
     # via -r requirements/../../../requirements/devenv.txt
 setuptools==80.9.0
-    # via pip-tools
+    # via
+    #   -c requirements/_test.txt
+    #   pip-tools
 tomlkit==0.13.2
     # via pylint
 typing-extensions==4.13.2

services/storage/src/simcore_service_storage/api/rpc/_async_jobs.py renamed to packages/celery-library/src/celery_library/routes/rpc.py

@@ -0,0 +1,144 @@
+from collections.abc import AsyncIterator, Awaitable, Callable
+from functools import partial
+from typing import Final
+
+import pytest
+from asgi_lifespan import LifespanManager
+from celery import Celery
+from celery.contrib.testing.worker import TestWorkController, start_worker
+from celery.signals import worker_init, worker_shutdown
+from celery.worker.worker import WorkController
+from celery_library import setup_celery_client
+from celery_library.routes.rpc import router as async_jobs_router
+from celery_library.signals import on_worker_init, on_worker_shutdown
+from celery_library.utils import get_celery_worker
+from celery_library.worker import CeleryTaskWorker
+from faker import Faker
+from fastapi import FastAPI
+from models_library.products import ProductName
+from models_library.rabbitmq_basic_types import RPCNamespace
+from models_library.users import UserID
+from pydantic import TypeAdapter
+from servicelib.rabbitmq import RabbitMQRPCClient
+from settings_library.celery import CelerySettings
+from settings_library.rabbit import RabbitSettings
+from settings_library.redis import RedisSettings
+
+pytest_simcore_core_services_selection = [
+    "rabbit",
+    "redis",
+    "postgres",
+]
+
+pytest_plugins = [
+    "pytest_simcore.docker_compose",
+    "pytest_simcore.docker_swarm",
+    "pytest_simcore.rabbit_service",
+    "pytest_simcore.redis_service",
+    "pytest_simcore.repository_paths",
+]
+
+_LIFESPAN_TIMEOUT: Final[int] = 10
+
+
+@pytest.fixture
+def rpc_namespace() -> RPCNamespace:
+    return TypeAdapter(RPCNamespace).validate_python("test")
+
+
+@pytest.fixture
+def celery_settings(
+    rabbit_service: RabbitSettings,
+    redis_service: RedisSettings,
+) -> CelerySettings:
+    return CelerySettings.create_from_envs()
+
+
+@pytest.fixture
+async def initialized_fast_api(
+    rabbitmq_rpc_client: Callable[[str], Awaitable[RabbitMQRPCClient]],
+    celery_settings: CelerySettings,
+    rpc_namespace: RPCNamespace,
+) -> AsyncIterator[FastAPI]:
+    app = FastAPI(
+        title="master_fastapi_app",
+        description="Service that manages osparc storage backend",
+        version="0.0.0",
+    )
+
+    setup_celery_client(app, celery_settings=celery_settings)
+    rpc_client = await rabbitmq_rpc_client("celery_test_client")
+    app.state.rabbitmq_rpc_client = rpc_client
+
+    async def startup() -> None:
+        rpc_server = app.state.rabbitmq_rpc_client
+        assert isinstance(rpc_server, RabbitMQRPCClient)
+        await rpc_server.register_router(async_jobs_router, rpc_namespace, app)
+
+    app.add_event_handler("startup", startup)
+
+    async with LifespanManager(
+        app, startup_timeout=_LIFESPAN_TIMEOUT, shutdown_timeout=_LIFESPAN_TIMEOUT
+    ):
+        yield app
+
+
+@pytest.fixture
+def register_celery_tasks() -> Callable[[Celery], None]:
+    """override if tasks are needed"""
+
+    def _(celery_app: Celery) -> None: ...
+
+    return _
+
+
+@pytest.fixture
+async def celery_worker_controller(
+    celery_settings: CelerySettings,
+    celery_app: Celery,
+    register_celery_tasks: Callable[[Celery], None],
+) -> AsyncIterator[TestWorkController]:
+
+    def _create_app() -> FastAPI:
+
+        return FastAPI(
+            title="worker_fastapi_app",
+            description="Test application for celery_library",
+            version="0.0.0",
+        )
+
+    def _on_worker_init_wrapper(sender: WorkController, **_kwargs) -> None:
+        return partial(on_worker_init, _create_app, celery_settings)(sender, **_kwargs)
+
+    worker_init.connect(_on_worker_init_wrapper)
+    worker_shutdown.connect(on_worker_shutdown)
+
+    register_celery_tasks(celery_app)
+
+    with start_worker(
+        celery_app,
+        pool="threads",
+        concurrency=1,
+        loglevel="info",
+        perform_ping_check=False,
+        queues="default,cpu_bound",
+    ) as worker:
+        yield worker
+
+
+@pytest.fixture
+def with_storage_celery_worker(
+    celery_worker_controller: TestWorkController,
+) -> CeleryTaskWorker:
+    assert isinstance(celery_worker_controller.app, Celery)
+    return get_celery_worker(celery_worker_controller.app)
+
+
+@pytest.fixture
+def user_id(faker: Faker) -> UserID:
+    return TypeAdapter(UserID).validate_python(faker.pyint(min_value=1, max_value=1000))
+
+
+@pytest.fixture
+def product_name() -> ProductName:
+    return TypeAdapter(ProductName).validate_python("pytest-product")