From daf740c5d394cedee49630fbd99eb1eba10965bf Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 6 Aug 2025 11:05:55 +0200 Subject: [PATCH 001/119] removed unrequired --- .../tests/long_running_tasks/test_long_running_tasks_task.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py index 902061cbbf2c..0477d6baa34f 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py @@ -7,7 +7,6 @@ import asyncio import urllib.parse from collections.abc import AsyncIterator, Awaitable, Callable -from contextlib import suppress from datetime import datetime, timedelta from typing import Any, Final @@ -105,8 +104,7 @@ async def _( yield _ for manager in managers: - with suppress(TimeoutError): # avoids tets hanging on teardown - await asyncio.wait_for(manager.teardown(), timeout=10) + await manager.teardown() @pytest.fixture From 392cc536999dab141fcbffeeafa75b3d8b1d4df3 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 6 Aug 2025 11:09:50 +0200 Subject: [PATCH 002/119] refactor fixture --- .../tests/long_running_tasks/conftest.py | 38 +++++++++++++++++++ .../test_long_running_tasks_task.py | 36 ++---------------- .../tests/long_running_tasks/utils.py | 3 ++ 3 files changed, 45 insertions(+), 32 deletions(-) create mode 100644 packages/service-library/tests/long_running_tasks/conftest.py create mode 100644 packages/service-library/tests/long_running_tasks/utils.py diff --git a/packages/service-library/tests/long_running_tasks/conftest.py b/packages/service-library/tests/long_running_tasks/conftest.py new file mode 100644 index 000000000000..5bfe5b10a39c --- /dev/null +++ b/packages/service-library/tests/long_running_tasks/conftest.py @@ -0,0 +1,38 @@ +from collections.abc import AsyncIterator, Awaitable, Callable +from datetime import timedelta + +import pytest +from faker import Faker +from servicelib.long_running_tasks.task import ( + RedisNamespace, + TasksManager, +) +from settings_library.redis import RedisSettings +from utils import TEST_CHECK_STALE_INTERVAL_S + + +@pytest.fixture +async def get_tasks_manager( + faker: Faker, +) -> AsyncIterator[ + Callable[[RedisSettings, RedisNamespace | None], Awaitable[TasksManager]] +]: + managers: list[TasksManager] = [] + + async def _( + redis_settings: RedisSettings, namespace: RedisNamespace | None + ) -> TasksManager: + tasks_manager = TasksManager( + stale_task_check_interval=timedelta(seconds=TEST_CHECK_STALE_INTERVAL_S), + stale_task_detect_timeout=timedelta(seconds=TEST_CHECK_STALE_INTERVAL_S), + redis_settings=redis_settings, + redis_namespace=namespace or f"test{faker.uuid4()}", + ) + await tasks_manager.setup() + managers.append(tasks_manager) + return tasks_manager + + yield _ + + for manager in managers: + await manager.teardown() diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py index 0477d6baa34f..e739fe491356 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py @@ -6,9 +6,9 @@ import asyncio import urllib.parse -from collections.abc import AsyncIterator, Awaitable, Callable -from datetime import datetime, timedelta -from typing import Any, Final +from collections.abc import Awaitable, Callable +from datetime import datetime +from typing import Any import pytest from faker import Faker @@ -32,6 +32,7 @@ from tenacity.retry import retry_if_exception_type from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed +from utils import TEST_CHECK_STALE_INTERVAL_S _RETRY_PARAMS: dict[str, Any] = { "reraise": True, @@ -72,41 +73,12 @@ async def failing_background_task(progress: TaskProgress): TaskRegistry.register(fast_background_task) TaskRegistry.register(failing_background_task) -TEST_CHECK_STALE_INTERVAL_S: Final[float] = 1 - @pytest.fixture def empty_context() -> TaskContext: return {} -@pytest.fixture -async def get_tasks_manager( - faker: Faker, -) -> AsyncIterator[ - Callable[[RedisSettings, RedisNamespace | None], Awaitable[TasksManager]] -]: - managers: list[TasksManager] = [] - - async def _( - redis_settings: RedisSettings, namespace: RedisNamespace | None - ) -> TasksManager: - tasks_manager = TasksManager( - stale_task_check_interval=timedelta(seconds=TEST_CHECK_STALE_INTERVAL_S), - stale_task_detect_timeout=timedelta(seconds=TEST_CHECK_STALE_INTERVAL_S), - redis_settings=redis_settings, - redis_namespace=namespace or f"test{faker.uuid4()}", - ) - await tasks_manager.setup() - managers.append(tasks_manager) - return tasks_manager - - yield _ - - for manager in managers: - await manager.teardown() - - @pytest.fixture async def tasks_manager( use_in_memory_redis: RedisSettings, diff --git a/packages/service-library/tests/long_running_tasks/utils.py b/packages/service-library/tests/long_running_tasks/utils.py new file mode 100644 index 000000000000..e473dd7e1daf --- /dev/null +++ b/packages/service-library/tests/long_running_tasks/utils.py @@ -0,0 +1,3 @@ +from typing import Final + +TEST_CHECK_STALE_INTERVAL_S: Final[float] = 1 From 6498e7a40f99fca6ef97a9e727686eb67d75ed9c Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 6 Aug 2025 14:49:16 +0200 Subject: [PATCH 003/119] enhanced removal and cancellation --- .../long_running_tasks/_store/base.py | 4 ++ .../long_running_tasks/_store/redis.py | 7 +++ .../src/servicelib/long_running_tasks/task.py | 44 ++++++++----------- 3 files changed, 29 insertions(+), 26 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/_store/base.py b/packages/service-library/src/servicelib/long_running_tasks/_store/base.py index 20829c01ca65..37944ef6ec39 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_store/base.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_store/base.py @@ -27,6 +27,10 @@ async def set_as_cancelled( ) -> None: """Mark a tracked task as cancelled.""" + @abstractmethod + async def delete_set_as_cancelled(self, task_id: TaskId) -> None: + """Remove a task from the cancelled tasks.""" + @abstractmethod async def get_cancelled(self) -> dict[TaskId, TaskContext]: """Get cancelled tasks.""" diff --git a/packages/service-library/src/servicelib/long_running_tasks/_store/redis.py b/packages/service-library/src/servicelib/long_running_tasks/_store/redis.py index 3ac1314e11c9..7ee0df1e1032 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_store/redis.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_store/redis.py @@ -80,6 +80,13 @@ async def set_as_cancelled( ) ) + async def delete_set_as_cancelled(self, task_id: TaskId) -> None: + await handle_redis_returns_union_types( + self._redis.hdel( + self._get_redis_hash_key(_STORE_TYPE_CANCELLED_TASKS), task_id + ) + ) + async def get_cancelled(self) -> dict[TaskId, TaskContext]: result: dict[str, str | None] = await handle_redis_returns_union_types( self._redis.hgetall(self._get_redis_hash_key(_STORE_TYPE_CANCELLED_TASKS)) diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index ca22d9ff6af3..987a086d8c3a 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -150,6 +150,7 @@ async def setup(self) -> None: lock_key=f"{__name__}_{self.redis_namespace}_stale_tasks_monitor", )(self._stale_tasks_monitor), interval=self.stale_task_check_interval, + wait_before_running=self.stale_task_check_interval, task_name=f"{__name__}.{self._stale_tasks_monitor.__name__}", ) await self._started_event_task_stale_tasks_monitor.wait() @@ -255,8 +256,8 @@ async def _cancelled_tasks_removal(self) -> None: self._started_event_task_cancelled_tasks_removal.set() cancelled_tasks = await self._tasks_data.get_cancelled() - for task_id, task_context in cancelled_tasks.items(): - await self.remove_task(task_id, task_context, reraise_errors=False) + for task_id in cancelled_tasks: + await self._cancel_and_remove_local_task(task_id) async def _status_update(self) -> None: """ @@ -368,19 +369,14 @@ async def get_task_result( return string_to_object(tracked_task.result_field.result) - async def _cancel_tracked_task( - self, task: asyncio.Task, task_id: TaskId, with_task_context: TaskContext - ) -> None: - try: - await self._tasks_data.set_as_cancelled(task_id, with_task_context) - await cancel_wait_task(task) - except Exception as e: # pylint:disable=broad-except - _logger.info( - "Task %s cancellation failed with error: %s", - task_id, - e, - stack_info=True, - ) + async def _cancel_and_remove_local_task(self, task_id: TaskId) -> None: + """cancels task and removes if from local tracker if this is the worke that started it""" + + task_to_cancel = self._created_tasks.pop(task_id, None) + if task_to_cancel is not None: + await cancel_wait_task(task_to_cancel) + await self._tasks_data.delete_set_as_cancelled(task_id) + await self._tasks_data.delete_task_data(task_id) async def remove_task( self, @@ -397,26 +393,22 @@ async def remove_task( raise return - if tracked_task.task_id in self._created_tasks: - task_to_cancel = self._created_tasks.pop(tracked_task.task_id, None) - if task_to_cancel is not None: - # canceling the task affects the worker that started it. - # awaiting the cancelled task is a must since if the CancelledError - # was intercepted, those actions need to finish - await cancel_wait_task(task_to_cancel) - - await self._tasks_data.delete_task_data(task_id) + await self._tasks_data.set_as_cancelled( + tracked_task.task_id, tracked_task.task_context + ) # wait for task to be removed since it might not have been running # in this process async for attempt in AsyncRetrying( - wait=wait_exponential(max=2), + wait=wait_exponential(max=1), stop=stop_after_delay(_TASK_REMOVAL_MAX_WAIT), retry=retry_if_exception_type(TryAgain), ): with attempt: try: - await self._get_tracked_task(task_id, with_task_context) + await self._get_tracked_task( + tracked_task.task_id, tracked_task.task_context + ) raise TryAgain except TaskNotFoundError: pass From 97e84660d05aa58aab074d4a69c6e72b97f84a24 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 6 Aug 2025 15:46:22 +0200 Subject: [PATCH 004/119] fixed shutdown --- .../src/servicelib/long_running_tasks/task.py | 37 ++++++++++++------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index 987a086d8c3a..6d6daa27dfc4 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -4,12 +4,14 @@ import inspect import logging import urllib.parse +from contextlib import suppress from typing import Any, ClassVar, Final, Protocol, TypeAlias from uuid import uuid4 from common_library.async_tools import cancel_wait_task from models_library.api_schemas_long_running_tasks.base import TaskProgress from pydantic import NonNegativeFloat, PositiveFloat +from servicelib.logging_utils import log_catch from settings_library.redis import RedisDatabase, RedisSettings from tenacity import ( AsyncRetrying, @@ -38,7 +40,7 @@ _CANCEL_TASKS_CHECK_INTERVAL: Final[datetime.timedelta] = datetime.timedelta(seconds=5) _STATUS_UPDATE_CHECK_INTERNAL: Final[datetime.timedelta] = datetime.timedelta(seconds=1) - +_MAX_EXCLUSIVE_TASK_CANCEL_TIMEOUT: Final[NonNegativeFloat] = 5 _TASK_REMOVAL_MAX_WAIT: Final[NonNegativeFloat] = 60 @@ -150,7 +152,6 @@ async def setup(self) -> None: lock_key=f"{__name__}_{self.redis_namespace}_stale_tasks_monitor", )(self._stale_tasks_monitor), interval=self.stale_task_check_interval, - wait_before_running=self.stale_task_check_interval, task_name=f"{__name__}.{self._stale_tasks_monitor.__name__}", ) await self._started_event_task_stale_tasks_monitor.wait() @@ -190,7 +191,14 @@ async def teardown(self) -> None: # stale_tasks_monitor if self._task_stale_tasks_monitor: - await cancel_wait_task(self._task_stale_tasks_monitor) + # since the task is using a redis lock, it might not have been started + # inside other processes, this helps to avoid getting stuck since the task + # might have never been created + with log_catch(_logger, reraise=False): + await cancel_wait_task( + self._task_stale_tasks_monitor, + max_delay=_MAX_EXCLUSIVE_TASK_CANCEL_TIMEOUT, + ) # cancelled_tasks_removal if self._task_cancelled_tasks_removal: @@ -235,16 +243,19 @@ async def _stale_tasks_monitor(self) -> None: # - finished with a result # - finished with errors # we just print the status from where one can infer the above - _logger.warning( - "Removing stale task '%s' with status '%s'", - task_id, - ( - await self.get_task_status(task_id, with_task_context=task_context) - ).model_dump_json(), - ) - await self.remove_task( - task_id, with_task_context=task_context, reraise_errors=False - ) + with suppress(TaskNotFoundError): + _logger.warning( + "Removing stale task '%s' with status '%s'", + task_id, + ( + await self.get_task_status( + task_id, with_task_context=task_context + ) + ).model_dump_json(), + ) + await self.remove_task( + task_id, with_task_context=task_context, reraise_errors=False + ) async def _cancelled_tasks_removal(self) -> None: """ From 3bfa2958990e982e1cdaf6c6cde80b263a9f8f25 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 6 Aug 2025 15:54:29 +0200 Subject: [PATCH 005/119] update comment --- .../src/servicelib/long_running_tasks/task.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index 6d6daa27dfc4..bba9355b138b 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -191,9 +191,9 @@ async def teardown(self) -> None: # stale_tasks_monitor if self._task_stale_tasks_monitor: - # since the task is using a redis lock, it might not have been started - # inside other processes, this helps to avoid getting stuck since the task - # might have never been created + # since the task is using a redis lock if the lock could not be acquired + # trying to cancel the task will hang, this avoids hanging + # there are no sideeffects in timing out this cancellation with log_catch(_logger, reraise=False): await cancel_wait_task( self._task_stale_tasks_monitor, From 2d44252785570417a7e5537f4f4aa310e4fac4b6 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 6 Aug 2025 15:54:54 +0200 Subject: [PATCH 006/119] avoid tests hanging --- .../tests/long_running_tasks/conftest.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/service-library/tests/long_running_tasks/conftest.py b/packages/service-library/tests/long_running_tasks/conftest.py index 5bfe5b10a39c..42eda8473aaa 100644 --- a/packages/service-library/tests/long_running_tasks/conftest.py +++ b/packages/service-library/tests/long_running_tasks/conftest.py @@ -1,8 +1,11 @@ +import logging from collections.abc import AsyncIterator, Awaitable, Callable from datetime import timedelta import pytest from faker import Faker +from pytest_mock import MockerFixture +from servicelib.logging_utils import log_catch from servicelib.long_running_tasks.task import ( RedisNamespace, TasksManager, @@ -10,13 +13,19 @@ from settings_library.redis import RedisSettings from utils import TEST_CHECK_STALE_INTERVAL_S +_logger = logging.getLogger(__name__) + @pytest.fixture async def get_tasks_manager( - faker: Faker, + faker: Faker, mocker: MockerFixture ) -> AsyncIterator[ Callable[[RedisSettings, RedisNamespace | None], Awaitable[TasksManager]] ]: + mocker.patch( + "servicelib.long_running_tasks.task._CANCEL_TASKS_CHECK_INTERVAL", + new=timedelta(seconds=TEST_CHECK_STALE_INTERVAL_S), + ) managers: list[TasksManager] = [] async def _( @@ -35,4 +44,5 @@ async def _( yield _ for manager in managers: - await manager.teardown() + with log_catch(_logger, reraise=False): + await manager.teardown() From a77d79d56b265f330ec55e58b24b19a9f02acaaf Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 6 Aug 2025 15:58:00 +0200 Subject: [PATCH 007/119] fixed test timing out --- .../tests/fastapi/long_running_tasks/test_long_running_tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py index 0f10b7a165f5..c34dc8e4e8f5 100644 --- a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py +++ b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py @@ -100,7 +100,7 @@ async def app( app.include_router(server_routes) setup_server(app, redis_settings=use_in_memory_redis, redis_namespace="test") setup_client(app) - async with LifespanManager(app): + async with LifespanManager(app, startup_timeout=30, shutdown_timeout=30): yield app From 5d60b6a81e498b1de97d84d5797a9daa0e1e42b4 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 7 Aug 2025 06:28:57 +0200 Subject: [PATCH 008/119] fixed tests and spedup --- .../tests/long_running_tasks/conftest.py | 18 +- .../test_long_running_tasks_lrt_api.py | 272 ++++++++++++++++++ 2 files changed, 285 insertions(+), 5 deletions(-) create mode 100644 packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py diff --git a/packages/service-library/tests/long_running_tasks/conftest.py b/packages/service-library/tests/long_running_tasks/conftest.py index 42eda8473aaa..515b34ef0a4e 100644 --- a/packages/service-library/tests/long_running_tasks/conftest.py +++ b/packages/service-library/tests/long_running_tasks/conftest.py @@ -1,3 +1,5 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument import logging from collections.abc import AsyncIterator, Awaitable, Callable from datetime import timedelta @@ -17,15 +19,21 @@ @pytest.fixture -async def get_tasks_manager( - faker: Faker, mocker: MockerFixture -) -> AsyncIterator[ - Callable[[RedisSettings, RedisNamespace | None], Awaitable[TasksManager]] -]: +async def mock_cancel_tasks_check_interval( + mocker: MockerFixture, +) -> None: mocker.patch( "servicelib.long_running_tasks.task._CANCEL_TASKS_CHECK_INTERVAL", new=timedelta(seconds=TEST_CHECK_STALE_INTERVAL_S), ) + + +@pytest.fixture +async def get_tasks_manager( + mock_cancel_tasks_check_interval: None, faker: Faker +) -> AsyncIterator[ + Callable[[RedisSettings, RedisNamespace | None], Awaitable[TasksManager]] +]: managers: list[TasksManager] = [] async def _( diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py new file mode 100644 index 000000000000..0a0770a72487 --- /dev/null +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py @@ -0,0 +1,272 @@ +# pylint: disable=protected-access +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument + +import asyncio +import secrets +from collections.abc import Awaitable, Callable +from typing import Any, Final + +import pytest +from models_library.api_schemas_long_running_tasks.base import TaskProgress +from pydantic import NonNegativeInt +from pytest_mock import MockerFixture +from servicelib.long_running_tasks import lrt_api +from servicelib.long_running_tasks.errors import TaskNotFoundError +from servicelib.long_running_tasks.models import TaskContext +from servicelib.long_running_tasks.task import ( + RedisNamespace, + TaskId, + TaskRegistry, + TasksManager, +) +from settings_library.redis import RedisSettings +from tenacity import ( + AsyncRetrying, + TryAgain, + retry_if_exception_type, + stop_after_delay, + wait_fixed, +) + +pytest_simcore_core_services_selection = [ + "redis", # TODO: remove when done with this part +] +pytest_simcore_ops_services_selection = [ + "redis-commander", +] + +_RETRY_PARAMS: dict[str, Any] = { + "reraise": True, + "wait": wait_fixed(0.1), + "stop": stop_after_delay(60), + "retry": retry_if_exception_type((AssertionError, TryAgain)), +} + + +async def _task_echo_input(progress: TaskProgress, to_return: Any) -> Any: + return to_return + + +async def _task_always_raise(progress: TaskProgress) -> None: + msg = "This task always raises an error" + raise RuntimeError(msg) + + +async def _task_takes_too_long(progress: TaskProgress) -> None: + # Simulate a long-running task that is taking too much time + await asyncio.sleep(1e9) + + +TaskRegistry.register(_task_echo_input) +TaskRegistry.register(_task_always_raise) +TaskRegistry.register(_task_takes_too_long) + + +@pytest.fixture +def managers_count() -> NonNegativeInt: + return 5 + + +@pytest.fixture +def disable_stale_tasks_monitor(mocker: MockerFixture) -> None: + # no need to autoremove stale tasks in these tests + async def _to_replace(self: TasksManager) -> None: + self._started_event_task_stale_tasks_monitor.set() + + mocker.patch.object( + TasksManager, + "_stale_tasks_monitor", + _to_replace, + ) + + +@pytest.fixture +async def tasks_managers( + disable_stale_tasks_monitor: None, + managers_count: NonNegativeInt, + redis_service: RedisSettings, + get_tasks_manager: Callable[ + [RedisSettings, RedisNamespace | None], Awaitable[TasksManager] + ], +) -> list[TasksManager]: + maanagers: list[TasksManager] = [] + for _ in range(managers_count): + manager = await get_tasks_manager(redis_service, "same-service") + maanagers.append(manager) + + return maanagers + + +def _get_task_manager(tasks_managers: list[TasksManager]) -> TasksManager: + return secrets.choice(tasks_managers) + + +async def _assert_task_status( + task_manager: TasksManager, task_id: TaskId, *, is_done: bool +) -> None: + result = await lrt_api.get_task_status(task_manager, TaskContext(), task_id) + assert result.done is is_done + + +async def _assert_task_status_on_random_manager( + tasks_managers: list[TasksManager], task_ids: list[TaskId], *, is_done: bool = True +) -> None: + for task_id in task_ids: + result = await lrt_api.get_task_status( + _get_task_manager(tasks_managers), TaskContext(), task_id + ) + assert result.done is is_done + + +async def _assert_task_status_done_on_all_managers( + tasks_managers: list[TasksManager], task_id: TaskId, *, is_done: bool = True +) -> None: + async for attempt in AsyncRetrying(**_RETRY_PARAMS): + with attempt: + await _assert_task_status( + _get_task_manager(tasks_managers), task_id, is_done=is_done + ) + + # check can do this form any task manager + for manager in tasks_managers: + await _assert_task_status(manager, task_id, is_done=is_done) + + +async def _assert_list_tasks_from_all_managers( + tasks_managers: list[TasksManager], task_context: TaskContext, task_count: int +) -> None: + for manager in tasks_managers: + tasks = await lrt_api.list_tasks(manager, task_context) + assert len(tasks) == task_count + + +async def _assert_task_is_no_longer_present( + tasks_managers: list[TasksManager], task_context: TaskContext, task_id: TaskId +) -> None: + with pytest.raises(TaskNotFoundError): + await lrt_api.get_task_status( + _get_task_manager(tasks_managers), task_context, task_id + ) + + +_TASK_CONTEXT: Final[list[TaskContext | None]] = [{"a": "context"}, None] +_IS_UNIQUE: Final[list[bool]] = [False, True] +_TASK_COUNT: Final[list[int]] = [5] + + +@pytest.mark.parametrize("task_count", _TASK_COUNT) +@pytest.mark.parametrize("task_context", _TASK_CONTEXT) +@pytest.mark.parametrize("is_unique", _IS_UNIQUE) +@pytest.mark.parametrize("to_return", [{"key": "value"}]) +async def test_workflow_with_result( + tasks_managers: list[TasksManager], + task_count: int, + is_unique: bool, + task_context: TaskContext | None, + to_return: Any, +): + saved_context = task_context or {} + task_count = 1 if is_unique else task_count + + task_ids: list[TaskId] = [] + for _ in range(task_count): + task_id = await lrt_api.start_task( + _get_task_manager(tasks_managers), + _task_echo_input.__name__, + unique=is_unique, + task_name=None, + task_context=task_context, + fire_and_forget=False, + to_return=to_return, + ) + task_ids.append(task_id) + + for task_id in task_ids: + await _assert_task_status_done_on_all_managers(tasks_managers, task_id) + + await _assert_list_tasks_from_all_managers( + tasks_managers, saved_context, task_count=task_count + ) + + # avoids tasks getting garbage collected + await _assert_task_status_on_random_manager(tasks_managers, task_ids, is_done=True) + + for task_id in task_ids: + result = await lrt_api.get_task_result( + _get_task_manager(tasks_managers), saved_context, task_id + ) + assert result == to_return + + await _assert_task_is_no_longer_present(tasks_managers, saved_context, task_id) + + +@pytest.mark.parametrize("task_count", _TASK_COUNT) +@pytest.mark.parametrize("task_context", _TASK_CONTEXT) +@pytest.mark.parametrize("is_unique", _IS_UNIQUE) +async def test_workflow_raises_error( + tasks_managers: list[TasksManager], + task_count: int, + is_unique: bool, + task_context: TaskContext | None, +): + saved_context = task_context or {} + task_count = 1 if is_unique else task_count + + task_ids: list[TaskId] = [] + for _ in range(task_count): + task_id = await lrt_api.start_task( + _get_task_manager(tasks_managers), + _task_always_raise.__name__, + unique=is_unique, + task_name=None, + task_context=task_context, + fire_and_forget=False, + ) + task_ids.append(task_id) + + for task_id in task_ids: + await _assert_task_status_done_on_all_managers(tasks_managers, task_id) + + await _assert_list_tasks_from_all_managers( + tasks_managers, saved_context, task_count=task_count + ) + + # avoids tasks getting garbage collected + await _assert_task_status_on_random_manager(tasks_managers, task_ids, is_done=True) + + for task_id in task_ids: + with pytest.raises(RuntimeError, match="This task always raises an error"): + await lrt_api.get_task_result( + _get_task_manager(tasks_managers), saved_context, task_id + ) + + await _assert_task_is_no_longer_present(tasks_managers, saved_context, task_id) + + +@pytest.mark.parametrize("task_count", _TASK_COUNT) +@pytest.mark.parametrize("task_context", _TASK_CONTEXT) +@pytest.mark.parametrize("is_unique", _IS_UNIQUE) +async def test_remove_task( + tasks_managers: list[TasksManager], + task_count: int, + is_unique: bool, + task_context: TaskContext | None, +): + task_id = await lrt_api.start_task( + _get_task_manager(tasks_managers), + _task_takes_too_long.__name__, + unique=is_unique, + task_name=None, + task_context=task_context, + fire_and_forget=False, + ) + saved_context = task_context or {} + + await _assert_task_status_done_on_all_managers( + tasks_managers, task_id, is_done=False + ) + + await lrt_api.remove_task(_get_task_manager(tasks_managers), saved_context, task_id) + + await _assert_task_is_no_longer_present(tasks_managers, saved_context, task_id) From cb90c9e8eb9b8bbbf000e75da2b63f4b5ae67240 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 7 Aug 2025 06:37:29 +0200 Subject: [PATCH 009/119] refactor --- .../long_running_tasks/test_long_running_tasks_lrt_api.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py index 0a0770a72487..b3f795fb4ae9 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py @@ -74,11 +74,7 @@ def disable_stale_tasks_monitor(mocker: MockerFixture) -> None: async def _to_replace(self: TasksManager) -> None: self._started_event_task_stale_tasks_monitor.set() - mocker.patch.object( - TasksManager, - "_stale_tasks_monitor", - _to_replace, - ) + mocker.patch.object(TasksManager, "_stale_tasks_monitor", _to_replace) @pytest.fixture From fd1cae6128451864b5c4d7e9d0925bee9169a085 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 7 Aug 2025 07:23:35 +0200 Subject: [PATCH 010/119] replaced TasksManager with BaseLongRunningManager in lrt_api --- .../aiohttp/long_running_tasks/_routes.py | 12 +- .../aiohttp/long_running_tasks/_server.py | 6 +- .../fastapi/long_running_tasks/_routes.py | 16 +- .../servicelib/long_running_tasks/lrt_api.py | 37 ++- .../test_long_running_tasks.py | 4 +- .../test_long_running_tasks.py | 6 +- ...test_long_running_tasks_context_manager.py | 6 +- .../tests/long_running_tasks/conftest.py | 19 +- .../test_long_running_tasks__rabbit.py | 0 .../test_long_running_tasks_lrt_api.py | 105 ++++--- .../test_long_running_tasks_task.py | 272 +++++++++++------- .../tests/long_running_tasks/utils.py | 21 ++ .../api/routes/dynamic_scheduler.py | 8 +- .../api/rest/containers_long_running_tasks.py | 18 +- .../simcore_service_webserver/tasks/_rest.py | 2 +- 15 files changed, 340 insertions(+), 192 deletions(-) create mode 100644 packages/service-library/tests/long_running_tasks/test_long_running_tasks__rabbit.py diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py index cb735779901a..210966f82725 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py @@ -29,7 +29,7 @@ async def list_tasks(request: web.Request) -> web.Response: abort_href=f"{request.app.router['cancel_and_delete_task'].url_for(task_id=t.task_id)}", ) for t in await lrt_api.list_tasks( - long_running_manager.tasks_manager, + long_running_manager, long_running_manager.get_task_context(request), ) ] @@ -42,7 +42,7 @@ async def get_task_status(request: web.Request) -> web.Response: long_running_manager = get_long_running_manager(request.app) task_status = await lrt_api.get_task_status( - long_running_manager.tasks_manager, + long_running_manager, long_running_manager.get_task_context(request), path_params.task_id, ) @@ -56,19 +56,19 @@ async def get_task_result(request: web.Request) -> web.Response | Any: # NOTE: this might raise an exception that will be catched by the _error_handlers return await lrt_api.get_task_result( - long_running_manager.tasks_manager, + long_running_manager, long_running_manager.get_task_context(request), path_params.task_id, ) -@routes.delete("/{task_id}", name="cancel_and_delete_task") -async def cancel_and_delete_task(request: web.Request) -> web.Response: +@routes.delete("/{task_id}", name="remove_task") +async def remove_task(request: web.Request) -> web.Response: path_params = parse_request_path_parameters_as(_PathParam, request) long_running_manager = get_long_running_manager(request.app) await lrt_api.remove_task( - long_running_manager.tasks_manager, + long_running_manager, long_running_manager.get_task_context(request), path_params.task_id, ) diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py index 68147fede027..e82cf4d841c7 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py @@ -63,7 +63,7 @@ async def start_long_running_task( task_id = None try: task_id = await lrt_api.start_task( - long_running_manager.tasks_manager, + long_running_manager, registerd_task_name, fire_and_forget=fire_and_forget, task_context=task_context, @@ -97,9 +97,7 @@ async def start_long_running_task( except asyncio.CancelledError: # remove the task, the client was disconnected if task_id: - await lrt_api.remove_task( - long_running_manager.tasks_manager, task_context, task_id - ) + await lrt_api.remove_task(long_running_manager, task_context, task_id) raise diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py index d3adbb1956db..3e58501bc361 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py @@ -29,9 +29,7 @@ async def list_tasks( request.url_for("cancel_and_delete_task", task_id=t.task_id) ), ) - for t in await lrt_api.list_tasks( - long_running_manager.tasks_manager, task_context={} - ) + for t in await lrt_api.list_tasks(long_running_manager, task_context={}) ] @@ -52,7 +50,7 @@ async def get_task_status( ) -> TaskStatus: assert request # nosec return await lrt_api.get_task_status( - long_running_manager.tasks_manager, task_context={}, task_id=task_id + long_running_manager, task_context={}, task_id=task_id ) @@ -75,13 +73,13 @@ async def get_task_result( ) -> TaskResult | Any: assert request # nosec return await lrt_api.get_task_result( - long_running_manager.tasks_manager, task_context={}, task_id=task_id + long_running_manager, task_context={}, task_id=task_id ) @router.delete( "/{task_id}", - summary="Cancel and deletes a task", + summary="Cancels and removes a task", response_model=None, status_code=status.HTTP_204_NO_CONTENT, responses={ @@ -89,7 +87,7 @@ async def get_task_result( }, ) @cancel_on_disconnect -async def cancel_and_delete_task( +async def remove_task( request: Request, task_id: TaskId, long_running_manager: Annotated[ @@ -97,6 +95,4 @@ async def cancel_and_delete_task( ], ) -> None: assert request # nosec - await lrt_api.remove_task( - long_running_manager.tasks_manager, task_context={}, task_id=task_id - ) + await lrt_api.remove_task(long_running_manager, task_context={}, task_id=task_id) diff --git a/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py b/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py index 6f732d49e49c..3df4f4e2d9b8 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py +++ b/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py @@ -4,15 +4,16 @@ from common_library.error_codes import create_error_code from ..logging_errors import create_troubleshootting_log_kwargs +from .base_long_running_manager import BaseLongRunningManager from .errors import TaskNotCompletedError, TaskNotFoundError from .models import TaskBase, TaskContext, TaskId, TaskStatus -from .task import RegisteredTaskName, TasksManager +from .task import RegisteredTaskName _logger = logging.getLogger(__name__) async def start_task( - tasks_manager: TasksManager, + long_running_manager: BaseLongRunningManager, registered_task_name: RegisteredTaskName, *, unique: bool = False, @@ -46,7 +47,7 @@ async def start_task( Returns: TaskId: the task unique identifier """ - return await tasks_manager.start_task( + return await long_running_manager.tasks_manager.start_task( registered_task_name, unique=unique, task_context=task_context, @@ -57,28 +58,34 @@ async def start_task( async def list_tasks( - tasks_manager: TasksManager, task_context: TaskContext + long_running_manager: BaseLongRunningManager, task_context: TaskContext ) -> list[TaskBase]: - return await tasks_manager.list_tasks(with_task_context=task_context) + return await long_running_manager.tasks_manager.list_tasks( + with_task_context=task_context + ) async def get_task_status( - tasks_manager: TasksManager, task_context: TaskContext, task_id: TaskId + long_running_manager: BaseLongRunningManager, + task_context: TaskContext, + task_id: TaskId, ) -> TaskStatus: """returns the status of a task""" - return await tasks_manager.get_task_status( + return await long_running_manager.tasks_manager.get_task_status( task_id=task_id, with_task_context=task_context ) async def get_task_result( - tasks_manager: TasksManager, task_context: TaskContext, task_id: TaskId + long_running_manager: BaseLongRunningManager, + task_context: TaskContext, + task_id: TaskId, ) -> Any: try: - task_result = await tasks_manager.get_task_result( + task_result = await long_running_manager.tasks_manager.get_task_result( task_id, with_task_context=task_context ) - await tasks_manager.remove_task( + await long_running_manager.tasks_manager.remove_task( task_id, with_task_context=task_context, reraise_errors=False ) return task_result @@ -94,14 +101,18 @@ async def get_task_result( ), ) # the task shall be removed in this case - await tasks_manager.remove_task( + await long_running_manager.tasks_manager.remove_task( task_id, with_task_context=task_context, reraise_errors=False ) raise async def remove_task( - tasks_manager: TasksManager, task_context: TaskContext, task_id: TaskId + long_running_manager: BaseLongRunningManager, + task_context: TaskContext, + task_id: TaskId, ) -> None: """cancels and removes the task""" - await tasks_manager.remove_task(task_id, with_task_context=task_context) + await long_running_manager.tasks_manager.remove_task( + task_id, with_task_context=task_context + ) diff --git a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py index 94eaecef7e30..80686b670a47 100644 --- a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py +++ b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py @@ -127,7 +127,7 @@ async def test_workflow( [ ("GET", "get_task_status"), ("GET", "get_task_result"), - ("DELETE", "cancel_and_delete_task"), + ("DELETE", "remove_task"), ], ) async def test_get_task_wrong_task_id_raises_not_found( @@ -188,7 +188,7 @@ async def test_cancel_task( task_id = await start_long_running_task(client) # cancel the task - delete_url = client.app.router["cancel_and_delete_task"].url_for(task_id=task_id) + delete_url = client.app.router["remove_task"].url_for(task_id=task_id) result = await client.delete(f"{delete_url}") data, error = await assert_status(result, status.HTTP_204_NO_CONTENT) assert not data diff --git a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py index c34dc8e4e8f5..e508ad92bc34 100644 --- a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py +++ b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py @@ -81,7 +81,7 @@ async def create_string_list_task( fail: bool = False, ) -> TaskId: return await lrt_api.start_task( - long_running_manager.tasks_manager, + long_running_manager, _string_list_task.__name__, num_strings=num_strings, sleep_time=sleep_time, @@ -205,7 +205,7 @@ async def test_workflow( [ ("GET", "get_task_status"), ("GET", "get_task_result"), - ("DELETE", "cancel_and_delete_task"), + ("DELETE", "remove_task"), ], ) async def test_get_task_wrong_task_id_raises_not_found( @@ -254,7 +254,7 @@ async def test_cancel_task( task_id = await start_long_running_task(app, client) # cancel the task - delete_url = app.url_path_for("cancel_and_delete_task", task_id=task_id) + delete_url = app.url_path_for("remove_task", task_id=task_id) result = await client.delete(f"{delete_url}") assert result.status_code == status.HTTP_204_NO_CONTENT diff --git a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py index 179c967088f1..d0a1cc2fe81a 100644 --- a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py +++ b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py @@ -71,9 +71,7 @@ async def create_task_user_defined_route( FastAPILongRunningManager, Depends(get_long_running_manager) ], ) -> TaskId: - return await lrt_api.start_task( - long_running_manager.tasks_manager, a_test_task.__name__ - ) + return await lrt_api.start_task(long_running_manager, a_test_task.__name__) @router.get("/api/failing", status_code=status.HTTP_200_OK) async def create_task_which_fails( @@ -82,7 +80,7 @@ async def create_task_which_fails( ], ) -> TaskId: return await lrt_api.start_task( - long_running_manager.tasks_manager, a_failing_test_task.__name__ + long_running_manager, a_failing_test_task.__name__ ) return router diff --git a/packages/service-library/tests/long_running_tasks/conftest.py b/packages/service-library/tests/long_running_tasks/conftest.py index 515b34ef0a4e..8ec2b090904b 100644 --- a/packages/service-library/tests/long_running_tasks/conftest.py +++ b/packages/service-library/tests/long_running_tasks/conftest.py @@ -13,7 +13,7 @@ TasksManager, ) from settings_library.redis import RedisSettings -from utils import TEST_CHECK_STALE_INTERVAL_S +from utils import TEST_CHECK_STALE_INTERVAL_S, NoWebAppLongRunningManager _logger = logging.getLogger(__name__) @@ -54,3 +54,20 @@ async def _( for manager in managers: with log_catch(_logger, reraise=False): await manager.teardown() + + +@pytest.fixture +def get_long_running_manager( + get_tasks_manager: Callable[ + [RedisSettings, RedisNamespace | None], Awaitable[TasksManager] + ], +) -> Callable[ + [RedisSettings, RedisNamespace | None], Awaitable[NoWebAppLongRunningManager] +]: + async def _( + redis_settings: RedisSettings, namespace: RedisNamespace | None + ) -> NoWebAppLongRunningManager: + tasks_manager = await get_tasks_manager(redis_settings, namespace) + return NoWebAppLongRunningManager(tasks_manager) + + return _ diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks__rabbit.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks__rabbit.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py index b3f795fb4ae9..b714f8529f6a 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py @@ -28,6 +28,7 @@ stop_after_delay, wait_fixed, ) +from utils import NoWebAppLongRunningManager pytest_simcore_core_services_selection = [ "redis", # TODO: remove when done with this part @@ -78,71 +79,85 @@ async def _to_replace(self: TasksManager) -> None: @pytest.fixture -async def tasks_managers( +async def long_running_managers( disable_stale_tasks_monitor: None, managers_count: NonNegativeInt, redis_service: RedisSettings, - get_tasks_manager: Callable[ - [RedisSettings, RedisNamespace | None], Awaitable[TasksManager] + get_long_running_manager: Callable[ + [RedisSettings, RedisNamespace | None], Awaitable[NoWebAppLongRunningManager] ], -) -> list[TasksManager]: - maanagers: list[TasksManager] = [] +) -> list[NoWebAppLongRunningManager]: + maanagers: list[NoWebAppLongRunningManager] = [] for _ in range(managers_count): - manager = await get_tasks_manager(redis_service, "same-service") - maanagers.append(manager) + long_running_manager = await get_long_running_manager( + redis_service, "same-service" + ) + maanagers.append(long_running_manager) return maanagers -def _get_task_manager(tasks_managers: list[TasksManager]) -> TasksManager: - return secrets.choice(tasks_managers) +def _get_task_manager( + long_running_managers: list[NoWebAppLongRunningManager], +) -> NoWebAppLongRunningManager: + return secrets.choice(long_running_managers) async def _assert_task_status( - task_manager: TasksManager, task_id: TaskId, *, is_done: bool + long_running_manager: NoWebAppLongRunningManager, task_id: TaskId, *, is_done: bool ) -> None: - result = await lrt_api.get_task_status(task_manager, TaskContext(), task_id) + result = await lrt_api.get_task_status(long_running_manager, TaskContext(), task_id) assert result.done is is_done async def _assert_task_status_on_random_manager( - tasks_managers: list[TasksManager], task_ids: list[TaskId], *, is_done: bool = True + long_running_managers: list[NoWebAppLongRunningManager], + task_ids: list[TaskId], + *, + is_done: bool = True ) -> None: for task_id in task_ids: result = await lrt_api.get_task_status( - _get_task_manager(tasks_managers), TaskContext(), task_id + _get_task_manager(long_running_managers), TaskContext(), task_id ) assert result.done is is_done async def _assert_task_status_done_on_all_managers( - tasks_managers: list[TasksManager], task_id: TaskId, *, is_done: bool = True + long_running_managers: list[NoWebAppLongRunningManager], + task_id: TaskId, + *, + is_done: bool = True ) -> None: async for attempt in AsyncRetrying(**_RETRY_PARAMS): with attempt: await _assert_task_status( - _get_task_manager(tasks_managers), task_id, is_done=is_done + _get_task_manager(long_running_managers), task_id, is_done=is_done ) # check can do this form any task manager - for manager in tasks_managers: + for manager in long_running_managers: await _assert_task_status(manager, task_id, is_done=is_done) async def _assert_list_tasks_from_all_managers( - tasks_managers: list[TasksManager], task_context: TaskContext, task_count: int + long_running_managers: list[NoWebAppLongRunningManager], + task_context: TaskContext, + task_count: int, ) -> None: - for manager in tasks_managers: + for manager in long_running_managers: tasks = await lrt_api.list_tasks(manager, task_context) assert len(tasks) == task_count async def _assert_task_is_no_longer_present( - tasks_managers: list[TasksManager], task_context: TaskContext, task_id: TaskId + long_running_managers: list[NoWebAppLongRunningManager], + task_context: TaskContext, + task_id: TaskId, ) -> None: with pytest.raises(TaskNotFoundError): await lrt_api.get_task_status( - _get_task_manager(tasks_managers), task_context, task_id + _get_task_manager(long_running_managers), task_context, task_id ) @@ -156,7 +171,7 @@ async def _assert_task_is_no_longer_present( @pytest.mark.parametrize("is_unique", _IS_UNIQUE) @pytest.mark.parametrize("to_return", [{"key": "value"}]) async def test_workflow_with_result( - tasks_managers: list[TasksManager], + long_running_managers: list[NoWebAppLongRunningManager], task_count: int, is_unique: bool, task_context: TaskContext | None, @@ -168,7 +183,7 @@ async def test_workflow_with_result( task_ids: list[TaskId] = [] for _ in range(task_count): task_id = await lrt_api.start_task( - _get_task_manager(tasks_managers), + _get_task_manager(long_running_managers), _task_echo_input.__name__, unique=is_unique, task_name=None, @@ -179,29 +194,33 @@ async def test_workflow_with_result( task_ids.append(task_id) for task_id in task_ids: - await _assert_task_status_done_on_all_managers(tasks_managers, task_id) + await _assert_task_status_done_on_all_managers(long_running_managers, task_id) await _assert_list_tasks_from_all_managers( - tasks_managers, saved_context, task_count=task_count + long_running_managers, saved_context, task_count=task_count ) # avoids tasks getting garbage collected - await _assert_task_status_on_random_manager(tasks_managers, task_ids, is_done=True) + await _assert_task_status_on_random_manager( + long_running_managers, task_ids, is_done=True + ) for task_id in task_ids: result = await lrt_api.get_task_result( - _get_task_manager(tasks_managers), saved_context, task_id + _get_task_manager(long_running_managers), saved_context, task_id ) assert result == to_return - await _assert_task_is_no_longer_present(tasks_managers, saved_context, task_id) + await _assert_task_is_no_longer_present( + long_running_managers, saved_context, task_id + ) @pytest.mark.parametrize("task_count", _TASK_COUNT) @pytest.mark.parametrize("task_context", _TASK_CONTEXT) @pytest.mark.parametrize("is_unique", _IS_UNIQUE) async def test_workflow_raises_error( - tasks_managers: list[TasksManager], + long_running_managers: list[NoWebAppLongRunningManager], task_count: int, is_unique: bool, task_context: TaskContext | None, @@ -212,7 +231,7 @@ async def test_workflow_raises_error( task_ids: list[TaskId] = [] for _ in range(task_count): task_id = await lrt_api.start_task( - _get_task_manager(tasks_managers), + _get_task_manager(long_running_managers), _task_always_raise.__name__, unique=is_unique, task_name=None, @@ -222,35 +241,39 @@ async def test_workflow_raises_error( task_ids.append(task_id) for task_id in task_ids: - await _assert_task_status_done_on_all_managers(tasks_managers, task_id) + await _assert_task_status_done_on_all_managers(long_running_managers, task_id) await _assert_list_tasks_from_all_managers( - tasks_managers, saved_context, task_count=task_count + long_running_managers, saved_context, task_count=task_count ) # avoids tasks getting garbage collected - await _assert_task_status_on_random_manager(tasks_managers, task_ids, is_done=True) + await _assert_task_status_on_random_manager( + long_running_managers, task_ids, is_done=True + ) for task_id in task_ids: with pytest.raises(RuntimeError, match="This task always raises an error"): await lrt_api.get_task_result( - _get_task_manager(tasks_managers), saved_context, task_id + _get_task_manager(long_running_managers), saved_context, task_id ) - await _assert_task_is_no_longer_present(tasks_managers, saved_context, task_id) + await _assert_task_is_no_longer_present( + long_running_managers, saved_context, task_id + ) @pytest.mark.parametrize("task_count", _TASK_COUNT) @pytest.mark.parametrize("task_context", _TASK_CONTEXT) @pytest.mark.parametrize("is_unique", _IS_UNIQUE) async def test_remove_task( - tasks_managers: list[TasksManager], + long_running_managers: list[NoWebAppLongRunningManager], task_count: int, is_unique: bool, task_context: TaskContext | None, ): task_id = await lrt_api.start_task( - _get_task_manager(tasks_managers), + _get_task_manager(long_running_managers), _task_takes_too_long.__name__, unique=is_unique, task_name=None, @@ -260,9 +283,13 @@ async def test_remove_task( saved_context = task_context or {} await _assert_task_status_done_on_all_managers( - tasks_managers, task_id, is_done=False + long_running_managers, task_id, is_done=False ) - await lrt_api.remove_task(_get_task_manager(tasks_managers), saved_context, task_id) + await lrt_api.remove_task( + _get_task_manager(long_running_managers), saved_context, task_id + ) - await _assert_task_is_no_longer_present(tasks_managers, saved_context, task_id) + await _assert_task_is_no_longer_present( + long_running_managers, saved_context, task_id + ) diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py index e739fe491356..c20dfbb4e0a5 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py @@ -21,18 +21,14 @@ TaskNotRegisteredError, ) from servicelib.long_running_tasks.models import TaskContext, TaskProgress, TaskStatus -from servicelib.long_running_tasks.task import ( - RedisNamespace, - TaskRegistry, - TasksManager, -) +from servicelib.long_running_tasks.task import RedisNamespace, TaskRegistry from settings_library.redis import RedisSettings from tenacity import TryAgain from tenacity.asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed -from utils import TEST_CHECK_STALE_INTERVAL_S +from utils import TEST_CHECK_STALE_INTERVAL_S, NoWebAppLongRunningManager _RETRY_PARAMS: dict[str, Any] = { "reraise": True, @@ -80,23 +76,23 @@ def empty_context() -> TaskContext: @pytest.fixture -async def tasks_manager( +async def long_running_manager( use_in_memory_redis: RedisSettings, - get_tasks_manager: Callable[ - [RedisSettings, RedisNamespace | None], Awaitable[TasksManager] + get_long_running_manager: Callable[ + [RedisSettings, RedisNamespace | None], Awaitable[NoWebAppLongRunningManager] ], -) -> TasksManager: - return await get_tasks_manager(use_in_memory_redis, None) +) -> NoWebAppLongRunningManager: + return await get_long_running_manager(use_in_memory_redis, None) @pytest.mark.parametrize("check_task_presence_before", [True, False]) async def test_task_is_auto_removed( - tasks_manager: TasksManager, + long_running_manager: NoWebAppLongRunningManager, check_task_presence_before: bool, empty_context: TaskContext, ): task_id = await lrt_api.start_task( - tasks_manager, + long_running_manager, a_background_task.__name__, raise_when_finished=False, total_sleep=10 * TEST_CHECK_STALE_INTERVAL_S, @@ -105,7 +101,7 @@ async def test_task_is_auto_removed( if check_task_presence_before: # immediately after starting the task is still there - task_status = await tasks_manager.get_task_status( + task_status = await long_running_manager.tasks_manager.get_task_status( task_id, with_task_context=empty_context ) assert task_status @@ -115,23 +111,29 @@ async def test_task_is_auto_removed( async for attempt in AsyncRetrying(**_RETRY_PARAMS): with attempt: if ( - await tasks_manager._tasks_data.get_task_data(task_id) # noqa: SLF001 + await long_running_manager.tasks_manager._tasks_data.get_task_data( + task_id + ) # noqa: SLF001 is not None ): msg = "wait till no element is found any longer" raise TryAgain(msg) with pytest.raises(TaskNotFoundError): - await tasks_manager.get_task_status(task_id, with_task_context=empty_context) + await long_running_manager.tasks_manager.get_task_status( + task_id, with_task_context=empty_context + ) with pytest.raises(TaskNotFoundError): - await tasks_manager.get_task_result(task_id, with_task_context=empty_context) + await long_running_manager.tasks_manager.get_task_result( + task_id, with_task_context=empty_context + ) async def test_checked_task_is_not_auto_removed( - tasks_manager: TasksManager, empty_context: TaskContext + long_running_manager: NoWebAppLongRunningManager, empty_context: TaskContext ): task_id = await lrt_api.start_task( - tasks_manager, + long_running_manager, a_background_task.__name__, raise_when_finished=False, total_sleep=5 * TEST_CHECK_STALE_INTERVAL_S, @@ -139,21 +141,21 @@ async def test_checked_task_is_not_auto_removed( ) async for attempt in AsyncRetrying(**_RETRY_PARAMS): with attempt: - status = await tasks_manager.get_task_status( + status = await long_running_manager.tasks_manager.get_task_status( task_id, with_task_context=empty_context ) assert status.done, f"task {task_id} not complete" - result = await tasks_manager.get_task_result( + result = await long_running_manager.tasks_manager.get_task_result( task_id, with_task_context=empty_context ) assert result async def test_fire_and_forget_task_is_not_auto_removed( - tasks_manager: TasksManager, empty_context: TaskContext + long_running_manager: NoWebAppLongRunningManager, empty_context: TaskContext ): task_id = await lrt_api.start_task( - tasks_manager, + long_running_manager, a_background_task.__name__, raise_when_finished=False, total_sleep=5 * TEST_CHECK_STALE_INTERVAL_S, @@ -162,35 +164,37 @@ async def test_fire_and_forget_task_is_not_auto_removed( ) await asyncio.sleep(3 * TEST_CHECK_STALE_INTERVAL_S) # the task shall still be present even if we did not check the status before - status = await tasks_manager.get_task_status( + status = await long_running_manager.tasks_manager.get_task_status( task_id, with_task_context=empty_context ) assert not status.done, "task was removed although it is fire and forget" # the task shall finish await asyncio.sleep(4 * TEST_CHECK_STALE_INTERVAL_S) # get the result - task_result = await tasks_manager.get_task_result( + task_result = await long_running_manager.tasks_manager.get_task_result( task_id, with_task_context=empty_context ) assert task_result == 42 async def test_get_result_of_unfinished_task_raises( - tasks_manager: TasksManager, empty_context: TaskContext + long_running_manager: NoWebAppLongRunningManager, empty_context: TaskContext ): task_id = await lrt_api.start_task( - tasks_manager, + long_running_manager, a_background_task.__name__, raise_when_finished=False, total_sleep=5 * TEST_CHECK_STALE_INTERVAL_S, task_context=empty_context, ) with pytest.raises(TaskNotCompletedError): - await tasks_manager.get_task_result(task_id, with_task_context=empty_context) + await long_running_manager.tasks_manager.get_task_result( + task_id, with_task_context=empty_context + ) async def test_unique_task_already_running( - tasks_manager: TasksManager, empty_context: TaskContext + long_running_manager: NoWebAppLongRunningManager, empty_context: TaskContext ): async def unique_task(progress: TaskProgress): _ = progress @@ -199,13 +203,19 @@ async def unique_task(progress: TaskProgress): TaskRegistry.register(unique_task) await lrt_api.start_task( - tasks_manager, unique_task.__name__, unique=True, task_context=empty_context + long_running_manager, + unique_task.__name__, + unique=True, + task_context=empty_context, ) # ensure unique running task regardless of how many times it gets started with pytest.raises(TaskAlreadyRunningError) as exec_info: await lrt_api.start_task( - tasks_manager, unique_task.__name__, unique=True, task_context=empty_context + long_running_manager, + unique_task.__name__, + unique=True, + task_context=empty_context, ) assert "must be unique, found: " in f"{exec_info.value}" @@ -213,7 +223,7 @@ async def unique_task(progress: TaskProgress): async def test_start_multiple_not_unique_tasks( - tasks_manager: TasksManager, empty_context: TaskContext + long_running_manager: NoWebAppLongRunningManager, empty_context: TaskContext ): async def not_unique_task(progress: TaskProgress): await asyncio.sleep(1) @@ -222,28 +232,36 @@ async def not_unique_task(progress: TaskProgress): for _ in range(5): await lrt_api.start_task( - tasks_manager, not_unique_task.__name__, task_context=empty_context + long_running_manager, not_unique_task.__name__, task_context=empty_context ) TaskRegistry.unregister(not_unique_task) @pytest.mark.parametrize("is_unique", [True, False]) -async def test_get_task_id(tasks_manager: TasksManager, faker: Faker, is_unique: bool): - obj1 = tasks_manager._get_task_id(faker.word(), is_unique=is_unique) # noqa: SLF001 - obj2 = tasks_manager._get_task_id(faker.word(), is_unique=is_unique) # noqa: SLF001 +async def test_get_task_id( + long_running_manager: NoWebAppLongRunningManager, faker: Faker, is_unique: bool +): + obj1 = long_running_manager.tasks_manager._get_task_id( + faker.word(), is_unique=is_unique + ) # noqa: SLF001 + obj2 = long_running_manager.tasks_manager._get_task_id( + faker.word(), is_unique=is_unique + ) # noqa: SLF001 assert obj1 != obj2 -async def test_get_status(tasks_manager: TasksManager, empty_context: TaskContext): +async def test_get_status( + long_running_manager: NoWebAppLongRunningManager, empty_context: TaskContext +): task_id = await lrt_api.start_task( - tasks_manager, + long_running_manager, a_background_task.__name__, raise_when_finished=False, total_sleep=10, task_context=empty_context, ) - task_status = await tasks_manager.get_task_status( + task_status = await long_running_manager.tasks_manager.get_task_status( task_id, with_task_context=empty_context ) assert isinstance(task_status, TaskStatus) @@ -254,72 +272,78 @@ async def test_get_status(tasks_manager: TasksManager, empty_context: TaskContex async def test_get_status_missing( - tasks_manager: TasksManager, empty_context: TaskContext + long_running_manager: NoWebAppLongRunningManager, empty_context: TaskContext ): with pytest.raises(TaskNotFoundError) as exec_info: - await tasks_manager.get_task_status( + await long_running_manager.tasks_manager.get_task_status( "missing_task_id", with_task_context=empty_context ) assert f"{exec_info.value}" == "No task with missing_task_id found" -async def test_get_result(tasks_manager: TasksManager, empty_context: TaskContext): +async def test_get_result( + long_running_manager: NoWebAppLongRunningManager, empty_context: TaskContext +): task_id = await lrt_api.start_task( - tasks_manager, fast_background_task.__name__, task_context=empty_context + long_running_manager, fast_background_task.__name__, task_context=empty_context ) async for attempt in AsyncRetrying(**_RETRY_PARAMS): with attempt: - status = await tasks_manager.get_task_status( + status = await long_running_manager.tasks_manager.get_task_status( task_id, with_task_context=empty_context ) assert status.done is True - result = await tasks_manager.get_task_result( + result = await long_running_manager.tasks_manager.get_task_result( task_id, with_task_context=empty_context ) assert result == 42 async def test_get_result_missing( - tasks_manager: TasksManager, empty_context: TaskContext + long_running_manager: NoWebAppLongRunningManager, empty_context: TaskContext ): with pytest.raises(TaskNotFoundError) as exec_info: - await tasks_manager.get_task_result( + await long_running_manager.tasks_manager.get_task_result( "missing_task_id", with_task_context=empty_context ) assert f"{exec_info.value}" == "No task with missing_task_id found" async def test_get_result_finished_with_error( - tasks_manager: TasksManager, empty_context: TaskContext + long_running_manager: NoWebAppLongRunningManager, empty_context: TaskContext ): task_id = await lrt_api.start_task( - tasks_manager, failing_background_task.__name__, task_context=empty_context + long_running_manager, + failing_background_task.__name__, + task_context=empty_context, ) # wait for result async for attempt in AsyncRetrying(**_RETRY_PARAMS): with attempt: assert ( - await tasks_manager.get_task_status( + await long_running_manager.tasks_manager.get_task_status( task_id, with_task_context=empty_context ) ).done with pytest.raises(RuntimeError, match="failing asap"): - await tasks_manager.get_task_result(task_id, with_task_context=empty_context) + await long_running_manager.tasks_manager.get_task_result( + task_id, with_task_context=empty_context + ) async def test_cancel_task_from_different_manager( use_in_memory_redis: RedisSettings, - get_tasks_manager: Callable[ - [RedisSettings, RedisNamespace | None], Awaitable[TasksManager] + get_long_running_manager: Callable[ + [RedisSettings, RedisNamespace | None], Awaitable[NoWebAppLongRunningManager] ], empty_context: TaskContext, ): - manager_1 = await get_tasks_manager(use_in_memory_redis, "test-namespace") - manager_2 = await get_tasks_manager(use_in_memory_redis, "test-namespace") - manager_3 = await get_tasks_manager(use_in_memory_redis, "test-namespace") + manager_1 = await get_long_running_manager(use_in_memory_redis, "test-namespace") + manager_2 = await get_long_running_manager(use_in_memory_redis, "test-namespace") + manager_3 = await get_long_running_manager(use_in_memory_redis, "test-namespace") task_id = await lrt_api.start_task( manager_1, @@ -331,42 +355,56 @@ async def test_cancel_task_from_different_manager( # wati for task to complete for manager in (manager_1, manager_2, manager_3): - status = await manager.get_task_status(task_id, empty_context) + status = await manager.tasks_manager.get_task_status(task_id, empty_context) assert status.done is False async for attempt in AsyncRetrying(**_RETRY_PARAMS): with attempt: for manager in (manager_1, manager_2, manager_3): - status = await manager.get_task_status(task_id, empty_context) + status = await manager.tasks_manager.get_task_status( + task_id, empty_context + ) assert status.done is True # check all provide the same result for manager in (manager_1, manager_2, manager_3): - task_result = await manager.get_task_result(task_id, empty_context) + task_result = await manager.tasks_manager.get_task_result( + task_id, empty_context + ) assert task_result == 42 -async def test_remove_task(tasks_manager: TasksManager, empty_context: TaskContext): +async def test_remove_task( + long_running_manager: NoWebAppLongRunningManager, empty_context: TaskContext +): task_id = await lrt_api.start_task( - tasks_manager, + long_running_manager, a_background_task.__name__, raise_when_finished=False, total_sleep=10, task_context=empty_context, ) - await tasks_manager.get_task_status(task_id, with_task_context=empty_context) - await tasks_manager.remove_task(task_id, with_task_context=empty_context) + await long_running_manager.tasks_manager.get_task_status( + task_id, with_task_context=empty_context + ) + await long_running_manager.tasks_manager.remove_task( + task_id, with_task_context=empty_context + ) with pytest.raises(TaskNotFoundError): - await tasks_manager.get_task_status(task_id, with_task_context=empty_context) + await long_running_manager.tasks_manager.get_task_status( + task_id, with_task_context=empty_context + ) with pytest.raises(TaskNotFoundError): - await tasks_manager.get_task_result(task_id, with_task_context=empty_context) + await long_running_manager.tasks_manager.get_task_result( + task_id, with_task_context=empty_context + ) async def test_remove_task_with_task_context( - tasks_manager: TasksManager, empty_context: TaskContext + long_running_manager: NoWebAppLongRunningManager, empty_context: TaskContext ): task_id = await lrt_api.start_task( - tasks_manager, + long_running_manager, a_background_task.__name__, raise_when_finished=False, total_sleep=10, @@ -374,41 +412,47 @@ async def test_remove_task_with_task_context( ) # getting status fails if wrong task context given with pytest.raises(TaskNotFoundError): - await tasks_manager.get_task_status( + await long_running_manager.tasks_manager.get_task_status( task_id, with_task_context={"wrong_task_context": 12} ) - await tasks_manager.get_task_status(task_id, with_task_context=empty_context) + await long_running_manager.tasks_manager.get_task_status( + task_id, with_task_context=empty_context + ) # removing task fails if wrong task context given with pytest.raises(TaskNotFoundError): - await tasks_manager.remove_task( + await long_running_manager.tasks_manager.remove_task( task_id, with_task_context={"wrong_task_context": 12} ) - await tasks_manager.remove_task(task_id, with_task_context=empty_context) + await long_running_manager.tasks_manager.remove_task( + task_id, with_task_context=empty_context + ) async def test_remove_unknown_task( - tasks_manager: TasksManager, empty_context: TaskContext + long_running_manager: NoWebAppLongRunningManager, empty_context: TaskContext ): with pytest.raises(TaskNotFoundError): - await tasks_manager.remove_task("invalid_id", with_task_context=empty_context) + await long_running_manager.tasks_manager.remove_task( + "invalid_id", with_task_context=empty_context + ) - await tasks_manager.remove_task( + await long_running_manager.tasks_manager.remove_task( "invalid_id", with_task_context=empty_context, reraise_errors=False ) async def test__cancelled_tasks_worker_equivalent_of_cancellation_from_a_different_process( - tasks_manager: TasksManager, empty_context: TaskContext + long_running_manager: NoWebAppLongRunningManager, empty_context: TaskContext ): task_id = await lrt_api.start_task( - tasks_manager, + long_running_manager, a_background_task.__name__, raise_when_finished=False, total_sleep=10, task_context=empty_context, ) - await tasks_manager._tasks_data.set_as_cancelled( # noqa: SLF001 + await long_running_manager.tasks_manager._tasks_data.set_as_cancelled( # noqa: SLF001 task_id, with_task_context=empty_context ) @@ -416,19 +460,29 @@ async def test__cancelled_tasks_worker_equivalent_of_cancellation_from_a_differe with attempt: # noqa: SIM117 with pytest.raises(TaskNotFoundError): assert ( - await tasks_manager.get_task_status(task_id, empty_context) is None + await long_running_manager.tasks_manager.get_task_status( + task_id, empty_context + ) + is None ) -async def test_list_tasks(tasks_manager: TasksManager, empty_context: TaskContext): - assert await tasks_manager.list_tasks(with_task_context=empty_context) == [] +async def test_list_tasks( + long_running_manager: NoWebAppLongRunningManager, empty_context: TaskContext +): + assert ( + await long_running_manager.tasks_manager.list_tasks( + with_task_context=empty_context + ) + == [] + ) # start a bunch of tasks NUM_TASKS = 10 task_ids = [] for _ in range(NUM_TASKS): task_ids.append( # noqa: PERF401 await lrt_api.start_task( - tasks_manager, + long_running_manager, a_background_task.__name__, raise_when_finished=False, total_sleep=10, @@ -436,45 +490,67 @@ async def test_list_tasks(tasks_manager: TasksManager, empty_context: TaskContex ) ) assert ( - len(await tasks_manager.list_tasks(with_task_context=empty_context)) + len( + await long_running_manager.tasks_manager.list_tasks( + with_task_context=empty_context + ) + ) == NUM_TASKS ) for task_index, task_id in enumerate(task_ids): - await tasks_manager.remove_task(task_id, with_task_context=empty_context) + await long_running_manager.tasks_manager.remove_task( + task_id, with_task_context=empty_context + ) assert len( - await tasks_manager.list_tasks(with_task_context=empty_context) + await long_running_manager.tasks_manager.list_tasks( + with_task_context=empty_context + ) ) == NUM_TASKS - (task_index + 1) async def test_list_tasks_filtering( - tasks_manager: TasksManager, empty_context: TaskContext + long_running_manager: NoWebAppLongRunningManager, empty_context: TaskContext ): await lrt_api.start_task( - tasks_manager, + long_running_manager, a_background_task.__name__, raise_when_finished=False, total_sleep=10, task_context=empty_context, ) await lrt_api.start_task( - tasks_manager, + long_running_manager, a_background_task.__name__, raise_when_finished=False, total_sleep=10, task_context={"user_id": 213}, ) await lrt_api.start_task( - tasks_manager, + long_running_manager, a_background_task.__name__, raise_when_finished=False, total_sleep=10, task_context={"user_id": 213, "product": "osparc"}, ) - assert len(await tasks_manager.list_tasks(with_task_context=empty_context)) == 3 - assert len(await tasks_manager.list_tasks(with_task_context={"user_id": 213})) == 1 assert ( len( - await tasks_manager.list_tasks( + await long_running_manager.tasks_manager.list_tasks( + with_task_context=empty_context + ) + ) + == 3 + ) + assert ( + len( + await long_running_manager.tasks_manager.list_tasks( + with_task_context={"user_id": 213} + ) + ) + == 1 + ) + assert ( + len( + await long_running_manager.tasks_manager.list_tasks( with_task_context={"user_id": 213, "product": "osparc"} ) ) @@ -482,7 +558,7 @@ async def test_list_tasks_filtering( ) assert ( len( - await tasks_manager.list_tasks( + await long_running_manager.tasks_manager.list_tasks( with_task_context={"user_id": 120, "product": "osparc"} ) ) @@ -490,10 +566,12 @@ async def test_list_tasks_filtering( ) -async def test_define_task_name(tasks_manager: TasksManager, faker: Faker): +async def test_define_task_name( + long_running_manager: NoWebAppLongRunningManager, faker: Faker +): task_name = faker.name() task_id = await lrt_api.start_task( - tasks_manager, + long_running_manager, a_background_task.__name__, raise_when_finished=False, total_sleep=10, @@ -502,6 +580,8 @@ async def test_define_task_name(tasks_manager: TasksManager, faker: Faker): assert urllib.parse.quote(task_name, safe="") in task_id -async def test_start_not_registered_task(tasks_manager: TasksManager): +async def test_start_not_registered_task( + long_running_manager: NoWebAppLongRunningManager, +): with pytest.raises(TaskNotRegisteredError): - await lrt_api.start_task(tasks_manager, "not_registered_task") + await lrt_api.start_task(long_running_manager, "not_registered_task") diff --git a/packages/service-library/tests/long_running_tasks/utils.py b/packages/service-library/tests/long_running_tasks/utils.py index e473dd7e1daf..32042e4bd501 100644 --- a/packages/service-library/tests/long_running_tasks/utils.py +++ b/packages/service-library/tests/long_running_tasks/utils.py @@ -1,3 +1,24 @@ from typing import Final +from servicelib.long_running_tasks.base_long_running_manager import ( + BaseLongRunningManager, +) +from servicelib.long_running_tasks.task import TasksManager + + +class NoWebAppLongRunningManager(BaseLongRunningManager): + def __init__(self, tasks_manager: TasksManager): + self._tasks_manager = tasks_manager + + @property + def tasks_manager(self) -> TasksManager: + return self._tasks_manager + + def setup(self) -> None: + pass + + async def teardown(self) -> None: + pass + + TEST_CHECK_STALE_INTERVAL_S: Final[float] = 1 diff --git a/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_scheduler.py b/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_scheduler.py index 2c84fd0bb363..6e1c2a09acaa 100644 --- a/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_scheduler.py @@ -116,7 +116,7 @@ async def _progress_callback( try: return await lrt_api.start_task( - long_running_manager.tasks_manager, + long_running_manager, _task_remove_service_containers.__name__, unique=True, node_uuid=node_uuid, @@ -181,7 +181,7 @@ async def _progress_callback( try: return await lrt_api.start_task( - long_running_manager.tasks_manager, + long_running_manager, _task_save_service_state.__name__, unique=True, node_uuid=node_uuid, @@ -228,7 +228,7 @@ async def _progress_callback( try: return await lrt_api.start_task( - long_running_manager.tasks_manager, + long_running_manager, _task_push_service_outputs.__name__, unique=True, node_uuid=node_uuid, @@ -270,7 +270,7 @@ async def _task_cleanup_service_docker_resources( try: return await lrt_api.start_task( - long_running_manager.tasks_manager, + long_running_manager, _task_cleanup_service_docker_resources.__name__, unique=True, node_uuid=node_uuid, diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_long_running_tasks.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_long_running_tasks.py index 7e9fdb3d0b8f..2fe01315f345 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_long_running_tasks.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_long_running_tasks.py @@ -59,7 +59,7 @@ async def pull_user_servcices_docker_images( try: return await lrt_api.start_task( - long_running_manager.tasks_manager, + long_running_manager, task_pull_user_servcices_docker_images.__name__, unique=True, app=app, @@ -100,7 +100,7 @@ async def create_service_containers_task( # pylint: disable=too-many-arguments try: return await lrt_api.start_task( - long_running_manager.tasks_manager, + long_running_manager, task_create_service_containers.__name__, unique=True, settings=settings, @@ -134,7 +134,7 @@ async def runs_docker_compose_down_task( try: return await lrt_api.start_task( - long_running_manager.tasks_manager, + long_running_manager, task_runs_docker_compose_down.__name__, unique=True, app=app, @@ -166,7 +166,7 @@ async def state_restore_task( try: return await lrt_api.start_task( - long_running_manager.tasks_manager, + long_running_manager, task_restore_state.__name__, unique=True, settings=settings, @@ -197,7 +197,7 @@ async def state_save_task( try: return await lrt_api.start_task( - long_running_manager.tasks_manager, + long_running_manager, task_save_state.__name__, unique=True, settings=settings, @@ -230,7 +230,7 @@ async def ports_inputs_pull_task( try: return await lrt_api.start_task( - long_running_manager.tasks_manager, + long_running_manager, task_ports_inputs_pull.__name__, unique=True, port_keys=port_keys, @@ -263,7 +263,7 @@ async def ports_outputs_pull_task( try: return await lrt_api.start_task( - long_running_manager.tasks_manager, + long_running_manager, task_ports_outputs_pull.__name__, unique=True, port_keys=port_keys, @@ -293,7 +293,7 @@ async def ports_outputs_push_task( try: return await lrt_api.start_task( - long_running_manager.tasks_manager, + long_running_manager, task_ports_outputs_push.__name__, unique=True, outputs_manager=outputs_manager, @@ -323,7 +323,7 @@ async def containers_restart_task( try: return await lrt_api.start_task( - long_running_manager.tasks_manager, + long_running_manager, task_containers_restart.__name__, unique=True, app=app, diff --git a/services/web/server/src/simcore_service_webserver/tasks/_rest.py b/services/web/server/src/simcore_service_webserver/tasks/_rest.py index 375165e7c775..79a53791d408 100644 --- a/services/web/server/src/simcore_service_webserver/tasks/_rest.py +++ b/services/web/server/src/simcore_service_webserver/tasks/_rest.py @@ -59,7 +59,7 @@ async def get_async_jobs(request: web.Request) -> web.Response: inprocess_long_running_manager = get_long_running_manager(request.app) inprocess_tracked_tasks = await lrt_api.list_tasks( - inprocess_long_running_manager.tasks_manager, + inprocess_long_running_manager, inprocess_long_running_manager.get_task_context(request), ) From a9befe42afeb84bcd0f95a3d0b60d517ce9f075f Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 7 Aug 2025 07:33:18 +0200 Subject: [PATCH 011/119] renamed cancellation to remove_task --- api/specs/web-server/_long_running_tasks.py | 4 ++-- api/specs/web-server/_long_running_tasks_legacy.py | 6 +++--- .../servicelib/aiohttp/long_running_tasks/_routes.py | 2 +- .../servicelib/aiohttp/long_running_tasks/_server.py | 2 +- .../servicelib/fastapi/long_running_tasks/_client.py | 2 +- .../fastapi/long_running_tasks/_context_manager.py | 2 +- .../servicelib/fastapi/long_running_tasks/_routes.py | 4 +--- .../servicelib/long_running_tasks/_rabbit/__init__.py | 0 .../long_running_tasks/_rabbit/lrt_clinet.py | 0 .../long_running_tasks/_rabbit/lrt_server.py | 0 .../test_long_running_tasks_with_task_context.py | 2 +- .../test_api_rest_containers_long_running_tasks.py | 2 +- .../src/simcore_service_webserver/api/v0/openapi.yaml | 10 +++++----- .../src/simcore_service_webserver/tasks/_rest.py | 2 +- .../tests/unit/with_dbs/01/test_long_running_tasks.py | 2 +- 15 files changed, 19 insertions(+), 21 deletions(-) create mode 100644 packages/service-library/src/servicelib/long_running_tasks/_rabbit/__init__.py create mode 100644 packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_clinet.py create mode 100644 packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_server.py diff --git a/api/specs/web-server/_long_running_tasks.py b/api/specs/web-server/_long_running_tasks.py index 1c6e033b867d..4183f3c5839a 100644 --- a/api/specs/web-server/_long_running_tasks.py +++ b/api/specs/web-server/_long_running_tasks.py @@ -53,8 +53,8 @@ def get_async_job_status( @router.delete( "/tasks/{task_id}", - name="cancel_and_delete_task", - description="Cancels and deletes a task", + name="remove_task", + description="Cancels and removes a task", responses=_export_data_responses, status_code=status.HTTP_204_NO_CONTENT, ) diff --git a/api/specs/web-server/_long_running_tasks_legacy.py b/api/specs/web-server/_long_running_tasks_legacy.py index d5fc487301ac..59bc8881b0c3 100644 --- a/api/specs/web-server/_long_running_tasks_legacy.py +++ b/api/specs/web-server/_long_running_tasks_legacy.py @@ -42,11 +42,11 @@ async def get_task_status( @router.delete( "/{task_id}", - name="cancel_and_delete_task", - description="Cancels and deletes a task", + name="remove_task", + description="Cancels and removes a task", status_code=status.HTTP_204_NO_CONTENT, ) -async def cancel_and_delete_task( +async def remove_task( _path_params: Annotated[_PathParam, Depends()], ): ... diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py index 210966f82725..fe8dfbf82f5d 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py @@ -26,7 +26,7 @@ async def list_tasks(request: web.Request) -> web.Response: task_id=t.task_id, status_href=f"{request.app.router['get_task_status'].url_for(task_id=t.task_id)}", result_href=f"{request.app.router['get_task_result'].url_for(task_id=t.task_id)}", - abort_href=f"{request.app.router['cancel_and_delete_task'].url_for(task_id=t.task_id)}", + abort_href=f"{request.app.router['remove_task'].url_for(task_id=t.task_id)}", ) for t in await lrt_api.list_tasks( long_running_manager, diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py index e82cf4d841c7..11131840a56d 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py @@ -81,7 +81,7 @@ async def start_long_running_task( f"http://{ip_addr}:{port}{request_.app.router['get_task_result'].url_for(task_id=task_id)}" # NOSONAR ) abort_url = TypeAdapter(AnyHttpUrl).validate_python( - f"http://{ip_addr}:{port}{request_.app.router['cancel_and_delete_task'].url_for(task_id=task_id)}" # NOSONAR + f"http://{ip_addr}:{port}{request_.app.router['remove_task'].url_for(task_id=task_id)}" # NOSONAR ) task_get = TaskGet( task_id=task_id, diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_client.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_client.py index 5a17014e209f..8b7d9b78207b 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_client.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_client.py @@ -171,7 +171,7 @@ async def get_task_result( return result.json() @retry_on_http_errors - async def cancel_and_delete_task( + async def remove_task( self, task_id: TaskId, *, timeout: PositiveFloat | None = None # noqa: ASYNC109 ) -> None: timeout = timeout or self._client_configuration.default_timeout diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_context_manager.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_context_manager.py index 6a7ff58814d4..2e618710ca07 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_context_manager.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_context_manager.py @@ -121,7 +121,7 @@ async def _wait_for_task_result() -> Any: yield result except TimeoutError as e: - await client.cancel_and_delete_task(task_id) + await client.remove_task(task_id) raise TaskClientTimeoutError( task_id=task_id, timeout=task_timeout, diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py index 3e58501bc361..0e8dbc16e8d2 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py @@ -25,9 +25,7 @@ async def list_tasks( task_id=t.task_id, status_href=str(request.url_for("get_task_status", task_id=t.task_id)), result_href=str(request.url_for("get_task_result", task_id=t.task_id)), - abort_href=str( - request.url_for("cancel_and_delete_task", task_id=t.task_id) - ), + abort_href=str(request.url_for("remove_task", task_id=t.task_id)), ) for t in await lrt_api.list_tasks(long_running_manager, task_context={}) ] diff --git a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/__init__.py b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_clinet.py b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_clinet.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_server.py b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_server.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py index e303d9362ada..2a8c76b2ca81 100644 --- a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py +++ b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py @@ -169,7 +169,7 @@ async def test_cancel_task( ): assert client_with_task_context.app task_id = await start_long_running_task(client_with_task_context) - cancel_url = client_with_task_context.app.router["cancel_and_delete_task"].url_for( + cancel_url = client_with_task_context.app.router["remove_task"].url_for( task_id=task_id ) # calling cancel without task context should find nothing diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_containers_long_running_tasks.py b/services/dynamic-sidecar/tests/unit/test_api_rest_containers_long_running_tasks.py index 75180ce4a00b..f5ca123d8d1b 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_containers_long_running_tasks.py +++ b/services/dynamic-sidecar/tests/unit/test_api_rest_containers_long_running_tasks.py @@ -106,7 +106,7 @@ async def auto_remove_task(client: Client, task_id: TaskId) -> AsyncIterator[Non try: yield finally: - await client.cancel_and_delete_task(task_id, timeout=10) + await client.remove_task(task_id, timeout=10) async def _get_container_timestamps( diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index a8b1de9be4de..d7402ba48604 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -3564,8 +3564,8 @@ paths: delete: tags: - long-running-tasks - summary: Cancel And Delete Task - description: Cancels and deletes a task + summary: Remove Task + description: Cancels and removes a task operationId: cancel_async_job parameters: - name: task_id @@ -3684,9 +3684,9 @@ paths: delete: tags: - long-running-tasks-legacy - summary: Cancel And Delete Task - description: Cancels and deletes a task - operationId: cancel_and_delete_task + summary: Remove Task + description: Cancels and removes a task + operationId: remove_task parameters: - name: task_id in: path diff --git a/services/web/server/src/simcore_service_webserver/tasks/_rest.py b/services/web/server/src/simcore_service_webserver/tasks/_rest.py index 79a53791d408..800d470221e9 100644 --- a/services/web/server/src/simcore_service_webserver/tasks/_rest.py +++ b/services/web/server/src/simcore_service_webserver/tasks/_rest.py @@ -92,7 +92,7 @@ async def get_async_jobs(request: web.Request) -> web.Response: TaskGet( task_id=f"{task.task_id}", status_href=f"{request.app.router['get_task_status'].url_for(task_id=task.task_id)}", - abort_href=f"{request.app.router['cancel_and_delete_task'].url_for(task_id=task.task_id)}", + abort_href=f"{request.app.router['remove_task'].url_for(task_id=task.task_id)}", result_href=f"{request.app.router['get_task_result'].url_for(task_id=task.task_id)}", ) for task in inprocess_tracked_tasks diff --git a/services/web/server/tests/unit/with_dbs/01/test_long_running_tasks.py b/services/web/server/tests/unit/with_dbs/01/test_long_running_tasks.py index 4a1f654e3787..7a06060369e2 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_long_running_tasks.py +++ b/services/web/server/tests/unit/with_dbs/01/test_long_running_tasks.py @@ -27,7 +27,7 @@ ("GET", "list_tasks", {}), ("GET", "get_task_status", {"task_id": "some_fake_task_id"}), ("GET", "get_task_result", {"task_id": "some_fake_task_id"}), - ("DELETE", "cancel_and_delete_task", {"task_id": "some_fake_task_id"}), + ("DELETE", "remove_task", {"task_id": "some_fake_task_id"}), ], ) async def test_long_running_tasks_access_restricted_to_logged_users( From df8636af73622d493c3cd6615ccf9a6c786cc544 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 7 Aug 2025 08:31:19 +0200 Subject: [PATCH 012/119] fixed hanging test --- packages/service-library/tests/conftest.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/service-library/tests/conftest.py b/packages/service-library/tests/conftest.py index 845a8565d226..e6b60300ae77 100644 --- a/packages/service-library/tests/conftest.py +++ b/packages/service-library/tests/conftest.py @@ -3,9 +3,10 @@ # pylint: disable=unused-argument # pylint: disable=unused-import +import asyncio import sys from collections.abc import AsyncIterable, AsyncIterator, Callable -from contextlib import AbstractAsyncContextManager, asynccontextmanager +from contextlib import AbstractAsyncContextManager, asynccontextmanager, suppress from copy import deepcopy from pathlib import Path from typing import Any @@ -91,7 +92,8 @@ async def _( yield client - await client.shutdown() + with suppress(TimeoutError): + await asyncio.wait_for(client.shutdown(), timeout=5.0) async def _cleanup_redis_data(clients_manager: RedisClientsManager) -> None: for db in RedisDatabase: From 897ca66650123afe2cf2987d06c126fe94862348 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 7 Aug 2025 08:35:38 +0200 Subject: [PATCH 013/119] fast cancellation of lrt --- .../src/pytest_simcore/long_running_tasks.py | 14 ++++++++++++++ packages/service-library/tests/conftest.py | 1 + .../tests/long_running_tasks/conftest.py | 13 +------------ services/dynamic-sidecar/tests/conftest.py | 3 +++ 4 files changed, 19 insertions(+), 12 deletions(-) create mode 100644 packages/pytest-simcore/src/pytest_simcore/long_running_tasks.py diff --git a/packages/pytest-simcore/src/pytest_simcore/long_running_tasks.py b/packages/pytest-simcore/src/pytest_simcore/long_running_tasks.py new file mode 100644 index 000000000000..e3911dc62f5a --- /dev/null +++ b/packages/pytest-simcore/src/pytest_simcore/long_running_tasks.py @@ -0,0 +1,14 @@ +from datetime import timedelta + +import pytest +from pytest_mock import MockerFixture + + +@pytest.fixture +async def fast_long_running_tasks_cancellation( + mocker: MockerFixture, +) -> None: + mocker.patch( + "servicelib.long_running_tasks.task._CANCEL_TASKS_CHECK_INTERVAL", + new=timedelta(seconds=1), + ) diff --git a/packages/service-library/tests/conftest.py b/packages/service-library/tests/conftest.py index e6b60300ae77..c4f63a18a1ba 100644 --- a/packages/service-library/tests/conftest.py +++ b/packages/service-library/tests/conftest.py @@ -26,6 +26,7 @@ "pytest_simcore.environment_configs", "pytest_simcore.file_extra", "pytest_simcore.logging", + "pytest_simcore.long_running_tasks", "pytest_simcore.pytest_global_environs", "pytest_simcore.rabbit_service", "pytest_simcore.redis_service", diff --git a/packages/service-library/tests/long_running_tasks/conftest.py b/packages/service-library/tests/long_running_tasks/conftest.py index 8ec2b090904b..ef248d82c5f8 100644 --- a/packages/service-library/tests/long_running_tasks/conftest.py +++ b/packages/service-library/tests/long_running_tasks/conftest.py @@ -6,7 +6,6 @@ import pytest from faker import Faker -from pytest_mock import MockerFixture from servicelib.logging_utils import log_catch from servicelib.long_running_tasks.task import ( RedisNamespace, @@ -18,19 +17,9 @@ _logger = logging.getLogger(__name__) -@pytest.fixture -async def mock_cancel_tasks_check_interval( - mocker: MockerFixture, -) -> None: - mocker.patch( - "servicelib.long_running_tasks.task._CANCEL_TASKS_CHECK_INTERVAL", - new=timedelta(seconds=TEST_CHECK_STALE_INTERVAL_S), - ) - - @pytest.fixture async def get_tasks_manager( - mock_cancel_tasks_check_interval: None, faker: Faker + fast_long_running_tasks_cancellation: None, faker: Faker ) -> AsyncIterator[ Callable[[RedisSettings, RedisNamespace | None], Awaitable[TasksManager]] ]: diff --git a/services/dynamic-sidecar/tests/conftest.py b/services/dynamic-sidecar/tests/conftest.py index 1bf41e23c834..6d9b16c9bc9b 100644 --- a/services/dynamic-sidecar/tests/conftest.py +++ b/services/dynamic-sidecar/tests/conftest.py @@ -42,6 +42,7 @@ "pytest_simcore.docker_swarm", "pytest_simcore.faker_users_data", "pytest_simcore.logging", + "pytest_simcore.long_running_tasks", "pytest_simcore.minio_service", "pytest_simcore.postgres_service", "pytest_simcore.pytest_global_environs", @@ -169,6 +170,7 @@ def mock_rabbit_check(mocker: MockerFixture) -> None: @pytest.fixture def base_mock_envs( + fast_long_running_tasks_cancellation: None, use_in_memory_redis: RedisSettings, dy_volumes: Path, shared_store_dir: Path, @@ -211,6 +213,7 @@ def base_mock_envs( @pytest.fixture def mock_environment( + fast_long_running_tasks_cancellation: None, use_in_memory_redis: RedisSettings, mock_storage_check: None, mock_postgres_check: None, From 18dde52d22ef7e42325e2a74bd93c4ccf0ac625f Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 11 Aug 2025 11:32:53 +0200 Subject: [PATCH 014/119] lrt api refactor --- .../long_running_tasks/_rabbit/lrt_client.py | 108 +++++++++++++++++ .../long_running_tasks/_rabbit/lrt_clinet.py | 0 .../long_running_tasks/_rabbit/lrt_server.py | 103 ++++++++++++++++ .../long_running_tasks/_rabbit/namespace.py | 8 ++ .../base_long_running_manager.py | 12 ++ .../servicelib/long_running_tasks/lrt_api.py | 70 ++++++++--- .../servicelib/long_running_tasks/models.py | 2 + .../test_long_running_tasks_lrt_api.py | 111 +++++++++++++----- 8 files changed, 369 insertions(+), 45 deletions(-) create mode 100644 packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py delete mode 100644 packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_clinet.py create mode 100644 packages/service-library/src/servicelib/long_running_tasks/_rabbit/namespace.py diff --git a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py new file mode 100644 index 000000000000..51f2449becf4 --- /dev/null +++ b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py @@ -0,0 +1,108 @@ +import logging +from typing import Any + +from models_library.rabbitmq_basic_types import RPCMethodName +from pydantic import TypeAdapter + +from ...logging_utils import log_decorator +from ...long_running_tasks.task import RegisteredTaskName +from ...rabbitmq._client_rpc import RabbitMQRPCClient +from ..models import RabbitNamespace, TaskBase, TaskContext, TaskId, TaskStatus +from .namespace import get_namespace + +_logger = logging.getLogger(__name__) + + +@log_decorator(_logger, level=logging.DEBUG) +async def start_task( + rabbitmq_rpc_client: RabbitMQRPCClient, + namespace: RabbitNamespace, + *, + registered_task_name: RegisteredTaskName, + unique: bool = False, + task_context: TaskContext | None = None, + task_name: str | None = None, + fire_and_forget: bool = False, + **task_kwargs: Any, +) -> TaskId: + result = await rabbitmq_rpc_client.request( + get_namespace(namespace), + TypeAdapter(RPCMethodName).validate_python("start_task"), + registered_task_name=registered_task_name, + unique=unique, + task_context=task_context, + task_name=task_name, + fire_and_forget=fire_and_forget, + **task_kwargs, + ) + assert isinstance(result, TaskId) # nosec + return result + + +@log_decorator(_logger, level=logging.DEBUG) +async def list_tasks( + rabbitmq_rpc_client: RabbitMQRPCClient, + namespace: RabbitNamespace, + *, + task_context: TaskContext, +) -> list[TaskBase]: + result = await rabbitmq_rpc_client.request( + get_namespace(namespace), + TypeAdapter(RPCMethodName).validate_python("list_tasks"), + task_context=task_context, + ) + assert TypeAdapter(list[TaskBase]).validate_python(result) # nosec + return result + + +@log_decorator(_logger, level=logging.DEBUG) +async def get_task_status( + rabbitmq_rpc_client: RabbitMQRPCClient, + namespace: RabbitNamespace, + *, + task_context: TaskContext, + task_id: TaskId, +) -> TaskStatus: + result = await rabbitmq_rpc_client.request( + get_namespace(namespace), + TypeAdapter(RPCMethodName).validate_python("get_task_status"), + task_context=task_context, + task_id=task_id, + ) + assert isinstance(result, TaskStatus) # nosec + return result + + +@log_decorator(_logger, level=logging.DEBUG) +async def get_task_result( + rabbitmq_rpc_client: RabbitMQRPCClient, + namespace: RabbitNamespace, + *, + task_context: TaskContext, + task_id: TaskId, +) -> Any: + return await rabbitmq_rpc_client.request( + get_namespace(namespace), + TypeAdapter(RPCMethodName).validate_python("get_task_result"), + task_context=task_context, + task_id=task_id, + ) + + +@log_decorator(_logger, level=logging.DEBUG) +async def remove_task( + rabbitmq_rpc_client: RabbitMQRPCClient, + namespace: RabbitNamespace, + *, + task_context: TaskContext, + task_id: TaskId, + reraise_errors: bool = True, +) -> None: + result = await rabbitmq_rpc_client.request( + get_namespace(namespace), + TypeAdapter(RPCMethodName).validate_python("remove_task"), + task_context=task_context, + task_id=task_id, + reraise_errors=reraise_errors, + ) + assert result is None # nosec diff --git a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_clinet.py b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_clinet.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_server.py b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_server.py index e69de29bb2d1..fd783658d40f 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_server.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_server.py @@ -0,0 +1,103 @@ +import logging +from typing import Any + +from common_library.error_codes import create_error_code + +from ...logging_errors import create_troubleshootting_log_kwargs +from ...rabbitmq import RPCRouter +from ..base_long_running_manager import BaseLongRunningManager +from ..errors import BaseLongRunningError, TaskNotCompletedError, TaskNotFoundError +from ..models import TaskBase, TaskContext, TaskId, TaskStatus +from ..task import RegisteredTaskName + +_logger = logging.getLogger(__name__) + +router = RPCRouter() + + +@router.expose(reraise_if_error_type=(BaseLongRunningError,)) +async def start_task( + long_running_manager: BaseLongRunningManager, + *, + registered_task_name: RegisteredTaskName, + unique: bool = False, + task_context: TaskContext | None = None, + task_name: str | None = None, + fire_and_forget: bool = False, + **task_kwargs: Any, +) -> TaskId: + return await long_running_manager.tasks_manager.start_task( + registered_task_name, + unique=unique, + task_context=task_context, + task_name=task_name, + fire_and_forget=fire_and_forget, + **task_kwargs, + ) + + +@router.expose(reraise_if_error_type=(BaseLongRunningError,)) +async def list_tasks( + long_running_manager: BaseLongRunningManager, *, task_context: TaskContext +) -> list[TaskBase]: + return await long_running_manager.tasks_manager.list_tasks( + with_task_context=task_context + ) + + +@router.expose(reraise_if_error_type=(BaseLongRunningError,)) +async def get_task_status( + long_running_manager: BaseLongRunningManager, + *, + task_context: TaskContext, + task_id: TaskId, +) -> TaskStatus: + return await long_running_manager.tasks_manager.get_task_status( + task_id=task_id, with_task_context=task_context + ) + + +@router.expose(reraise_if_error_type=(BaseLongRunningError, Exception)) +async def get_task_result( + long_running_manager: BaseLongRunningManager, + *, + task_context: TaskContext, + task_id: TaskId, +) -> Any: + try: + task_result = await long_running_manager.tasks_manager.get_task_result( + task_id, with_task_context=task_context + ) + await long_running_manager.tasks_manager.remove_task( + task_id, with_task_context=task_context, reraise_errors=False + ) + return task_result + except (TaskNotFoundError, TaskNotCompletedError): + raise + except Exception as exc: + _logger.exception( + **create_troubleshootting_log_kwargs( + user_error_msg=f"{task_id=} raised an exception while getting its result", + error=exc, + error_code=create_error_code(exc), + error_context={"task_context": task_context, "task_id": task_id}, + ), + ) + # the task shall be removed in this case + await long_running_manager.tasks_manager.remove_task( + task_id, with_task_context=task_context, reraise_errors=False + ) + raise + + +@router.expose(reraise_if_error_type=(BaseLongRunningError,)) +async def remove_task( + long_running_manager: BaseLongRunningManager, + *, + task_context: TaskContext, + task_id: TaskId, + reraise_errors: bool, +) -> None: + await long_running_manager.tasks_manager.remove_task( + task_id, with_task_context=task_context, reraise_errors=reraise_errors + ) diff --git a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/namespace.py b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/namespace.py new file mode 100644 index 000000000000..fd3a9dc638c2 --- /dev/null +++ b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/namespace.py @@ -0,0 +1,8 @@ +from models_library.rabbitmq_basic_types import RPCNamespace +from pydantic import TypeAdapter + +from ..models import RabbitNamespace + + +def get_namespace(namespace: RabbitNamespace) -> RPCNamespace: + return TypeAdapter(RPCNamespace).validate_python(f"lrt-{namespace}") diff --git a/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py b/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py index d09428d3aa22..55b9a4cd69d2 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py +++ b/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py @@ -1,5 +1,7 @@ from abc import ABC, abstractmethod +from ..rabbitmq._client_rpc import RabbitMQRPCClient +from .models import RabbitNamespace from .task import TasksManager @@ -13,6 +15,16 @@ class BaseLongRunningManager(ABC): def tasks_manager(self) -> TasksManager: pass + @property + @abstractmethod + def rpc_server(self) -> RabbitMQRPCClient: + pass + + @property + @abstractmethod + def rabbit_namespace(self) -> RabbitNamespace: + pass + @abstractmethod async def setup(self) -> None: pass diff --git a/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py b/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py index 3df4f4e2d9b8..0ae1eaf080cc 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py +++ b/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py @@ -2,8 +2,11 @@ from typing import Any from common_library.error_codes import create_error_code +from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient from ..logging_errors import create_troubleshootting_log_kwargs +from ._rabbit import lrt_client, lrt_server +from ._rabbit.namespace import get_namespace from .base_long_running_manager import BaseLongRunningManager from .errors import TaskNotCompletedError, TaskNotFoundError from .models import TaskBase, TaskContext, TaskId, TaskStatus @@ -13,6 +16,7 @@ async def start_task( + rabbitmq_rpc_client: RabbitMQRPCClient, long_running_manager: BaseLongRunningManager, registered_task_name: RegisteredTaskName, *, @@ -47,8 +51,11 @@ async def start_task( Returns: TaskId: the task unique identifier """ - return await long_running_manager.tasks_manager.start_task( - registered_task_name, + + return await lrt_client.start_task( + rabbitmq_rpc_client, + long_running_manager.rabbit_namespace, + registered_task_name=registered_task_name, unique=unique, task_context=task_context, task_name=task_name, @@ -58,35 +65,51 @@ async def start_task( async def list_tasks( - long_running_manager: BaseLongRunningManager, task_context: TaskContext + rabbitmq_rpc_client: RabbitMQRPCClient, + long_running_manager: BaseLongRunningManager, + task_context: TaskContext, ) -> list[TaskBase]: - return await long_running_manager.tasks_manager.list_tasks( - with_task_context=task_context + return await lrt_client.list_tasks( + rabbitmq_rpc_client, + long_running_manager.rabbit_namespace, + task_context=task_context, ) async def get_task_status( + rabbitmq_rpc_client: RabbitMQRPCClient, long_running_manager: BaseLongRunningManager, task_context: TaskContext, task_id: TaskId, ) -> TaskStatus: """returns the status of a task""" - return await long_running_manager.tasks_manager.get_task_status( - task_id=task_id, with_task_context=task_context + return await lrt_client.get_task_status( + rabbitmq_rpc_client, + long_running_manager.rabbit_namespace, + task_id=task_id, + task_context=task_context, ) async def get_task_result( + rabbitmq_rpc_client: RabbitMQRPCClient, long_running_manager: BaseLongRunningManager, task_context: TaskContext, task_id: TaskId, ) -> Any: try: - task_result = await long_running_manager.tasks_manager.get_task_result( - task_id, with_task_context=task_context + task_result = await lrt_client.get_task_result( + rabbitmq_rpc_client, + long_running_manager.rabbit_namespace, + task_context=task_context, + task_id=task_id, ) - await long_running_manager.tasks_manager.remove_task( - task_id, with_task_context=task_context, reraise_errors=False + await lrt_client.remove_task( + rabbitmq_rpc_client, + long_running_manager.rabbit_namespace, + task_id=task_id, + task_context=task_context, + reraise_errors=False, ) return task_result except (TaskNotFoundError, TaskNotCompletedError): @@ -101,18 +124,35 @@ async def get_task_result( ), ) # the task shall be removed in this case - await long_running_manager.tasks_manager.remove_task( - task_id, with_task_context=task_context, reraise_errors=False + await lrt_client.remove_task( + rabbitmq_rpc_client, + long_running_manager.rabbit_namespace, + task_id=task_id, + task_context=task_context, + reraise_errors=False, ) raise async def remove_task( + rabbitmq_rpc_client: RabbitMQRPCClient, long_running_manager: BaseLongRunningManager, task_context: TaskContext, task_id: TaskId, ) -> None: """cancels and removes the task""" - await long_running_manager.tasks_manager.remove_task( - task_id, with_task_context=task_context + await lrt_client.remove_task( + rabbitmq_rpc_client, + long_running_manager.rabbit_namespace, + task_id=task_id, + task_context=task_context, + ) + + +async def register_rabbit_routes(long_running_manager: BaseLongRunningManager) -> None: + rpc_server = long_running_manager.rpc_server + await rpc_server.register_router( + lrt_server.router, + get_namespace(long_running_manager.rabbit_namespace), + long_running_manager, ) diff --git a/packages/service-library/src/servicelib/long_running_tasks/models.py b/packages/service-library/src/servicelib/long_running_tasks/models.py index a8c626714c1b..8cfdc7e55c01 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/models.py +++ b/packages/service-library/src/servicelib/long_running_tasks/models.py @@ -28,6 +28,8 @@ RequestBody: TypeAlias = Any TaskContext: TypeAlias = dict[str, Any] +RabbitNamespace: TypeAlias = str + class ResultField(BaseModel): result: str | None = None diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py index b714f8529f6a..43613bd2a6f8 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py @@ -4,7 +4,7 @@ import asyncio import secrets -from collections.abc import Awaitable, Callable +from collections.abc import AsyncIterable, Awaitable, Callable from typing import Any, Final import pytest @@ -12,6 +12,7 @@ from pydantic import NonNegativeInt from pytest_mock import MockerFixture from servicelib.long_running_tasks import lrt_api +from servicelib.long_running_tasks._rabbit.lrt_client import RabbitNamespace from servicelib.long_running_tasks.errors import TaskNotFoundError from servicelib.long_running_tasks.models import TaskContext from servicelib.long_running_tasks.task import ( @@ -20,6 +21,8 @@ TaskRegistry, TasksManager, ) +from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient +from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings from tenacity import ( AsyncRetrying, @@ -31,10 +34,7 @@ from utils import NoWebAppLongRunningManager pytest_simcore_core_services_selection = [ - "redis", # TODO: remove when done with this part -] -pytest_simcore_ops_services_selection = [ - "redis-commander", + "rabbit", ] _RETRY_PARAMS: dict[str, Any] = { @@ -78,19 +78,32 @@ async def _to_replace(self: TasksManager) -> None: mocker.patch.object(TasksManager, "_stale_tasks_monitor", _to_replace) +@pytest.fixture +async def rabbitmq_rpc_client( + rabbit_service: RabbitSettings, +) -> AsyncIterable[RabbitMQRPCClient]: + client = await RabbitMQRPCClient.create( + client_name="test-lrt-rpc-client", settings=rabbit_service + ) + yield client + await client.close() + + @pytest.fixture async def long_running_managers( disable_stale_tasks_monitor: None, managers_count: NonNegativeInt, - redis_service: RedisSettings, + use_in_memory_redis: RedisSettings, + rabbit_service: RabbitSettings, get_long_running_manager: Callable[ - [RedisSettings, RedisNamespace | None], Awaitable[NoWebAppLongRunningManager] + [RedisSettings, RedisNamespace | None, RabbitSettings, RabbitNamespace], + Awaitable[NoWebAppLongRunningManager], ], ) -> list[NoWebAppLongRunningManager]: maanagers: list[NoWebAppLongRunningManager] = [] for _ in range(managers_count): long_running_manager = await get_long_running_manager( - redis_service, "same-service" + use_in_memory_redis, "same-service", rabbit_service, "some-service" ) maanagers.append(long_running_manager) @@ -104,13 +117,20 @@ def _get_task_manager( async def _assert_task_status( - long_running_manager: NoWebAppLongRunningManager, task_id: TaskId, *, is_done: bool + rabbitmq_rpc_client: RabbitMQRPCClient, + long_running_manager: NoWebAppLongRunningManager, + task_id: TaskId, + *, + is_done: bool ) -> None: - result = await lrt_api.get_task_status(long_running_manager, TaskContext(), task_id) + result = await lrt_api.get_task_status( + rabbitmq_rpc_client, long_running_manager, TaskContext(), task_id + ) assert result.done is is_done async def _assert_task_status_on_random_manager( + rabbitmq_rpc_client: RabbitMQRPCClient, long_running_managers: list[NoWebAppLongRunningManager], task_ids: list[TaskId], *, @@ -118,12 +138,16 @@ async def _assert_task_status_on_random_manager( ) -> None: for task_id in task_ids: result = await lrt_api.get_task_status( - _get_task_manager(long_running_managers), TaskContext(), task_id + rabbitmq_rpc_client, + _get_task_manager(long_running_managers), + TaskContext(), + task_id, ) assert result.done is is_done async def _assert_task_status_done_on_all_managers( + rabbitmq_rpc_client: RabbitMQRPCClient, long_running_managers: list[NoWebAppLongRunningManager], task_id: TaskId, *, @@ -132,32 +156,42 @@ async def _assert_task_status_done_on_all_managers( async for attempt in AsyncRetrying(**_RETRY_PARAMS): with attempt: await _assert_task_status( - _get_task_manager(long_running_managers), task_id, is_done=is_done + rabbitmq_rpc_client, + _get_task_manager(long_running_managers), + task_id, + is_done=is_done, ) # check can do this form any task manager for manager in long_running_managers: - await _assert_task_status(manager, task_id, is_done=is_done) + await _assert_task_status( + rabbitmq_rpc_client, manager, task_id, is_done=is_done + ) async def _assert_list_tasks_from_all_managers( + rabbitmq_rpc_client: RabbitMQRPCClient, long_running_managers: list[NoWebAppLongRunningManager], task_context: TaskContext, task_count: int, ) -> None: for manager in long_running_managers: - tasks = await lrt_api.list_tasks(manager, task_context) + tasks = await lrt_api.list_tasks(rabbitmq_rpc_client, manager, task_context) assert len(tasks) == task_count async def _assert_task_is_no_longer_present( + rabbitmq_rpc_client: RabbitMQRPCClient, long_running_managers: list[NoWebAppLongRunningManager], task_context: TaskContext, task_id: TaskId, ) -> None: with pytest.raises(TaskNotFoundError): await lrt_api.get_task_status( - _get_task_manager(long_running_managers), task_context, task_id + rabbitmq_rpc_client, + _get_task_manager(long_running_managers), + task_context, + task_id, ) @@ -172,6 +206,7 @@ async def _assert_task_is_no_longer_present( @pytest.mark.parametrize("to_return", [{"key": "value"}]) async def test_workflow_with_result( long_running_managers: list[NoWebAppLongRunningManager], + rabbitmq_rpc_client: RabbitMQRPCClient, task_count: int, is_unique: bool, task_context: TaskContext | None, @@ -183,6 +218,7 @@ async def test_workflow_with_result( task_ids: list[TaskId] = [] for _ in range(task_count): task_id = await lrt_api.start_task( + rabbitmq_rpc_client, _get_task_manager(long_running_managers), _task_echo_input.__name__, unique=is_unique, @@ -194,25 +230,30 @@ async def test_workflow_with_result( task_ids.append(task_id) for task_id in task_ids: - await _assert_task_status_done_on_all_managers(long_running_managers, task_id) + await _assert_task_status_done_on_all_managers( + rabbitmq_rpc_client, long_running_managers, task_id + ) await _assert_list_tasks_from_all_managers( - long_running_managers, saved_context, task_count=task_count + rabbitmq_rpc_client, long_running_managers, saved_context, task_count=task_count ) # avoids tasks getting garbage collected await _assert_task_status_on_random_manager( - long_running_managers, task_ids, is_done=True + rabbitmq_rpc_client, long_running_managers, task_ids, is_done=True ) for task_id in task_ids: result = await lrt_api.get_task_result( - _get_task_manager(long_running_managers), saved_context, task_id + rabbitmq_rpc_client, + _get_task_manager(long_running_managers), + saved_context, + task_id, ) assert result == to_return await _assert_task_is_no_longer_present( - long_running_managers, saved_context, task_id + rabbitmq_rpc_client, long_running_managers, saved_context, task_id ) @@ -221,6 +262,7 @@ async def test_workflow_with_result( @pytest.mark.parametrize("is_unique", _IS_UNIQUE) async def test_workflow_raises_error( long_running_managers: list[NoWebAppLongRunningManager], + rabbitmq_rpc_client: RabbitMQRPCClient, task_count: int, is_unique: bool, task_context: TaskContext | None, @@ -231,6 +273,7 @@ async def test_workflow_raises_error( task_ids: list[TaskId] = [] for _ in range(task_count): task_id = await lrt_api.start_task( + rabbitmq_rpc_client, _get_task_manager(long_running_managers), _task_always_raise.__name__, unique=is_unique, @@ -241,38 +284,43 @@ async def test_workflow_raises_error( task_ids.append(task_id) for task_id in task_ids: - await _assert_task_status_done_on_all_managers(long_running_managers, task_id) + await _assert_task_status_done_on_all_managers( + rabbitmq_rpc_client, long_running_managers, task_id + ) await _assert_list_tasks_from_all_managers( - long_running_managers, saved_context, task_count=task_count + rabbitmq_rpc_client, long_running_managers, saved_context, task_count=task_count ) # avoids tasks getting garbage collected await _assert_task_status_on_random_manager( - long_running_managers, task_ids, is_done=True + rabbitmq_rpc_client, long_running_managers, task_ids, is_done=True ) for task_id in task_ids: with pytest.raises(RuntimeError, match="This task always raises an error"): await lrt_api.get_task_result( - _get_task_manager(long_running_managers), saved_context, task_id + rabbitmq_rpc_client, + _get_task_manager(long_running_managers), + saved_context, + task_id, ) await _assert_task_is_no_longer_present( - long_running_managers, saved_context, task_id + rabbitmq_rpc_client, long_running_managers, saved_context, task_id ) -@pytest.mark.parametrize("task_count", _TASK_COUNT) @pytest.mark.parametrize("task_context", _TASK_CONTEXT) @pytest.mark.parametrize("is_unique", _IS_UNIQUE) async def test_remove_task( long_running_managers: list[NoWebAppLongRunningManager], - task_count: int, + rabbitmq_rpc_client: RabbitMQRPCClient, is_unique: bool, task_context: TaskContext | None, ): task_id = await lrt_api.start_task( + rabbitmq_rpc_client, _get_task_manager(long_running_managers), _task_takes_too_long.__name__, unique=is_unique, @@ -283,13 +331,16 @@ async def test_remove_task( saved_context = task_context or {} await _assert_task_status_done_on_all_managers( - long_running_managers, task_id, is_done=False + rabbitmq_rpc_client, long_running_managers, task_id, is_done=False ) await lrt_api.remove_task( - _get_task_manager(long_running_managers), saved_context, task_id + rabbitmq_rpc_client, + _get_task_manager(long_running_managers), + saved_context, + task_id, ) await _assert_task_is_no_longer_present( - long_running_managers, saved_context, task_id + rabbitmq_rpc_client, long_running_managers, saved_context, task_id ) From c84389da8d5ed25f630775dd6a374749ac5afad2 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 11 Aug 2025 11:33:24 +0200 Subject: [PATCH 015/119] extended tests long_running_manager --- .../tests/long_running_tasks/conftest.py | 31 ++++++++++++---- .../tests/long_running_tasks/utils.py | 35 ++++++++++++++++--- 2 files changed, 56 insertions(+), 10 deletions(-) diff --git a/packages/service-library/tests/long_running_tasks/conftest.py b/packages/service-library/tests/long_running_tasks/conftest.py index ef248d82c5f8..ecfe2e356b81 100644 --- a/packages/service-library/tests/long_running_tasks/conftest.py +++ b/packages/service-library/tests/long_running_tasks/conftest.py @@ -7,10 +7,12 @@ import pytest from faker import Faker from servicelib.logging_utils import log_catch +from servicelib.long_running_tasks._rabbit.lrt_client import RabbitNamespace from servicelib.long_running_tasks.task import ( RedisNamespace, TasksManager, ) +from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings from utils import TEST_CHECK_STALE_INTERVAL_S, NoWebAppLongRunningManager @@ -46,17 +48,34 @@ async def _( @pytest.fixture -def get_long_running_manager( +async def get_long_running_manager( get_tasks_manager: Callable[ [RedisSettings, RedisNamespace | None], Awaitable[TasksManager] ], -) -> Callable[ - [RedisSettings, RedisNamespace | None], Awaitable[NoWebAppLongRunningManager] +) -> AsyncIterator[ + Callable[ + [RedisSettings, RedisNamespace | None, RabbitSettings, RabbitNamespace], + Awaitable[NoWebAppLongRunningManager], + ] ]: + managers: list[NoWebAppLongRunningManager] = [] + async def _( - redis_settings: RedisSettings, namespace: RedisNamespace | None + redis_settings: RedisSettings, + namespace: RedisNamespace | None, + rabbit_settings: RabbitSettings, + rabbit_namespace: RabbitNamespace, ) -> NoWebAppLongRunningManager: tasks_manager = await get_tasks_manager(redis_settings, namespace) - return NoWebAppLongRunningManager(tasks_manager) + manager = NoWebAppLongRunningManager( + tasks_manager, rabbit_settings, rabbit_namespace + ) + await manager.setup() + managers.append(manager) + return manager - return _ + yield _ + + for manager in managers: + with log_catch(_logger, reraise=False): + await manager.teardown() diff --git a/packages/service-library/tests/long_running_tasks/utils.py b/packages/service-library/tests/long_running_tasks/utils.py index 32042e4bd501..24318947e0e6 100644 --- a/packages/service-library/tests/long_running_tasks/utils.py +++ b/packages/service-library/tests/long_running_tasks/utils.py @@ -1,24 +1,51 @@ from typing import Final +from servicelib.long_running_tasks import lrt_api from servicelib.long_running_tasks.base_long_running_manager import ( BaseLongRunningManager, ) +from servicelib.long_running_tasks.models import RabbitNamespace from servicelib.long_running_tasks.task import TasksManager +from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient +from settings_library.rabbit import RabbitSettings class NoWebAppLongRunningManager(BaseLongRunningManager): - def __init__(self, tasks_manager: TasksManager): + def __init__( + self, + tasks_manager: TasksManager, + rabbit_settings: RabbitSettings, + rabbit_namespace: RabbitNamespace, + ): self._tasks_manager = tasks_manager + self._rabbit_namespace = rabbit_namespace + self.rabbit_settings = rabbit_settings + self._rpc_server: RabbitMQRPCClient | None = None + @property def tasks_manager(self) -> TasksManager: return self._tasks_manager - def setup(self) -> None: - pass + @property + def rpc_server(self): + assert self._rpc_server is not None # nosec + return self._rpc_server + + @property + def rabbit_namespace(self) -> str: + return self._rabbit_namespace + + async def setup(self) -> None: + self._rpc_server = await RabbitMQRPCClient.create( + client_name=f"lrt-{self.rabbit_namespace}", settings=self.rabbit_settings + ) + await lrt_api.register_rabbit_routes(self) async def teardown(self) -> None: - pass + if self._rpc_server is not None: + await self._rpc_server.close() + self._rpc_server = None TEST_CHECK_STALE_INTERVAL_S: Final[float] = 1 From 6ff969d69081500856576bc34d7c9f8d78beb01a Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 11 Aug 2025 11:52:42 +0200 Subject: [PATCH 016/119] fixed tests --- .../tests/long_running_tasks/conftest.py | 14 +- .../test_long_running_tasks_lrt_api.py | 13 +- .../test_long_running_tasks_task.py | 121 ++++++++++++++---- 3 files changed, 112 insertions(+), 36 deletions(-) diff --git a/packages/service-library/tests/long_running_tasks/conftest.py b/packages/service-library/tests/long_running_tasks/conftest.py index ecfe2e356b81..5b184dd7b7e9 100644 --- a/packages/service-library/tests/long_running_tasks/conftest.py +++ b/packages/service-library/tests/long_running_tasks/conftest.py @@ -1,7 +1,7 @@ # pylint: disable=redefined-outer-name # pylint: disable=unused-argument import logging -from collections.abc import AsyncIterator, Awaitable, Callable +from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable from datetime import timedelta import pytest @@ -12,6 +12,7 @@ RedisNamespace, TasksManager, ) +from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings from utils import TEST_CHECK_STALE_INTERVAL_S, NoWebAppLongRunningManager @@ -79,3 +80,14 @@ async def _( for manager in managers: with log_catch(_logger, reraise=False): await manager.teardown() + + +@pytest.fixture +async def rabbitmq_rpc_client( + rabbit_service: RabbitSettings, +) -> AsyncIterable[RabbitMQRPCClient]: + client = await RabbitMQRPCClient.create( + client_name="test-lrt-rpc-client", settings=rabbit_service + ) + yield client + await client.close() diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py index 43613bd2a6f8..f57f63d802e7 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py @@ -4,7 +4,7 @@ import asyncio import secrets -from collections.abc import AsyncIterable, Awaitable, Callable +from collections.abc import Awaitable, Callable from typing import Any, Final import pytest @@ -78,17 +78,6 @@ async def _to_replace(self: TasksManager) -> None: mocker.patch.object(TasksManager, "_stale_tasks_monitor", _to_replace) -@pytest.fixture -async def rabbitmq_rpc_client( - rabbit_service: RabbitSettings, -) -> AsyncIterable[RabbitMQRPCClient]: - client = await RabbitMQRPCClient.create( - client_name="test-lrt-rpc-client", settings=rabbit_service - ) - yield client - await client.close() - - @pytest.fixture async def long_running_managers( disable_stale_tasks_monitor: None, diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py index c20dfbb4e0a5..4d3bab21c1ba 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py @@ -14,6 +14,7 @@ from faker import Faker from models_library.api_schemas_long_running_tasks.base import ProgressMessage from servicelib.long_running_tasks import lrt_api +from servicelib.long_running_tasks._rabbit.lrt_client import RabbitNamespace from servicelib.long_running_tasks.errors import ( TaskAlreadyRunningError, TaskNotCompletedError, @@ -22,6 +23,8 @@ ) from servicelib.long_running_tasks.models import TaskContext, TaskProgress, TaskStatus from servicelib.long_running_tasks.task import RedisNamespace, TaskRegistry +from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient +from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings from tenacity import TryAgain from tenacity.asyncio import AsyncRetrying @@ -30,6 +33,10 @@ from tenacity.wait import wait_fixed from utils import TEST_CHECK_STALE_INTERVAL_S, NoWebAppLongRunningManager +pytest_simcore_core_services_selection = [ + "rabbit", +] + _RETRY_PARAMS: dict[str, Any] = { "reraise": True, "wait": wait_fixed(0.1), @@ -78,20 +85,26 @@ def empty_context() -> TaskContext: @pytest.fixture async def long_running_manager( use_in_memory_redis: RedisSettings, + rabbit_service: RabbitSettings, get_long_running_manager: Callable[ - [RedisSettings, RedisNamespace | None], Awaitable[NoWebAppLongRunningManager] + [RedisSettings, RedisNamespace | None, RabbitSettings, RabbitNamespace], + Awaitable[NoWebAppLongRunningManager], ], ) -> NoWebAppLongRunningManager: - return await get_long_running_manager(use_in_memory_redis, None) + return await get_long_running_manager( + use_in_memory_redis, None, rabbit_service, "rabbit-namespace" + ) @pytest.mark.parametrize("check_task_presence_before", [True, False]) async def test_task_is_auto_removed( + rabbitmq_rpc_client: RabbitMQRPCClient, long_running_manager: NoWebAppLongRunningManager, check_task_presence_before: bool, empty_context: TaskContext, ): task_id = await lrt_api.start_task( + rabbitmq_rpc_client, long_running_manager, a_background_task.__name__, raise_when_finished=False, @@ -130,9 +143,12 @@ async def test_task_is_auto_removed( async def test_checked_task_is_not_auto_removed( - long_running_manager: NoWebAppLongRunningManager, empty_context: TaskContext + rabbitmq_rpc_client: RabbitMQRPCClient, + long_running_manager: NoWebAppLongRunningManager, + empty_context: TaskContext, ): task_id = await lrt_api.start_task( + rabbitmq_rpc_client, long_running_manager, a_background_task.__name__, raise_when_finished=False, @@ -152,9 +168,12 @@ async def test_checked_task_is_not_auto_removed( async def test_fire_and_forget_task_is_not_auto_removed( - long_running_manager: NoWebAppLongRunningManager, empty_context: TaskContext + rabbitmq_rpc_client: RabbitMQRPCClient, + long_running_manager: NoWebAppLongRunningManager, + empty_context: TaskContext, ): task_id = await lrt_api.start_task( + rabbitmq_rpc_client, long_running_manager, a_background_task.__name__, raise_when_finished=False, @@ -178,9 +197,12 @@ async def test_fire_and_forget_task_is_not_auto_removed( async def test_get_result_of_unfinished_task_raises( - long_running_manager: NoWebAppLongRunningManager, empty_context: TaskContext + rabbitmq_rpc_client: RabbitMQRPCClient, + long_running_manager: NoWebAppLongRunningManager, + empty_context: TaskContext, ): task_id = await lrt_api.start_task( + rabbitmq_rpc_client, long_running_manager, a_background_task.__name__, raise_when_finished=False, @@ -194,7 +216,9 @@ async def test_get_result_of_unfinished_task_raises( async def test_unique_task_already_running( - long_running_manager: NoWebAppLongRunningManager, empty_context: TaskContext + rabbitmq_rpc_client: RabbitMQRPCClient, + long_running_manager: NoWebAppLongRunningManager, + empty_context: TaskContext, ): async def unique_task(progress: TaskProgress): _ = progress @@ -203,6 +227,7 @@ async def unique_task(progress: TaskProgress): TaskRegistry.register(unique_task) await lrt_api.start_task( + rabbitmq_rpc_client, long_running_manager, unique_task.__name__, unique=True, @@ -212,6 +237,7 @@ async def unique_task(progress: TaskProgress): # ensure unique running task regardless of how many times it gets started with pytest.raises(TaskAlreadyRunningError) as exec_info: await lrt_api.start_task( + rabbitmq_rpc_client, long_running_manager, unique_task.__name__, unique=True, @@ -223,7 +249,9 @@ async def unique_task(progress: TaskProgress): async def test_start_multiple_not_unique_tasks( - long_running_manager: NoWebAppLongRunningManager, empty_context: TaskContext + rabbitmq_rpc_client: RabbitMQRPCClient, + long_running_manager: NoWebAppLongRunningManager, + empty_context: TaskContext, ): async def not_unique_task(progress: TaskProgress): await asyncio.sleep(1) @@ -232,7 +260,10 @@ async def not_unique_task(progress: TaskProgress): for _ in range(5): await lrt_api.start_task( - long_running_manager, not_unique_task.__name__, task_context=empty_context + rabbitmq_rpc_client, + long_running_manager, + not_unique_task.__name__, + task_context=empty_context, ) TaskRegistry.unregister(not_unique_task) @@ -252,9 +283,12 @@ async def test_get_task_id( async def test_get_status( - long_running_manager: NoWebAppLongRunningManager, empty_context: TaskContext + rabbitmq_rpc_client: RabbitMQRPCClient, + long_running_manager: NoWebAppLongRunningManager, + empty_context: TaskContext, ): task_id = await lrt_api.start_task( + rabbitmq_rpc_client, long_running_manager, a_background_task.__name__, raise_when_finished=False, @@ -282,10 +316,15 @@ async def test_get_status_missing( async def test_get_result( - long_running_manager: NoWebAppLongRunningManager, empty_context: TaskContext + rabbitmq_rpc_client: RabbitMQRPCClient, + long_running_manager: NoWebAppLongRunningManager, + empty_context: TaskContext, ): task_id = await lrt_api.start_task( - long_running_manager, fast_background_task.__name__, task_context=empty_context + rabbitmq_rpc_client, + long_running_manager, + fast_background_task.__name__, + task_context=empty_context, ) async for attempt in AsyncRetrying(**_RETRY_PARAMS): @@ -312,9 +351,12 @@ async def test_get_result_missing( async def test_get_result_finished_with_error( - long_running_manager: NoWebAppLongRunningManager, empty_context: TaskContext + rabbitmq_rpc_client: RabbitMQRPCClient, + long_running_manager: NoWebAppLongRunningManager, + empty_context: TaskContext, ): task_id = await lrt_api.start_task( + rabbitmq_rpc_client, long_running_manager, failing_background_task.__name__, task_context=empty_context, @@ -335,17 +377,27 @@ async def test_get_result_finished_with_error( async def test_cancel_task_from_different_manager( + rabbitmq_rpc_client: RabbitMQRPCClient, + rabbit_service: RabbitSettings, use_in_memory_redis: RedisSettings, get_long_running_manager: Callable[ - [RedisSettings, RedisNamespace | None], Awaitable[NoWebAppLongRunningManager] + [RedisSettings, RedisNamespace | None, RabbitSettings, RabbitNamespace], + Awaitable[NoWebAppLongRunningManager], ], empty_context: TaskContext, ): - manager_1 = await get_long_running_manager(use_in_memory_redis, "test-namespace") - manager_2 = await get_long_running_manager(use_in_memory_redis, "test-namespace") - manager_3 = await get_long_running_manager(use_in_memory_redis, "test-namespace") + manager_1 = await get_long_running_manager( + use_in_memory_redis, "test-namespace", rabbit_service, "test-namespace" + ) + manager_2 = await get_long_running_manager( + use_in_memory_redis, "test-namespace", rabbit_service, "test-namespace" + ) + manager_3 = await get_long_running_manager( + use_in_memory_redis, "test-namespace", rabbit_service, "test-namespace" + ) task_id = await lrt_api.start_task( + rabbitmq_rpc_client, manager_1, a_background_task.__name__, raise_when_finished=False, @@ -375,9 +427,12 @@ async def test_cancel_task_from_different_manager( async def test_remove_task( - long_running_manager: NoWebAppLongRunningManager, empty_context: TaskContext + rabbitmq_rpc_client: RabbitMQRPCClient, + long_running_manager: NoWebAppLongRunningManager, + empty_context: TaskContext, ): task_id = await lrt_api.start_task( + rabbitmq_rpc_client, long_running_manager, a_background_task.__name__, raise_when_finished=False, @@ -401,9 +456,12 @@ async def test_remove_task( async def test_remove_task_with_task_context( - long_running_manager: NoWebAppLongRunningManager, empty_context: TaskContext + rabbitmq_rpc_client: RabbitMQRPCClient, + long_running_manager: NoWebAppLongRunningManager, + empty_context: TaskContext, ): task_id = await lrt_api.start_task( + rabbitmq_rpc_client, long_running_manager, a_background_task.__name__, raise_when_finished=False, @@ -443,9 +501,12 @@ async def test_remove_unknown_task( async def test__cancelled_tasks_worker_equivalent_of_cancellation_from_a_different_process( - long_running_manager: NoWebAppLongRunningManager, empty_context: TaskContext + rabbitmq_rpc_client: RabbitMQRPCClient, + long_running_manager: NoWebAppLongRunningManager, + empty_context: TaskContext, ): task_id = await lrt_api.start_task( + rabbitmq_rpc_client, long_running_manager, a_background_task.__name__, raise_when_finished=False, @@ -468,7 +529,9 @@ async def test__cancelled_tasks_worker_equivalent_of_cancellation_from_a_differe async def test_list_tasks( - long_running_manager: NoWebAppLongRunningManager, empty_context: TaskContext + rabbitmq_rpc_client: RabbitMQRPCClient, + long_running_manager: NoWebAppLongRunningManager, + empty_context: TaskContext, ): assert ( await long_running_manager.tasks_manager.list_tasks( @@ -482,6 +545,7 @@ async def test_list_tasks( for _ in range(NUM_TASKS): task_ids.append( # noqa: PERF401 await lrt_api.start_task( + rabbitmq_rpc_client, long_running_manager, a_background_task.__name__, raise_when_finished=False, @@ -509,9 +573,12 @@ async def test_list_tasks( async def test_list_tasks_filtering( - long_running_manager: NoWebAppLongRunningManager, empty_context: TaskContext + rabbitmq_rpc_client: RabbitMQRPCClient, + long_running_manager: NoWebAppLongRunningManager, + empty_context: TaskContext, ): await lrt_api.start_task( + rabbitmq_rpc_client, long_running_manager, a_background_task.__name__, raise_when_finished=False, @@ -519,6 +586,7 @@ async def test_list_tasks_filtering( task_context=empty_context, ) await lrt_api.start_task( + rabbitmq_rpc_client, long_running_manager, a_background_task.__name__, raise_when_finished=False, @@ -526,6 +594,7 @@ async def test_list_tasks_filtering( task_context={"user_id": 213}, ) await lrt_api.start_task( + rabbitmq_rpc_client, long_running_manager, a_background_task.__name__, raise_when_finished=False, @@ -567,10 +636,13 @@ async def test_list_tasks_filtering( async def test_define_task_name( - long_running_manager: NoWebAppLongRunningManager, faker: Faker + rabbitmq_rpc_client: RabbitMQRPCClient, + long_running_manager: NoWebAppLongRunningManager, + faker: Faker, ): task_name = faker.name() task_id = await lrt_api.start_task( + rabbitmq_rpc_client, long_running_manager, a_background_task.__name__, raise_when_finished=False, @@ -581,7 +653,10 @@ async def test_define_task_name( async def test_start_not_registered_task( + rabbitmq_rpc_client: RabbitMQRPCClient, long_running_manager: NoWebAppLongRunningManager, ): with pytest.raises(TaskNotRegisteredError): - await lrt_api.start_task(long_running_manager, "not_registered_task") + await lrt_api.start_task( + rabbitmq_rpc_client, long_running_manager, "not_registered_task" + ) From 08f82cc76780201a661726694c4476479098f50f Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 11 Aug 2025 13:34:39 +0200 Subject: [PATCH 017/119] removed unused --- .../tests/long_running_tasks/test_long_running_tasks__rabbit.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 packages/service-library/tests/long_running_tasks/test_long_running_tasks__rabbit.py diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks__rabbit.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks__rabbit.py deleted file mode 100644 index e69de29bb2d1..000000000000 From 7f24c6505bb1250fa9d6d5db11c11084f0b8f090 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 11 Aug 2025 13:56:45 +0200 Subject: [PATCH 018/119] fixed fastapi tests --- .../fastapi/long_running_tasks/_manager.py | 41 +++++++++++++++++++ .../fastapi/long_running_tasks/_routes.py | 21 ++++++++-- .../fastapi/long_running_tasks/_server.py | 6 +++ .../long_running_tasks/_rabbit/lrt_client.py | 20 +++++++-- .../fastapi/long_running_tasks/conftest.py | 19 ++++++++- .../test_long_running_client.py | 2 +- .../test_long_running_tasks.py | 28 +++++++++++-- ...test_long_running_tasks_context_manager.py | 20 +++++++-- 8 files changed, 141 insertions(+), 16 deletions(-) diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_manager.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_manager.py index 6f37eb40825d..42aa15b13e04 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_manager.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_manager.py @@ -1,10 +1,14 @@ import datetime from fastapi import FastAPI +from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings +from ...long_running_tasks import lrt_api from ...long_running_tasks.base_long_running_manager import BaseLongRunningManager +from ...long_running_tasks.models import RabbitNamespace from ...long_running_tasks.task import RedisNamespace, TasksManager +from ...rabbitmq._client_rpc import RabbitMQRPCClient class FastAPILongRunningManager(BaseLongRunningManager): @@ -14,7 +18,9 @@ def __init__( stale_task_check_interval: datetime.timedelta, stale_task_detect_timeout: datetime.timedelta, redis_settings: RedisSettings, + rabbit_settings: RabbitSettings, redis_namespace: RedisNamespace, + rabbit_namespace: RabbitNamespace, ): self._app = app self._tasks_manager = TasksManager( @@ -23,13 +29,48 @@ def __init__( redis_settings=redis_settings, redis_namespace=redis_namespace, ) + self._rabbit_namespace = rabbit_namespace + self.rabbit_settings = rabbit_settings + self._rpc_server: RabbitMQRPCClient | None = None + self._rpc_client: RabbitMQRPCClient | None = None @property def tasks_manager(self) -> TasksManager: return self._tasks_manager + @property + def rpc_server(self) -> RabbitMQRPCClient: + assert self._rpc_server is not None # nosec + return self._rpc_server + + @property + def rpc_client(self) -> RabbitMQRPCClient: + assert self._rpc_client is not None # nosec + return self._rpc_client + + @property + def rabbit_namespace(self) -> str: + return self._rabbit_namespace + async def setup(self) -> None: await self._tasks_manager.setup() + self._rpc_server = await RabbitMQRPCClient.create( + client_name=f"lrt-server-{self.rabbit_namespace}", + settings=self.rabbit_settings, + ) + self._rpc_client = await RabbitMQRPCClient.create( + client_name=f"lrt-client-{self.rabbit_namespace}", + settings=self.rabbit_settings, + ) + await lrt_api.register_rabbit_routes(self) async def teardown(self) -> None: await self._tasks_manager.teardown() + + if self._rpc_server is not None: + await self._rpc_server.close() + self._rpc_server = None + + if self._rpc_client is not None: + await self._rpc_client.close() + self._rpc_client = None diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py index 0e8dbc16e8d2..e4caf671cb76 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py @@ -27,7 +27,9 @@ async def list_tasks( result_href=str(request.url_for("get_task_result", task_id=t.task_id)), abort_href=str(request.url_for("remove_task", task_id=t.task_id)), ) - for t in await lrt_api.list_tasks(long_running_manager, task_context={}) + for t in await lrt_api.list_tasks( + long_running_manager.rpc_client, long_running_manager, task_context={} + ) ] @@ -48,7 +50,10 @@ async def get_task_status( ) -> TaskStatus: assert request # nosec return await lrt_api.get_task_status( - long_running_manager, task_context={}, task_id=task_id + long_running_manager.rpc_client, + long_running_manager, + task_context={}, + task_id=task_id, ) @@ -71,7 +76,10 @@ async def get_task_result( ) -> TaskResult | Any: assert request # nosec return await lrt_api.get_task_result( - long_running_manager, task_context={}, task_id=task_id + long_running_manager.rpc_client, + long_running_manager, + task_context={}, + task_id=task_id, ) @@ -93,4 +101,9 @@ async def remove_task( ], ) -> None: assert request # nosec - await lrt_api.remove_task(long_running_manager, task_context={}, task_id=task_id) + await lrt_api.remove_task( + long_running_manager.rpc_client, + long_running_manager, + task_context={}, + task_id=task_id, + ) diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py index f00e6c8f5215..a1c3da20a74d 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py @@ -1,6 +1,7 @@ import datetime from fastapi import APIRouter, FastAPI +from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings from ...long_running_tasks.constants import ( @@ -8,6 +9,7 @@ DEFAULT_STALE_TASK_DETECT_TIMEOUT, ) from ...long_running_tasks.errors import BaseLongRunningError +from ...long_running_tasks.models import RabbitNamespace from ...long_running_tasks.task import RedisNamespace from ._error_handlers import base_long_running_error_handler from ._manager import FastAPILongRunningManager @@ -20,6 +22,8 @@ def setup( router_prefix: str = "", redis_settings: RedisSettings, redis_namespace: RedisNamespace, + rabbit_settings: RabbitSettings, + rabbit_namespace: RabbitNamespace, stale_task_check_interval: datetime.timedelta = DEFAULT_STALE_TASK_CHECK_INTERVAL, stale_task_detect_timeout: datetime.timedelta = DEFAULT_STALE_TASK_DETECT_TIMEOUT, ) -> None: @@ -47,6 +51,8 @@ async def on_startup() -> None: stale_task_detect_timeout=stale_task_detect_timeout, redis_settings=redis_settings, redis_namespace=redis_namespace, + rabbit_settings=rabbit_settings, + rabbit_namespace=rabbit_namespace, ) ) await long_running_manager.setup() diff --git a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py index 51f2449becf4..1f222187e22d 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py @@ -1,8 +1,9 @@ import logging -from typing import Any +from datetime import timedelta +from typing import Any, Final from models_library.rabbitmq_basic_types import RPCMethodName -from pydantic import TypeAdapter +from pydantic import PositiveInt, TypeAdapter from ...logging_utils import log_decorator from ...long_running_tasks.task import RegisteredTaskName @@ -12,6 +13,13 @@ _logger = logging.getLogger(__name__) +_RPC_TIMEOUT_VERY_LONG_REQUEST: Final[PositiveInt] = int( + timedelta(minutes=60).total_seconds() +) +_RPC_TIMEOUT_NORMAL_REQUEST: Final[PositiveInt] = int( + timedelta(seconds=30).total_seconds() +) + @log_decorator(_logger, level=logging.DEBUG) async def start_task( @@ -34,6 +42,7 @@ async def start_task( task_name=task_name, fire_and_forget=fire_and_forget, **task_kwargs, + timeout_s=_RPC_TIMEOUT_NORMAL_REQUEST, ) assert isinstance(result, TaskId) # nosec return result @@ -50,9 +59,9 @@ async def list_tasks( get_namespace(namespace), TypeAdapter(RPCMethodName).validate_python("list_tasks"), task_context=task_context, + timeout_s=_RPC_TIMEOUT_NORMAL_REQUEST, ) - assert TypeAdapter(list[TaskBase]).validate_python(result) # nosec - return result + return TypeAdapter(list[TaskBase]).validate_python(result) @log_decorator(_logger, level=logging.DEBUG) @@ -68,6 +77,7 @@ async def get_task_status( TypeAdapter(RPCMethodName).validate_python("get_task_status"), task_context=task_context, task_id=task_id, + timeout_s=_RPC_TIMEOUT_NORMAL_REQUEST, ) assert isinstance(result, TaskStatus) # nosec return result @@ -86,6 +96,7 @@ async def get_task_result( TypeAdapter(RPCMethodName).validate_python("get_task_result"), task_context=task_context, task_id=task_id, + timeout_s=_RPC_TIMEOUT_NORMAL_REQUEST, ) @@ -104,5 +115,6 @@ async def remove_task( task_context=task_context, task_id=task_id, reraise_errors=reraise_errors, + timeout_s=_RPC_TIMEOUT_VERY_LONG_REQUEST, ) assert result is None # nosec diff --git a/packages/service-library/tests/fastapi/long_running_tasks/conftest.py b/packages/service-library/tests/fastapi/long_running_tasks/conftest.py index 0cab1161c096..c2bdbb70c166 100644 --- a/packages/service-library/tests/fastapi/long_running_tasks/conftest.py +++ b/packages/service-library/tests/fastapi/long_running_tasks/conftest.py @@ -9,17 +9,23 @@ from fastapi import FastAPI from httpx import ASGITransport, AsyncClient from servicelib.fastapi import long_running_tasks +from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient +from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings @pytest.fixture -async def bg_task_app(router_prefix: str, redis_service: RedisSettings) -> FastAPI: +async def bg_task_app( + router_prefix: str, redis_service: RedisSettings, rabbit_service: RabbitSettings +) -> FastAPI: app = FastAPI() long_running_tasks.server.setup( app, redis_settings=redis_service, redis_namespace="test", + rabbit_settings=rabbit_service, + rabbit_namespace="test", router_prefix=router_prefix, ) return app @@ -33,3 +39,14 @@ async def async_client(bg_task_app: FastAPI) -> AsyncIterable[AsyncClient]: headers={"Content-Type": "application/json"}, ) as client: yield client + + +@pytest.fixture +async def rabbitmq_rpc_client( + rabbit_service: RabbitSettings, +) -> AsyncIterable[RabbitMQRPCClient]: + client = await RabbitMQRPCClient.create( + client_name="test-lrt-rpc-client", settings=rabbit_service + ) + yield client + await client.close() diff --git a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_client.py b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_client.py index 02d392126cbf..42f76a58f724 100644 --- a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_client.py +++ b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_client.py @@ -8,7 +8,7 @@ @pytest.mark.parametrize( "error_class, error_args", [ - (HTTPError, dict(message="")), + (HTTPError, {"message": ""}), ], ) async def test_retry_on_errors( diff --git a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py index e508ad92bc34..a7d07739119f 100644 --- a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py +++ b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py @@ -33,6 +33,7 @@ TaskStatus, ) from servicelib.long_running_tasks.task import TaskContext, TaskRegistry +from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings from tenacity.asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type @@ -40,6 +41,10 @@ from tenacity.wait import wait_fixed from yarl import URL +pytest_simcore_core_services_selection = [ + "rabbit", +] + ITEM_PUBLISH_SLEEP: Final[float] = 0.1 @@ -81,6 +86,7 @@ async def create_string_list_task( fail: bool = False, ) -> TaskId: return await lrt_api.start_task( + long_running_manager.rpc_client, long_running_manager, _string_list_task.__name__, num_strings=num_strings, @@ -93,12 +99,20 @@ async def create_string_list_task( @pytest.fixture async def app( - server_routes: APIRouter, use_in_memory_redis: RedisSettings + server_routes: APIRouter, + use_in_memory_redis: RedisSettings, + rabbit_service: RabbitSettings, ) -> AsyncIterator[FastAPI]: # overrides fastapi/conftest.py:app app = FastAPI(title="test app") app.include_router(server_routes) - setup_server(app, redis_settings=use_in_memory_redis, redis_namespace="test") + setup_server( + app, + redis_settings=use_in_memory_redis, + redis_namespace="test", + rabbit_settings=rabbit_service, + rabbit_namespace="test", + ) setup_client(app) async with LifespanManager(app, startup_timeout=30, shutdown_timeout=30): yield app @@ -188,7 +202,15 @@ async def test_workflow( ("generated item", 0.8), ("finished", 1.0), ] - assert all(x in progress_updates for x in EXPECTED_MESSAGES) + + async for attempt in AsyncRetrying( + wait=wait_fixed(0.1), + stop=stop_after_delay(10), + reraise=True, + retry=retry_if_exception_type(AssertionError), + ): + with attempt: + assert all(x in progress_updates for x in EXPECTED_MESSAGES) # now check the result result_url = app.url_path_for("get_task_result", task_id=task_id) result = await client.get(f"{result_url}") diff --git a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py index d0a1cc2fe81a..dedaa0e4ffc4 100644 --- a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py +++ b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py @@ -28,8 +28,13 @@ TaskProgress, ) from servicelib.long_running_tasks.task import TaskRegistry +from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings +pytest_simcore_core_services_selection = [ + "rabbit", +] + TASK_SLEEP_INTERVAL: Final[PositiveFloat] = 0.1 # UTILS @@ -71,7 +76,9 @@ async def create_task_user_defined_route( FastAPILongRunningManager, Depends(get_long_running_manager) ], ) -> TaskId: - return await lrt_api.start_task(long_running_manager, a_test_task.__name__) + return await lrt_api.start_task( + long_running_manager.rpc_client, long_running_manager, a_test_task.__name__ + ) @router.get("/api/failing", status_code=status.HTTP_200_OK) async def create_task_which_fails( @@ -80,7 +87,9 @@ async def create_task_which_fails( ], ) -> TaskId: return await lrt_api.start_task( - long_running_manager, a_failing_test_task.__name__ + long_running_manager.rpc_client, + long_running_manager, + a_failing_test_task.__name__, ) return router @@ -88,7 +97,10 @@ async def create_task_which_fails( @pytest.fixture async def bg_task_app( - user_routes: APIRouter, router_prefix: str, use_in_memory_redis: RedisSettings + user_routes: APIRouter, + router_prefix: str, + use_in_memory_redis: RedisSettings, + rabbit_service: RabbitSettings, ) -> AsyncIterable[FastAPI]: app = FastAPI() @@ -99,6 +111,8 @@ async def bg_task_app( router_prefix=router_prefix, redis_settings=use_in_memory_redis, redis_namespace="test", + rabbit_settings=rabbit_service, + rabbit_namespace="test", ) setup_client(app, router_prefix=router_prefix) From ae2f4f17b9cf6e5712fe48f21f04fbf86ae830bc Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 11 Aug 2025 14:19:46 +0200 Subject: [PATCH 019/119] fixed most tests --- .../aiohttp/long_running_tasks/_manager.py | 42 ++++++++++++++++++- .../aiohttp/long_running_tasks/_routes.py | 4 ++ .../aiohttp/long_running_tasks/_server.py | 15 ++++++- .../test_long_running_tasks.py | 24 +++++++---- .../test_long_running_tasks_client.py | 15 +++++-- ...st_long_running_tasks_with_task_context.py | 10 +++-- .../long_running_tasks.py | 11 +++-- 7 files changed, 101 insertions(+), 20 deletions(-) diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_manager.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_manager.py index f03945126bd0..53d4ec06a25d 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_manager.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_manager.py @@ -1,11 +1,14 @@ import datetime from aiohttp import web +from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings +from ...long_running_tasks import lrt_api from ...long_running_tasks.base_long_running_manager import BaseLongRunningManager -from ...long_running_tasks.models import TaskContext +from ...long_running_tasks.models import RabbitNamespace, TaskContext from ...long_running_tasks.task import RedisNamespace, TasksManager +from ...rabbitmq._client_rpc import RabbitMQRPCClient from ._constants import APP_LONG_RUNNING_MANAGER_KEY from ._request import get_task_context @@ -17,7 +20,9 @@ def __init__( stale_task_check_interval: datetime.timedelta, stale_task_detect_timeout: datetime.timedelta, redis_settings: RedisSettings, + rabbit_settings: RabbitSettings, redis_namespace: RedisNamespace, + rabbit_namespace: RabbitNamespace, ): self._app = app self._tasks_manager = TasksManager( @@ -26,17 +31,52 @@ def __init__( redis_settings=redis_settings, redis_namespace=redis_namespace, ) + self._rabbit_namespace = rabbit_namespace + self.rabbit_settings = rabbit_settings + self._rpc_server: RabbitMQRPCClient | None = None + self._rpc_client: RabbitMQRPCClient | None = None @property def tasks_manager(self) -> TasksManager: return self._tasks_manager + @property + def rpc_server(self) -> RabbitMQRPCClient: + assert self._rpc_server is not None # nosec + return self._rpc_server + + @property + def rpc_client(self) -> RabbitMQRPCClient: + assert self._rpc_client is not None # nosec + return self._rpc_client + + @property + def rabbit_namespace(self) -> RabbitNamespace: + return self._rabbit_namespace + async def setup(self) -> None: await self._tasks_manager.setup() + self._rpc_server = await RabbitMQRPCClient.create( + client_name=f"lrt-server-{self.rabbit_namespace}", + settings=self.rabbit_settings, + ) + self._rpc_client = await RabbitMQRPCClient.create( + client_name=f"lrt-client-{self.rabbit_namespace}", + settings=self.rabbit_settings, + ) + await lrt_api.register_rabbit_routes(self) async def teardown(self) -> None: await self._tasks_manager.teardown() + if self._rpc_server is not None: + await self._rpc_server.close() + self._rpc_server = None + + if self._rpc_client is not None: + await self._rpc_client.close() + self._rpc_client = None + @staticmethod def get_task_context(request: web.Request) -> TaskContext: return get_task_context(request) diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py index fe8dfbf82f5d..5c4ea1ca9ce0 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py @@ -29,6 +29,7 @@ async def list_tasks(request: web.Request) -> web.Response: abort_href=f"{request.app.router['remove_task'].url_for(task_id=t.task_id)}", ) for t in await lrt_api.list_tasks( + long_running_manager.rpc_client, long_running_manager, long_running_manager.get_task_context(request), ) @@ -42,6 +43,7 @@ async def get_task_status(request: web.Request) -> web.Response: long_running_manager = get_long_running_manager(request.app) task_status = await lrt_api.get_task_status( + long_running_manager.rpc_client, long_running_manager, long_running_manager.get_task_context(request), path_params.task_id, @@ -56,6 +58,7 @@ async def get_task_result(request: web.Request) -> web.Response | Any: # NOTE: this might raise an exception that will be catched by the _error_handlers return await lrt_api.get_task_result( + long_running_manager.rpc_client, long_running_manager, long_running_manager.get_task_context(request), path_params.task_id, @@ -68,6 +71,7 @@ async def remove_task(request: web.Request) -> web.Response: long_running_manager = get_long_running_manager(request.app) await lrt_api.remove_task( + long_running_manager.rpc_client, long_running_manager, long_running_manager.get_task_context(request), path_params.task_id, diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py index 11131840a56d..1d177d56e145 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py @@ -8,6 +8,7 @@ from aiohttp.web import HTTPException from common_library.json_serialization import json_dumps from pydantic import AnyHttpUrl, TypeAdapter +from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings from ...aiohttp import status @@ -20,7 +21,7 @@ DEFAULT_STALE_TASK_CHECK_INTERVAL, DEFAULT_STALE_TASK_DETECT_TIMEOUT, ) -from ...long_running_tasks.models import TaskContext, TaskGet +from ...long_running_tasks.models import RabbitNamespace, TaskContext, TaskGet from ...long_running_tasks.task import RedisNamespace, RegisteredTaskName from ..typing_extension import Handler from . import _routes @@ -63,6 +64,7 @@ async def start_long_running_task( task_id = None try: task_id = await lrt_api.start_task( + long_running_manager.rpc_client, long_running_manager, registerd_task_name, fire_and_forget=fire_and_forget, @@ -97,7 +99,12 @@ async def start_long_running_task( except asyncio.CancelledError: # remove the task, the client was disconnected if task_id: - await lrt_api.remove_task(long_running_manager, task_context, task_id) + await lrt_api.remove_task( + long_running_manager.rpc_client, + long_running_manager, + task_context, + task_id, + ) raise @@ -142,6 +149,8 @@ def setup( router_prefix: str, redis_settings: RedisSettings, redis_namespace: RedisNamespace, + rabbit_settings: RabbitSettings, + rabbit_namespace: RabbitNamespace, handler_check_decorator: Callable = _no_ops_decorator, task_request_context_decorator: Callable = _no_task_context_decorator, stale_task_check_interval: datetime.timedelta = DEFAULT_STALE_TASK_CHECK_INTERVAL, @@ -170,7 +179,9 @@ async def on_cleanup_ctx(app: web.Application) -> AsyncGenerator[None, None]: stale_task_check_interval=stale_task_check_interval, stale_task_detect_timeout=stale_task_detect_timeout, redis_settings=redis_settings, + rabbit_settings=rabbit_settings, redis_namespace=redis_namespace, + rabbit_namespace=rabbit_namespace, ) ) diff --git a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py index 80686b670a47..3157bc6d4259 100644 --- a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py +++ b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py @@ -23,6 +23,7 @@ from servicelib.aiohttp.rest_middlewares import append_rest_middlewares from servicelib.long_running_tasks.models import TaskGet, TaskId, TaskStatus from servicelib.long_running_tasks.task import TaskContext +from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings from tenacity.asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type @@ -30,17 +31,15 @@ from tenacity.wait import wait_fixed pytest_simcore_core_services_selection = [ - "redis", -] - -pytest_simcore_ops_services_selection = [ - "redis-commander", + "rabbit", ] @pytest.fixture def app( - server_routes: web.RouteTableDef, redis_service: RedisSettings + server_routes: web.RouteTableDef, + use_in_memory_redis: RedisSettings, + rabbit_service: RabbitSettings, ) -> web.Application: app = web.Application() app.add_routes(server_routes) @@ -48,8 +47,10 @@ def app( append_rest_middlewares(app, api_version="") long_running_tasks.server.setup( app, - redis_settings=redis_service, + redis_settings=use_in_memory_redis, redis_namespace="test", + rabbit_settings=rabbit_service, + rabbit_namespace="test", router_prefix="/futures", ) @@ -109,7 +110,14 @@ async def test_workflow( ("generated item", 0.8), ("finished", 1.0), ] - assert all(x in progress_updates for x in EXPECTED_MESSAGES) + async for attempt in AsyncRetrying( + wait=wait_fixed(0.1), + stop=stop_after_delay(10), + reraise=True, + retry=retry_if_exception_type(AssertionError), + ): + with attempt: + assert all(x in progress_updates for x in EXPECTED_MESSAGES) # now get the result result_url = client.app.router["get_task_result"].url_for(task_id=task_id) result = await client.get(f"{result_url}") diff --git a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_client.py b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_client.py index 0d6265197345..6310b679ac8b 100644 --- a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_client.py +++ b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_client.py @@ -15,13 +15,20 @@ long_running_task_request, ) from servicelib.aiohttp.rest_middlewares import append_rest_middlewares +from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings from yarl import URL +pytest_simcore_core_services_selection = [ + "rabbit", +] + @pytest.fixture def app( - server_routes: web.RouteTableDef, use_in_memory_redis: RedisSettings + server_routes: web.RouteTableDef, + use_in_memory_redis: RedisSettings, + rabbit_service: RabbitSettings, ) -> web.Application: app = web.Application() app.add_routes(server_routes) @@ -31,6 +38,8 @@ def app( app, redis_settings=use_in_memory_redis, redis_namespace="test", + rabbit_settings=rabbit_service, + rabbit_namespace="test", router_prefix="/futures", ) @@ -58,7 +67,7 @@ async def test_long_running_task_request_raises_400( client: TestClient, long_running_task_url: URL ): # missing parameters raises - with pytest.raises(ClientResponseError): + with pytest.raises(ClientResponseError): # noqa: PT012 async for _ in long_running_task_request( client.session, long_running_task_url, None ): @@ -95,7 +104,7 @@ async def test_long_running_task_request_timeout( ): assert client.app task: LRTask | None = None - with pytest.raises(asyncio.TimeoutError): + with pytest.raises(asyncio.TimeoutError): # noqa: PT012 async for task in long_running_task_request( client.session, long_running_task_url.with_query(num_strings=10, sleep_time=1), diff --git a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py index 2a8c76b2ca81..925735540e4b 100644 --- a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py +++ b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py @@ -27,10 +27,11 @@ from servicelib.aiohttp.typing_extension import Handler from servicelib.long_running_tasks.models import TaskGet, TaskId from servicelib.long_running_tasks.task import TaskContext +from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings pytest_simcore_core_services_selection = [ - "redis", + "rabbit", ] # WITH TASK CONTEXT # NOTE: as the long running task framework may be used in any number of services @@ -67,7 +68,8 @@ async def _test_task_context_decorator( def app_with_task_context( server_routes: web.RouteTableDef, task_context_decorator, - redis_service: RedisSettings, + use_in_memory_redis: RedisSettings, + rabbit_service: RabbitSettings, ) -> web.Application: app = web.Application() app.add_routes(server_routes) @@ -75,8 +77,10 @@ def app_with_task_context( append_rest_middlewares(app, api_version="") long_running_tasks.server.setup( app, - redis_settings=redis_service, + redis_settings=use_in_memory_redis, redis_namespace="test", + rabbit_settings=rabbit_service, + rabbit_namespace="test", router_prefix="/futures_with_task_context", task_request_context_decorator=task_context_decorator, ) diff --git a/services/web/server/src/simcore_service_webserver/long_running_tasks.py b/services/web/server/src/simcore_service_webserver/long_running_tasks.py index a97c82a5852d..ffa8c1a8b190 100644 --- a/services/web/server/src/simcore_service_webserver/long_running_tasks.py +++ b/services/web/server/src/simcore_service_webserver/long_running_tasks.py @@ -10,16 +10,18 @@ ) from servicelib.aiohttp.long_running_tasks.server import setup from servicelib.aiohttp.typing_extension import Handler +from servicelib.long_running_tasks.models import RabbitNamespace from servicelib.long_running_tasks.task import RedisNamespace -from . import redis +from . import rabbitmq, redis from ._meta import API_VTAG from .login.decorators import login_required from .models import AuthenticatedRequestContext _logger = logging.getLogger(__name__) -_LONG_RUNNING_TASKS_NAMESPACE: Final[RedisNamespace] = "webserver-legacy" +_LRT_REDIS_NAMESPACE: Final[RedisNamespace] = "webserver-legacy" +_LRT_RABBIT_NAMESPACE: Final[RabbitNamespace] = "webserver-legacy" def webserver_request_context_decorator(handler: Handler): @@ -37,10 +39,13 @@ async def _test_task_context_decorator( @ensure_single_setup(__name__, logger=_logger) def setup_long_running_tasks(app: web.Application) -> None: + setup( app, redis_settings=redis.get_plugin_settings(app), - redis_namespace=_LONG_RUNNING_TASKS_NAMESPACE, + rabbit_settings=rabbitmq.get_plugin_settings(app), + redis_namespace=_LRT_REDIS_NAMESPACE, + rabbit_namespace=_LRT_RABBIT_NAMESPACE, router_prefix=f"/{API_VTAG}/tasks-legacy", handler_check_decorator=login_required, task_request_context_decorator=webserver_request_context_decorator, From de9805bd1e4e9e4deec3ab4146c3db5c10c51c8e Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 11 Aug 2025 14:31:41 +0200 Subject: [PATCH 020/119] fixed interfaces --- .../api/rest/containers_long_running_tasks.py | 9 +++++++++ .../simcore_service_dynamic_sidecar/core/application.py | 2 ++ 2 files changed, 11 insertions(+) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_long_running_tasks.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_long_running_tasks.py index 2fe01315f345..a908720f8788 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_long_running_tasks.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_long_running_tasks.py @@ -59,6 +59,7 @@ async def pull_user_servcices_docker_images( try: return await lrt_api.start_task( + long_running_manager.rpc_client, long_running_manager, task_pull_user_servcices_docker_images.__name__, unique=True, @@ -100,6 +101,7 @@ async def create_service_containers_task( # pylint: disable=too-many-arguments try: return await lrt_api.start_task( + long_running_manager.rpc_client, long_running_manager, task_create_service_containers.__name__, unique=True, @@ -134,6 +136,7 @@ async def runs_docker_compose_down_task( try: return await lrt_api.start_task( + long_running_manager.rpc_client, long_running_manager, task_runs_docker_compose_down.__name__, unique=True, @@ -166,6 +169,7 @@ async def state_restore_task( try: return await lrt_api.start_task( + long_running_manager.rpc_client, long_running_manager, task_restore_state.__name__, unique=True, @@ -197,6 +201,7 @@ async def state_save_task( try: return await lrt_api.start_task( + long_running_manager.rpc_client, long_running_manager, task_save_state.__name__, unique=True, @@ -230,6 +235,7 @@ async def ports_inputs_pull_task( try: return await lrt_api.start_task( + long_running_manager.rpc_client, long_running_manager, task_ports_inputs_pull.__name__, unique=True, @@ -263,6 +269,7 @@ async def ports_outputs_pull_task( try: return await lrt_api.start_task( + long_running_manager.rpc_client, long_running_manager, task_ports_outputs_pull.__name__, unique=True, @@ -293,6 +300,7 @@ async def ports_outputs_push_task( try: return await lrt_api.start_task( + long_running_manager.rpc_client, long_running_manager, task_ports_outputs_push.__name__, unique=True, @@ -323,6 +331,7 @@ async def containers_restart_task( try: return await lrt_api.start_task( + long_running_manager.rpc_client, long_running_manager, task_containers_restart.__name__, unique=True, diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py index 46cf61b0ebd7..87788ed88bec 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py @@ -150,6 +150,8 @@ def create_base_app() -> FastAPI: app, redis_settings=app_settings.REDIS_SETTINGS, redis_namespace=f"dy_sidecar-{app_settings.DY_SIDECAR_RUN_ID}", + rabbit_settings=app_settings.RABBIT_SETTINGS, + rabbit_namespace=f"dy_sidecar-{app_settings.DY_SIDECAR_RUN_ID}", ) app.include_router(get_main_router(app)) From 14eede71f2fad96821a4ffe1da1805b678eb72a4 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 11 Aug 2025 14:33:43 +0200 Subject: [PATCH 021/119] fixed interface --- .../api/routes/dynamic_scheduler.py | 4 ++++ .../modules/dynamic_sidecar/module_setup.py | 8 ++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_scheduler.py b/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_scheduler.py index 6e1c2a09acaa..edd06ce9d8e2 100644 --- a/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_scheduler.py @@ -116,6 +116,7 @@ async def _progress_callback( try: return await lrt_api.start_task( + long_running_manager.rpc_client, long_running_manager, _task_remove_service_containers.__name__, unique=True, @@ -181,6 +182,7 @@ async def _progress_callback( try: return await lrt_api.start_task( + long_running_manager.rpc_client, long_running_manager, _task_save_service_state.__name__, unique=True, @@ -228,6 +230,7 @@ async def _progress_callback( try: return await lrt_api.start_task( + long_running_manager.rpc_client, long_running_manager, _task_push_service_outputs.__name__, unique=True, @@ -270,6 +273,7 @@ async def _task_cleanup_service_docker_resources( try: return await lrt_api.start_task( + long_running_manager.rpc_client, long_running_manager, _task_cleanup_service_docker_resources.__name__, unique=True, diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/module_setup.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/module_setup.py index 253f9be601df..d022f92fa279 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/module_setup.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/module_setup.py @@ -1,11 +1,13 @@ from fastapi import FastAPI from servicelib.fastapi import long_running_tasks +from servicelib.long_running_tasks.models import RabbitNamespace from servicelib.long_running_tasks.task import RedisNamespace from ...core.settings import AppSettings from . import api_client, scheduler -_LONG_RUNNING_TASKS_NAMESPACE: RedisNamespace = "director-v2" +_LRT_REDIS_NAMESPACE: RedisNamespace = "director-v2" +_LRT_RABBIT_NAMESPACE: RabbitNamespace = "director-v2" def setup(app: FastAPI) -> None: @@ -15,7 +17,9 @@ def setup(app: FastAPI) -> None: long_running_tasks.server.setup( app, redis_settings=settings.REDIS, - redis_namespace=_LONG_RUNNING_TASKS_NAMESPACE, + redis_namespace=_LRT_REDIS_NAMESPACE, + rabbit_settings=settings.DIRECTOR_V2_RABBITMQ, + rabbit_namespace=_LRT_RABBIT_NAMESPACE, ) async def on_startup() -> None: From 119b892a12c9d8aea1f1c531fa52fbb319f17f7a Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 11 Aug 2025 14:36:40 +0200 Subject: [PATCH 022/119] makes settings manadatory --- .../src/simcore_service_storage/core/settings.py | 5 +---- .../modules/long_running_tasks.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/services/storage/src/simcore_service_storage/core/settings.py b/services/storage/src/simcore_service_storage/core/settings.py index a3725ac48576..9d49f4660ba8 100644 --- a/services/storage/src/simcore_service_storage/core/settings.py +++ b/services/storage/src/simcore_service_storage/core/settings.py @@ -75,10 +75,7 @@ class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): ] STORAGE_RABBITMQ: Annotated[ - RabbitSettings | None, - Field( - json_schema_extra={"auto_default_from_env": True}, - ), + RabbitSettings, Field(json_schema_extra={"auto_default_from_env": True}) ] STORAGE_S3_CLIENT_MAX_TRANSFER_CONCURRENCY: Annotated[ diff --git a/services/storage/src/simcore_service_storage/modules/long_running_tasks.py b/services/storage/src/simcore_service_storage/modules/long_running_tasks.py index 834ec2dcbb76..88e81fa75d94 100644 --- a/services/storage/src/simcore_service_storage/modules/long_running_tasks.py +++ b/services/storage/src/simcore_service_storage/modules/long_running_tasks.py @@ -2,18 +2,23 @@ from fastapi import FastAPI from servicelib.fastapi.long_running_tasks._server import setup +from servicelib.long_running_tasks.models import RabbitNamespace from servicelib.long_running_tasks.task import RedisNamespace from .._meta import API_VTAG from ..core.settings import get_application_settings -_LONG_RUNNING_TASKS_NAMESPACE: Final[RedisNamespace] = "storage" +_LRT_REDIS_NAMESPACE: Final[RedisNamespace] = "storage" +_LRT_RABBIT_NAMESPACE: Final[RabbitNamespace] = "storage" def setup_rest_api_long_running_tasks_for_uploads(app: FastAPI) -> None: + settings = get_application_settings(app) setup( app, router_prefix=f"/{API_VTAG}/futures", - redis_settings=get_application_settings(app).STORAGE_REDIS, - redis_namespace=_LONG_RUNNING_TASKS_NAMESPACE, + redis_settings=settings.STORAGE_REDIS, + redis_namespace=_LRT_REDIS_NAMESPACE, + rabbit_settings=settings.STORAGE_RABBITMQ, + rabbit_namespace=_LRT_RABBIT_NAMESPACE, ) From f5a8f1bf0584f4d03e71f5201d882d6235b1c6cd Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 11 Aug 2025 14:39:59 +0200 Subject: [PATCH 023/119] fixed interface --- .../src/simcore_service_webserver/long_running_tasks.py | 4 ++-- .../web/server/src/simcore_service_webserver/tasks/_rest.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/long_running_tasks.py b/services/web/server/src/simcore_service_webserver/long_running_tasks.py index ffa8c1a8b190..86951d25ab78 100644 --- a/services/web/server/src/simcore_service_webserver/long_running_tasks.py +++ b/services/web/server/src/simcore_service_webserver/long_running_tasks.py @@ -13,7 +13,7 @@ from servicelib.long_running_tasks.models import RabbitNamespace from servicelib.long_running_tasks.task import RedisNamespace -from . import rabbitmq, redis +from . import rabbitmq_settings, redis from ._meta import API_VTAG from .login.decorators import login_required from .models import AuthenticatedRequestContext @@ -43,7 +43,7 @@ def setup_long_running_tasks(app: web.Application) -> None: setup( app, redis_settings=redis.get_plugin_settings(app), - rabbit_settings=rabbitmq.get_plugin_settings(app), + rabbit_settings=rabbitmq_settings.get_plugin_settings(app), redis_namespace=_LRT_REDIS_NAMESPACE, rabbit_namespace=_LRT_RABBIT_NAMESPACE, router_prefix=f"/{API_VTAG}/tasks-legacy", diff --git a/services/web/server/src/simcore_service_webserver/tasks/_rest.py b/services/web/server/src/simcore_service_webserver/tasks/_rest.py index 800d470221e9..6b3e567a3f6c 100644 --- a/services/web/server/src/simcore_service_webserver/tasks/_rest.py +++ b/services/web/server/src/simcore_service_webserver/tasks/_rest.py @@ -59,6 +59,7 @@ async def get_async_jobs(request: web.Request) -> web.Response: inprocess_long_running_manager = get_long_running_manager(request.app) inprocess_tracked_tasks = await lrt_api.list_tasks( + inprocess_long_running_manager.rpc_client, inprocess_long_running_manager, inprocess_long_running_manager.get_task_context(request), ) From e664ba84f927ea681e17f55cf4502ad2a8952092 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 11 Aug 2025 15:01:28 +0200 Subject: [PATCH 024/119] fixed broken test --- .../tests/deferred_tasks/test__base_deferred_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/service-library/tests/deferred_tasks/test__base_deferred_handler.py b/packages/service-library/tests/deferred_tasks/test__base_deferred_handler.py index 6a3a872c8616..a26156f2c3f2 100644 --- a/packages/service-library/tests/deferred_tasks/test__base_deferred_handler.py +++ b/packages/service-library/tests/deferred_tasks/test__base_deferred_handler.py @@ -50,10 +50,10 @@ class MockKeys(StrAutoEnum): @pytest.fixture async def redis_client_sdk( - redis_service: RedisSettings, + use_in_memory_redis: RedisSettings, ) -> AsyncIterable[RedisClientSDK]: sdk = RedisClientSDK( - redis_service.build_redis_dsn(RedisDatabase.DEFERRED_TASKS), + use_in_memory_redis.build_redis_dsn(RedisDatabase.DEFERRED_TASKS), decode_responses=False, client_name="pytest", ) From 5738b656cf6235337aa84bf25bc8cfe984c91d89 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 11 Aug 2025 15:16:43 +0200 Subject: [PATCH 025/119] fixed tests director-v2 --- .../unit/test_api_route_dynamic_scheduler.py | 13 ++++++++++++- .../test_modules_dynamic_sidecar_scheduler.py | 17 +++++++++++++---- ...st_modules_dynamic_sidecar_scheduler_task.py | 13 ++++++++++++- 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/services/director-v2/tests/unit/test_api_route_dynamic_scheduler.py b/services/director-v2/tests/unit/test_api_route_dynamic_scheduler.py index aa0eea7c7d7c..941d83bd97a6 100644 --- a/services/director-v2/tests/unit/test_api_route_dynamic_scheduler.py +++ b/services/director-v2/tests/unit/test_api_route_dynamic_scheduler.py @@ -7,6 +7,8 @@ import pytest import respx +from common_library.json_serialization import json_dumps +from common_library.serialization import model_dump_with_secrets from faker import Faker from fastapi import status from httpx import Response @@ -15,6 +17,7 @@ from models_library.service_settings_labels import SimcoreServiceLabels from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict +from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings from simcore_service_director_v2.models.dynamic_services_scheduler import SchedulerData from simcore_service_director_v2.modules.dynamic_sidecar.errors import ( @@ -25,12 +28,16 @@ ) from starlette.testclient import TestClient +pytest_simcore_core_services_selection = [ + "rabbit", +] + @pytest.fixture def mock_env( use_in_memory_redis: RedisSettings, mock_exclusive: None, - disable_rabbitmq: None, + rabbit_service: RabbitSettings, disable_postgres: None, mock_env: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, @@ -50,6 +57,10 @@ def mock_env( monkeypatch.setenv("S3_REGION", faker.pystr()) monkeypatch.setenv("S3_SECRET_KEY", faker.pystr()) monkeypatch.setenv("S3_BUCKET_NAME", faker.pystr()) + monkeypatch.setenv( + "DIRECTOR_V2_RABBITMQ", + json_dumps(model_dump_with_secrets(rabbit_service, show_secrets=True)), + ) @pytest.fixture diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py index 07deb1aeb8e6..e2c79a8287d1 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py @@ -12,6 +12,8 @@ import pytest import respx +from common_library.json_serialization import json_dumps +from common_library.serialization import model_dump_with_secrets from faker import Faker from fastapi import FastAPI from models_library.api_schemas_directorv2.dynamic_services_service import ( @@ -23,6 +25,7 @@ from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from respx.router import MockRouter +from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings from simcore_service_director_v2.models.dynamic_services_scheduler import ( DockerContainerInspect, @@ -54,10 +57,12 @@ # and ensure faster tests _TEST_SCHEDULER_INTERVAL_SECONDS: Final[NonNegativeFloat] = 0.1 -log = logging.getLogger(__name__) +_logger = logging.getLogger(__name__) -pytest_simcore_core_services_selection = ["postgres"] +pytest_simcore_core_services_selection = [ + "rabbit", +] pytest_simcore_ops_services_selection = ["adminer"] @@ -128,7 +133,7 @@ def mock_env( use_in_memory_redis: RedisSettings, mock_exclusive: None, disable_postgres: None, - disable_rabbitmq: None, + rabbit_service: RabbitSettings, mock_env: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, simcore_services_network_name: str, @@ -146,6 +151,10 @@ def mock_env( monkeypatch.setenv("S3_REGION", faker.pystr()) monkeypatch.setenv("S3_SECRET_KEY", faker.pystr()) monkeypatch.setenv("S3_BUCKET_NAME", faker.pystr()) + monkeypatch.setenv( + "DIRECTOR_V2_RABBITMQ", + json_dumps(model_dump_with_secrets(rabbit_service, show_secrets=True)), + ) @pytest.fixture @@ -166,7 +175,7 @@ async def action( scheduler_data: SchedulerData, # noqa: ARG003 ) -> None: message = f"{cls.__name__} action triggered" - log.warning(message) + _logger.warning(message) # replace REGISTERED EVENTS mocker.patch( diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler_task.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler_task.py index e8ed258bbea5..4ce2d40ed545 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler_task.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler_task.py @@ -12,6 +12,8 @@ import httpx import pytest import respx +from common_library.json_serialization import json_dumps +from common_library.serialization import model_dump_with_secrets from faker import Faker from fastapi import FastAPI from models_library.docker import DockerNodeID @@ -20,6 +22,7 @@ from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from respx.router import MockRouter +from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings from simcore_service_director_v2.models.dynamic_services_scheduler import SchedulerData from simcore_service_director_v2.modules.dynamic_sidecar.api_client._public import ( @@ -39,6 +42,10 @@ DynamicSidecarsScheduler, ) +pytest_simcore_core_services_selection = [ + "rabbit", +] + SCHEDULER_INTERVAL_SECONDS: Final[float] = 0.1 @@ -46,7 +53,7 @@ def mock_env( use_in_memory_redis: RedisSettings, disable_postgres: None, - disable_rabbitmq: None, + rabbit_service: RabbitSettings, mock_env: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, simcore_services_network_name: str, @@ -64,7 +71,11 @@ def mock_env( "POSTGRES_USER": "", "POSTGRES_PASSWORD": "", "POSTGRES_DB": "", + "DIRECTOR_V2_RABBITMQ": json_dumps( + model_dump_with_secrets(rabbit_service, show_secrets=True) + ), } + setenvs_from_dict(monkeypatch, disabled_services_envs) monkeypatch.setenv("DIRECTOR_V2_DYNAMIC_SCHEDULER_ENABLED", "true") From 84b17df09f9bcef80aa7a302a2ac5a1fe6adf0ae Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 12 Aug 2025 09:12:59 +0200 Subject: [PATCH 026/119] fixed serialization of result and enhanced registration --- .../aiohttp/long_running_tasks/_server.py | 2 +- .../long_running_tasks/_rabbit/lrt_client.py | 9 +++++++- .../long_running_tasks/_rabbit/lrt_server.py | 21 +++++++++++++++++-- ...dis_serialization.py => _serialization.py} | 0 .../src/servicelib/long_running_tasks/task.py | 10 ++++++++- ...long_running_tasks__redis_serialization.py | 2 +- ...test_long_running_tasks__serialization.py} | 2 +- 7 files changed, 39 insertions(+), 7 deletions(-) rename packages/service-library/src/servicelib/long_running_tasks/{_redis_serialization.py => _serialization.py} (100%) rename packages/service-library/tests/long_running_tasks/{test_long_running_tasks__error_serialization.py => test_long_running_tasks__serialization.py} (95%) diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py index 1d177d56e145..3b843490f24e 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py @@ -13,7 +13,7 @@ from ...aiohttp import status from ...long_running_tasks import lrt_api -from ...long_running_tasks._redis_serialization import ( +from ...long_running_tasks._serialization import ( BaseObjectSerializer, register_custom_serialization, ) diff --git a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py index 1f222187e22d..f0722627ccf9 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py @@ -8,6 +8,7 @@ from ...logging_utils import log_decorator from ...long_running_tasks.task import RegisteredTaskName from ...rabbitmq._client_rpc import RabbitMQRPCClient +from .._serialization import string_to_object from ..models import RabbitNamespace, TaskBase, TaskContext, TaskId, TaskStatus from .namespace import get_namespace @@ -91,13 +92,19 @@ async def get_task_result( task_context: TaskContext, task_id: TaskId, ) -> Any: - return await rabbitmq_rpc_client.request( + serialized_result = await rabbitmq_rpc_client.request( get_namespace(namespace), TypeAdapter(RPCMethodName).validate_python("get_task_result"), task_context=task_context, task_id=task_id, timeout_s=_RPC_TIMEOUT_NORMAL_REQUEST, ) + assert isinstance(serialized_result, str) # nosec + task_result = string_to_object(serialized_result) + + if isinstance(task_result, Exception): + raise task_result + return task_result @log_decorator(_logger, level=logging.DEBUG) diff --git a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_server.py b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_server.py index fd783658d40f..d03ce38fd8e1 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_server.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_server.py @@ -5,6 +5,7 @@ from ...logging_errors import create_troubleshootting_log_kwargs from ...rabbitmq import RPCRouter +from .._serialization import object_to_string from ..base_long_running_manager import BaseLongRunningManager from ..errors import BaseLongRunningError, TaskNotCompletedError, TaskNotFoundError from ..models import TaskBase, TaskContext, TaskId, TaskStatus @@ -57,8 +58,7 @@ async def get_task_status( ) -@router.expose(reraise_if_error_type=(BaseLongRunningError, Exception)) -async def get_task_result( +async def _get_transferarble_task_result( long_running_manager: BaseLongRunningManager, *, task_context: TaskContext, @@ -90,6 +90,23 @@ async def get_task_result( raise +@router.expose(reraise_if_error_type=(BaseLongRunningError, Exception)) +async def get_task_result( + long_running_manager: BaseLongRunningManager, + *, + task_context: TaskContext, + task_id: TaskId, +) -> str: + try: + return object_to_string( + await _get_transferarble_task_result( + long_running_manager, task_context=task_context, task_id=task_id + ) + ) + except Exception as exc: # pylint:disable=broad-exception-caught + return object_to_string(exc) + + @router.expose(reraise_if_error_type=(BaseLongRunningError,)) async def remove_task( long_running_manager: BaseLongRunningManager, diff --git a/packages/service-library/src/servicelib/long_running_tasks/_redis_serialization.py b/packages/service-library/src/servicelib/long_running_tasks/_serialization.py similarity index 100% rename from packages/service-library/src/servicelib/long_running_tasks/_redis_serialization.py rename to packages/service-library/src/servicelib/long_running_tasks/_serialization.py diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index bba9355b138b..c38b784e7b4e 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -23,7 +23,7 @@ from ..background_task import create_periodic_task from ..redis import RedisClientSDK, exclusive -from ._redis_serialization import object_to_string, string_to_object +from ._serialization import object_to_string, string_to_object from ._store.base import BaseStore from ._store.redis import RedisStore from .errors import ( @@ -64,6 +64,14 @@ class TaskRegistry: def register(cls, task: TaskProtocol) -> None: cls.REGISTERED_TASKS[task.__name__] = task + @classmethod + def register_partial( + cls, task: TaskProtocol, *partial_args, **partial_kwargs + ) -> None: + cls.REGISTERED_TASKS[task.__name__] = functools.partial( + task, *partial_args, **partial_kwargs + ) + @classmethod def unregister(cls, task: TaskProtocol) -> None: if task.__name__ in cls.REGISTERED_TASKS: diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks__redis_serialization.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks__redis_serialization.py index f0d0d14f165b..a4d84f8873e7 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks__redis_serialization.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks__redis_serialization.py @@ -3,7 +3,7 @@ import pytest from aiohttp.web import HTTPException, HTTPInternalServerError from servicelib.aiohttp.long_running_tasks._server import AiohttpHTTPExceptionSerializer -from servicelib.long_running_tasks._redis_serialization import ( +from servicelib.long_running_tasks._serialization import ( object_to_string, register_custom_serialization, string_to_object, diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks__error_serialization.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks__serialization.py similarity index 95% rename from packages/service-library/tests/long_running_tasks/test_long_running_tasks__error_serialization.py rename to packages/service-library/tests/long_running_tasks/test_long_running_tasks__serialization.py index f0d0d14f165b..a4d84f8873e7 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks__error_serialization.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks__serialization.py @@ -3,7 +3,7 @@ import pytest from aiohttp.web import HTTPException, HTTPInternalServerError from servicelib.aiohttp.long_running_tasks._server import AiohttpHTTPExceptionSerializer -from servicelib.long_running_tasks._redis_serialization import ( +from servicelib.long_running_tasks._serialization import ( object_to_string, register_custom_serialization, string_to_object, From fb9ff382ba425915620b4af4e43d69cc24a38fbb Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 12 Aug 2025 11:45:51 +0200 Subject: [PATCH 027/119] refactor interface for registration --- .../src/servicelib/long_running_tasks/task.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index c38b784e7b4e..6b7d38bdedf9 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -65,12 +65,10 @@ def register(cls, task: TaskProtocol) -> None: cls.REGISTERED_TASKS[task.__name__] = task @classmethod - def register_partial( - cls, task: TaskProtocol, *partial_args, **partial_kwargs - ) -> None: + def register_partial(cls, task: TaskProtocol, **partial_kwargs) -> None: cls.REGISTERED_TASKS[task.__name__] = functools.partial( - task, *partial_args, **partial_kwargs - ) + task, **partial_kwargs + ) # type: ignore[assignment] @classmethod def unregister(cls, task: TaskProtocol) -> None: From 258a979c85a55b632a5a400ab66f81b746d769e4 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 12 Aug 2025 11:46:19 +0200 Subject: [PATCH 028/119] fixed tests --- .../projects/_controller/_rest_utils.py | 9 +++- .../projects/_controller/nodes_rest.py | 4 +- .../projects/_controller/projects_rest.py | 18 +++++--- .../projects/_crud_api_create.py | 41 ++++++++++--------- .../projects/_permalink_service.py | 26 +++++++----- .../projects/plugin.py | 5 +++ .../_projects_permalinks.py | 15 ++++--- .../utils_aiohttp.py | 13 +++--- .../02/test_projects_cancellations.py | 11 +++-- .../02/test_projects_crud_handlers.py | 6 +++ .../02/test_projects_metadata_handlers.py | 7 ++++ .../02/test_projects_nodes_handler.py | 6 +++ .../02/test_projects_states_handlers.py | 7 ++++ 13 files changed, 114 insertions(+), 54 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/_rest_utils.py b/services/web/server/src/simcore_service_webserver/projects/_controller/_rest_utils.py index beab5959668f..077761083a23 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/_rest_utils.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/_rest_utils.py @@ -4,6 +4,7 @@ from models_library.rest_pagination_utils import paginate_data from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON from servicelib.rest_constants import RESPONSE_MODEL_POLICY +from yarl import URL from .. import _permalink_service from .._crud_api_read import _paralell_update @@ -11,13 +12,17 @@ async def aggregate_data_to_projects_from_request( - request: web.Request, + app: web.Application, + url: URL, + headers: dict[str, str], projects: list[ProjectDict], ) -> list[ProjectDict]: update_permalink_per_project = [ # permalink - _permalink_service.aggregate_permalink_in_project(request, project=prj) + _permalink_service.aggregate_permalink_in_project( + app, url, headers, project=prj + ) for prj in projects ] diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/nodes_rest.py b/services/web/server/src/simcore_service_webserver/projects/_controller/nodes_rest.py index f0637cc256ea..277b80991374 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/nodes_rest.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/nodes_rest.py @@ -322,7 +322,8 @@ async def _stop_dynamic_service_task( return web.json_response(status=status.HTTP_204_NO_CONTENT) -TaskRegistry.register(_stop_dynamic_service_task) +def register_stop_dynamic_service_task(app: web.Application) -> None: + TaskRegistry.register_partial(_stop_dynamic_service_task, app=app) @routes.post( @@ -352,7 +353,6 @@ async def stop_node(request: web.Request) -> web.Response: _stop_dynamic_service_task.__name__, task_context=jsonable_encoder(req_ctx), # task arguments from here on --- - app=request.app, dynamic_service_stop=DynamicServiceStop( user_id=req_ctx.user_id, project_id=path_params.project_id, diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py index 538644fc2a20..2b0b4a6f3eee 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py @@ -101,7 +101,8 @@ async def create_project(request: web.Request): fire_and_forget=True, task_context=jsonable_encoder(req_ctx), # arguments - request=request, + url=request.url, + headers=dict(request.headers), new_project_was_hidden_before_data_was_copied=query_params.hidden, from_study=query_params.from_study, as_template=query_params.as_template, @@ -158,7 +159,7 @@ async def list_projects(request: web.Request): ) projects = await _rest_utils.aggregate_data_to_projects_from_request( - request, projects + request.app, request.url, dict(request.headers), projects ) return _rest_utils.create_page_response( @@ -197,7 +198,7 @@ async def list_projects_full_search(request: web.Request): ) projects = await _rest_utils.aggregate_data_to_projects_from_request( - request, projects + request.app, request.url, dict(request.headers), projects ) return _rest_utils.create_page_response( @@ -247,7 +248,9 @@ async def get_active_project(request: web.Request) -> web.Response: ) # updates project's permalink field - await update_or_pop_permalink_in_project(request, project) + await update_or_pop_permalink_in_project( + request.app, request.url, dict(request.headers), project + ) data = ProjectGet.from_domain_model(project).data(exclude_unset=True) @@ -280,7 +283,9 @@ async def get_project(request: web.Request): ) # Adds permalink - await update_or_pop_permalink_in_project(request, project) + await update_or_pop_permalink_in_project( + request.app, request.url, dict(request.headers), project + ) data = ProjectGet.from_domain_model(project).data(exclude_unset=True) return envelope_json_response(data) @@ -419,7 +424,8 @@ async def clone_project(request: web.Request): fire_and_forget=True, task_context=jsonable_encoder(req_ctx), # arguments - request=request, + url=request.url, + headers=dict(request.headers), new_project_was_hidden_before_data_was_copied=False, from_study=path_params.project_id, as_template=False, diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py index 7761e17ad13c..5ae9663defc9 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py @@ -27,6 +27,7 @@ ProjectNodeCreate, ) from simcore_postgres_database.webserver_models import ProjectType as ProjectTypeDB +from yarl import URL from ..application_settings import get_application_settings from ..catalog import catalog_service @@ -249,7 +250,9 @@ async def _compose_project_data( async def create_project( # pylint: disable=too-many-arguments,too-many-branches,too-many-statements # noqa: C901, PLR0913 progress: TaskProgress, *, - request: web.Request, + app: web.Application, + url: URL, + headers: dict[str, str], new_project_was_hidden_before_data_was_copied: bool, from_study: ProjectID | None, as_template: bool, @@ -281,7 +284,6 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche web.HTTPUnauthorized: """ - assert request.app # nosec _logger.info( "create_project for '%s' with %s %s %s", f"{user_id=}", @@ -290,7 +292,7 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche f"{from_study=}", ) - _projects_repository_legacy = ProjectDBAPI.get_from_app_context(request.app) + _projects_repository_legacy = ProjectDBAPI.get_from_app_context(app) new_project: ProjectDict = {} copy_file_coro = None @@ -303,7 +305,7 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche if predefined_project: if workspace_id := predefined_project.get("workspaceId", None): await check_user_workspace_access( - request.app, + app, user_id=user_id, workspace_id=workspace_id, product_name=product_name, @@ -312,7 +314,7 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche if folder_id := predefined_project.get("folderId", None): # Check user has access to folder await folders_folders_repository.get_for_user_or_workspace( - request.app, + app, folder_id=folder_id, product_name=product_name, user_id=user_id if workspace_id is None else None, @@ -328,7 +330,7 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche project_node_coro, copy_file_coro, ) = await _prepare_project_copy( - request.app, + app, user_id=user_id, product_name=product_name, src_project_uuid=from_study, @@ -342,7 +344,7 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche # 1.2 does project belong to some folder? workspace_id = new_project["workspaceId"] prj_to_folder_db = await _folders_repository.get_project_to_folder( - request.app, + app, project_id=from_study, private_workspace_user_id_or_none=( user_id if workspace_id is None else None @@ -361,7 +363,7 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche if predefined_project: # 2. overrides with optional body and re-validate new_project, project_nodes = await _compose_project_data( - request.app, + app, user_id=user_id, new_project=new_project, predefined_project=predefined_project, @@ -378,7 +380,7 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche ) # add parent linking if needed await set_project_ancestors( - request.app, + app, user_id=user_id, project_uuid=new_project["uuid"], parent_project_uuid=parent_project_uuid, @@ -389,7 +391,7 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche # 3.2 move project to proper folder if folder_id: await _folders_repository.insert_project_to_folder( - request.app, + app, project_id=new_project["uuid"], folder_id=folder_id, private_workspace_user_id_or_none=( @@ -405,20 +407,20 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche # 5. unhide the project if needed since it is now complete if not new_project_was_hidden_before_data_was_copied: await _projects_repository.patch_project( - request.app, + app, project_uuid=new_project["uuid"], new_partial_project_data={"hidden": False}, ) # update the network information in director-v2 await dynamic_scheduler_service.update_projects_networks( - request.app, project_id=ProjectID(new_project["uuid"]) + app, project_id=ProjectID(new_project["uuid"]) ) await progress.update() # This is a new project and every new graph needs to be reflected in the pipeline tables await director_v2_service.create_or_update_pipeline( - request.app, + app, user_id, new_project["uuid"], product_name, @@ -433,12 +435,12 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche user_id=user_id, project=new_project, is_template=as_template, - app=request.app, + app=app, ) await progress.update() # Adds permalink - await update_or_pop_permalink_in_project(request, new_project) + await update_or_pop_permalink_in_project(app, url, headers, new_project) # Adds folderId user_specific_project_data_db = ( @@ -454,7 +456,7 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche # Overwrite project access rights if workspace_id: workspace: UserWorkspaceWithAccessRights = await get_user_workspace( - request.app, + app, user_id=user_id, workspace_id=workspace_id, product_name=product_name, @@ -497,7 +499,7 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche except (ParentProjectNotFoundError, ParentNodeNotFoundError) as exc: if project_uuid := new_project.get("uuid"): await _projects_service.submit_delete_project_task( - app=request.app, + app=app, project_uuid=project_uuid, user_id=user_id, simcore_user_agent=simcore_user_agent, @@ -511,7 +513,7 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche ) if project_uuid := new_project.get("uuid"): await _projects_service.submit_delete_project_task( - app=request.app, + app=app, project_uuid=project_uuid, user_id=user_id, simcore_user_agent=simcore_user_agent, @@ -519,4 +521,5 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche raise -TaskRegistry.register(create_project) +def register_create_project_task(app: web.Application) -> None: + TaskRegistry.register_partial(create_project, app=app) diff --git a/services/web/server/src/simcore_service_webserver/projects/_permalink_service.py b/services/web/server/src/simcore_service_webserver/projects/_permalink_service.py index e6fa6e61a8b2..bb1a470e9546 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_permalink_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_permalink_service.py @@ -5,6 +5,7 @@ from aiohttp import web from models_library.api_schemas_webserver.permalinks import ProjectPermalink from models_library.projects import ProjectID +from yarl import URL from .exceptions import PermalinkFactoryError, PermalinkNotAllowedError from .models import ProjectDict @@ -15,9 +16,12 @@ class CreateLinkCoroutine(Protocol): async def __call__( - self, request: web.Request, project_uuid: ProjectID - ) -> ProjectPermalink: - ... + self, + app: web.Application, + url: URL, + headers: dict[str, str], + project_uuid: ProjectID, + ) -> ProjectPermalink: ... def register_factory(app: web.Application, factory_coro: CreateLinkCoroutine): @@ -39,13 +43,13 @@ def _get_factory(app: web.Application) -> CreateLinkCoroutine: async def _create_permalink( - request: web.Request, project_uuid: ProjectID + app: web.Application, url: URL, headers: dict[str, str], project_uuid: ProjectID ) -> ProjectPermalink: - create_coro: CreateLinkCoroutine = _get_factory(request.app) + create_coro: CreateLinkCoroutine = _get_factory(app) try: permalink: ProjectPermalink = await asyncio.wait_for( - create_coro(request=request, project_uuid=project_uuid), + create_coro(app, url, headers, project_uuid), timeout=_PERMALINK_CREATE_TIMEOUT_S, ) return permalink @@ -55,7 +59,7 @@ async def _create_permalink( async def update_or_pop_permalink_in_project( - request: web.Request, project: ProjectDict + app: web.Application, url: URL, headers: dict[str, str], project: ProjectDict ) -> ProjectPermalink | None: """Updates permalink entry in project @@ -64,7 +68,9 @@ async def update_or_pop_permalink_in_project( If fails, it pops it from project (so it is not set in the pydantic model. SEE ProjectGet.permalink) """ try: - permalink = await _create_permalink(request, project_uuid=project["uuid"]) + permalink = await _create_permalink( + app, url, headers, project_uuid=project["uuid"] + ) assert permalink # nosec project["permalink"] = permalink @@ -78,12 +84,12 @@ async def update_or_pop_permalink_in_project( async def aggregate_permalink_in_project( - request: web.Request, project: ProjectDict + app: web.Application, url: URL, headers: dict[str, str], project: ProjectDict ) -> ProjectDict: """ Adapter to use in parallel aggregation of fields in a project dataset """ - await update_or_pop_permalink_in_project(request, project) + await update_or_pop_permalink_in_project(app, url, headers, project) return project diff --git a/services/web/server/src/simcore_service_webserver/projects/plugin.py b/services/web/server/src/simcore_service_webserver/projects/plugin.py index 5028739d881b..275a15a164ee 100644 --- a/services/web/server/src/simcore_service_webserver/projects/plugin.py +++ b/services/web/server/src/simcore_service_webserver/projects/plugin.py @@ -29,6 +29,8 @@ wallets_rest, workspaces_rest, ) +from ._controller.nodes_rest import register_stop_dynamic_service_task +from ._crud_api_create import register_create_project_task from ._projects_repository_legacy import setup_projects_db from ._security_service import setup_projects_access @@ -75,4 +77,7 @@ def setup_projects(app: web.Application) -> bool: app.router.add_routes(workspaces_rest.routes) app.router.add_routes(trash_rest.routes) + register_create_project_task(app) + register_stop_dynamic_service_task(app) + return True diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects_permalinks.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects_permalinks.py index 92206532c6be..37ee33ba34a8 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects_permalinks.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects_permalinks.py @@ -9,6 +9,7 @@ from typing_extensions import ( # https://docs.pydantic.dev/latest/api/standard_library_types/#typeddict TypedDict, ) +from yarl import URL from ..db.plugin import get_database_engine_legacy from ..projects.exceptions import PermalinkNotAllowedError, ProjectNotFoundError @@ -33,8 +34,10 @@ class _GroupAccessRightsDict(TypedDict): def create_permalink_for_study( - request: web.Request, + app: web.Application, *, + url: URL, + headers: dict[str, str], project_uuid: ProjectID | ProjectIDStr, project_type: ProjectType, project_access_rights: dict[_GroupID, _GroupAccessRightsDict], @@ -65,7 +68,7 @@ def create_permalink_for_study( raise PermalinkNotAllowedError(msg) # create - url_for = create_url_for_function(request) + url_for = create_url_for_function(app, url, headers) permalink = TypeAdapter(HttpUrl).validate_python( url_for(route_name="get_redirection_to_study_page", id=f"{project_uuid}"), ) @@ -77,14 +80,14 @@ def create_permalink_for_study( async def permalink_factory( - request: web.Request, project_uuid: ProjectID + app: web.Application, url: URL, headers: dict[str, str], project_uuid: ProjectID ) -> ProjectPermalink: """ - Assumes project_id is up-to-date in the database """ # NOTE: next iterations will mobe this as part of the project repository pattern - engine = get_database_engine_legacy(request.app) + engine = get_database_engine_legacy(app) async with engine.acquire() as conn: access_rights_subquery = ( sa.select( @@ -121,7 +124,9 @@ async def permalink_factory( raise ProjectNotFoundError(project_uuid=project_uuid) return create_permalink_for_study( - request, + app, + url=url, + headers=headers, project_uuid=row.uuid, project_type=row.type, project_access_rights=row.access_rights, diff --git a/services/web/server/src/simcore_service_webserver/utils_aiohttp.py b/services/web/server/src/simcore_service_webserver/utils_aiohttp.py index b70a6c6897aa..0458897f12a8 100644 --- a/services/web/server/src/simcore_service_webserver/utils_aiohttp.py +++ b/services/web/server/src/simcore_service_webserver/utils_aiohttp.py @@ -35,8 +35,9 @@ def get_routes_view(routes: RouteTableDef) -> str: return fh.getvalue() -def create_url_for_function(request: web.Request) -> Callable: - app = request.app +def create_url_for_function( + app: web.Application, url: URL, headers: dict[str, str] +) -> Callable: def _url_for(route_name: str, **params: dict[str, Any]) -> str: """Reverse URL constructing using named resources""" @@ -44,16 +45,16 @@ def _url_for(route_name: str, **params: dict[str, Any]) -> str: rel_url: URL = app.router[route_name].url_for( **{k: f"{v}" for k, v in params.items()} ) - url: URL = ( - request.url.origin() + _url: URL = ( + url.origin() .with_scheme( # Custom header by traefik. See labels in docker-compose as: # - traefik.http.middlewares.${SWARM_STACK_NAME_NO_HYPHEN}_sslheader.headers.customrequestheaders.X-Forwarded-Proto=http - request.headers.get(X_FORWARDED_PROTO, request.url.scheme) + headers.get(X_FORWARDED_PROTO, url.scheme) ) .with_path(str(rel_url)) ) - return f"{url}" + return f"{_url}" except KeyError as err: msg = f"Cannot find URL because there is no resource registered as {route_name=}Check name spelling or whether the router was not registered" diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_cancellations.py b/services/web/server/tests/unit/with_dbs/02/test_projects_cancellations.py index fdd9ec0549b9..409459eac39d 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_cancellations.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_cancellations.py @@ -27,6 +27,7 @@ from servicelib.rabbitmq.rpc_interfaces.async_jobs.async_jobs import ( AsyncJobComposedResult, ) +from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings from simcore_postgres_database.models.users import UserRole from simcore_service_webserver._meta import api_version_prefix @@ -36,19 +37,21 @@ from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed +pytest_simcore_core_services_selection = [ + "rabbit", +] + API_PREFIX = "/" + api_version_prefix @pytest.fixture def app_environment( use_in_memory_redis: RedisSettings, + rabbit_settings: RabbitSettings, app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, ) -> EnvVarsDict: - envs_plugins = setenvs_from_dict( - monkeypatch, - {}, - ) + envs_plugins = setenvs_from_dict(monkeypatch, {}) return app_environment | envs_plugins diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py index 4ec2ccf7eefa..70962c0af5fb 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py @@ -32,6 +32,7 @@ from pytest_simcore.helpers.webserver_users import UserInfoDict from servicelib.aiohttp import status from servicelib.rest_constants import X_PRODUCT_NAME_HEADER +from settings_library.rabbit import RabbitSettings from simcore_postgres_database.models.products import products from simcore_postgres_database.models.projects_to_products import projects_to_products from simcore_service_webserver._meta import api_version_prefix @@ -47,6 +48,10 @@ from simcore_service_webserver.utils import to_datetime from yarl import URL +pytest_simcore_core_services_selection = [ + "rabbit", +] + API_PREFIX = "/" + api_version_prefix @@ -189,6 +194,7 @@ async def _assert_get_same_project( ], ) async def test_list_projects( + rabbit_settings: RabbitSettings, client: TestClient, logged_user: dict[str, Any], user_project: dict[str, Any], diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py index 9aa4f2b8161d..ee94a5640aab 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py @@ -28,12 +28,17 @@ ) from pytest_simcore.helpers.webserver_users import UserInfoDict from servicelib.aiohttp import status +from settings_library.rabbit import RabbitSettings from simcore_postgres_database.utils_projects_metadata import ( get as get_db_project_metadata, ) from simcore_service_webserver.projects import _crud_api_delete from simcore_service_webserver.projects.models import ProjectDict +pytest_simcore_core_services_selection = [ + "rabbit", +] + @pytest.mark.acceptance_test( "For https://github.com/ITISFoundation/osparc-simcore/issues/4313" @@ -113,6 +118,7 @@ async def _wait_until_deleted(): @pytest.mark.parametrize(*standard_user_role_response()) async def test_new_project_with_parent_project_node( + rabbit_settings: RabbitSettings, mock_dynamic_scheduler: None, # for deletion mocked_dynamic_services_interface: dict[str, MagicMock], @@ -269,6 +275,7 @@ async def test_new_project_with_invalid_parent_project_node( @pytest.mark.parametrize(*standard_user_role_response()) async def test_set_project_parent_backward_compatibility( + rabbit_settings: RabbitSettings, mock_dynamic_scheduler: None, # for deletion mocked_dynamic_services_interface: dict[str, MagicMock], diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handler.py b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handler.py index ef3fd0c98bd9..29986444c841 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handler.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handler.py @@ -47,6 +47,7 @@ ) from servicelib.aiohttp import status from servicelib.common_headers import UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE +from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings from simcore_postgres_database.models.projects import projects as projects_db_model from simcore_service_webserver.db.models import UserRole @@ -63,6 +64,10 @@ wait_fixed, ) +pytest_simcore_core_services_selection = [ + "rabbit", +] + @pytest.mark.parametrize( "user_role,expected", @@ -926,6 +931,7 @@ async def test_start_node_raises_if_called_with_wrong_data( @pytest.mark.parametrize(*standard_role_response(), ids=str) async def test_stop_node( + rabbit_settings: RabbitSettings, use_in_memory_redis: RedisSettings, client: TestClient, user_project_with_num_dynamic_services: Callable[[int], Awaitable[ProjectDict]], diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py index 67845662e0b0..9ccf777fdc9d 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py @@ -66,6 +66,7 @@ from pytest_simcore.helpers.webserver_users import UserInfoDict from servicelib.aiohttp import status from servicelib.common_headers import UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE +from settings_library.rabbit import RabbitSettings from simcore_postgres_database.models.products import products from simcore_postgres_database.models.wallets import wallets from simcore_service_webserver._meta import API_VTAG @@ -84,6 +85,11 @@ wait_fixed, ) +pytest_simcore_core_services_selection = [ + "rabbit", +] + + RESOURCE_NAME = "projects" API_PREFIX = f"/{API_VTAG}" @@ -304,6 +310,7 @@ async def _delete_project(client: TestClient, project: dict) -> ClientResponse: @pytest.mark.parametrize(*standard_role_response()) async def test_share_project_user_roles( + rabbit_service: RabbitSettings, mock_dynamic_scheduler: None, client: TestClient, logged_user: dict, From 7815f8d6574022f3784bdb74c95935c7890f68d8 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 12 Aug 2025 11:58:09 +0200 Subject: [PATCH 029/119] fixeed tests --- .../04/studies_dispatcher/conftest.py | 4 ++- .../test_studies_dispatcher_handlers.py | 29 ++++++++++++++++++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/conftest.py b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/conftest.py index 1b4b8e20ff28..93ec781cab84 100644 --- a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/conftest.py +++ b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/conftest.py @@ -13,7 +13,9 @@ @pytest.fixture -def app_environment(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch): +def app_environment( + app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch +) -> EnvVarsDict: envs_plugins = setenvs_from_dict( monkeypatch, { diff --git a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_handlers.py b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_handlers.py index e213da910cf9..9c4fec4309d2 100644 --- a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_handlers.py +++ b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_handlers.py @@ -15,17 +15,22 @@ from aiohttp import ClientResponse, ClientSession from aiohttp.test_utils import TestClient, TestServer from aioresponses import aioresponses +from common_library.json_serialization import json_dumps +from common_library.serialization import model_dump_with_secrets from common_library.users_enums import UserRole from models_library.projects_state import ProjectShareState, ProjectStatus from pydantic import BaseModel, ByteSize, TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.assert_checks import assert_status +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.webserver_users import UserInfoDict from pytest_simcore.pydantic_models import ( assert_validation_model, walk_model_examples_in_package, ) from servicelib.aiohttp import status +from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings from settings_library.utils_session import DEFAULT_SESSION_COOKIE_NAME from simcore_service_webserver.studies_dispatcher._core import ViewerInfo @@ -33,6 +38,10 @@ from sqlalchemy.sql import text from yarl import URL +pytest_simcore_core_services_selection = [ + "rabbit", +] + # # FIXTURES OVERRIDES # @@ -76,7 +85,25 @@ def postgres_db(postgres_db: sa.engine.Engine) -> sa.engine.Engine: @pytest.fixture -def web_server(redis_service: RedisSettings, web_server: TestServer) -> TestServer: +def app_environment( + app_environment: EnvVarsDict, + monkeypatch: pytest.MonkeyPatch, + rabbit_service: RabbitSettings, +) -> EnvVarsDict: + return setenvs_from_dict( + monkeypatch, + { + "WEBSERVER_RABBITMQ": json_dumps( + model_dump_with_secrets(rabbit_service, show_secrets=True) + ) + }, + ) + + +@pytest.fixture +def web_server( + redis_service: RedisSettings, rabbit_service: RabbitSettings, web_server: TestServer +) -> TestServer: # # Extends web_server to start redis_service # From 372e668a4e3d0905e11269a64b407a5bf165cebc Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 12 Aug 2025 11:58:34 +0200 Subject: [PATCH 030/119] added todo --- .../service-library/src/servicelib/long_running_tasks/task.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index 6b7d38bdedf9..d5b6fc684ad0 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -60,6 +60,7 @@ def __name__(self) -> str: ... class TaskRegistry: REGISTERED_TASKS: ClassVar[dict[RegisteredTaskName, TaskProtocol]] = {} + # TODO: maybe only use one method to register @classmethod def register(cls, task: TaskProtocol) -> None: cls.REGISTERED_TASKS[task.__name__] = task From 51c455ffaaf96745c4669505ae88cc7e3162d462 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 12 Aug 2025 12:09:51 +0200 Subject: [PATCH 031/119] fixede failing tests --- ...est_studies_dispatcher_projects_permalinks.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/services/web/server/tests/unit/isolated/test_studies_dispatcher_projects_permalinks.py b/services/web/server/tests/unit/isolated/test_studies_dispatcher_projects_permalinks.py index def692faeccd..b448e806a681 100644 --- a/services/web/server/tests/unit/isolated/test_studies_dispatcher_projects_permalinks.py +++ b/services/web/server/tests/unit/isolated/test_studies_dispatcher_projects_permalinks.py @@ -91,7 +91,9 @@ def test_create_permalink(fake_get_project_request: web.Request, is_public: bool project_uuid: str = fake_get_project_request.match_info["project_uuid"] permalink = create_permalink_for_study( - fake_get_project_request, + fake_get_project_request.app, + url=fake_get_project_request.url, + headers=dict(fake_get_project_request.headers), project_uuid=project_uuid, project_type=ProjectType.TEMPLATE, project_access_rights={"1": {"read": True, "write": False, "delete": False}}, @@ -119,7 +121,9 @@ def test_permalink_only_for_template_projects( ): with pytest.raises(PermalinkNotAllowedError): create_permalink_for_study( - fake_get_project_request, + fake_get_project_request.app, + url=fake_get_project_request.url, + headers=dict(fake_get_project_request.headers), **{**valid_project_kwargs, "project_type": ProjectType.STANDARD} ) @@ -129,7 +133,9 @@ def test_permalink_only_when_read_access_to_everyone( ): with pytest.raises(PermalinkNotAllowedError): create_permalink_for_study( - fake_get_project_request, + fake_get_project_request.app, + url=fake_get_project_request.url, + headers=dict(fake_get_project_request.headers), **{ **valid_project_kwargs, "project_access_rights": { @@ -140,7 +146,9 @@ def test_permalink_only_when_read_access_to_everyone( with pytest.raises(PermalinkNotAllowedError): create_permalink_for_study( - fake_get_project_request, + fake_get_project_request.app, + url=fake_get_project_request.url, + headers=dict(fake_get_project_request.headers), **{ **valid_project_kwargs, "project_access_rights": { From fa308707d02e43d1951f56e19305683d70faadc4 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 12 Aug 2025 13:25:06 +0200 Subject: [PATCH 032/119] renamed --- .../src/servicelib/fastapi/long_running_tasks/_server.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py index a1c3da20a74d..53d21173bf1e 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py @@ -59,8 +59,10 @@ async def on_startup() -> None: async def on_shutdown() -> None: if app.state.long_running_manager: - task_manager: FastAPILongRunningManager = app.state.long_running_manager - await task_manager.teardown() + long_running_manager: FastAPILongRunningManager = ( + app.state.long_running_manager + ) + await long_running_manager.teardown() app.add_event_handler("startup", on_startup) app.add_event_handler("shutdown", on_shutdown) From 451e442c9946aa156275619a289c5705b6e1791a Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 13 Aug 2025 08:15:55 +0200 Subject: [PATCH 033/119] fixed tests --- .../src/servicelib/long_running_tasks/task.py | 8 ++- .../api/rest/containers_long_running_tasks.py | 47 +------------ .../core/application.py | 12 +--- .../models/schemas/application_health.py | 4 +- .../modules/long_running_tasks.py | 66 +++++++++++++++---- services/dynamic-sidecar/tests/conftest.py | 4 +- .../tests/unit/api/rest/test_disk.py | 6 +- .../tests/unit/api/rest/test_volumes.py | 3 + .../dynamic-sidecar/tests/unit/conftest.py | 6 +- .../tests/unit/test_api_rest_containers.py | 24 +++++-- ..._api_rest_containers_long_running_tasks.py | 24 ++++++- .../tests/unit/test_api_rest_health.py | 14 +++- .../unit/test_api_rest_prometheus_metrics.py | 39 +++++++---- .../tests/unit/test_models_shared_store.py | 2 + 14 files changed, 154 insertions(+), 105 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index d5b6fc684ad0..700600997d78 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -67,9 +67,11 @@ def register(cls, task: TaskProtocol) -> None: @classmethod def register_partial(cls, task: TaskProtocol, **partial_kwargs) -> None: - cls.REGISTERED_TASKS[task.__name__] = functools.partial( - task, **partial_kwargs - ) # type: ignore[assignment] + partail_task = functools.partial(task, **partial_kwargs) + partail_task.__name__ = ( + task.__name__ + ) # allows to call via the partial of via the orignal method + cls.REGISTERED_TASKS[task.__name__] = partail_task # type: ignore[assignment] @classmethod def unregister(cls, task: TaskProtocol) -> None: diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_long_running_tasks.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_long_running_tasks.py index a908720f8788..f7943f89f923 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_long_running_tasks.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_long_running_tasks.py @@ -1,7 +1,7 @@ from textwrap import dedent from typing import Annotated, cast -from fastapi import APIRouter, Depends, FastAPI, Request, status +from fastapi import APIRouter, Depends, Request, status from servicelib.fastapi.long_running_tasks._manager import FastAPILongRunningManager from servicelib.fastapi.long_running_tasks.server import get_long_running_manager from servicelib.fastapi.requests_decorators import cancel_on_disconnect @@ -12,7 +12,6 @@ from ...core.settings import ApplicationSettings from ...models.schemas.application_health import ApplicationHealth from ...models.schemas.containers import ContainersCreate -from ...models.shared_store import SharedStore from ...modules.inputs import InputsState from ...modules.long_running_tasks import ( task_containers_restart, @@ -25,16 +24,10 @@ task_runs_docker_compose_down, task_save_state, ) -from ...modules.mounted_fs import MountedVolumes -from ...modules.outputs import OutputsManager from ._dependencies import ( - get_application, get_application_health, get_inputs_state, - get_mounted_volumes, - get_outputs_manager, get_settings, - get_shared_store, ) router = APIRouter() @@ -52,8 +45,6 @@ async def pull_user_servcices_docker_images( long_running_manager: Annotated[ FastAPILongRunningManager, Depends(get_long_running_manager) ], - shared_store: Annotated[SharedStore, Depends(get_shared_store)], - app: Annotated[FastAPI, Depends(get_application)], ) -> TaskId: assert request # nosec @@ -63,8 +54,6 @@ async def pull_user_servcices_docker_images( long_running_manager, task_pull_user_servcices_docker_images.__name__, unique=True, - app=app, - shared_store=shared_store, ) except TaskAlreadyRunningError as e: return cast(str, e.managed_task.task_id) # type: ignore[attr-defined] # pylint:disable=no-member @@ -93,8 +82,6 @@ async def create_service_containers_task( # pylint: disable=too-many-arguments FastAPILongRunningManager, Depends(get_long_running_manager) ], settings: Annotated[ApplicationSettings, Depends(get_settings)], - shared_store: Annotated[SharedStore, Depends(get_shared_store)], - app: Annotated[FastAPI, Depends(get_application)], application_health: Annotated[ApplicationHealth, Depends(get_application_health)], ) -> TaskId: assert request # nosec @@ -107,8 +94,6 @@ async def create_service_containers_task( # pylint: disable=too-many-arguments unique=True, settings=settings, containers_create=containers_create, - shared_store=shared_store, - app=app, application_health=application_health, ) except TaskAlreadyRunningError as e: @@ -128,9 +113,6 @@ async def runs_docker_compose_down_task( FastAPILongRunningManager, Depends(get_long_running_manager) ], settings: Annotated[ApplicationSettings, Depends(get_settings)], - shared_store: Annotated[SharedStore, Depends(get_shared_store)], - app: Annotated[FastAPI, Depends(get_application)], - mounted_volumes: Annotated[MountedVolumes, Depends(get_mounted_volumes)], ) -> TaskId: assert request # nosec @@ -140,10 +122,7 @@ async def runs_docker_compose_down_task( long_running_manager, task_runs_docker_compose_down.__name__, unique=True, - app=app, - shared_store=shared_store, settings=settings, - mounted_volumes=mounted_volumes, ) except TaskAlreadyRunningError as e: return cast(str, e.managed_task.task_id) # type: ignore[attr-defined] # pylint:disable=no-member @@ -162,8 +141,6 @@ async def state_restore_task( FastAPILongRunningManager, Depends(get_long_running_manager) ], settings: Annotated[ApplicationSettings, Depends(get_settings)], - mounted_volumes: Annotated[MountedVolumes, Depends(get_mounted_volumes)], - app: Annotated[FastAPI, Depends(get_application)], ) -> TaskId: assert request # nosec @@ -174,8 +151,6 @@ async def state_restore_task( task_restore_state.__name__, unique=True, settings=settings, - mounted_volumes=mounted_volumes, - app=app, ) except TaskAlreadyRunningError as e: return cast(str, e.managed_task.task_id) # type: ignore[attr-defined] # pylint:disable=no-member @@ -193,8 +168,6 @@ async def state_save_task( long_running_manager: Annotated[ FastAPILongRunningManager, Depends(get_long_running_manager) ], - app: Annotated[FastAPI, Depends(get_application)], - mounted_volumes: Annotated[MountedVolumes, Depends(get_mounted_volumes)], settings: Annotated[ApplicationSettings, Depends(get_settings)], ) -> TaskId: assert request # nosec @@ -206,8 +179,6 @@ async def state_save_task( task_save_state.__name__, unique=True, settings=settings, - mounted_volumes=mounted_volumes, - app=app, ) except TaskAlreadyRunningError as e: return cast(str, e.managed_task.task_id) # type: ignore[attr-defined] # pylint:disable=no-member @@ -225,9 +196,7 @@ async def ports_inputs_pull_task( long_running_manager: Annotated[ FastAPILongRunningManager, Depends(get_long_running_manager) ], - app: Annotated[FastAPI, Depends(get_application)], settings: Annotated[ApplicationSettings, Depends(get_settings)], - mounted_volumes: Annotated[MountedVolumes, Depends(get_mounted_volumes)], inputs_state: Annotated[InputsState, Depends(get_inputs_state)], port_keys: list[str] | None = None, ) -> TaskId: @@ -240,8 +209,6 @@ async def ports_inputs_pull_task( task_ports_inputs_pull.__name__, unique=True, port_keys=port_keys, - mounted_volumes=mounted_volumes, - app=app, settings=settings, inputs_pulling_enabled=inputs_state.inputs_pulling_enabled, ) @@ -261,8 +228,6 @@ async def ports_outputs_pull_task( long_running_manager: Annotated[ FastAPILongRunningManager, Depends(get_long_running_manager) ], - app: Annotated[FastAPI, Depends(get_application)], - mounted_volumes: Annotated[MountedVolumes, Depends(get_mounted_volumes)], port_keys: list[str] | None = None, ) -> TaskId: assert request # nosec @@ -274,8 +239,6 @@ async def ports_outputs_pull_task( task_ports_outputs_pull.__name__, unique=True, port_keys=port_keys, - mounted_volumes=mounted_volumes, - app=app, ) except TaskAlreadyRunningError as e: return cast(str, e.managed_task.task_id) # type: ignore[attr-defined] # pylint:disable=no-member @@ -293,8 +256,6 @@ async def ports_outputs_push_task( long_running_manager: Annotated[ FastAPILongRunningManager, Depends(get_long_running_manager) ], - outputs_manager: Annotated[OutputsManager, Depends(get_outputs_manager)], - app: Annotated[FastAPI, Depends(get_application)], ) -> TaskId: assert request # nosec @@ -304,8 +265,6 @@ async def ports_outputs_push_task( long_running_manager, task_ports_outputs_push.__name__, unique=True, - outputs_manager=outputs_manager, - app=app, ) except TaskAlreadyRunningError as e: return cast(str, e.managed_task.task_id) # type: ignore[attr-defined] # pylint:disable=no-member @@ -323,9 +282,7 @@ async def containers_restart_task( long_running_manager: Annotated[ FastAPILongRunningManager, Depends(get_long_running_manager) ], - app: Annotated[FastAPI, Depends(get_application)], settings: Annotated[ApplicationSettings, Depends(get_settings)], - shared_store: Annotated[SharedStore, Depends(get_shared_store)], ) -> TaskId: assert request # nosec @@ -335,9 +292,7 @@ async def containers_restart_task( long_running_manager, task_containers_restart.__name__, unique=True, - app=app, settings=settings, - shared_store=shared_store, ) except TaskAlreadyRunningError as e: return cast(str, e.managed_task.task_id) # type: ignore[attr-defined] # pylint:disable=no-member diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py index 87788ed88bec..e5bf46098967 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py @@ -5,7 +5,6 @@ from common_library.json_serialization import json_dumps from fastapi import FastAPI from servicelib.async_utils import cancel_sequential_workers -from servicelib.fastapi import long_running_tasks from servicelib.fastapi.logging_lifespan import create_logging_shutdown_event from servicelib.fastapi.openapi import ( get_common_oas_options, @@ -24,6 +23,7 @@ from ..models.shared_store import SharedStore, setup_shared_store from ..modules.attribute_monitor import setup_attribute_monitor from ..modules.inputs import setup_inputs +from ..modules.long_running_tasks import setup_long_running_tasks from ..modules.mounted_fs import MountedVolumes, setup_mounted_fs from ..modules.notifications import setup_notifications from ..modules.outputs import setup_outputs @@ -146,14 +146,6 @@ def create_base_app() -> FastAPI: override_fastapi_openapi_method(app) app.state.settings = app_settings - long_running_tasks.server.setup( - app, - redis_settings=app_settings.REDIS_SETTINGS, - redis_namespace=f"dy_sidecar-{app_settings.DY_SIDECAR_RUN_ID}", - rabbit_settings=app_settings.RABBIT_SETTINGS, - rabbit_namespace=f"dy_sidecar-{app_settings.DY_SIDECAR_RUN_ID}", - ) - app.include_router(get_main_router(app)) setup_reserved_space(app) @@ -193,6 +185,8 @@ def create_app() -> FastAPI: setup_inputs(app) setup_outputs(app) + setup_long_running_tasks(app) + setup_attribute_monitor(app) setup_user_services_preferences(app) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/schemas/application_health.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/schemas/application_health.py index 4da644858b9d..72413188e4b7 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/schemas/application_health.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/models/schemas/application_health.py @@ -1,5 +1,3 @@ -from typing import Optional - from pydantic import BaseModel, Field @@ -7,6 +5,6 @@ class ApplicationHealth(BaseModel): is_healthy: bool = Field( default=True, description="returns True if the service sis running correctly" ) - error_message: Optional[str] = Field( + error_message: str | None = Field( default=None, description="in case of error this gets set" ) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py index 2e326a11258d..0e42bdd4d8bf 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py @@ -3,7 +3,7 @@ from collections.abc import AsyncGenerator from contextlib import asynccontextmanager from pathlib import Path -from typing import Final +from typing import Any, Final from fastapi import FastAPI from models_library.api_schemas_long_running_tasks.base import TaskProgress @@ -11,9 +11,10 @@ from models_library.rabbitmq_messages import ProgressType, SimcorePlatformStatus from models_library.service_settings_labels import LegacyState from pydantic import PositiveInt +from servicelib.fastapi import long_running_tasks from servicelib.file_utils import log_directory_changes from servicelib.logging_utils import log_context -from servicelib.long_running_tasks.task import TaskRegistry +from servicelib.long_running_tasks.task import TaskProtocol, TaskRegistry from servicelib.progress_bar import ProgressBarData from servicelib.utils import logged_gather from simcore_sdk.node_data import data_manager @@ -625,15 +626,52 @@ async def task_containers_restart( await progress.update(message="started log fetching", percent=0.99) -for task in ( - task_pull_user_servcices_docker_images, - task_create_service_containers, - task_runs_docker_compose_down, - task_restore_state, - task_save_state, - task_ports_inputs_pull, - task_ports_outputs_pull, - task_ports_outputs_push, - task_containers_restart, -): - TaskRegistry.register(task) +def setup_long_running_tasks(app: FastAPI) -> None: + app_settings: ApplicationSettings = app.state.settings + long_running_tasks.server.setup( + app, + redis_settings=app_settings.REDIS_SETTINGS, + redis_namespace=f"dy_sidecar-{app_settings.DY_SIDECAR_RUN_ID}", + rabbit_settings=app_settings.RABBIT_SETTINGS, + rabbit_namespace=f"dy_sidecar-{app_settings.DY_SIDECAR_RUN_ID}", + ) + + async def on_startup() -> None: + shared_store: SharedStore = app.state.shared_store + mounted_volumes: MountedVolumes = app.state.mounted_volumes + outputs_manager: OutputsManager = app.state.outputs_manager + + context_app_store: dict[str, Any] = { + "app": app, + "shared_store": shared_store, + } + context_app_store_volumes: dict[str, Any] = { + "app": app, + "shared_store": shared_store, + "mounted_volumes": mounted_volumes, + } + context_app_volumes: dict[str, Any] = { + "app": app, + "mounted_volumes": mounted_volumes, + } + context_app_outputs: dict[str, Any] = { + "app": app, + "outputs_manager": outputs_manager, + } + + task_context: dict[TaskProtocol, dict[str, Any]] = { + task_pull_user_servcices_docker_images: context_app_store, + task_create_service_containers: context_app_store, + task_runs_docker_compose_down: context_app_store_volumes, + task_restore_state: context_app_volumes, + task_save_state: context_app_volumes, + task_ports_inputs_pull: context_app_volumes, + task_ports_outputs_pull: context_app_volumes, + task_ports_outputs_push: context_app_outputs, + task_containers_restart: context_app_store, + } + + for handler, context in task_context.items(): + TaskRegistry.register_partial(handler, **context) + + app.add_event_handler("startup", on_startup) diff --git a/services/dynamic-sidecar/tests/conftest.py b/services/dynamic-sidecar/tests/conftest.py index 6d9b16c9bc9b..3943692efdb9 100644 --- a/services/dynamic-sidecar/tests/conftest.py +++ b/services/dynamic-sidecar/tests/conftest.py @@ -360,9 +360,7 @@ def mock_stop_heart_beat_task(mocker: MockerFixture) -> AsyncMock: @pytest.fixture def mock_metrics_params(faker: Faker) -> CreateServiceMetricsAdditionalParams: return TypeAdapter(CreateServiceMetricsAdditionalParams).validate_python( - CreateServiceMetricsAdditionalParams.model_config["json_schema_extra"][ - "example" - ], + CreateServiceMetricsAdditionalParams.model_json_schema()["example"] ) diff --git a/services/dynamic-sidecar/tests/unit/api/rest/test_disk.py b/services/dynamic-sidecar/tests/unit/api/rest/test_disk.py index 3d6bda8d8f1b..fa466827e0dd 100644 --- a/services/dynamic-sidecar/tests/unit/api/rest/test_disk.py +++ b/services/dynamic-sidecar/tests/unit/api/rest/test_disk.py @@ -1,5 +1,7 @@ # pylint:disable=unused-argument +from unittest.mock import AsyncMock + from async_asgi_testclient import TestClient from fastapi import status from simcore_service_dynamic_sidecar._meta import API_VTAG @@ -9,7 +11,9 @@ async def test_reserved_disk_space_freed( - cleanup_reserved_disk_space: None, test_client: TestClient + mock_core_rabbitmq: dict[str, AsyncMock], + cleanup_reserved_disk_space: None, + test_client: TestClient, ): assert _RESERVED_DISK_SPACE_NAME.exists() response = await test_client.post(f"/{API_VTAG}/disk/reserved:free") diff --git a/services/dynamic-sidecar/tests/unit/api/rest/test_volumes.py b/services/dynamic-sidecar/tests/unit/api/rest/test_volumes.py index 40eab12336a3..5bf729dfe0db 100644 --- a/services/dynamic-sidecar/tests/unit/api/rest/test_volumes.py +++ b/services/dynamic-sidecar/tests/unit/api/rest/test_volumes.py @@ -1,6 +1,7 @@ # pylint: disable=unused-argument from pathlib import Path +from unittest.mock import AsyncMock import pytest from async_asgi_testclient import TestClient @@ -20,6 +21,7 @@ ], ) async def test_volumes_state_saved_ok( + mock_core_rabbitmq: dict[str, AsyncMock], ensure_shared_store_dir: Path, test_client: TestClient, volume_category: VolumeCategory, @@ -46,6 +48,7 @@ async def test_volumes_state_saved_ok( @pytest.mark.parametrize("invalid_volume_category", ["outputs", "outputS"]) async def test_volumes_state_saved_error( + mock_core_rabbitmq: dict[str, AsyncMock], ensure_shared_store_dir: Path, test_client: TestClient, invalid_volume_category: VolumeCategory, diff --git a/services/dynamic-sidecar/tests/unit/conftest.py b/services/dynamic-sidecar/tests/unit/conftest.py index 75b9d316c103..3bcc0d3847f9 100644 --- a/services/dynamic-sidecar/tests/unit/conftest.py +++ b/services/dynamic-sidecar/tests/unit/conftest.py @@ -40,11 +40,7 @@ @pytest.fixture -def app( - mock_environment: EnvVarsDict, - mock_registry_service: AsyncMock, - mock_core_rabbitmq: dict[str, AsyncMock], -) -> FastAPI: +def app(mock_environment: EnvVarsDict, mock_registry_service: AsyncMock) -> FastAPI: """creates app with registry and rabbitMQ services mocked""" return create_app() diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_containers.py b/services/dynamic-sidecar/tests/unit/test_api_rest_containers.py index 970f8aeb67e4..ba0e3f629ceb 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_containers.py +++ b/services/dynamic-sidecar/tests/unit/test_api_rest_containers.py @@ -19,6 +19,7 @@ from aiodocker.volumes import DockerVolume from aiofiles.os import mkdir from async_asgi_testclient import TestClient +from common_library.serialization import model_dump_with_secrets from faker import Faker from fastapi import FastAPI, status from models_library.api_schemas_dynamic_sidecar.containers import ActivityInfo @@ -29,6 +30,7 @@ from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict from servicelib.docker_constants import SUFFIX_EGRESS_PROXY_NAME from servicelib.long_running_tasks.models import TaskId +from settings_library.rabbit import RabbitSettings from simcore_service_dynamic_sidecar._meta import API_VTAG from simcore_service_dynamic_sidecar.api.rest.containers import _INACTIVE_FOR_LONG_TIME from simcore_service_dynamic_sidecar.core.application import AppState @@ -47,6 +49,10 @@ from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed +pytest_simcore_core_services_selection = [ + "rabbit", +] + WAIT_FOR_OUTPUTS_WATCHER: Final[float] = 0.1 FAST_POLLING_INTERVAL: Final[float] = 0.1 @@ -162,9 +168,19 @@ async def _assert_compose_spec_pulled(compose_spec: str, settings: ApplicationSe @pytest.fixture def mock_environment( - mock_environment: EnvVarsDict, mock_rabbitmq_envs: EnvVarsDict + monkeypatch: pytest.MonkeyPatch, + rabbit_service: RabbitSettings, + mock_environment: EnvVarsDict, ) -> EnvVarsDict: - return mock_rabbitmq_envs + return setenvs_from_dict( + monkeypatch, + { + **mock_environment, + "RABBIT_SETTINGS": json.dumps( + model_dump_with_secrets(rabbit_service, show_secrets=True) + ), + }, + ) @pytest.fixture @@ -267,10 +283,10 @@ def not_started_containers() -> list[str]: def mock_outputs_labels() -> dict[str, ServiceOutput]: return { "output_port_1": TypeAdapter(ServiceOutput).validate_python( - ServiceOutput.model_config["json_schema_extra"]["examples"][3] + ServiceOutput.model_json_schema()["examples"][3] ), "output_port_2": TypeAdapter(ServiceOutput).validate_python( - ServiceOutput.model_config["json_schema_extra"]["examples"][3] + ServiceOutput.model_json_schema()["examples"][3] ), } diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_containers_long_running_tasks.py b/services/dynamic-sidecar/tests/unit/test_api_rest_containers_long_running_tasks.py index f5ca123d8d1b..15e78861aca9 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_containers_long_running_tasks.py +++ b/services/dynamic-sidecar/tests/unit/test_api_rest_containers_long_running_tasks.py @@ -17,6 +17,7 @@ from aiodocker.containers import DockerContainer from aiodocker.volumes import DockerVolume from asgi_lifespan import LifespanManager +from common_library.serialization import model_dump_with_secrets from fastapi import FastAPI from fastapi.routing import APIRoute from httpx import ASGITransport, AsyncClient @@ -28,12 +29,13 @@ from models_library.services_creation import CreateServiceMetricsAdditionalParams from pydantic import AnyHttpUrl, TypeAdapter from pytest_mock.plugin import MockerFixture -from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict +from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict from servicelib.fastapi.long_running_tasks.client import Client, periodic_task_result from servicelib.fastapi.long_running_tasks.client import setup as client_setup from servicelib.long_running_tasks.errors import TaskExceptionError from servicelib.long_running_tasks.models import TaskId from servicelib.long_running_tasks.task import TaskRegistry +from settings_library.rabbit import RabbitSettings from simcore_sdk.node_ports_common.exceptions import NodeNotFound from simcore_service_dynamic_sidecar._meta import API_VTAG from simcore_service_dynamic_sidecar.api.rest import containers_long_running_tasks @@ -53,6 +55,10 @@ wait_fixed, ) +pytest_simcore_core_services_selection = [ + "rabbit", +] + FAST_STATUS_POLL: Final[float] = 0.1 CREATE_SERVICE_CONTAINERS_TIMEOUT: Final[float] = 60 DEFAULT_COMMAND_TIMEOUT: Final[int] = 5 @@ -171,8 +177,20 @@ def backend_url() -> AnyHttpUrl: @pytest.fixture -def mock_environment(mock_rabbitmq_envs: EnvVarsDict) -> EnvVarsDict: - return mock_rabbitmq_envs +def mock_environment( + monkeypatch: pytest.MonkeyPatch, + rabbit_service: RabbitSettings, + mock_environment: EnvVarsDict, +) -> EnvVarsDict: + return setenvs_from_dict( + monkeypatch, + { + **mock_environment, + "RABBIT_SETTINGS": json.dumps( + model_dump_with_secrets(rabbit_service, show_secrets=True) + ), + }, + ) @pytest.fixture diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_health.py b/services/dynamic-sidecar/tests/unit/test_api_rest_health.py index 987ddbf1e636..a5542917b117 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_health.py +++ b/services/dynamic-sidecar/tests/unit/test_api_rest_health.py @@ -1,6 +1,8 @@ # pylint: disable=redefined-outer-name # pylint: disable=unused-argument +from unittest.mock import AsyncMock + from async_asgi_testclient import TestClient from fastapi import status from simcore_service_dynamic_sidecar.models.schemas.application_health import ( @@ -8,14 +10,18 @@ ) -async def test_is_healthy(test_client: TestClient) -> None: +async def test_is_healthy( + mock_core_rabbitmq: dict[str, AsyncMock], test_client: TestClient +) -> None: test_client.application.state.application_health.is_healthy = True response = await test_client.get("/health") assert response.status_code == status.HTTP_200_OK, response assert response.json() == ApplicationHealth(is_healthy=True).model_dump() -async def test_is_unhealthy(test_client: TestClient) -> None: +async def test_is_unhealthy( + mock_core_rabbitmq: dict[str, AsyncMock], test_client: TestClient +) -> None: test_client.application.state.application_health.is_healthy = False response = await test_client.get("/health") assert response.status_code == status.HTTP_503_SERVICE_UNAVAILABLE, response @@ -24,7 +30,9 @@ async def test_is_unhealthy(test_client: TestClient) -> None: } -async def test_is_unhealthy_via_rabbitmq(test_client: TestClient) -> None: +async def test_is_unhealthy_via_rabbitmq( + mock_core_rabbitmq: dict[str, AsyncMock], test_client: TestClient +) -> None: # pylint: disable=protected-access test_client.application.state.rabbitmq_client._healthy_state = False # noqa: SLF001 response = await test_client.get("/health") diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_prometheus_metrics.py b/services/dynamic-sidecar/tests/unit/test_api_rest_prometheus_metrics.py index 78e5b22046ea..8a653fad77d1 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_prometheus_metrics.py +++ b/services/dynamic-sidecar/tests/unit/test_api_rest_prometheus_metrics.py @@ -5,11 +5,11 @@ import json from collections.abc import AsyncIterable from typing import Final -from unittest.mock import AsyncMock import pytest from aiodocker.volumes import DockerVolume from asgi_lifespan import LifespanManager +from common_library.serialization import model_dump_with_secrets from fastapi import FastAPI, status from httpx import ASGITransport, AsyncClient from models_library.api_schemas_dynamic_sidecar.containers import DockerComposeYamlStr @@ -20,6 +20,7 @@ from servicelib.fastapi.long_running_tasks.client import Client, periodic_task_result from servicelib.fastapi.long_running_tasks.client import setup as client_setup from servicelib.long_running_tasks.models import TaskId +from settings_library.rabbit import RabbitSettings from simcore_service_dynamic_sidecar._meta import API_VTAG from simcore_service_dynamic_sidecar.models.schemas.containers import ( ContainersComposeSpec, @@ -30,10 +31,31 @@ UserServicesMetrics, ) +pytest_simcore_core_services_selection = [ + "rabbit", +] + _FAST_STATUS_POLL: Final[float] = 0.1 _CREATE_SERVICE_CONTAINERS_TIMEOUT: Final[float] = 60 +@pytest.fixture +def mock_environment( + monkeypatch: pytest.MonkeyPatch, + rabbit_service: RabbitSettings, + mock_environment: EnvVarsDict, +) -> EnvVarsDict: + return setenvs_from_dict( + monkeypatch, + { + **mock_environment, + "RABBIT_SETTINGS": json.dumps( + model_dump_with_secrets(rabbit_service, show_secrets=True) + ), + }, + ) + + @pytest.fixture async def enable_prometheus_metrics( monkeypatch: pytest.MonkeyPatch, mock_environment: EnvVarsDict @@ -42,14 +64,14 @@ async def enable_prometheus_metrics( monkeypatch, { "DY_SIDECAR_CALLBACKS_MAPPING": json.dumps( - CallbacksMapping.model_config["json_schema_extra"]["examples"][2] - ) + CallbacksMapping.model_json_schema()["examples"][2] + ), }, ) @pytest.fixture -async def app(mock_rabbitmq_envs: EnvVarsDict, app: FastAPI) -> AsyncIterable[FastAPI]: +async def app(app: FastAPI) -> AsyncIterable[FastAPI]: client_setup(app) async with LifespanManager(app): yield app @@ -118,17 +140,13 @@ async def _get_task_id_create_service_containers( return task_id -async def test_metrics_disabled( - mock_core_rabbitmq: dict[str, AsyncMock], httpx_async_client: AsyncClient -) -> None: +async def test_metrics_disabled(httpx_async_client: AsyncClient) -> None: response = await httpx_async_client.get("/metrics") assert response.status_code == status.HTTP_404_NOT_FOUND, response async def test_metrics_enabled_no_containers_running( - enable_prometheus_metrics: None, - mock_core_rabbitmq: dict[str, AsyncMock], - httpx_async_client: AsyncClient, + enable_prometheus_metrics: None, httpx_async_client: AsyncClient ) -> None: response = await httpx_async_client.get("/metrics") assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR, response @@ -137,7 +155,6 @@ async def test_metrics_enabled_no_containers_running( async def test_metrics_enabled_containers_will_start( enable_prometheus_metrics: None, - mock_core_rabbitmq: dict[str, AsyncMock], app: FastAPI, httpx_async_client: AsyncClient, client: Client, diff --git a/services/dynamic-sidecar/tests/unit/test_models_shared_store.py b/services/dynamic-sidecar/tests/unit/test_models_shared_store.py index 2c2b474a0290..7ecf24a2d33f 100644 --- a/services/dynamic-sidecar/tests/unit/test_models_shared_store.py +++ b/services/dynamic-sidecar/tests/unit/test_models_shared_store.py @@ -5,6 +5,7 @@ from copy import deepcopy from pathlib import Path from typing import Any +from unittest.mock import AsyncMock import arrow import pytest @@ -23,6 +24,7 @@ @pytest.fixture def trigger_setup_shutdown_events( + mock_core_rabbitmq: dict[str, AsyncMock], shared_store_dir: Path, app: FastAPI, test_client: TestClient, From 9d8b9ee34d34ff6af9763eccaefacdbab05caaea Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 13 Aug 2025 08:21:32 +0200 Subject: [PATCH 034/119] mypy --- .../src/servicelib/long_running_tasks/task.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index 700600997d78..56cacf226bad 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -68,9 +68,8 @@ def register(cls, task: TaskProtocol) -> None: @classmethod def register_partial(cls, task: TaskProtocol, **partial_kwargs) -> None: partail_task = functools.partial(task, **partial_kwargs) - partail_task.__name__ = ( - task.__name__ - ) # allows to call via the partial of via the orignal method + # allows to call via the partial of via the orignal method + partail_task.__name__ = task.__name__ # type: ignore[attr-defined] cls.REGISTERED_TASKS[task.__name__] = partail_task # type: ignore[assignment] @classmethod From 87939ab949dc61eabb99ff5fa10b008a250b2459 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 13 Aug 2025 09:23:45 +0200 Subject: [PATCH 035/119] fixeed tests --- .../test_studies_dispatcher_projects.py | 26 +++++++++++++++++++ .../test_studies_dispatcher_studies_access.py | 25 ++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_projects.py b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_projects.py index 75e09bf72fc8..a268b839f478 100644 --- a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_projects.py +++ b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_projects.py @@ -10,13 +10,18 @@ import pytest from aiohttp.test_utils import TestClient +from common_library.json_serialization import json_dumps +from common_library.serialization import model_dump_with_secrets from faker import Faker from models_library.projects import Project, ProjectID from models_library.projects_nodes_io import NodeID from pytest_mock import MockerFixture +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.webserver_fake_services_data import list_fake_file_consumers from pytest_simcore.helpers.webserver_login import NewUser from pytest_simcore.helpers.webserver_projects import delete_all_projects +from settings_library.rabbit import RabbitSettings from simcore_service_webserver.groups.api import auto_add_user_to_groups from simcore_service_webserver.projects._projects_service import get_project_for_user from simcore_service_webserver.studies_dispatcher._models import ServiceInfo @@ -29,9 +34,30 @@ ) from simcore_service_webserver.users.users_service import get_user +pytest_simcore_core_services_selection = [ + "rabbit", +] + + FAKE_FILE_VIEWS = list_fake_file_consumers() +@pytest.fixture +def app_environment( + app_environment: EnvVarsDict, + monkeypatch: pytest.MonkeyPatch, + rabbit_service: RabbitSettings, +) -> EnvVarsDict: + return setenvs_from_dict( + monkeypatch, + { + "WEBSERVER_RABBITMQ": json_dumps( + model_dump_with_secrets(rabbit_service, show_secrets=True) + ) + }, + ) + + @pytest.fixture async def user(client: TestClient) -> AsyncIterator[UserInfo]: async with NewUser(app=client.app) as user_db: diff --git a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py index f120ab0c23cc..e6acbc36a733 100644 --- a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py +++ b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py @@ -17,6 +17,8 @@ import redis.asyncio as aioredis from aiohttp import ClientResponse, ClientSession, web from aiohttp.test_utils import TestClient, TestServer +from common_library.json_serialization import json_dumps +from common_library.serialization import model_dump_with_secrets from common_library.users_enums import UserRole from faker import Faker from models_library.api_schemas_rpc_async_jobs.async_jobs import AsyncJobStatus @@ -29,6 +31,8 @@ from pytest_mock import MockerFixture from pytest_simcore.aioresponses_mocker import AioResponsesMock from pytest_simcore.helpers.assert_checks import assert_status +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.webserver_parametrizations import MockedStorageSubsystem from pytest_simcore.helpers.webserver_projects import NewProject, delete_all_projects from pytest_simcore.helpers.webserver_users import UserInfoDict @@ -38,6 +42,7 @@ AsyncJobComposedResult, ) from servicelib.rest_responses import unwrap_envelope +from settings_library.rabbit import RabbitSettings from settings_library.utils_session import DEFAULT_SESSION_COOKIE_NAME from simcore_service_webserver.projects._projects_service import ( submit_delete_project_task, @@ -50,6 +55,10 @@ ) from tenacity import retry, stop_after_attempt, wait_fixed +pytest_simcore_core_services_selection = [ + "rabbit", +] + async def _get_user_projects(client) -> list[ProjectDict]: url = client.app.router["list_projects"].url_for() @@ -88,6 +97,22 @@ def _is_user_authenticated(session: ClientSession) -> bool: return DEFAULT_SESSION_COOKIE_NAME in [c.key for c in session.cookie_jar] +@pytest.fixture +def app_environment( + app_environment: EnvVarsDict, + monkeypatch: pytest.MonkeyPatch, + rabbit_service: RabbitSettings, +) -> EnvVarsDict: + return setenvs_from_dict( + monkeypatch, + { + "WEBSERVER_RABBITMQ": json_dumps( + model_dump_with_secrets(rabbit_service, show_secrets=True) + ) + }, + ) + + @pytest.fixture async def published_project( client: TestClient, From 1695f75e5419224b824578991529e7c732c67b03 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 13 Aug 2025 09:28:30 +0200 Subject: [PATCH 036/119] fixed tests --- .../unit/with_dbs/test_api_route_dynamic_services.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/services/director-v2/tests/unit/with_dbs/test_api_route_dynamic_services.py b/services/director-v2/tests/unit/with_dbs/test_api_route_dynamic_services.py index ba858c5940cc..8a70f85eb170 100644 --- a/services/director-v2/tests/unit/with_dbs/test_api_route_dynamic_services.py +++ b/services/director-v2/tests/unit/with_dbs/test_api_route_dynamic_services.py @@ -40,6 +40,7 @@ X_DYNAMIC_SIDECAR_REQUEST_SCHEME, X_SIMCORE_USER_AGENT, ) +from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings from simcore_service_director_v2.models.dynamic_services_scheduler import SchedulerData from simcore_service_director_v2.modules.dynamic_sidecar.errors import ( @@ -53,6 +54,7 @@ pytest_simcore_core_services_selection = [ "postgres", + "rabbit", "redis", ] pytest_simcore_ops_services_selection = [ @@ -72,7 +74,6 @@ class ServiceParams(NamedTuple): @pytest.fixture def minimal_config( - disable_rabbitmq: None, mock_env: EnvVarsDict, postgres_host_config: dict[str, str], monkeypatch: pytest.MonkeyPatch, @@ -100,8 +101,8 @@ def mock_env( mock_env: EnvVarsDict, mock_exclusive: None, disable_postgres: None, - disable_rabbitmq: None, redis_service: RedisSettings, + rabbit_service: RabbitSettings, monkeypatch: pytest.MonkeyPatch, faker: Faker, ) -> None: @@ -126,11 +127,6 @@ def mock_env( monkeypatch.setenv("COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH", "{}") monkeypatch.setenv("DIRECTOR_V2_DYNAMIC_SCHEDULER_ENABLED", "true") - monkeypatch.setenv("RABBIT_HOST", "mocked_host") - monkeypatch.setenv("RABBIT_SECURE", "false") - monkeypatch.setenv("RABBIT_USER", "mocked_user") - monkeypatch.setenv("RABBIT_PASSWORD", "mocked_password") - monkeypatch.setenv("REGISTRY_AUTH", "false") monkeypatch.setenv("REGISTRY_USER", "test") monkeypatch.setenv("REGISTRY_PW", "test") From d8904deba1113e3ccf846a1647003d744ad9bd0a Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 13 Aug 2025 09:42:26 +0200 Subject: [PATCH 037/119] fixed tests --- services/director-v2/tests/unit/with_dbs/test_cli.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/director-v2/tests/unit/with_dbs/test_cli.py b/services/director-v2/tests/unit/with_dbs/test_cli.py index cce3c67968dc..a4a231984a4c 100644 --- a/services/director-v2/tests/unit/with_dbs/test_cli.py +++ b/services/director-v2/tests/unit/with_dbs/test_cli.py @@ -23,6 +23,7 @@ from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.long_running_tasks.models import ProgressCallback +from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings from simcore_service_director_v2.cli import DEFAULT_NODE_SAVE_ATTEMPTS, main from simcore_service_director_v2.cli._close_and_save_service import ( @@ -32,6 +33,7 @@ pytest_simcore_core_services_selection = [ "postgres", + "rabbit", "redis", ] pytest_simcore_ops_services_selection = [ @@ -44,6 +46,7 @@ def minimal_configuration( mock_env: EnvVarsDict, postgres_host_config: dict[str, str], redis_service: RedisSettings, + rabbit_service: RabbitSettings, monkeypatch: pytest.MonkeyPatch, faker: Faker, with_product: dict[str, Any], From 6b13f67447a5291f2cfa8ae230084ac724d4e34f Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 13 Aug 2025 10:10:13 +0200 Subject: [PATCH 038/119] removed unused --- .../dynamic-sidecar/tests/unit/conftest.py | 20 +------------------ 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/services/dynamic-sidecar/tests/unit/conftest.py b/services/dynamic-sidecar/tests/unit/conftest.py index 3bcc0d3847f9..fc113f1c674a 100644 --- a/services/dynamic-sidecar/tests/unit/conftest.py +++ b/services/dynamic-sidecar/tests/unit/conftest.py @@ -12,7 +12,7 @@ from async_asgi_testclient import TestClient from fastapi import FastAPI from pytest_mock.plugin import MockerFixture -from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict +from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict from simcore_service_dynamic_sidecar.core.application import AppState, create_app from simcore_service_dynamic_sidecar.core.docker_compose_utils import ( docker_compose_down, @@ -127,24 +127,6 @@ async def cleanup_containers(app: FastAPI) -> AsyncIterator[None]: await docker_compose_down(app_state.compose_spec, app_state.settings) -@pytest.fixture -def mock_rabbitmq_envs( - mock_core_rabbitmq: dict[str, AsyncMock], - monkeypatch: pytest.MonkeyPatch, - mock_environment: EnvVarsDict, -) -> EnvVarsDict: - setenvs_from_dict( - monkeypatch, - { - "RABBIT_HOST": "mocked_host", - "RABBIT_SECURE": "false", - "RABBIT_USER": "mocked_user", - "RABBIT_PASSWORD": "mocked_password", - }, - ) - return mock_environment - - @pytest.fixture def port_notifier(app: FastAPI) -> PortNotifier: settings: ApplicationSettings = app.state.settings From a02f6e3ef8ed06562e3f2d43fd5097ece90b5e50 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 13 Aug 2025 10:13:10 +0200 Subject: [PATCH 039/119] fixed issues with loops in test --- .../test_api_rest_workflow_service_metrics.py | 82 +++++++++++++------ 1 file changed, 59 insertions(+), 23 deletions(-) diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_workflow_service_metrics.py b/services/dynamic-sidecar/tests/unit/test_api_rest_workflow_service_metrics.py index 145fd791fd3a..15362b0fd19e 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_workflow_service_metrics.py +++ b/services/dynamic-sidecar/tests/unit/test_api_rest_workflow_service_metrics.py @@ -16,6 +16,7 @@ from aiodocker.utils import clean_filters from aiodocker.volumes import DockerVolume from asgi_lifespan import LifespanManager +from common_library.serialization import model_dump_with_secrets from fastapi import FastAPI from httpx import ASGITransport, AsyncClient from models_library.api_schemas_dynamic_sidecar.containers import DockerComposeYamlStr @@ -36,7 +37,9 @@ from servicelib.fastapi.long_running_tasks.client import setup as client_setup from servicelib.long_running_tasks.errors import TaskExceptionError from servicelib.long_running_tasks.models import TaskId +from settings_library.rabbit import RabbitSettings from simcore_service_dynamic_sidecar._meta import API_VTAG +from simcore_service_dynamic_sidecar.core.application import create_app from simcore_service_dynamic_sidecar.core.docker_utils import get_container_states from simcore_service_dynamic_sidecar.models.schemas.containers import ( ContainersComposeSpec, @@ -47,6 +50,10 @@ from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed +pytest_simcore_core_services_selection = [ + "rabbit", +] + _FAST_STATUS_POLL: Final[float] = 0.1 _CREATE_SERVICE_CONTAINERS_TIMEOUT: Final[float] = 60 _BASE_HEART_BEAT_INTERVAL: Final[float] = 0.1 @@ -81,26 +88,34 @@ def backend_url() -> AnyHttpUrl: @pytest.fixture -def mock_environment( +async def mock_environment( mock_postgres_check: None, + mock_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, - mock_rabbitmq_envs: EnvVarsDict, + rabbit_service: RabbitSettings, + mock_registry_service: AsyncMock, ) -> EnvVarsDict: - setenvs_from_dict( + return setenvs_from_dict( monkeypatch, - {"RESOURCE_TRACKING_HEARTBEAT_INTERVAL": f"{_BASE_HEART_BEAT_INTERVAL}"}, + { + **mock_environment, + "RESOURCE_TRACKING_HEARTBEAT_INTERVAL": f"{_BASE_HEART_BEAT_INTERVAL}", + "RABBIT_SETTINGS": json.dumps( + model_dump_with_secrets(rabbit_service, show_secrets=True) + ), + }, ) - return mock_rabbitmq_envs @pytest.fixture -async def app(app: FastAPI) -> AsyncIterable[FastAPI]: +async def app(mock_environment: EnvVarsDict) -> AsyncIterable[FastAPI]: + lcal_app = create_app() # add the client setup to the same application # this is only required for testing, in reality # this will be in a different process - client_setup(app) - async with LifespanManager(app): - yield app + client_setup(lcal_app) + async with LifespanManager(lcal_app): + yield lcal_app @pytest.fixture @@ -122,7 +137,7 @@ async def httpx_async_client( @pytest.fixture -def client( +async def client( app: FastAPI, httpx_async_client: AsyncClient, backend_url: AnyHttpUrl ) -> Client: return Client(app=app, async_client=httpx_async_client, base_url=f"{backend_url}") @@ -144,6 +159,15 @@ def mock_user_services_fail_to_stop(mocker: MockerFixture) -> None: ) +@pytest.fixture +def mock_post_rabbit_message(mocker: MockerFixture) -> AsyncMock: + return mocker.patch( + "simcore_service_dynamic_sidecar.core.rabbitmq._post_rabbit_message", + return_value=None, + autospec=True, + ) + + async def _get_task_id_create_service_containers( httpx_async_client: AsyncClient, compose_spec: DockerComposeYamlStr, @@ -173,11 +197,11 @@ async def _get_task_id_docker_compose_down(httpx_async_client: AsyncClient) -> T def _get_resource_tracking_messages( - mock_core_rabbitmq: dict[str, AsyncMock], + mock_post_rabbit_message: AsyncMock, ) -> list[RabbitResourceTrackingMessages]: return [ x[0][1] - for x in mock_core_rabbitmq["post_rabbit_message"].call_args_list + for x in mock_post_rabbit_message.call_args_list if isinstance(x[0][1], RabbitResourceTrackingMessages) ] @@ -201,7 +225,7 @@ async def _wait_for_containers_to_be_running(app: FastAPI) -> None: async def test_service_starts_and_closes_as_expected( - mock_core_rabbitmq: dict[str, AsyncMock], + mock_post_rabbit_message: AsyncMock, app: FastAPI, httpx_async_client: AsyncClient, client: Client, @@ -235,7 +259,9 @@ async def test_service_starts_and_closes_as_expected( await asyncio.sleep(_BASE_HEART_BEAT_INTERVAL * 10) # Ensure messages arrive in the expected order - resource_tracking_messages = _get_resource_tracking_messages(mock_core_rabbitmq) + resource_tracking_messages = _get_resource_tracking_messages( + mock_post_rabbit_message + ) assert len(resource_tracking_messages) >= 3 start_message = resource_tracking_messages[0] @@ -252,7 +278,7 @@ async def test_service_starts_and_closes_as_expected( @pytest.mark.parametrize("with_compose_down", [True, False]) async def test_user_services_fail_to_start( - mock_core_rabbitmq: dict[str, AsyncMock], + mock_post_rabbit_message: AsyncMock, app: FastAPI, httpx_async_client: AsyncClient, client: Client, @@ -284,12 +310,14 @@ async def test_user_services_fail_to_start( assert result is None # no messages were sent - resource_tracking_messages = _get_resource_tracking_messages(mock_core_rabbitmq) + resource_tracking_messages = _get_resource_tracking_messages( + mock_post_rabbit_message + ) assert len(resource_tracking_messages) == 0 async def test_user_services_fail_to_stop_or_save_data( - mock_core_rabbitmq: dict[str, AsyncMock], + mock_post_rabbit_message: AsyncMock, app: FastAPI, httpx_async_client: AsyncClient, client: Client, @@ -327,7 +355,9 @@ async def test_user_services_fail_to_stop_or_save_data( ... # Ensure messages arrive in the expected order - resource_tracking_messages = _get_resource_tracking_messages(mock_core_rabbitmq) + resource_tracking_messages = _get_resource_tracking_messages( + mock_post_rabbit_message + ) assert len(resource_tracking_messages) >= 3 start_message = resource_tracking_messages[0] @@ -384,7 +414,7 @@ async def _mocked_get_container_states( @pytest.mark.parametrize("expected_platform_state", SimcorePlatformStatus) async def test_user_services_crash_when_running( - mock_core_rabbitmq: dict[str, AsyncMock], + mock_post_rabbit_message: AsyncMock, app: FastAPI, httpx_async_client: AsyncClient, client: Client, @@ -419,7 +449,9 @@ async def test_user_services_crash_when_running( await _simulate_container_crash(container_names) # check only start and heartbeats are present - resource_tracking_messages = _get_resource_tracking_messages(mock_core_rabbitmq) + resource_tracking_messages = _get_resource_tracking_messages( + mock_post_rabbit_message + ) assert len(resource_tracking_messages) >= 2 start_message = resource_tracking_messages[0] @@ -431,11 +463,13 @@ async def test_user_services_crash_when_running( # reset mock await asyncio.sleep(_BASE_HEART_BEAT_INTERVAL * 2) - mock_core_rabbitmq["post_rabbit_message"].reset_mock() + mock_post_rabbit_message.reset_mock() # wait a bit more and check no further heartbeats are sent await asyncio.sleep(_BASE_HEART_BEAT_INTERVAL * 2) - new_resource_tracking_messages = _get_resource_tracking_messages(mock_core_rabbitmq) + new_resource_tracking_messages = _get_resource_tracking_messages( + mock_post_rabbit_message + ) assert len(new_resource_tracking_messages) == 0 # sending stop events, and since there was an issue multiple stops @@ -450,7 +484,9 @@ async def test_user_services_crash_when_running( ) as result: assert result is None - resource_tracking_messages = _get_resource_tracking_messages(mock_core_rabbitmq) + resource_tracking_messages = _get_resource_tracking_messages( + mock_post_rabbit_message + ) # NOTE: only 1 stop event arrives here since the stopping of the containers # was successful assert len(resource_tracking_messages) == 1 From 7f6ecda3cb617fd5485e2f01dd9486346f752ecb Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 13 Aug 2025 10:15:56 +0200 Subject: [PATCH 040/119] rename --- .../unit/test_api_rest_workflow_service_metrics.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/services/dynamic-sidecar/tests/unit/test_api_rest_workflow_service_metrics.py b/services/dynamic-sidecar/tests/unit/test_api_rest_workflow_service_metrics.py index 15362b0fd19e..520a01ad46c3 100644 --- a/services/dynamic-sidecar/tests/unit/test_api_rest_workflow_service_metrics.py +++ b/services/dynamic-sidecar/tests/unit/test_api_rest_workflow_service_metrics.py @@ -90,10 +90,10 @@ def backend_url() -> AnyHttpUrl: @pytest.fixture async def mock_environment( mock_postgres_check: None, + mock_registry_service: AsyncMock, mock_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, rabbit_service: RabbitSettings, - mock_registry_service: AsyncMock, ) -> EnvVarsDict: return setenvs_from_dict( monkeypatch, @@ -109,13 +109,14 @@ async def mock_environment( @pytest.fixture async def app(mock_environment: EnvVarsDict) -> AsyncIterable[FastAPI]: - lcal_app = create_app() + local_app = create_app() # add the client setup to the same application # this is only required for testing, in reality # this will be in a different process - client_setup(lcal_app) - async with LifespanManager(lcal_app): - yield lcal_app + client_setup(local_app) + + async with LifespanManager(local_app): + yield local_app @pytest.fixture From 730f20d6d534f7a548b3cfcf4180cac65e6290b3 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 13 Aug 2025 11:21:09 +0200 Subject: [PATCH 041/119] extended timeout period --- .github/workflows/ci-testing-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml index b130c5254ba4..b225a782d11f 100644 --- a/.github/workflows/ci-testing-deploy.yml +++ b/.github/workflows/ci-testing-deploy.yml @@ -1252,7 +1252,7 @@ jobs: unit-test-service-library: needs: changes if: ${{ needs.changes.outputs.service-library == 'true' || github.event_name == 'push' || github.event.inputs.force_all_builds == 'true' }} - timeout-minutes: 18 # if this timeout gets too small, then split the tests + timeout-minutes: 20 # if this timeout gets too small, then split the tests name: "[unit] service-library" runs-on: ${{ matrix.os }} strategy: From 60ee2726cd47dabd774420578efb8b046356ca54 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 13 Aug 2025 13:03:33 +0200 Subject: [PATCH 042/119] enhanced errors --- .../src/servicelib/long_running_tasks/errors.py | 2 +- .../service-library/src/servicelib/long_running_tasks/task.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/errors.py b/packages/service-library/src/servicelib/long_running_tasks/errors.py index 75e46da5b0c2..01aad0c81569 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/errors.py +++ b/packages/service-library/src/servicelib/long_running_tasks/errors.py @@ -7,7 +7,7 @@ class BaseLongRunningError(OsparcErrorMixin, Exception): class TaskNotRegisteredError(BaseLongRunningError): msg_template: str = ( - "no task with task_name='{task_name}' was found in the task registry. " + "no task with task_name='{task_name}' was found in the task registry tasks={tasks}. " "Make sure it's registered before starting it." ) diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index 56cacf226bad..d06ea1bafd20 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -466,7 +466,9 @@ async def start_task( **task_kwargs: Any, ) -> TaskId: if registered_task_name not in TaskRegistry.REGISTERED_TASKS: - raise TaskNotRegisteredError(task_name=registered_task_name) + raise TaskNotRegisteredError( + task_name=registered_task_name, tasks=TaskRegistry.REGISTERED_TASKS + ) task = TaskRegistry.REGISTERED_TASKS[registered_task_name] From e9ad6dc74c37cc8b7600afbca812e18b5eb183a9 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 13 Aug 2025 13:03:45 +0200 Subject: [PATCH 043/119] added message --- .../simcore_service_webserver/projects/_crud_api_create.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py index 5ae9663defc9..b780b8c404f0 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py @@ -523,3 +523,7 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche def register_create_project_task(app: web.Application) -> None: TaskRegistry.register_partial(create_project, app=app) + _logger.debug( + "I've manged to register the create_project task %s", + TaskRegistry.REGISTERED_TASKS, + ) From 2380c61cf6a003e07b4dffaf2135c9d6c57b7e28 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 13 Aug 2025 15:27:00 +0200 Subject: [PATCH 044/119] removed uncecessary code --- .../servicelib/long_running_tasks/lrt_api.py | 44 +++---------------- 1 file changed, 6 insertions(+), 38 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py b/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py index 0ae1eaf080cc..fc200bf22125 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py +++ b/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py @@ -1,14 +1,11 @@ import logging from typing import Any -from common_library.error_codes import create_error_code from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient -from ..logging_errors import create_troubleshootting_log_kwargs from ._rabbit import lrt_client, lrt_server from ._rabbit.namespace import get_namespace from .base_long_running_manager import BaseLongRunningManager -from .errors import TaskNotCompletedError, TaskNotFoundError from .models import TaskBase, TaskContext, TaskId, TaskStatus from .task import RegisteredTaskName @@ -97,41 +94,12 @@ async def get_task_result( task_context: TaskContext, task_id: TaskId, ) -> Any: - try: - task_result = await lrt_client.get_task_result( - rabbitmq_rpc_client, - long_running_manager.rabbit_namespace, - task_context=task_context, - task_id=task_id, - ) - await lrt_client.remove_task( - rabbitmq_rpc_client, - long_running_manager.rabbit_namespace, - task_id=task_id, - task_context=task_context, - reraise_errors=False, - ) - return task_result - except (TaskNotFoundError, TaskNotCompletedError): - raise - except Exception as exc: - _logger.exception( - **create_troubleshootting_log_kwargs( - user_error_msg=f"{task_id=} raised an exception while getting its result", - error=exc, - error_code=create_error_code(exc), - error_context={"task_context": task_context, "task_id": task_id}, - ), - ) - # the task shall be removed in this case - await lrt_client.remove_task( - rabbitmq_rpc_client, - long_running_manager.rabbit_namespace, - task_id=task_id, - task_context=task_context, - reraise_errors=False, - ) - raise + return await lrt_client.get_task_result( + rabbitmq_rpc_client, + long_running_manager.rabbit_namespace, + task_context=task_context, + task_id=task_id, + ) async def remove_task( From 82861ce65c23539e51305686f91b054255ec1cf1 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 14 Aug 2025 12:50:31 +0200 Subject: [PATCH 045/119] fixed issue with method registration --- .../long_running_tasks/_rabbit/lrt_server.py | 13 +------------ .../src/servicelib/long_running_tasks/task.py | 9 ++------- .../modules/long_running_tasks.py | 2 +- .../long_running_tasks.py | 19 +++++++++++++------ .../projects/_controller/nodes_rest.py | 2 +- .../projects/_crud_api_create.py | 6 +----- .../projects/plugin.py | 8 +++++--- 7 files changed, 24 insertions(+), 35 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_server.py b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_server.py index d03ce38fd8e1..f3285da4caa8 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_server.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_server.py @@ -1,9 +1,6 @@ import logging from typing import Any -from common_library.error_codes import create_error_code - -from ...logging_errors import create_troubleshootting_log_kwargs from ...rabbitmq import RPCRouter from .._serialization import object_to_string from ..base_long_running_manager import BaseLongRunningManager @@ -74,15 +71,7 @@ async def _get_transferarble_task_result( return task_result except (TaskNotFoundError, TaskNotCompletedError): raise - except Exception as exc: - _logger.exception( - **create_troubleshootting_log_kwargs( - user_error_msg=f"{task_id=} raised an exception while getting its result", - error=exc, - error_code=create_error_code(exc), - error_context={"task_context": task_context, "task_id": task_id}, - ), - ) + except Exception: # the task shall be removed in this case await long_running_manager.tasks_manager.remove_task( task_id, with_task_context=task_context, reraise_errors=False diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index d06ea1bafd20..56c42ad529e4 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -60,15 +60,10 @@ def __name__(self) -> str: ... class TaskRegistry: REGISTERED_TASKS: ClassVar[dict[RegisteredTaskName, TaskProtocol]] = {} - # TODO: maybe only use one method to register @classmethod - def register(cls, task: TaskProtocol) -> None: - cls.REGISTERED_TASKS[task.__name__] = task - - @classmethod - def register_partial(cls, task: TaskProtocol, **partial_kwargs) -> None: + def register(cls, task: TaskProtocol, **partial_kwargs) -> None: partail_task = functools.partial(task, **partial_kwargs) - # allows to call via the partial of via the orignal method + # allows to call the partial via it's original name partail_task.__name__ = task.__name__ # type: ignore[attr-defined] cls.REGISTERED_TASKS[task.__name__] = partail_task # type: ignore[assignment] diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py index 0e42bdd4d8bf..5e03cdbe2d6d 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py @@ -672,6 +672,6 @@ async def on_startup() -> None: } for handler, context in task_context.items(): - TaskRegistry.register_partial(handler, **context) + TaskRegistry.register(handler, **context) app.add_event_handler("startup", on_startup) diff --git a/services/web/server/src/simcore_service_webserver/long_running_tasks.py b/services/web/server/src/simcore_service_webserver/long_running_tasks.py index 86951d25ab78..e9ec7af5af8e 100644 --- a/services/web/server/src/simcore_service_webserver/long_running_tasks.py +++ b/services/web/server/src/simcore_service_webserver/long_running_tasks.py @@ -10,18 +10,21 @@ ) from servicelib.aiohttp.long_running_tasks.server import setup from servicelib.aiohttp.typing_extension import Handler -from servicelib.long_running_tasks.models import RabbitNamespace -from servicelib.long_running_tasks.task import RedisNamespace from . import rabbitmq_settings, redis from ._meta import API_VTAG from .login.decorators import login_required from .models import AuthenticatedRequestContext +from .projects.plugin import register_projects_long_running_tasks _logger = logging.getLogger(__name__) -_LRT_REDIS_NAMESPACE: Final[RedisNamespace] = "webserver-legacy" -_LRT_RABBIT_NAMESPACE: Final[RabbitNamespace] = "webserver-legacy" + +_LRT_NAMESPACE_PREFIX: Final[str] = "webserver-legacy" + + +def _get_namespace(suffix: str) -> str: + return f"{_LRT_NAMESPACE_PREFIX}-{suffix}" def webserver_request_context_decorator(handler: Handler): @@ -39,13 +42,17 @@ async def _test_task_context_decorator( @ensure_single_setup(__name__, logger=_logger) def setup_long_running_tasks(app: web.Application) -> None: + # register all long-running tasks from different modules + register_projects_long_running_tasks(app) + + namespace_suffix = "TODO" # TODO recover from settings setup( app, redis_settings=redis.get_plugin_settings(app), rabbit_settings=rabbitmq_settings.get_plugin_settings(app), - redis_namespace=_LRT_REDIS_NAMESPACE, - rabbit_namespace=_LRT_RABBIT_NAMESPACE, + redis_namespace=_get_namespace(namespace_suffix), + rabbit_namespace=_get_namespace(namespace_suffix), router_prefix=f"/{API_VTAG}/tasks-legacy", handler_check_decorator=login_required, task_request_context_decorator=webserver_request_context_decorator, diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/nodes_rest.py b/services/web/server/src/simcore_service_webserver/projects/_controller/nodes_rest.py index 277b80991374..29fd9e903efe 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/nodes_rest.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/nodes_rest.py @@ -323,7 +323,7 @@ async def _stop_dynamic_service_task( def register_stop_dynamic_service_task(app: web.Application) -> None: - TaskRegistry.register_partial(_stop_dynamic_service_task, app=app) + TaskRegistry.register(_stop_dynamic_service_task, app=app) @routes.post( diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py index b780b8c404f0..3486e58f53bb 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py @@ -522,8 +522,4 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche def register_create_project_task(app: web.Application) -> None: - TaskRegistry.register_partial(create_project, app=app) - _logger.debug( - "I've manged to register the create_project task %s", - TaskRegistry.REGISTERED_TASKS, - ) + TaskRegistry.register(create_project, app=app) diff --git a/services/web/server/src/simcore_service_webserver/projects/plugin.py b/services/web/server/src/simcore_service_webserver/projects/plugin.py index 275a15a164ee..e714ed350d73 100644 --- a/services/web/server/src/simcore_service_webserver/projects/plugin.py +++ b/services/web/server/src/simcore_service_webserver/projects/plugin.py @@ -37,6 +37,11 @@ logger = logging.getLogger(__name__) +def register_projects_long_running_tasks(app: web.Application) -> None: + register_create_project_task(app) + register_stop_dynamic_service_task(app) + + @app_module_setup( "simcore_service_webserver.projects", ModuleCategory.ADDON, @@ -77,7 +82,4 @@ def setup_projects(app: web.Application) -> bool: app.router.add_routes(workspaces_rest.routes) app.router.add_routes(trash_rest.routes) - register_create_project_task(app) - register_stop_dynamic_service_task(app) - return True From d8cfe2e3b723d7db3db3c8614ba3a302f3853148 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 14 Aug 2025 14:01:21 +0200 Subject: [PATCH 046/119] using correct namespaces to handle requests --- services/docker-compose.yml | 10 +++++++ .../simcore_service_webserver/application.py | 2 +- .../application_settings.py | 9 +++++++ .../long_running_tasks/__init__.py | 0 .../plugin.py} | 17 ++++++------ .../long_running_tasks/settings.py | 26 +++++++++++++++++++ .../simcore_service_webserver/tasks/_rest.py | 2 +- 7 files changed, 56 insertions(+), 10 deletions(-) create mode 100644 services/web/server/src/simcore_service_webserver/long_running_tasks/__init__.py rename services/web/server/src/simcore_service_webserver/{long_running_tasks.py => long_running_tasks/plugin.py} (76%) create mode 100644 services/web/server/src/simcore_service_webserver/long_running_tasks/settings.py diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 8ed41a82657e..6574f75d060b 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -699,6 +699,8 @@ services: WEBSERVER_LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} WEBSERVER_LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} + LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: webserver + # WEBSERVER_SERVER_HOST WEBSERVER_HOST: ${WEBSERVER_HOST} @@ -927,6 +929,8 @@ services: WEBSERVER_STATICWEB: "null" WEBSERVER_FUNCTIONS: ${WEBSERVER_FUNCTIONS} # needed for api-server + LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: wb-api-server + networks: *webserver_networks wb-db-event-listener: @@ -936,6 +940,8 @@ services: environment: WEBSERVER_LOGLEVEL: ${WB_DB_EL_LOGLEVEL} + LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: wb-db-event-listener + WEBSERVER_HOST: ${WEBSERVER_HOST} WEBSERVER_PORT: ${WEBSERVER_PORT} @@ -1032,6 +1038,8 @@ services: LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: wb-garbage-collector + # WEBSERVER_DB POSTGRES_DB: ${POSTGRES_DB} POSTGRES_ENDPOINT: ${POSTGRES_ENDPOINT} @@ -1124,6 +1132,8 @@ services: WEBSERVER_APP_FACTORY_NAME: WEBSERVER_AUTHZ_APP_FACTORY WEBSERVER_LOGLEVEL: ${WB_AUTH_LOGLEVEL} + LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: wb-auth + GUNICORN_CMD_ARGS: ${WEBSERVER_GUNICORN_CMD_ARGS} # WEBSERVER_DB diff --git a/services/web/server/src/simcore_service_webserver/application.py b/services/web/server/src/simcore_service_webserver/application.py index 32e96663433b..1d6721823fdd 100644 --- a/services/web/server/src/simcore_service_webserver/application.py +++ b/services/web/server/src/simcore_service_webserver/application.py @@ -40,7 +40,7 @@ from .licenses.plugin import setup_licenses from .login.plugin import setup_login from .login_auth.plugin import setup_login_auth -from .long_running_tasks import setup_long_running_tasks +from .long_running_tasks.plugin import setup_long_running_tasks from .notifications.plugin import setup_notifications from .payments.plugin import setup_payments from .products.plugin import setup_products diff --git a/services/web/server/src/simcore_service_webserver/application_settings.py b/services/web/server/src/simcore_service_webserver/application_settings.py index 919b3947f9b0..b9335185de1f 100644 --- a/services/web/server/src/simcore_service_webserver/application_settings.py +++ b/services/web/server/src/simcore_service_webserver/application_settings.py @@ -40,6 +40,7 @@ from .invitations.settings import InvitationsSettings from .licenses.settings import LicensesSettings from .login.settings import LoginSettings +from .long_running_tasks.settings import LongRunningTasksSettings from .payments.settings import PaymentsSettings from .projects.settings import ProjectsSettings from .resource_manager.settings import ResourceManagerSettings @@ -266,6 +267,14 @@ class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): ), ] + WEBSERVER_LONG_RUNNING_TASKS: Annotated[ + LongRunningTasksSettings | None, + Field( + json_schema_extra={"auto_default_from_env": True}, + description="login plugin", + ), + ] + WEBSERVER_PAYMENTS: Annotated[ PaymentsSettings | None, Field( diff --git a/services/web/server/src/simcore_service_webserver/long_running_tasks/__init__.py b/services/web/server/src/simcore_service_webserver/long_running_tasks/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/services/web/server/src/simcore_service_webserver/long_running_tasks.py b/services/web/server/src/simcore_service_webserver/long_running_tasks/plugin.py similarity index 76% rename from services/web/server/src/simcore_service_webserver/long_running_tasks.py rename to services/web/server/src/simcore_service_webserver/long_running_tasks/plugin.py index e9ec7af5af8e..a7a6cabad85a 100644 --- a/services/web/server/src/simcore_service_webserver/long_running_tasks.py +++ b/services/web/server/src/simcore_service_webserver/long_running_tasks/plugin.py @@ -11,11 +11,12 @@ from servicelib.aiohttp.long_running_tasks.server import setup from servicelib.aiohttp.typing_extension import Handler -from . import rabbitmq_settings, redis -from ._meta import API_VTAG -from .login.decorators import login_required -from .models import AuthenticatedRequestContext -from .projects.plugin import register_projects_long_running_tasks +from .. import rabbitmq_settings, redis +from .._meta import API_VTAG +from ..login.decorators import login_required +from ..models import AuthenticatedRequestContext +from ..projects.plugin import register_projects_long_running_tasks +from . import settings as long_running_tasks_settings _logger = logging.getLogger(__name__) @@ -45,14 +46,14 @@ def setup_long_running_tasks(app: web.Application) -> None: # register all long-running tasks from different modules register_projects_long_running_tasks(app) - namespace_suffix = "TODO" # TODO recover from settings + settings = long_running_tasks_settings.get_plugin_settings(app) setup( app, redis_settings=redis.get_plugin_settings(app), rabbit_settings=rabbitmq_settings.get_plugin_settings(app), - redis_namespace=_get_namespace(namespace_suffix), - rabbit_namespace=_get_namespace(namespace_suffix), + redis_namespace=_get_namespace(settings.LONG_RUNNING_TASKS_NAMESPACE_SUFFIX), + rabbit_namespace=_get_namespace(settings.LONG_RUNNING_TASKS_NAMESPACE_SUFFIX), router_prefix=f"/{API_VTAG}/tasks-legacy", handler_check_decorator=login_required, task_request_context_decorator=webserver_request_context_decorator, diff --git a/services/web/server/src/simcore_service_webserver/long_running_tasks/settings.py b/services/web/server/src/simcore_service_webserver/long_running_tasks/settings.py new file mode 100644 index 000000000000..891148f03348 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/long_running_tasks/settings.py @@ -0,0 +1,26 @@ +from typing import Annotated + +from aiohttp import web +from pydantic import Field +from settings_library.base import BaseCustomSettings + +from ..constants import APP_SETTINGS_KEY + + +class LongRunningTasksSettings(BaseCustomSettings): + LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: Annotated[ + str, + Field( + description=( + "suffic to distinguis the service inside " + "the long_running_tasks framework" + ), + ), + ] + + +def get_plugin_settings(app: web.Application) -> LongRunningTasksSettings: + settings = app[APP_SETTINGS_KEY].WEBSERVER_LONG_RUNNING_TASKS + assert settings, "setup_settings not called?" # nosec + assert isinstance(settings, LongRunningTasksSettings) # nosec + return settings diff --git a/services/web/server/src/simcore_service_webserver/tasks/_rest.py b/services/web/server/src/simcore_service_webserver/tasks/_rest.py index 6b3e567a3f6c..288945d0ea6a 100644 --- a/services/web/server/src/simcore_service_webserver/tasks/_rest.py +++ b/services/web/server/src/simcore_service_webserver/tasks/_rest.py @@ -34,7 +34,7 @@ from .._meta import API_VTAG from ..constants import ASYNC_JOB_CLIENT_NAME from ..login.decorators import login_required -from ..long_running_tasks import webserver_request_context_decorator +from ..long_running_tasks.plugin import webserver_request_context_decorator from ..models import AuthenticatedRequestContext from ..rabbitmq import get_rabbitmq_rpc_client from ..security.decorators import permission_required From b2f3ea56ea57bbbec3e5c073a9a9b87f6c0891da Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 14 Aug 2025 14:14:17 +0200 Subject: [PATCH 047/119] fixed issues with failing services --- .../long_running_tasks/_rabbit/_models.py | 6 ++++++ .../long_running_tasks/_rabbit/lrt_client.py | 17 ++++++++++++----- .../long_running_tasks/_rabbit/lrt_server.py | 9 +++++++-- 3 files changed, 25 insertions(+), 7 deletions(-) create mode 100644 packages/service-library/src/servicelib/long_running_tasks/_rabbit/_models.py diff --git a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/_models.py b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/_models.py new file mode 100644 index 000000000000..5b3fa9f3a994 --- /dev/null +++ b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/_models.py @@ -0,0 +1,6 @@ +from pydantic import BaseModel + + +class RPCErrorResponse(BaseModel): + str_traceback: str + error_object: str diff --git a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py index f0722627ccf9..1d9517fe5dbf 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py @@ -10,6 +10,7 @@ from ...rabbitmq._client_rpc import RabbitMQRPCClient from .._serialization import string_to_object from ..models import RabbitNamespace, TaskBase, TaskContext, TaskId, TaskStatus +from ._models import RPCErrorResponse from .namespace import get_namespace _logger = logging.getLogger(__name__) @@ -99,12 +100,18 @@ async def get_task_result( task_id=task_id, timeout_s=_RPC_TIMEOUT_NORMAL_REQUEST, ) - assert isinstance(serialized_result, str) # nosec - task_result = string_to_object(serialized_result) + assert isinstance(serialized_result, RPCErrorResponse | str) # nosec + if isinstance(serialized_result, RPCErrorResponse): + error = string_to_object(serialized_result.error_object) + _logger.warning( + "Remote task finished with error: '%s: %s'\n%s", + error.__class__.__name__, + error, + serialized_result.str_traceback, + ) + raise error - if isinstance(task_result, Exception): - raise task_result - return task_result + return string_to_object(serialized_result) @log_decorator(_logger, level=logging.DEBUG) diff --git a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_server.py b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_server.py index f3285da4caa8..7bd4e772e460 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_server.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_server.py @@ -1,4 +1,5 @@ import logging +import traceback from typing import Any from ...rabbitmq import RPCRouter @@ -7,6 +8,7 @@ from ..errors import BaseLongRunningError, TaskNotCompletedError, TaskNotFoundError from ..models import TaskBase, TaskContext, TaskId, TaskStatus from ..task import RegisteredTaskName +from ._models import RPCErrorResponse _logger = logging.getLogger(__name__) @@ -85,7 +87,7 @@ async def get_task_result( *, task_context: TaskContext, task_id: TaskId, -) -> str: +) -> RPCErrorResponse | str: try: return object_to_string( await _get_transferarble_task_result( @@ -93,7 +95,10 @@ async def get_task_result( ) ) except Exception as exc: # pylint:disable=broad-exception-caught - return object_to_string(exc) + return RPCErrorResponse( + str_traceback="".join(traceback.format_tb(exc.__traceback__)), + error_object=object_to_string(exc), + ) @router.expose(reraise_if_error_type=(BaseLongRunningError,)) From 37b672ed47c4b4e19cacb5cbc750f1c267592777 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 14 Aug 2025 14:32:28 +0200 Subject: [PATCH 048/119] refactor setup --- .../simcore_service_webserver/application_settings.py | 3 +++ .../long_running_tasks/plugin.py | 10 ++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/application_settings.py b/services/web/server/src/simcore_service_webserver/application_settings.py index b9335185de1f..eb1369448728 100644 --- a/services/web/server/src/simcore_service_webserver/application_settings.py +++ b/services/web/server/src/simcore_service_webserver/application_settings.py @@ -588,6 +588,9 @@ def to_client_statics(self) -> dict[str, Any]: "WEBSERVER_TRASH": { "TRASH_RETENTION_DAYS", }, + "WEBSERVER_LONG_RUNNING_TASKS": { + "LONG_RUNNING_TASKS_NAMESPACE_SUFFIX", + }, }, exclude_none=True, ) diff --git a/services/web/server/src/simcore_service_webserver/long_running_tasks/plugin.py b/services/web/server/src/simcore_service_webserver/long_running_tasks/plugin.py index a7a6cabad85a..448f8295ea13 100644 --- a/services/web/server/src/simcore_service_webserver/long_running_tasks/plugin.py +++ b/services/web/server/src/simcore_service_webserver/long_running_tasks/plugin.py @@ -4,12 +4,13 @@ from aiohttp import web from models_library.utils.fastapi_encoders import jsonable_encoder -from servicelib.aiohttp.application_setup import ensure_single_setup +from servicelib.aiohttp.application_setup import app_module_setup from servicelib.aiohttp.long_running_tasks._constants import ( RQT_LONG_RUNNING_TASKS_CONTEXT_KEY, ) from servicelib.aiohttp.long_running_tasks.server import setup from servicelib.aiohttp.typing_extension import Handler +from simcore_service_webserver.rabbitmq import ModuleCategory from .. import rabbitmq_settings, redis from .._meta import API_VTAG @@ -41,7 +42,12 @@ async def _test_task_context_decorator( return _test_task_context_decorator -@ensure_single_setup(__name__, logger=_logger) +@app_module_setup( + __name__, + ModuleCategory.ADDON, + settings_name="WEBSERVER_LONG_RUNNING_TASKS", + logger=_logger, +) def setup_long_running_tasks(app: web.Application) -> None: # register all long-running tasks from different modules register_projects_long_running_tasks(app) From ba2e2af2aa12e7d44b6c84e59e9f88a2e20479da Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 14 Aug 2025 14:53:02 +0200 Subject: [PATCH 049/119] fixed tests --- .../tests/unit/test_api_route_dynamic_scheduler.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/services/director-v2/tests/unit/test_api_route_dynamic_scheduler.py b/services/director-v2/tests/unit/test_api_route_dynamic_scheduler.py index 941d83bd97a6..b573f4292fa8 100644 --- a/services/director-v2/tests/unit/test_api_route_dynamic_scheduler.py +++ b/services/director-v2/tests/unit/test_api_route_dynamic_scheduler.py @@ -214,10 +214,7 @@ async def test_409_response( ) assert response.status_code == status.HTTP_202_ACCEPTED task_id = response.json() - assert ( - f"simcore_service_director_v2.api.routes.dynamic_scheduler.{task_name}" - in task_id - ) + assert f"director-v2.functools.{task_name}" in task_id response = client.request( method, From 817929e50763449366fb7a8fd9210100048d3518 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 14 Aug 2025 15:07:08 +0200 Subject: [PATCH 050/119] fixed flaky tests --- .env-devel | 1 + .../src/simcore_service_webserver/long_running_tasks/plugin.py | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.env-devel b/.env-devel index f47839476369..0ea0874d3359 100644 --- a/.env-devel +++ b/.env-devel @@ -147,6 +147,7 @@ LICENSES_ITIS_VIP_API_URL=https://replace-with-itis-api/{category} LICENSES_ITIS_VIP_CATEGORIES='{"HumanWholeBody": "Humans", "HumanBodyRegion": "Humans (Region)", "AnimalWholeBody": "Animal"}' LICENSES_SPEAG_PHANTOMS_API_URL=https://replace-with-speag-api/{category} LICENSES_SPEAG_PHANTOMS_CATEGORIES='{"ComputationalPhantom": "Phantom of the Opera"}' +LONG_RUNNING_TASKS_NAMESPACE_SUFFIX=development # Can use 'docker run -it itisfoundation/invitations:latest simcore-service-invitations generate-dotenv --auto-password' INVITATIONS_DEFAULT_PRODUCT=osparc diff --git a/services/web/server/src/simcore_service_webserver/long_running_tasks/plugin.py b/services/web/server/src/simcore_service_webserver/long_running_tasks/plugin.py index 448f8295ea13..0ff8a6906438 100644 --- a/services/web/server/src/simcore_service_webserver/long_running_tasks/plugin.py +++ b/services/web/server/src/simcore_service_webserver/long_running_tasks/plugin.py @@ -4,13 +4,12 @@ from aiohttp import web from models_library.utils.fastapi_encoders import jsonable_encoder -from servicelib.aiohttp.application_setup import app_module_setup +from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup from servicelib.aiohttp.long_running_tasks._constants import ( RQT_LONG_RUNNING_TASKS_CONTEXT_KEY, ) from servicelib.aiohttp.long_running_tasks.server import setup from servicelib.aiohttp.typing_extension import Handler -from simcore_service_webserver.rabbitmq import ModuleCategory from .. import rabbitmq_settings, redis from .._meta import API_VTAG From 9b14fe56cecb3bb0f7ae39428ba972ed7a5318c5 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 14 Aug 2025 15:53:23 +0200 Subject: [PATCH 051/119] fixed typos --- .../src/servicelib/long_running_tasks/task.py | 6 +++--- .../long_running_tasks/settings.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index 56c42ad529e4..01bbe742b8c7 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -62,10 +62,10 @@ class TaskRegistry: @classmethod def register(cls, task: TaskProtocol, **partial_kwargs) -> None: - partail_task = functools.partial(task, **partial_kwargs) + partial_task = functools.partial(task, **partial_kwargs) # allows to call the partial via it's original name - partail_task.__name__ = task.__name__ # type: ignore[attr-defined] - cls.REGISTERED_TASKS[task.__name__] = partail_task # type: ignore[assignment] + partial_task.__name__ = task.__name__ # type: ignore[attr-defined] + cls.REGISTERED_TASKS[task.__name__] = partial_task # type: ignore[assignment] @classmethod def unregister(cls, task: TaskProtocol) -> None: diff --git a/services/web/server/src/simcore_service_webserver/long_running_tasks/settings.py b/services/web/server/src/simcore_service_webserver/long_running_tasks/settings.py index 891148f03348..ac3feb588005 100644 --- a/services/web/server/src/simcore_service_webserver/long_running_tasks/settings.py +++ b/services/web/server/src/simcore_service_webserver/long_running_tasks/settings.py @@ -12,8 +12,8 @@ class LongRunningTasksSettings(BaseCustomSettings): str, Field( description=( - "suffic to distinguis the service inside " - "the long_running_tasks framework" + "suffix to distinguish between the various services based on this image " + "inside the long_running_tasks framework" ), ), ] From 7bc16d10676ac1e70490e8bd247a5efe4735b3bd Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 15 Aug 2025 08:34:55 +0200 Subject: [PATCH 052/119] renamed --- .../long_running_tasks/_rabbit/lrt_client.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py index 1d9517fe5dbf..024f35d80cb9 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py @@ -18,8 +18,8 @@ _RPC_TIMEOUT_VERY_LONG_REQUEST: Final[PositiveInt] = int( timedelta(minutes=60).total_seconds() ) -_RPC_TIMEOUT_NORMAL_REQUEST: Final[PositiveInt] = int( - timedelta(seconds=30).total_seconds() +_RPC_TIMEOUT_SHORT_REQUESTS: Final[PositiveInt] = int( + timedelta(seconds=20).total_seconds() ) @@ -44,7 +44,7 @@ async def start_task( task_name=task_name, fire_and_forget=fire_and_forget, **task_kwargs, - timeout_s=_RPC_TIMEOUT_NORMAL_REQUEST, + timeout_s=_RPC_TIMEOUT_SHORT_REQUESTS, ) assert isinstance(result, TaskId) # nosec return result @@ -61,7 +61,7 @@ async def list_tasks( get_namespace(namespace), TypeAdapter(RPCMethodName).validate_python("list_tasks"), task_context=task_context, - timeout_s=_RPC_TIMEOUT_NORMAL_REQUEST, + timeout_s=_RPC_TIMEOUT_SHORT_REQUESTS, ) return TypeAdapter(list[TaskBase]).validate_python(result) @@ -79,7 +79,7 @@ async def get_task_status( TypeAdapter(RPCMethodName).validate_python("get_task_status"), task_context=task_context, task_id=task_id, - timeout_s=_RPC_TIMEOUT_NORMAL_REQUEST, + timeout_s=_RPC_TIMEOUT_SHORT_REQUESTS, ) assert isinstance(result, TaskStatus) # nosec return result @@ -98,7 +98,7 @@ async def get_task_result( TypeAdapter(RPCMethodName).validate_python("get_task_result"), task_context=task_context, task_id=task_id, - timeout_s=_RPC_TIMEOUT_NORMAL_REQUEST, + timeout_s=_RPC_TIMEOUT_SHORT_REQUESTS, ) assert isinstance(serialized_result, RPCErrorResponse | str) # nosec if isinstance(serialized_result, RPCErrorResponse): From 159ab96a6c471187ccb2752db5dc094ce39945aa Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 15 Aug 2025 14:10:50 +0200 Subject: [PATCH 053/119] fixes concurrency issue with tests in CI --- .../long_running_tasks/_store/base.py | 9 +- .../long_running_tasks/_store/redis.py | 92 ++++++++++++++----- .../src/servicelib/long_running_tasks/task.py | 32 ++++--- .../tests/long_running_tasks/conftest.py | 12 +++ .../test_long_running_tasks__store.py | 4 +- .../test_long_running_tasks_lrt_api.py | 18 +--- .../test_long_running_tasks_task.py | 6 +- 7 files changed, 116 insertions(+), 57 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/_store/base.py b/packages/service-library/src/servicelib/long_running_tasks/_store/base.py index 37944ef6ec39..6fc5df31f489 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_store/base.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_store/base.py @@ -1,4 +1,5 @@ from abc import abstractmethod +from typing import Any from ..models import TaskContext, TaskData, TaskId @@ -10,9 +11,15 @@ async def get_task_data(self, task_id: TaskId) -> TaskData | None: """Retrieve a tracked task""" @abstractmethod - async def set_task_data(self, task_id: TaskId, value: TaskData) -> None: + async def add_task_data(self, task_id: TaskId, value: TaskData) -> None: """Set a tracked task's data""" + @abstractmethod + async def update_task_data( + self, task_id: TaskId, *, updates: dict[str, Any] + ) -> None: + """Update a tracked task's data by specifying each single field to update""" + @abstractmethod async def list_tasks_data(self) -> list[TaskData]: """List all tracked tasks.""" diff --git a/packages/service-library/src/servicelib/long_running_tasks/_store/redis.py b/packages/service-library/src/servicelib/long_running_tasks/_store/redis.py index 7ee0df1e1032..edeea3a79ccd 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_store/redis.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_store/redis.py @@ -1,8 +1,10 @@ +import json from typing import Any, Final import redis.asyncio as aioredis from common_library.json_serialization import json_dumps, json_loads from pydantic import TypeAdapter +from servicelib.utils import limited_gather from settings_library.redis import RedisDatabase, RedisSettings from ...redis._client import RedisClientSDK @@ -12,6 +14,17 @@ _STORE_TYPE_TASK_DATA: Final[str] = "TD" _STORE_TYPE_CANCELLED_TASKS: Final[str] = "CT" +_LIST_CONCURRENCY: Final[int] = 2 + + +def _encode_dict(data: dict[str, Any]) -> dict[str, str]: + """replaces dict with a JSON-serializable dict""" + return {k: json_dumps(v) for k, v in data.items()} + + +def _decode_dict(data: dict[str, str]) -> dict[str, Any]: + """replaces dict with a JSON-deserialized dict""" + return {k: json.loads(v) for k, v in data.items()} class RedisStore(BaseStore): @@ -37,58 +50,93 @@ def _redis(self) -> aioredis.Redis: assert self._client # nosec return self._client.redis - def _get_redis_hash_key(self, store_type: str) -> str: - return f"{self.namespace}:{store_type}" + def _get_redis_key_task_data_match(self) -> str: + return f"{self.namespace}:{_STORE_TYPE_TASK_DATA}*" + + def _get_redis_key_task_data_hash(self, task_id: TaskId) -> str: + return f"{self.namespace}:{_STORE_TYPE_TASK_DATA}:{task_id}" - def _get_key(self, store_type: str, name: str) -> str: - return f"{self.namespace}:{store_type}:{name}" + def _get_key_cancelled_tasks(self) -> str: + return f"{self.namespace}:{_STORE_TYPE_CANCELLED_TASKS}" + + # TaskData async def get_task_data(self, task_id: TaskId) -> TaskData | None: - result: Any | None = await handle_redis_returns_union_types( - self._redis.hget(self._get_redis_hash_key(_STORE_TYPE_TASK_DATA), task_id) + result: dict[str, Any] = await handle_redis_returns_union_types( + self._redis.hgetall( + self._get_redis_key_task_data_hash(task_id), + ) + ) + return ( + TypeAdapter(TaskData).validate_python(_decode_dict(result)) + if result and len(result) + else None ) - return TypeAdapter(TaskData).validate_json(result) if result else None - async def set_task_data(self, task_id: TaskId, value: TaskData) -> None: + async def add_task_data(self, task_id: TaskId, value: TaskData) -> None: await handle_redis_returns_union_types( self._redis.hset( - self._get_redis_hash_key(_STORE_TYPE_TASK_DATA), - task_id, - value.model_dump_json(), + self._get_redis_key_task_data_hash(task_id), + mapping=_encode_dict(value.model_dump()), + ) + ) + + async def update_task_data( + self, + task_id: TaskId, + *, + updates: dict[str, Any], + ) -> None: + await handle_redis_returns_union_types( + self._redis.hset( + self._get_redis_key_task_data_hash(task_id), + mapping=_encode_dict(updates), ) ) async def list_tasks_data(self) -> list[TaskData]: - result: list[Any] = await handle_redis_returns_union_types( - self._redis.hvals(self._get_redis_hash_key(_STORE_TYPE_TASK_DATA)) + hash_keys: list[str] = [ + x + async for x in self._redis.scan_iter(self._get_redis_key_task_data_match()) + ] + + result = await limited_gather( + *[ + handle_redis_returns_union_types(self._redis.hgetall(key)) + for key in hash_keys + ], + limit=_LIST_CONCURRENCY, ) - return [TypeAdapter(TaskData).validate_json(item) for item in result] + + return [ + TypeAdapter(TaskData).validate_python(_decode_dict(item)) + for item in result + if item + ] async def delete_task_data(self, task_id: TaskId) -> None: await handle_redis_returns_union_types( - self._redis.hdel(self._get_redis_hash_key(_STORE_TYPE_TASK_DATA), task_id) + self._redis.delete(self._get_redis_key_task_data_hash(task_id)) ) + # cancelled + async def set_as_cancelled( self, task_id: TaskId, with_task_context: TaskContext ) -> None: await handle_redis_returns_union_types( self._redis.hset( - self._get_redis_hash_key(_STORE_TYPE_CANCELLED_TASKS), - task_id, - json_dumps(with_task_context), + self._get_key_cancelled_tasks(), task_id, json_dumps(with_task_context) ) ) async def delete_set_as_cancelled(self, task_id: TaskId) -> None: await handle_redis_returns_union_types( - self._redis.hdel( - self._get_redis_hash_key(_STORE_TYPE_CANCELLED_TASKS), task_id - ) + self._redis.hdel(self._get_key_cancelled_tasks(), task_id) ) async def get_cancelled(self) -> dict[TaskId, TaskContext]: result: dict[str, str | None] = await handle_redis_returns_union_types( - self._redis.hgetall(self._get_redis_hash_key(_STORE_TYPE_CANCELLED_TASKS)) + self._redis.hgetall(self._get_key_cancelled_tasks()) ) return {task_id: json_loads(context) for task_id, context in result.items()} diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index 01bbe742b8c7..89cb7dc1a056 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -293,25 +293,25 @@ async def _status_update(self) -> None: # already done and updatet data in redis continue - # update and store in Redis - task_data.is_done = is_done - + result_field: ResultField | None = None # get task result try: - task_data.result_field = ResultField( - result=object_to_string(task.result()) - ) + result_field = ResultField(result=object_to_string(task.result())) except asyncio.InvalidStateError: # task was not completed try again next time and see if it is done continue except asyncio.CancelledError: - task_data.result_field = ResultField( + result_field = ResultField( error=object_to_string(TaskCancelledError(task_id=task_id)) ) except Exception as e: # pylint:disable=broad-except - task_data.result_field = ResultField(error=object_to_string(e)) + result_field = ResultField(error=object_to_string(e)) - await self._tasks_data.set_task_data(task_id, task_data) + # update and store in Redis + updates = {"is_done": is_done, "result_field": task_data.result_field} + if result_field is not None: + updates["result_field"] = result_field + await self._tasks_data.update_task_data(task_id, updates=updates) async def list_tasks(self, with_task_context: TaskContext | None) -> list[TaskBase]: if not with_task_context: @@ -348,10 +348,12 @@ async def get_task_status( raises TaskNotFoundError if the task cannot be found """ - task_data: TaskData = await self._get_tracked_task(task_id, with_task_context) - task_data.last_status_check = datetime.datetime.now(tz=datetime.UTC) - await self._tasks_data.set_task_data(task_id, task_data) + task_data = await self._get_tracked_task(task_id, with_task_context) + await self._tasks_data.update_task_data( + task_id, + updates={"last_status_check": datetime.datetime.now(tz=datetime.UTC)}, + ) return TaskStatus.model_validate( { "task_progress": task_data.task_progress, @@ -442,7 +444,9 @@ async def _update_progress( try: tracked_data = await self._get_tracked_task(task_id, task_context) tracked_data.task_progress = task_progress - await self._tasks_data.set_task_data(task_id=task_id, value=tracked_data) + await self._tasks_data.update_task_data( + task_id, updates={"task_progress": task_progress.model_dump()} + ) except TaskNotFoundError: _logger.debug( "Task '%s' not found while updating progress %s", @@ -508,7 +512,7 @@ async def _task_with_progress(progress: TaskProgress, handler: TaskProtocol): task_context=context_to_use, fire_and_forget=fire_and_forget, ) - await self._tasks_data.set_task_data(task_id, tracked_task) + await self._tasks_data.add_task_data(task_id, tracked_task) return tracked_task.task_id diff --git a/packages/service-library/tests/long_running_tasks/conftest.py b/packages/service-library/tests/long_running_tasks/conftest.py index 5b184dd7b7e9..28705adab747 100644 --- a/packages/service-library/tests/long_running_tasks/conftest.py +++ b/packages/service-library/tests/long_running_tasks/conftest.py @@ -1,11 +1,14 @@ +# pylint: disable=protected-access # pylint: disable=redefined-outer-name # pylint: disable=unused-argument + import logging from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable from datetime import timedelta import pytest from faker import Faker +from pytest_mock import MockerFixture from servicelib.logging_utils import log_catch from servicelib.long_running_tasks._rabbit.lrt_client import RabbitNamespace from servicelib.long_running_tasks.task import ( @@ -91,3 +94,12 @@ async def rabbitmq_rpc_client( ) yield client await client.close() + + +@pytest.fixture +def disable_stale_tasks_monitor(mocker: MockerFixture) -> None: + # no need to autoremove stale tasks in these tests + async def _to_replace(self: TasksManager) -> None: + self._started_event_task_stale_tasks_monitor.set() + + mocker.patch.object(TasksManager, "_stale_tasks_monitor", _to_replace) diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks__store.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks__store.py index bd4586be6487..41e499ac5b86 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks__store.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks__store.py @@ -42,7 +42,7 @@ async def test_workflow(store: BaseStore, task_data: TaskData) -> None: assert await store.list_tasks_data() == [] assert await store.get_task_data("missing") is None - await store.set_task_data(task_data.task_id, task_data) + await store.add_task_data(task_data.task_id, task_data) assert await store.list_tasks_data() == [task_data] @@ -91,7 +91,7 @@ async def test_workflow_multiple_redis_stores_with_different_namespaces( assert await store.get_cancelled() == {} for store in redis_stores: - await store.set_task_data(task_data.task_id, task_data) + await store.add_task_data(task_data.task_id, task_data) await store.set_as_cancelled(task_data.task_id, None) for store in redis_stores: diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py index f57f63d802e7..5454131c7d37 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py @@ -1,4 +1,3 @@ -# pylint: disable=protected-access # pylint: disable=redefined-outer-name # pylint: disable=unused-argument @@ -10,17 +9,11 @@ import pytest from models_library.api_schemas_long_running_tasks.base import TaskProgress from pydantic import NonNegativeInt -from pytest_mock import MockerFixture from servicelib.long_running_tasks import lrt_api from servicelib.long_running_tasks._rabbit.lrt_client import RabbitNamespace from servicelib.long_running_tasks.errors import TaskNotFoundError from servicelib.long_running_tasks.models import TaskContext -from servicelib.long_running_tasks.task import ( - RedisNamespace, - TaskId, - TaskRegistry, - TasksManager, -) +from servicelib.long_running_tasks.task import RedisNamespace, TaskId, TaskRegistry from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings @@ -69,15 +62,6 @@ def managers_count() -> NonNegativeInt: return 5 -@pytest.fixture -def disable_stale_tasks_monitor(mocker: MockerFixture) -> None: - # no need to autoremove stale tasks in these tests - async def _to_replace(self: TasksManager) -> None: - self._started_event_task_stale_tasks_monitor.set() - - mocker.patch.object(TasksManager, "_stale_tasks_monitor", _to_replace) - - @pytest.fixture async def long_running_managers( disable_stale_tasks_monitor: None, diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py index 4d3bab21c1ba..dd646941ce22 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py @@ -22,7 +22,10 @@ TaskNotRegisteredError, ) from servicelib.long_running_tasks.models import TaskContext, TaskProgress, TaskStatus -from servicelib.long_running_tasks.task import RedisNamespace, TaskRegistry +from servicelib.long_running_tasks.task import ( + RedisNamespace, + TaskRegistry, +) from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings @@ -529,6 +532,7 @@ async def test__cancelled_tasks_worker_equivalent_of_cancellation_from_a_differe async def test_list_tasks( + disable_stale_tasks_monitor: None, rabbitmq_rpc_client: RabbitMQRPCClient, long_running_manager: NoWebAppLongRunningManager, empty_context: TaskContext, From 2ae799d3807165e08f63d5d6332903a8b5144a69 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 18 Aug 2025 09:17:46 +0200 Subject: [PATCH 054/119] refactor long running manager --- .../aiohttp/long_running_tasks/_manager.py | 71 +----------------- .../aiohttp/long_running_tasks/_server.py | 1 - .../fastapi/long_running_tasks/_manager.py | 73 +----------------- .../fastapi/long_running_tasks/_server.py | 1 - .../long_running_tasks/_rabbit/lrt_server.py | 3 - .../base_long_running_manager.py | 75 +++++++++++++++---- .../servicelib/long_running_tasks/lrt_api.py | 15 +--- .../tests/long_running_tasks/conftest.py | 53 ++++--------- .../test_long_running_tasks_lrt_api.py | 30 ++++---- .../test_long_running_tasks_task.py | 55 +++++++------- .../tests/long_running_tasks/utils.py | 48 ------------ 11 files changed, 125 insertions(+), 300 deletions(-) diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_manager.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_manager.py index 53d4ec06a25d..b2023c980681 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_manager.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_manager.py @@ -1,81 +1,12 @@ -import datetime - from aiohttp import web -from settings_library.rabbit import RabbitSettings -from settings_library.redis import RedisSettings -from ...long_running_tasks import lrt_api from ...long_running_tasks.base_long_running_manager import BaseLongRunningManager -from ...long_running_tasks.models import RabbitNamespace, TaskContext -from ...long_running_tasks.task import RedisNamespace, TasksManager -from ...rabbitmq._client_rpc import RabbitMQRPCClient +from ...long_running_tasks.models import TaskContext from ._constants import APP_LONG_RUNNING_MANAGER_KEY from ._request import get_task_context class AiohttpLongRunningManager(BaseLongRunningManager): - def __init__( - self, - app: web.Application, - stale_task_check_interval: datetime.timedelta, - stale_task_detect_timeout: datetime.timedelta, - redis_settings: RedisSettings, - rabbit_settings: RabbitSettings, - redis_namespace: RedisNamespace, - rabbit_namespace: RabbitNamespace, - ): - self._app = app - self._tasks_manager = TasksManager( - stale_task_check_interval=stale_task_check_interval, - stale_task_detect_timeout=stale_task_detect_timeout, - redis_settings=redis_settings, - redis_namespace=redis_namespace, - ) - self._rabbit_namespace = rabbit_namespace - self.rabbit_settings = rabbit_settings - self._rpc_server: RabbitMQRPCClient | None = None - self._rpc_client: RabbitMQRPCClient | None = None - - @property - def tasks_manager(self) -> TasksManager: - return self._tasks_manager - - @property - def rpc_server(self) -> RabbitMQRPCClient: - assert self._rpc_server is not None # nosec - return self._rpc_server - - @property - def rpc_client(self) -> RabbitMQRPCClient: - assert self._rpc_client is not None # nosec - return self._rpc_client - - @property - def rabbit_namespace(self) -> RabbitNamespace: - return self._rabbit_namespace - - async def setup(self) -> None: - await self._tasks_manager.setup() - self._rpc_server = await RabbitMQRPCClient.create( - client_name=f"lrt-server-{self.rabbit_namespace}", - settings=self.rabbit_settings, - ) - self._rpc_client = await RabbitMQRPCClient.create( - client_name=f"lrt-client-{self.rabbit_namespace}", - settings=self.rabbit_settings, - ) - await lrt_api.register_rabbit_routes(self) - - async def teardown(self) -> None: - await self._tasks_manager.teardown() - - if self._rpc_server is not None: - await self._rpc_server.close() - self._rpc_server = None - - if self._rpc_client is not None: - await self._rpc_client.close() - self._rpc_client = None @staticmethod def get_task_context(request: web.Request) -> TaskContext: diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py index 3b843490f24e..757f783abd35 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py @@ -175,7 +175,6 @@ async def on_cleanup_ctx(app: web.Application) -> AsyncGenerator[None, None]: # add components to state app[APP_LONG_RUNNING_MANAGER_KEY] = long_running_manager = ( AiohttpLongRunningManager( - app=app, stale_task_check_interval=stale_task_check_interval, stale_task_detect_timeout=stale_task_detect_timeout, redis_settings=redis_settings, diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_manager.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_manager.py index 42aa15b13e04..54215c0fe015 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_manager.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_manager.py @@ -1,76 +1,5 @@ -import datetime - -from fastapi import FastAPI -from settings_library.rabbit import RabbitSettings -from settings_library.redis import RedisSettings - -from ...long_running_tasks import lrt_api from ...long_running_tasks.base_long_running_manager import BaseLongRunningManager -from ...long_running_tasks.models import RabbitNamespace -from ...long_running_tasks.task import RedisNamespace, TasksManager -from ...rabbitmq._client_rpc import RabbitMQRPCClient class FastAPILongRunningManager(BaseLongRunningManager): - def __init__( - self, - app: FastAPI, - stale_task_check_interval: datetime.timedelta, - stale_task_detect_timeout: datetime.timedelta, - redis_settings: RedisSettings, - rabbit_settings: RabbitSettings, - redis_namespace: RedisNamespace, - rabbit_namespace: RabbitNamespace, - ): - self._app = app - self._tasks_manager = TasksManager( - stale_task_check_interval=stale_task_check_interval, - stale_task_detect_timeout=stale_task_detect_timeout, - redis_settings=redis_settings, - redis_namespace=redis_namespace, - ) - self._rabbit_namespace = rabbit_namespace - self.rabbit_settings = rabbit_settings - self._rpc_server: RabbitMQRPCClient | None = None - self._rpc_client: RabbitMQRPCClient | None = None - - @property - def tasks_manager(self) -> TasksManager: - return self._tasks_manager - - @property - def rpc_server(self) -> RabbitMQRPCClient: - assert self._rpc_server is not None # nosec - return self._rpc_server - - @property - def rpc_client(self) -> RabbitMQRPCClient: - assert self._rpc_client is not None # nosec - return self._rpc_client - - @property - def rabbit_namespace(self) -> str: - return self._rabbit_namespace - - async def setup(self) -> None: - await self._tasks_manager.setup() - self._rpc_server = await RabbitMQRPCClient.create( - client_name=f"lrt-server-{self.rabbit_namespace}", - settings=self.rabbit_settings, - ) - self._rpc_client = await RabbitMQRPCClient.create( - client_name=f"lrt-client-{self.rabbit_namespace}", - settings=self.rabbit_settings, - ) - await lrt_api.register_rabbit_routes(self) - - async def teardown(self) -> None: - await self._tasks_manager.teardown() - - if self._rpc_server is not None: - await self._rpc_server.close() - self._rpc_server = None - - if self._rpc_client is not None: - await self._rpc_client.close() - self._rpc_client = None + pass diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py index 53d21173bf1e..75e4c06f34a0 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py @@ -46,7 +46,6 @@ async def on_startup() -> None: # add components to state app.state.long_running_manager = long_running_manager = ( FastAPILongRunningManager( - app=app, stale_task_check_interval=stale_task_check_interval, stale_task_detect_timeout=stale_task_detect_timeout, redis_settings=redis_settings, diff --git a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_server.py b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_server.py index 7bd4e772e460..451c74bd75f9 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_server.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_server.py @@ -1,4 +1,3 @@ -import logging import traceback from typing import Any @@ -10,8 +9,6 @@ from ..task import RegisteredTaskName from ._models import RPCErrorResponse -_logger = logging.getLogger(__name__) - router = RPCRouter() diff --git a/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py b/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py index 55b9a4cd69d2..9a7071261f5a 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py +++ b/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py @@ -1,34 +1,83 @@ -from abc import ABC, abstractmethod +import datetime + +from settings_library.rabbit import RabbitSettings +from settings_library.redis import RedisSettings from ..rabbitmq._client_rpc import RabbitMQRPCClient +from ._rabbit.namespace import get_namespace from .models import RabbitNamespace -from .task import TasksManager +from .task import RedisNamespace, TasksManager -class BaseLongRunningManager(ABC): +class BaseLongRunningManager: """ Provides a commond inteface for aiohttp and fastapi services """ + def __init__( + self, + stale_task_check_interval: datetime.timedelta, + stale_task_detect_timeout: datetime.timedelta, + redis_settings: RedisSettings, + rabbit_settings: RabbitSettings, + redis_namespace: RedisNamespace, + rabbit_namespace: RabbitNamespace, + ): + self._tasks_manager = TasksManager( + stale_task_check_interval=stale_task_check_interval, + stale_task_detect_timeout=stale_task_detect_timeout, + redis_settings=redis_settings, + redis_namespace=redis_namespace, + ) + self._rabbit_namespace = rabbit_namespace + self.rabbit_settings = rabbit_settings + self._rpc_server: RabbitMQRPCClient | None = None + self._rpc_client: RabbitMQRPCClient | None = None + @property - @abstractmethod def tasks_manager(self) -> TasksManager: - pass + return self._tasks_manager @property - @abstractmethod def rpc_server(self) -> RabbitMQRPCClient: - pass + assert self._rpc_server is not None # nosec + return self._rpc_server + + @property + def rpc_client(self) -> RabbitMQRPCClient: + assert self._rpc_client is not None # nosec + return self._rpc_client @property - @abstractmethod def rabbit_namespace(self) -> RabbitNamespace: - pass + return self._rabbit_namespace - @abstractmethod async def setup(self) -> None: - pass + await self._tasks_manager.setup() + self._rpc_server = await RabbitMQRPCClient.create( + client_name=f"lrt-server-{self.rabbit_namespace}", + settings=self.rabbit_settings, + ) + self._rpc_client = await RabbitMQRPCClient.create( + client_name=f"lrt-client-{self.rabbit_namespace}", + settings=self.rabbit_settings, + ) + + from ._rabbit.lrt_server import router + + await self.rpc_server.register_router( + router, + get_namespace(self.rabbit_namespace), + self, + ) - @abstractmethod async def teardown(self) -> None: - pass + await self._tasks_manager.teardown() + + if self._rpc_server is not None: + await self._rpc_server.close() + self._rpc_server = None + + if self._rpc_client is not None: + await self._rpc_client.close() + self._rpc_client = None diff --git a/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py b/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py index fc200bf22125..e3fc442a79e2 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py +++ b/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py @@ -1,16 +1,12 @@ -import logging from typing import Any from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient -from ._rabbit import lrt_client, lrt_server -from ._rabbit.namespace import get_namespace +from ._rabbit import lrt_client from .base_long_running_manager import BaseLongRunningManager from .models import TaskBase, TaskContext, TaskId, TaskStatus from .task import RegisteredTaskName -_logger = logging.getLogger(__name__) - async def start_task( rabbitmq_rpc_client: RabbitMQRPCClient, @@ -115,12 +111,3 @@ async def remove_task( task_id=task_id, task_context=task_context, ) - - -async def register_rabbit_routes(long_running_manager: BaseLongRunningManager) -> None: - rpc_server = long_running_manager.rpc_server - await rpc_server.register_router( - lrt_server.router, - get_namespace(long_running_manager.rabbit_namespace), - long_running_manager, - ) diff --git a/packages/service-library/tests/long_running_tasks/conftest.py b/packages/service-library/tests/long_running_tasks/conftest.py index 28705adab747..d5aaf9ccf8e7 100644 --- a/packages/service-library/tests/long_running_tasks/conftest.py +++ b/packages/service-library/tests/long_running_tasks/conftest.py @@ -11,6 +11,9 @@ from pytest_mock import MockerFixture from servicelib.logging_utils import log_catch from servicelib.long_running_tasks._rabbit.lrt_client import RabbitNamespace +from servicelib.long_running_tasks.base_long_running_manager import ( + BaseLongRunningManager, +) from servicelib.long_running_tasks.task import ( RedisNamespace, TasksManager, @@ -18,61 +21,35 @@ from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings -from utils import TEST_CHECK_STALE_INTERVAL_S, NoWebAppLongRunningManager +from utils import TEST_CHECK_STALE_INTERVAL_S _logger = logging.getLogger(__name__) -@pytest.fixture -async def get_tasks_manager( - fast_long_running_tasks_cancellation: None, faker: Faker -) -> AsyncIterator[ - Callable[[RedisSettings, RedisNamespace | None], Awaitable[TasksManager]] -]: - managers: list[TasksManager] = [] - - async def _( - redis_settings: RedisSettings, namespace: RedisNamespace | None - ) -> TasksManager: - tasks_manager = TasksManager( - stale_task_check_interval=timedelta(seconds=TEST_CHECK_STALE_INTERVAL_S), - stale_task_detect_timeout=timedelta(seconds=TEST_CHECK_STALE_INTERVAL_S), - redis_settings=redis_settings, - redis_namespace=namespace or f"test{faker.uuid4()}", - ) - await tasks_manager.setup() - managers.append(tasks_manager) - return tasks_manager - - yield _ - - for manager in managers: - with log_catch(_logger, reraise=False): - await manager.teardown() - - @pytest.fixture async def get_long_running_manager( - get_tasks_manager: Callable[ - [RedisSettings, RedisNamespace | None], Awaitable[TasksManager] - ], + fast_long_running_tasks_cancellation: None, faker: Faker ) -> AsyncIterator[ Callable[ [RedisSettings, RedisNamespace | None, RabbitSettings, RabbitNamespace], - Awaitable[NoWebAppLongRunningManager], + Awaitable[BaseLongRunningManager], ] ]: - managers: list[NoWebAppLongRunningManager] = [] + managers: list[BaseLongRunningManager] = [] async def _( redis_settings: RedisSettings, namespace: RedisNamespace | None, rabbit_settings: RabbitSettings, rabbit_namespace: RabbitNamespace, - ) -> NoWebAppLongRunningManager: - tasks_manager = await get_tasks_manager(redis_settings, namespace) - manager = NoWebAppLongRunningManager( - tasks_manager, rabbit_settings, rabbit_namespace + ) -> BaseLongRunningManager: + manager = BaseLongRunningManager( + stale_task_check_interval=timedelta(seconds=TEST_CHECK_STALE_INTERVAL_S), + stale_task_detect_timeout=timedelta(seconds=TEST_CHECK_STALE_INTERVAL_S), + redis_namespace=namespace or f"test{faker.uuid4()}", + redis_settings=redis_settings, + rabbit_namespace=rabbit_namespace, + rabbit_settings=rabbit_settings, ) await manager.setup() managers.append(manager) diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py index 5454131c7d37..ba0719263bb2 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py @@ -11,6 +11,9 @@ from pydantic import NonNegativeInt from servicelib.long_running_tasks import lrt_api from servicelib.long_running_tasks._rabbit.lrt_client import RabbitNamespace +from servicelib.long_running_tasks.base_long_running_manager import ( + BaseLongRunningManager, +) from servicelib.long_running_tasks.errors import TaskNotFoundError from servicelib.long_running_tasks.models import TaskContext from servicelib.long_running_tasks.task import RedisNamespace, TaskId, TaskRegistry @@ -24,7 +27,6 @@ stop_after_delay, wait_fixed, ) -from utils import NoWebAppLongRunningManager pytest_simcore_core_services_selection = [ "rabbit", @@ -70,10 +72,10 @@ async def long_running_managers( rabbit_service: RabbitSettings, get_long_running_manager: Callable[ [RedisSettings, RedisNamespace | None, RabbitSettings, RabbitNamespace], - Awaitable[NoWebAppLongRunningManager], + Awaitable[BaseLongRunningManager], ], -) -> list[NoWebAppLongRunningManager]: - maanagers: list[NoWebAppLongRunningManager] = [] +) -> list[BaseLongRunningManager]: + maanagers: list[BaseLongRunningManager] = [] for _ in range(managers_count): long_running_manager = await get_long_running_manager( use_in_memory_redis, "same-service", rabbit_service, "some-service" @@ -84,14 +86,14 @@ async def long_running_managers( def _get_task_manager( - long_running_managers: list[NoWebAppLongRunningManager], -) -> NoWebAppLongRunningManager: + long_running_managers: list[BaseLongRunningManager], +) -> BaseLongRunningManager: return secrets.choice(long_running_managers) async def _assert_task_status( rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_manager: NoWebAppLongRunningManager, + long_running_manager: BaseLongRunningManager, task_id: TaskId, *, is_done: bool @@ -104,7 +106,7 @@ async def _assert_task_status( async def _assert_task_status_on_random_manager( rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_managers: list[NoWebAppLongRunningManager], + long_running_managers: list[BaseLongRunningManager], task_ids: list[TaskId], *, is_done: bool = True @@ -121,7 +123,7 @@ async def _assert_task_status_on_random_manager( async def _assert_task_status_done_on_all_managers( rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_managers: list[NoWebAppLongRunningManager], + long_running_managers: list[BaseLongRunningManager], task_id: TaskId, *, is_done: bool = True @@ -144,7 +146,7 @@ async def _assert_task_status_done_on_all_managers( async def _assert_list_tasks_from_all_managers( rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_managers: list[NoWebAppLongRunningManager], + long_running_managers: list[BaseLongRunningManager], task_context: TaskContext, task_count: int, ) -> None: @@ -155,7 +157,7 @@ async def _assert_list_tasks_from_all_managers( async def _assert_task_is_no_longer_present( rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_managers: list[NoWebAppLongRunningManager], + long_running_managers: list[BaseLongRunningManager], task_context: TaskContext, task_id: TaskId, ) -> None: @@ -178,7 +180,7 @@ async def _assert_task_is_no_longer_present( @pytest.mark.parametrize("is_unique", _IS_UNIQUE) @pytest.mark.parametrize("to_return", [{"key": "value"}]) async def test_workflow_with_result( - long_running_managers: list[NoWebAppLongRunningManager], + long_running_managers: list[BaseLongRunningManager], rabbitmq_rpc_client: RabbitMQRPCClient, task_count: int, is_unique: bool, @@ -234,7 +236,7 @@ async def test_workflow_with_result( @pytest.mark.parametrize("task_context", _TASK_CONTEXT) @pytest.mark.parametrize("is_unique", _IS_UNIQUE) async def test_workflow_raises_error( - long_running_managers: list[NoWebAppLongRunningManager], + long_running_managers: list[BaseLongRunningManager], rabbitmq_rpc_client: RabbitMQRPCClient, task_count: int, is_unique: bool, @@ -287,7 +289,7 @@ async def test_workflow_raises_error( @pytest.mark.parametrize("task_context", _TASK_CONTEXT) @pytest.mark.parametrize("is_unique", _IS_UNIQUE) async def test_remove_task( - long_running_managers: list[NoWebAppLongRunningManager], + long_running_managers: list[BaseLongRunningManager], rabbitmq_rpc_client: RabbitMQRPCClient, is_unique: bool, task_context: TaskContext | None, diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py index dd646941ce22..09ccc2919e54 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py @@ -15,6 +15,9 @@ from models_library.api_schemas_long_running_tasks.base import ProgressMessage from servicelib.long_running_tasks import lrt_api from servicelib.long_running_tasks._rabbit.lrt_client import RabbitNamespace +from servicelib.long_running_tasks.base_long_running_manager import ( + BaseLongRunningManager, +) from servicelib.long_running_tasks.errors import ( TaskAlreadyRunningError, TaskNotCompletedError, @@ -34,7 +37,7 @@ from tenacity.retry import retry_if_exception_type from tenacity.stop import stop_after_delay from tenacity.wait import wait_fixed -from utils import TEST_CHECK_STALE_INTERVAL_S, NoWebAppLongRunningManager +from utils import TEST_CHECK_STALE_INTERVAL_S pytest_simcore_core_services_selection = [ "rabbit", @@ -91,9 +94,9 @@ async def long_running_manager( rabbit_service: RabbitSettings, get_long_running_manager: Callable[ [RedisSettings, RedisNamespace | None, RabbitSettings, RabbitNamespace], - Awaitable[NoWebAppLongRunningManager], + Awaitable[BaseLongRunningManager], ], -) -> NoWebAppLongRunningManager: +) -> BaseLongRunningManager: return await get_long_running_manager( use_in_memory_redis, None, rabbit_service, "rabbit-namespace" ) @@ -102,7 +105,7 @@ async def long_running_manager( @pytest.mark.parametrize("check_task_presence_before", [True, False]) async def test_task_is_auto_removed( rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_manager: NoWebAppLongRunningManager, + long_running_manager: BaseLongRunningManager, check_task_presence_before: bool, empty_context: TaskContext, ): @@ -127,9 +130,9 @@ async def test_task_is_auto_removed( async for attempt in AsyncRetrying(**_RETRY_PARAMS): with attempt: if ( - await long_running_manager.tasks_manager._tasks_data.get_task_data( + await long_running_manager.tasks_manager._tasks_data.get_task_data( # noqa: SLF001 task_id - ) # noqa: SLF001 + ) is not None ): msg = "wait till no element is found any longer" @@ -147,7 +150,7 @@ async def test_task_is_auto_removed( async def test_checked_task_is_not_auto_removed( rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_manager: NoWebAppLongRunningManager, + long_running_manager: BaseLongRunningManager, empty_context: TaskContext, ): task_id = await lrt_api.start_task( @@ -172,7 +175,7 @@ async def test_checked_task_is_not_auto_removed( async def test_fire_and_forget_task_is_not_auto_removed( rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_manager: NoWebAppLongRunningManager, + long_running_manager: BaseLongRunningManager, empty_context: TaskContext, ): task_id = await lrt_api.start_task( @@ -201,7 +204,7 @@ async def test_fire_and_forget_task_is_not_auto_removed( async def test_get_result_of_unfinished_task_raises( rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_manager: NoWebAppLongRunningManager, + long_running_manager: BaseLongRunningManager, empty_context: TaskContext, ): task_id = await lrt_api.start_task( @@ -220,7 +223,7 @@ async def test_get_result_of_unfinished_task_raises( async def test_unique_task_already_running( rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_manager: NoWebAppLongRunningManager, + long_running_manager: BaseLongRunningManager, empty_context: TaskContext, ): async def unique_task(progress: TaskProgress): @@ -253,7 +256,7 @@ async def unique_task(progress: TaskProgress): async def test_start_multiple_not_unique_tasks( rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_manager: NoWebAppLongRunningManager, + long_running_manager: BaseLongRunningManager, empty_context: TaskContext, ): async def not_unique_task(progress: TaskProgress): @@ -274,7 +277,7 @@ async def not_unique_task(progress: TaskProgress): @pytest.mark.parametrize("is_unique", [True, False]) async def test_get_task_id( - long_running_manager: NoWebAppLongRunningManager, faker: Faker, is_unique: bool + long_running_manager: BaseLongRunningManager, faker: Faker, is_unique: bool ): obj1 = long_running_manager.tasks_manager._get_task_id( faker.word(), is_unique=is_unique @@ -287,7 +290,7 @@ async def test_get_task_id( async def test_get_status( rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_manager: NoWebAppLongRunningManager, + long_running_manager: BaseLongRunningManager, empty_context: TaskContext, ): task_id = await lrt_api.start_task( @@ -309,7 +312,7 @@ async def test_get_status( async def test_get_status_missing( - long_running_manager: NoWebAppLongRunningManager, empty_context: TaskContext + long_running_manager: BaseLongRunningManager, empty_context: TaskContext ): with pytest.raises(TaskNotFoundError) as exec_info: await long_running_manager.tasks_manager.get_task_status( @@ -320,7 +323,7 @@ async def test_get_status_missing( async def test_get_result( rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_manager: NoWebAppLongRunningManager, + long_running_manager: BaseLongRunningManager, empty_context: TaskContext, ): task_id = await lrt_api.start_task( @@ -344,7 +347,7 @@ async def test_get_result( async def test_get_result_missing( - long_running_manager: NoWebAppLongRunningManager, empty_context: TaskContext + long_running_manager: BaseLongRunningManager, empty_context: TaskContext ): with pytest.raises(TaskNotFoundError) as exec_info: await long_running_manager.tasks_manager.get_task_result( @@ -355,7 +358,7 @@ async def test_get_result_missing( async def test_get_result_finished_with_error( rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_manager: NoWebAppLongRunningManager, + long_running_manager: BaseLongRunningManager, empty_context: TaskContext, ): task_id = await lrt_api.start_task( @@ -385,7 +388,7 @@ async def test_cancel_task_from_different_manager( use_in_memory_redis: RedisSettings, get_long_running_manager: Callable[ [RedisSettings, RedisNamespace | None, RabbitSettings, RabbitNamespace], - Awaitable[NoWebAppLongRunningManager], + Awaitable[BaseLongRunningManager], ], empty_context: TaskContext, ): @@ -431,7 +434,7 @@ async def test_cancel_task_from_different_manager( async def test_remove_task( rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_manager: NoWebAppLongRunningManager, + long_running_manager: BaseLongRunningManager, empty_context: TaskContext, ): task_id = await lrt_api.start_task( @@ -460,7 +463,7 @@ async def test_remove_task( async def test_remove_task_with_task_context( rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_manager: NoWebAppLongRunningManager, + long_running_manager: BaseLongRunningManager, empty_context: TaskContext, ): task_id = await lrt_api.start_task( @@ -491,7 +494,7 @@ async def test_remove_task_with_task_context( async def test_remove_unknown_task( - long_running_manager: NoWebAppLongRunningManager, empty_context: TaskContext + long_running_manager: BaseLongRunningManager, empty_context: TaskContext ): with pytest.raises(TaskNotFoundError): await long_running_manager.tasks_manager.remove_task( @@ -505,7 +508,7 @@ async def test_remove_unknown_task( async def test__cancelled_tasks_worker_equivalent_of_cancellation_from_a_different_process( rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_manager: NoWebAppLongRunningManager, + long_running_manager: BaseLongRunningManager, empty_context: TaskContext, ): task_id = await lrt_api.start_task( @@ -534,7 +537,7 @@ async def test__cancelled_tasks_worker_equivalent_of_cancellation_from_a_differe async def test_list_tasks( disable_stale_tasks_monitor: None, rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_manager: NoWebAppLongRunningManager, + long_running_manager: BaseLongRunningManager, empty_context: TaskContext, ): assert ( @@ -578,7 +581,7 @@ async def test_list_tasks( async def test_list_tasks_filtering( rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_manager: NoWebAppLongRunningManager, + long_running_manager: BaseLongRunningManager, empty_context: TaskContext, ): await lrt_api.start_task( @@ -641,7 +644,7 @@ async def test_list_tasks_filtering( async def test_define_task_name( rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_manager: NoWebAppLongRunningManager, + long_running_manager: BaseLongRunningManager, faker: Faker, ): task_name = faker.name() @@ -658,7 +661,7 @@ async def test_define_task_name( async def test_start_not_registered_task( rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_manager: NoWebAppLongRunningManager, + long_running_manager: BaseLongRunningManager, ): with pytest.raises(TaskNotRegisteredError): await lrt_api.start_task( diff --git a/packages/service-library/tests/long_running_tasks/utils.py b/packages/service-library/tests/long_running_tasks/utils.py index 24318947e0e6..e473dd7e1daf 100644 --- a/packages/service-library/tests/long_running_tasks/utils.py +++ b/packages/service-library/tests/long_running_tasks/utils.py @@ -1,51 +1,3 @@ from typing import Final -from servicelib.long_running_tasks import lrt_api -from servicelib.long_running_tasks.base_long_running_manager import ( - BaseLongRunningManager, -) -from servicelib.long_running_tasks.models import RabbitNamespace -from servicelib.long_running_tasks.task import TasksManager -from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient -from settings_library.rabbit import RabbitSettings - - -class NoWebAppLongRunningManager(BaseLongRunningManager): - def __init__( - self, - tasks_manager: TasksManager, - rabbit_settings: RabbitSettings, - rabbit_namespace: RabbitNamespace, - ): - self._tasks_manager = tasks_manager - - self._rabbit_namespace = rabbit_namespace - self.rabbit_settings = rabbit_settings - self._rpc_server: RabbitMQRPCClient | None = None - - @property - def tasks_manager(self) -> TasksManager: - return self._tasks_manager - - @property - def rpc_server(self): - assert self._rpc_server is not None # nosec - return self._rpc_server - - @property - def rabbit_namespace(self) -> str: - return self._rabbit_namespace - - async def setup(self) -> None: - self._rpc_server = await RabbitMQRPCClient.create( - client_name=f"lrt-{self.rabbit_namespace}", settings=self.rabbit_settings - ) - await lrt_api.register_rabbit_routes(self) - - async def teardown(self) -> None: - if self._rpc_server is not None: - await self._rpc_server.close() - self._rpc_server = None - - TEST_CHECK_STALE_INTERVAL_S: Final[float] = 1 From f95b2f765b7ba3aff242f11cfc3afc3ab558ea5d Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 18 Aug 2025 09:26:18 +0200 Subject: [PATCH 055/119] alliged all names as in #8220 --- services/docker-compose.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 095bc65cc623..dd9130d467c0 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -699,7 +699,7 @@ services: WEBSERVER_LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} WEBSERVER_LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} - LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: webserver + LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: wb # WEBSERVER_SERVER_HOST @@ -931,7 +931,7 @@ services: WEBSERVER_STATICWEB: "null" WEBSERVER_FUNCTIONS: ${WEBSERVER_FUNCTIONS} # needed for api-server - LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: wb-api-server + LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: api networks: *webserver_networks @@ -942,7 +942,7 @@ services: environment: WEBSERVER_LOGLEVEL: ${WB_DB_EL_LOGLEVEL} - LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: wb-db-event-listener + LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: db WEBSERVER_HOST: ${WEBSERVER_HOST} WEBSERVER_PORT: ${WEBSERVER_PORT} @@ -1040,7 +1040,7 @@ services: LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} - LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: wb-garbage-collector + LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: gc # WEBSERVER_DB POSTGRES_DB: ${POSTGRES_DB} @@ -1134,7 +1134,7 @@ services: WEBSERVER_APP_FACTORY_NAME: WEBSERVER_AUTHZ_APP_FACTORY WEBSERVER_LOGLEVEL: ${WB_AUTH_LOGLEVEL} - LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: wb-auth + LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: auth GUNICORN_CMD_ARGS: ${WEBSERVER_GUNICORN_CMD_ARGS} From 5edf2f2e7ac627af91f2666b895452091730ccb0 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 18 Aug 2025 09:31:29 +0200 Subject: [PATCH 056/119] using relative imports --- .../src/servicelib/aiohttp/profiler_middleware.py | 5 +---- .../src/servicelib/long_running_tasks/_store/redis.py | 2 +- .../src/servicelib/long_running_tasks/lrt_api.py | 3 +-- .../src/servicelib/long_running_tasks/task.py | 2 +- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/service-library/src/servicelib/aiohttp/profiler_middleware.py b/packages/service-library/src/servicelib/aiohttp/profiler_middleware.py index 4256820b4b30..07d3c7127297 100644 --- a/packages/service-library/src/servicelib/aiohttp/profiler_middleware.py +++ b/packages/service-library/src/servicelib/aiohttp/profiler_middleware.py @@ -1,9 +1,6 @@ from aiohttp.web import HTTPInternalServerError, Request, StreamResponse, middleware -from servicelib.mimetype_constants import ( - MIMETYPE_APPLICATION_JSON, - MIMETYPE_APPLICATION_ND_JSON, -) +from ..mimetype_constants import MIMETYPE_APPLICATION_JSON, MIMETYPE_APPLICATION_ND_JSON from ..utils_profiling_middleware import _is_profiling, _profiler, append_profile diff --git a/packages/service-library/src/servicelib/long_running_tasks/_store/redis.py b/packages/service-library/src/servicelib/long_running_tasks/_store/redis.py index edeea3a79ccd..6ec443619318 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_store/redis.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_store/redis.py @@ -4,11 +4,11 @@ import redis.asyncio as aioredis from common_library.json_serialization import json_dumps, json_loads from pydantic import TypeAdapter -from servicelib.utils import limited_gather from settings_library.redis import RedisDatabase, RedisSettings from ...redis._client import RedisClientSDK from ...redis._utils import handle_redis_returns_union_types +from ...utils import limited_gather from ..models import TaskContext, TaskData, TaskId from .base import BaseStore diff --git a/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py b/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py index e3fc442a79e2..bf2e1a319361 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py +++ b/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py @@ -1,7 +1,6 @@ from typing import Any -from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient - +from ..rabbitmq._client_rpc import RabbitMQRPCClient from ._rabbit import lrt_client from .base_long_running_manager import BaseLongRunningManager from .models import TaskBase, TaskContext, TaskId, TaskStatus diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index 89cb7dc1a056..56e5c7a893b6 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -11,7 +11,6 @@ from common_library.async_tools import cancel_wait_task from models_library.api_schemas_long_running_tasks.base import TaskProgress from pydantic import NonNegativeFloat, PositiveFloat -from servicelib.logging_utils import log_catch from settings_library.redis import RedisDatabase, RedisSettings from tenacity import ( AsyncRetrying, @@ -22,6 +21,7 @@ ) from ..background_task import create_periodic_task +from ..logging_utils import log_catch from ..redis import RedisClientSDK, exclusive from ._serialization import object_to_string, string_to_object from ._store.base import BaseStore From e52694347d252df8a60b79421fcaed576ec6960e Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 18 Aug 2025 09:48:03 +0200 Subject: [PATCH 057/119] simplified start_task interface --- .../aiohttp/long_running_tasks/_server.py | 1 - .../servicelib/long_running_tasks/lrt_api.py | 3 +- .../test_long_running_tasks.py | 1 - ...test_long_running_tasks_context_manager.py | 8 +- .../test_long_running_tasks_lrt_api.py | 3 - .../test_long_running_tasks_task.py | 82 ++++--------------- .../api/routes/dynamic_scheduler.py | 4 - .../api/rest/containers_long_running_tasks.py | 9 -- 8 files changed, 18 insertions(+), 93 deletions(-) diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py index 757f783abd35..2658f6d67bc8 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py @@ -64,7 +64,6 @@ async def start_long_running_task( task_id = None try: task_id = await lrt_api.start_task( - long_running_manager.rpc_client, long_running_manager, registerd_task_name, fire_and_forget=fire_and_forget, diff --git a/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py b/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py index bf2e1a319361..4d4a2c8b4f9e 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py +++ b/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py @@ -8,7 +8,6 @@ async def start_task( - rabbitmq_rpc_client: RabbitMQRPCClient, long_running_manager: BaseLongRunningManager, registered_task_name: RegisteredTaskName, *, @@ -45,7 +44,7 @@ async def start_task( """ return await lrt_client.start_task( - rabbitmq_rpc_client, + long_running_manager.rpc_client, long_running_manager.rabbit_namespace, registered_task_name=registered_task_name, unique=unique, diff --git a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py index a7d07739119f..dce021286aff 100644 --- a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py +++ b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py @@ -86,7 +86,6 @@ async def create_string_list_task( fail: bool = False, ) -> TaskId: return await lrt_api.start_task( - long_running_manager.rpc_client, long_running_manager, _string_list_task.__name__, num_strings=num_strings, diff --git a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py index dedaa0e4ffc4..880d8fe65c25 100644 --- a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py +++ b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py @@ -76,9 +76,7 @@ async def create_task_user_defined_route( FastAPILongRunningManager, Depends(get_long_running_manager) ], ) -> TaskId: - return await lrt_api.start_task( - long_running_manager.rpc_client, long_running_manager, a_test_task.__name__ - ) + return await lrt_api.start_task(long_running_manager, a_test_task.__name__) @router.get("/api/failing", status_code=status.HTTP_200_OK) async def create_task_which_fails( @@ -87,9 +85,7 @@ async def create_task_which_fails( ], ) -> TaskId: return await lrt_api.start_task( - long_running_manager.rpc_client, - long_running_manager, - a_failing_test_task.__name__, + long_running_manager, a_failing_test_task.__name__ ) return router diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py index ba0719263bb2..3c4021ad6573 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py @@ -193,7 +193,6 @@ async def test_workflow_with_result( task_ids: list[TaskId] = [] for _ in range(task_count): task_id = await lrt_api.start_task( - rabbitmq_rpc_client, _get_task_manager(long_running_managers), _task_echo_input.__name__, unique=is_unique, @@ -248,7 +247,6 @@ async def test_workflow_raises_error( task_ids: list[TaskId] = [] for _ in range(task_count): task_id = await lrt_api.start_task( - rabbitmq_rpc_client, _get_task_manager(long_running_managers), _task_always_raise.__name__, unique=is_unique, @@ -295,7 +293,6 @@ async def test_remove_task( task_context: TaskContext | None, ): task_id = await lrt_api.start_task( - rabbitmq_rpc_client, _get_task_manager(long_running_managers), _task_takes_too_long.__name__, unique=is_unique, diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py index 09ccc2919e54..36199655bf6b 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py @@ -104,13 +104,11 @@ async def long_running_manager( @pytest.mark.parametrize("check_task_presence_before", [True, False]) async def test_task_is_auto_removed( - rabbitmq_rpc_client: RabbitMQRPCClient, long_running_manager: BaseLongRunningManager, check_task_presence_before: bool, empty_context: TaskContext, ): task_id = await lrt_api.start_task( - rabbitmq_rpc_client, long_running_manager, a_background_task.__name__, raise_when_finished=False, @@ -149,12 +147,9 @@ async def test_task_is_auto_removed( async def test_checked_task_is_not_auto_removed( - rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_manager: BaseLongRunningManager, - empty_context: TaskContext, + long_running_manager: BaseLongRunningManager, empty_context: TaskContext ): task_id = await lrt_api.start_task( - rabbitmq_rpc_client, long_running_manager, a_background_task.__name__, raise_when_finished=False, @@ -174,12 +169,9 @@ async def test_checked_task_is_not_auto_removed( async def test_fire_and_forget_task_is_not_auto_removed( - rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_manager: BaseLongRunningManager, - empty_context: TaskContext, + long_running_manager: BaseLongRunningManager, empty_context: TaskContext ): task_id = await lrt_api.start_task( - rabbitmq_rpc_client, long_running_manager, a_background_task.__name__, raise_when_finished=False, @@ -203,12 +195,9 @@ async def test_fire_and_forget_task_is_not_auto_removed( async def test_get_result_of_unfinished_task_raises( - rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_manager: BaseLongRunningManager, - empty_context: TaskContext, + long_running_manager: BaseLongRunningManager, empty_context: TaskContext ): task_id = await lrt_api.start_task( - rabbitmq_rpc_client, long_running_manager, a_background_task.__name__, raise_when_finished=False, @@ -222,9 +211,7 @@ async def test_get_result_of_unfinished_task_raises( async def test_unique_task_already_running( - rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_manager: BaseLongRunningManager, - empty_context: TaskContext, + long_running_manager: BaseLongRunningManager, empty_context: TaskContext ): async def unique_task(progress: TaskProgress): _ = progress @@ -233,7 +220,6 @@ async def unique_task(progress: TaskProgress): TaskRegistry.register(unique_task) await lrt_api.start_task( - rabbitmq_rpc_client, long_running_manager, unique_task.__name__, unique=True, @@ -243,7 +229,6 @@ async def unique_task(progress: TaskProgress): # ensure unique running task regardless of how many times it gets started with pytest.raises(TaskAlreadyRunningError) as exec_info: await lrt_api.start_task( - rabbitmq_rpc_client, long_running_manager, unique_task.__name__, unique=True, @@ -255,9 +240,7 @@ async def unique_task(progress: TaskProgress): async def test_start_multiple_not_unique_tasks( - rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_manager: BaseLongRunningManager, - empty_context: TaskContext, + long_running_manager: BaseLongRunningManager, empty_context: TaskContext ): async def not_unique_task(progress: TaskProgress): await asyncio.sleep(1) @@ -266,7 +249,6 @@ async def not_unique_task(progress: TaskProgress): for _ in range(5): await lrt_api.start_task( - rabbitmq_rpc_client, long_running_manager, not_unique_task.__name__, task_context=empty_context, @@ -289,12 +271,9 @@ async def test_get_task_id( async def test_get_status( - rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_manager: BaseLongRunningManager, - empty_context: TaskContext, + long_running_manager: BaseLongRunningManager, empty_context: TaskContext ): task_id = await lrt_api.start_task( - rabbitmq_rpc_client, long_running_manager, a_background_task.__name__, raise_when_finished=False, @@ -322,15 +301,10 @@ async def test_get_status_missing( async def test_get_result( - rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_manager: BaseLongRunningManager, - empty_context: TaskContext, + long_running_manager: BaseLongRunningManager, empty_context: TaskContext ): task_id = await lrt_api.start_task( - rabbitmq_rpc_client, - long_running_manager, - fast_background_task.__name__, - task_context=empty_context, + long_running_manager, fast_background_task.__name__, task_context=empty_context ) async for attempt in AsyncRetrying(**_RETRY_PARAMS): @@ -357,12 +331,9 @@ async def test_get_result_missing( async def test_get_result_finished_with_error( - rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_manager: BaseLongRunningManager, - empty_context: TaskContext, + long_running_manager: BaseLongRunningManager, empty_context: TaskContext ): task_id = await lrt_api.start_task( - rabbitmq_rpc_client, long_running_manager, failing_background_task.__name__, task_context=empty_context, @@ -383,7 +354,6 @@ async def test_get_result_finished_with_error( async def test_cancel_task_from_different_manager( - rabbitmq_rpc_client: RabbitMQRPCClient, rabbit_service: RabbitSettings, use_in_memory_redis: RedisSettings, get_long_running_manager: Callable[ @@ -403,7 +373,6 @@ async def test_cancel_task_from_different_manager( ) task_id = await lrt_api.start_task( - rabbitmq_rpc_client, manager_1, a_background_task.__name__, raise_when_finished=False, @@ -433,12 +402,9 @@ async def test_cancel_task_from_different_manager( async def test_remove_task( - rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_manager: BaseLongRunningManager, - empty_context: TaskContext, + long_running_manager: BaseLongRunningManager, empty_context: TaskContext ): task_id = await lrt_api.start_task( - rabbitmq_rpc_client, long_running_manager, a_background_task.__name__, raise_when_finished=False, @@ -462,12 +428,9 @@ async def test_remove_task( async def test_remove_task_with_task_context( - rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_manager: BaseLongRunningManager, - empty_context: TaskContext, + long_running_manager: BaseLongRunningManager, empty_context: TaskContext ): task_id = await lrt_api.start_task( - rabbitmq_rpc_client, long_running_manager, a_background_task.__name__, raise_when_finished=False, @@ -507,12 +470,9 @@ async def test_remove_unknown_task( async def test__cancelled_tasks_worker_equivalent_of_cancellation_from_a_different_process( - rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_manager: BaseLongRunningManager, - empty_context: TaskContext, + long_running_manager: BaseLongRunningManager, empty_context: TaskContext ): task_id = await lrt_api.start_task( - rabbitmq_rpc_client, long_running_manager, a_background_task.__name__, raise_when_finished=False, @@ -536,7 +496,6 @@ async def test__cancelled_tasks_worker_equivalent_of_cancellation_from_a_differe async def test_list_tasks( disable_stale_tasks_monitor: None, - rabbitmq_rpc_client: RabbitMQRPCClient, long_running_manager: BaseLongRunningManager, empty_context: TaskContext, ): @@ -552,7 +511,6 @@ async def test_list_tasks( for _ in range(NUM_TASKS): task_ids.append( # noqa: PERF401 await lrt_api.start_task( - rabbitmq_rpc_client, long_running_manager, a_background_task.__name__, raise_when_finished=False, @@ -580,12 +538,9 @@ async def test_list_tasks( async def test_list_tasks_filtering( - rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_manager: BaseLongRunningManager, - empty_context: TaskContext, + long_running_manager: BaseLongRunningManager, empty_context: TaskContext ): await lrt_api.start_task( - rabbitmq_rpc_client, long_running_manager, a_background_task.__name__, raise_when_finished=False, @@ -593,7 +548,6 @@ async def test_list_tasks_filtering( task_context=empty_context, ) await lrt_api.start_task( - rabbitmq_rpc_client, long_running_manager, a_background_task.__name__, raise_when_finished=False, @@ -601,7 +555,6 @@ async def test_list_tasks_filtering( task_context={"user_id": 213}, ) await lrt_api.start_task( - rabbitmq_rpc_client, long_running_manager, a_background_task.__name__, raise_when_finished=False, @@ -643,13 +596,10 @@ async def test_list_tasks_filtering( async def test_define_task_name( - rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_manager: BaseLongRunningManager, - faker: Faker, + long_running_manager: BaseLongRunningManager, faker: Faker ): task_name = faker.name() task_id = await lrt_api.start_task( - rabbitmq_rpc_client, long_running_manager, a_background_task.__name__, raise_when_finished=False, @@ -664,6 +614,4 @@ async def test_start_not_registered_task( long_running_manager: BaseLongRunningManager, ): with pytest.raises(TaskNotRegisteredError): - await lrt_api.start_task( - rabbitmq_rpc_client, long_running_manager, "not_registered_task" - ) + await lrt_api.start_task(long_running_manager, "not_registered_task") diff --git a/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_scheduler.py b/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_scheduler.py index edd06ce9d8e2..6e1c2a09acaa 100644 --- a/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_scheduler.py @@ -116,7 +116,6 @@ async def _progress_callback( try: return await lrt_api.start_task( - long_running_manager.rpc_client, long_running_manager, _task_remove_service_containers.__name__, unique=True, @@ -182,7 +181,6 @@ async def _progress_callback( try: return await lrt_api.start_task( - long_running_manager.rpc_client, long_running_manager, _task_save_service_state.__name__, unique=True, @@ -230,7 +228,6 @@ async def _progress_callback( try: return await lrt_api.start_task( - long_running_manager.rpc_client, long_running_manager, _task_push_service_outputs.__name__, unique=True, @@ -273,7 +270,6 @@ async def _task_cleanup_service_docker_resources( try: return await lrt_api.start_task( - long_running_manager.rpc_client, long_running_manager, _task_cleanup_service_docker_resources.__name__, unique=True, diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_long_running_tasks.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_long_running_tasks.py index f7943f89f923..91900a0a0e41 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_long_running_tasks.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_long_running_tasks.py @@ -50,7 +50,6 @@ async def pull_user_servcices_docker_images( try: return await lrt_api.start_task( - long_running_manager.rpc_client, long_running_manager, task_pull_user_servcices_docker_images.__name__, unique=True, @@ -88,7 +87,6 @@ async def create_service_containers_task( # pylint: disable=too-many-arguments try: return await lrt_api.start_task( - long_running_manager.rpc_client, long_running_manager, task_create_service_containers.__name__, unique=True, @@ -118,7 +116,6 @@ async def runs_docker_compose_down_task( try: return await lrt_api.start_task( - long_running_manager.rpc_client, long_running_manager, task_runs_docker_compose_down.__name__, unique=True, @@ -146,7 +143,6 @@ async def state_restore_task( try: return await lrt_api.start_task( - long_running_manager.rpc_client, long_running_manager, task_restore_state.__name__, unique=True, @@ -174,7 +170,6 @@ async def state_save_task( try: return await lrt_api.start_task( - long_running_manager.rpc_client, long_running_manager, task_save_state.__name__, unique=True, @@ -204,7 +199,6 @@ async def ports_inputs_pull_task( try: return await lrt_api.start_task( - long_running_manager.rpc_client, long_running_manager, task_ports_inputs_pull.__name__, unique=True, @@ -234,7 +228,6 @@ async def ports_outputs_pull_task( try: return await lrt_api.start_task( - long_running_manager.rpc_client, long_running_manager, task_ports_outputs_pull.__name__, unique=True, @@ -261,7 +254,6 @@ async def ports_outputs_push_task( try: return await lrt_api.start_task( - long_running_manager.rpc_client, long_running_manager, task_ports_outputs_push.__name__, unique=True, @@ -288,7 +280,6 @@ async def containers_restart_task( try: return await lrt_api.start_task( - long_running_manager.rpc_client, long_running_manager, task_containers_restart.__name__, unique=True, From c30c1225d24b19ef7eea8c98341ff3068259769b Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 18 Aug 2025 09:50:43 +0200 Subject: [PATCH 058/119] fixed description --- .../src/simcore_service_webserver/application_settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/application_settings.py b/services/web/server/src/simcore_service_webserver/application_settings.py index eb1369448728..8b0174f867a9 100644 --- a/services/web/server/src/simcore_service_webserver/application_settings.py +++ b/services/web/server/src/simcore_service_webserver/application_settings.py @@ -271,7 +271,7 @@ class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): LongRunningTasksSettings | None, Field( json_schema_extra={"auto_default_from_env": True}, - description="login plugin", + description="long running tasks plugin", ), ] From 28f9e620cd13cdb429ab593886c6867cc17c595f Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 18 Aug 2025 09:58:15 +0200 Subject: [PATCH 059/119] moved RedisNamespace in models --- .../servicelib/aiohttp/long_running_tasks/_server.py | 9 +++++++-- .../servicelib/fastapi/long_running_tasks/_server.py | 3 +-- .../long_running_tasks/base_long_running_manager.py | 4 ++-- .../src/servicelib/long_running_tasks/models.py | 1 + .../src/servicelib/long_running_tasks/task.py | 11 +++++++++-- .../tests/long_running_tasks/conftest.py | 6 ++---- .../test_long_running_tasks_lrt_api.py | 4 ++-- .../test_long_running_tasks_task.py | 8 +++++--- .../modules/dynamic_sidecar/module_setup.py | 3 +-- .../modules/long_running_tasks.py | 3 +-- 10 files changed, 31 insertions(+), 21 deletions(-) diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py index 2658f6d67bc8..cc6dcd106687 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py @@ -21,8 +21,13 @@ DEFAULT_STALE_TASK_CHECK_INTERVAL, DEFAULT_STALE_TASK_DETECT_TIMEOUT, ) -from ...long_running_tasks.models import RabbitNamespace, TaskContext, TaskGet -from ...long_running_tasks.task import RedisNamespace, RegisteredTaskName +from ...long_running_tasks.models import ( + RabbitNamespace, + RedisNamespace, + TaskContext, + TaskGet, +) +from ...long_running_tasks.task import RegisteredTaskName from ..typing_extension import Handler from . import _routes from ._constants import ( diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py index 75e4c06f34a0..487d0d740011 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py @@ -9,8 +9,7 @@ DEFAULT_STALE_TASK_DETECT_TIMEOUT, ) from ...long_running_tasks.errors import BaseLongRunningError -from ...long_running_tasks.models import RabbitNamespace -from ...long_running_tasks.task import RedisNamespace +from ...long_running_tasks.models import RabbitNamespace, RedisNamespace from ._error_handlers import base_long_running_error_handler from ._manager import FastAPILongRunningManager from ._routes import router diff --git a/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py b/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py index 9a7071261f5a..8fa7852895c3 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py +++ b/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py @@ -5,8 +5,8 @@ from ..rabbitmq._client_rpc import RabbitMQRPCClient from ._rabbit.namespace import get_namespace -from .models import RabbitNamespace -from .task import RedisNamespace, TasksManager +from .models import RabbitNamespace, RedisNamespace +from .task import TasksManager class BaseLongRunningManager: diff --git a/packages/service-library/src/servicelib/long_running_tasks/models.py b/packages/service-library/src/servicelib/long_running_tasks/models.py index 8cfdc7e55c01..ab196bde2688 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/models.py +++ b/packages/service-library/src/servicelib/long_running_tasks/models.py @@ -29,6 +29,7 @@ TaskContext: TypeAlias = dict[str, Any] RabbitNamespace: TypeAlias = str +RedisNamespace: TypeAlias = str class ResultField(BaseModel): diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index 56e5c7a893b6..5be6555d31e3 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -33,7 +33,15 @@ TaskNotFoundError, TaskNotRegisteredError, ) -from .models import ResultField, TaskBase, TaskContext, TaskData, TaskId, TaskStatus +from .models import ( + RedisNamespace, + ResultField, + TaskBase, + TaskContext, + TaskData, + TaskId, + TaskStatus, +) _logger = logging.getLogger(__name__) @@ -45,7 +53,6 @@ RegisteredTaskName: TypeAlias = str -RedisNamespace: TypeAlias = str class TaskProtocol(Protocol): diff --git a/packages/service-library/tests/long_running_tasks/conftest.py b/packages/service-library/tests/long_running_tasks/conftest.py index d5aaf9ccf8e7..83b62d4075f7 100644 --- a/packages/service-library/tests/long_running_tasks/conftest.py +++ b/packages/service-library/tests/long_running_tasks/conftest.py @@ -14,10 +14,8 @@ from servicelib.long_running_tasks.base_long_running_manager import ( BaseLongRunningManager, ) -from servicelib.long_running_tasks.task import ( - RedisNamespace, - TasksManager, -) +from servicelib.long_running_tasks.models import RedisNamespace +from servicelib.long_running_tasks.task import TasksManager from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py index 3c4021ad6573..4f0a21167954 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py @@ -15,8 +15,8 @@ BaseLongRunningManager, ) from servicelib.long_running_tasks.errors import TaskNotFoundError -from servicelib.long_running_tasks.models import TaskContext -from servicelib.long_running_tasks.task import RedisNamespace, TaskId, TaskRegistry +from servicelib.long_running_tasks.models import RedisNamespace, TaskContext +from servicelib.long_running_tasks.task import TaskId, TaskRegistry from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py index 36199655bf6b..0db0a8e9f268 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py @@ -24,11 +24,13 @@ TaskNotFoundError, TaskNotRegisteredError, ) -from servicelib.long_running_tasks.models import TaskContext, TaskProgress, TaskStatus -from servicelib.long_running_tasks.task import ( +from servicelib.long_running_tasks.models import ( RedisNamespace, - TaskRegistry, + TaskContext, + TaskProgress, + TaskStatus, ) +from servicelib.long_running_tasks.task import TaskRegistry from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/module_setup.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/module_setup.py index d022f92fa279..aa2f1bf7b81d 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/module_setup.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/module_setup.py @@ -1,7 +1,6 @@ from fastapi import FastAPI from servicelib.fastapi import long_running_tasks -from servicelib.long_running_tasks.models import RabbitNamespace -from servicelib.long_running_tasks.task import RedisNamespace +from servicelib.long_running_tasks.models import RabbitNamespace, RedisNamespace from ...core.settings import AppSettings from . import api_client, scheduler diff --git a/services/storage/src/simcore_service_storage/modules/long_running_tasks.py b/services/storage/src/simcore_service_storage/modules/long_running_tasks.py index 88e81fa75d94..be8a5ead85be 100644 --- a/services/storage/src/simcore_service_storage/modules/long_running_tasks.py +++ b/services/storage/src/simcore_service_storage/modules/long_running_tasks.py @@ -2,8 +2,7 @@ from fastapi import FastAPI from servicelib.fastapi.long_running_tasks._server import setup -from servicelib.long_running_tasks.models import RabbitNamespace -from servicelib.long_running_tasks.task import RedisNamespace +from servicelib.long_running_tasks.models import RabbitNamespace, RedisNamespace from .._meta import API_VTAG from ..core.settings import get_application_settings From a9cc987c5396bb44a58999bf87fe6d7dd87a70f1 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 18 Aug 2025 10:30:21 +0200 Subject: [PATCH 060/119] replaced both namespaces with only a single one --- .../aiohttp/long_running_tasks/_server.py | 13 +++------- .../fastapi/long_running_tasks/_server.py | 8 +++---- .../long_running_tasks/_rabbit/lrt_client.py | 24 +++++++++---------- .../long_running_tasks/_rabbit/namespace.py | 4 ++-- .../base_long_running_manager.py | 21 ++++++++-------- .../servicelib/long_running_tasks/lrt_api.py | 10 ++++---- .../servicelib/long_running_tasks/models.py | 3 +-- .../src/servicelib/long_running_tasks/task.py | 16 ++++++------- .../test_long_running_tasks.py | 3 +-- .../test_long_running_tasks_client.py | 3 +-- ...st_long_running_tasks_with_task_context.py | 3 +-- .../fastapi/long_running_tasks/conftest.py | 3 +-- .../test_long_running_tasks.py | 3 +-- ...test_long_running_tasks_context_manager.py | 3 +-- .../tests/long_running_tasks/conftest.py | 11 ++++----- .../test_long_running_tasks_lrt_api.py | 7 +++--- .../test_long_running_tasks_task.py | 15 ++++++------ .../modules/dynamic_sidecar/module_setup.py | 8 ++----- .../modules/long_running_tasks.py | 3 +-- .../modules/long_running_tasks.py | 11 ++------- .../long_running_tasks/plugin.py | 13 ++++------ 21 files changed, 73 insertions(+), 112 deletions(-) diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py index cc6dcd106687..ff9bffbf6f4d 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py @@ -21,12 +21,7 @@ DEFAULT_STALE_TASK_CHECK_INTERVAL, DEFAULT_STALE_TASK_DETECT_TIMEOUT, ) -from ...long_running_tasks.models import ( - RabbitNamespace, - RedisNamespace, - TaskContext, - TaskGet, -) +from ...long_running_tasks.models import LRTNamespace, TaskContext, TaskGet from ...long_running_tasks.task import RegisteredTaskName from ..typing_extension import Handler from . import _routes @@ -152,9 +147,8 @@ def setup( *, router_prefix: str, redis_settings: RedisSettings, - redis_namespace: RedisNamespace, rabbit_settings: RabbitSettings, - rabbit_namespace: RabbitNamespace, + lrt_namespace: LRTNamespace, handler_check_decorator: Callable = _no_ops_decorator, task_request_context_decorator: Callable = _no_task_context_decorator, stale_task_check_interval: datetime.timedelta = DEFAULT_STALE_TASK_CHECK_INTERVAL, @@ -183,8 +177,7 @@ async def on_cleanup_ctx(app: web.Application) -> AsyncGenerator[None, None]: stale_task_detect_timeout=stale_task_detect_timeout, redis_settings=redis_settings, rabbit_settings=rabbit_settings, - redis_namespace=redis_namespace, - rabbit_namespace=rabbit_namespace, + lrt_namespace=lrt_namespace, ) ) diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py index 487d0d740011..81e2bb745f80 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py @@ -9,7 +9,7 @@ DEFAULT_STALE_TASK_DETECT_TIMEOUT, ) from ...long_running_tasks.errors import BaseLongRunningError -from ...long_running_tasks.models import RabbitNamespace, RedisNamespace +from ...long_running_tasks.models import LRTNamespace from ._error_handlers import base_long_running_error_handler from ._manager import FastAPILongRunningManager from ._routes import router @@ -20,9 +20,8 @@ def setup( *, router_prefix: str = "", redis_settings: RedisSettings, - redis_namespace: RedisNamespace, rabbit_settings: RabbitSettings, - rabbit_namespace: RabbitNamespace, + lrt_namespace: LRTNamespace, stale_task_check_interval: datetime.timedelta = DEFAULT_STALE_TASK_CHECK_INTERVAL, stale_task_detect_timeout: datetime.timedelta = DEFAULT_STALE_TASK_DETECT_TIMEOUT, ) -> None: @@ -48,9 +47,8 @@ async def on_startup() -> None: stale_task_check_interval=stale_task_check_interval, stale_task_detect_timeout=stale_task_detect_timeout, redis_settings=redis_settings, - redis_namespace=redis_namespace, rabbit_settings=rabbit_settings, - rabbit_namespace=rabbit_namespace, + lrt_namespace=lrt_namespace, ) ) await long_running_manager.setup() diff --git a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py index 024f35d80cb9..94184e6af050 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py @@ -9,9 +9,9 @@ from ...long_running_tasks.task import RegisteredTaskName from ...rabbitmq._client_rpc import RabbitMQRPCClient from .._serialization import string_to_object -from ..models import RabbitNamespace, TaskBase, TaskContext, TaskId, TaskStatus +from ..models import LRTNamespace, TaskBase, TaskContext, TaskId, TaskStatus from ._models import RPCErrorResponse -from .namespace import get_namespace +from .namespace import get_rabbit_namespace _logger = logging.getLogger(__name__) @@ -26,7 +26,7 @@ @log_decorator(_logger, level=logging.DEBUG) async def start_task( rabbitmq_rpc_client: RabbitMQRPCClient, - namespace: RabbitNamespace, + namespace: LRTNamespace, *, registered_task_name: RegisteredTaskName, unique: bool = False, @@ -36,7 +36,7 @@ async def start_task( **task_kwargs: Any, ) -> TaskId: result = await rabbitmq_rpc_client.request( - get_namespace(namespace), + get_rabbit_namespace(namespace), TypeAdapter(RPCMethodName).validate_python("start_task"), registered_task_name=registered_task_name, unique=unique, @@ -53,12 +53,12 @@ async def start_task( @log_decorator(_logger, level=logging.DEBUG) async def list_tasks( rabbitmq_rpc_client: RabbitMQRPCClient, - namespace: RabbitNamespace, + namespace: LRTNamespace, *, task_context: TaskContext, ) -> list[TaskBase]: result = await rabbitmq_rpc_client.request( - get_namespace(namespace), + get_rabbit_namespace(namespace), TypeAdapter(RPCMethodName).validate_python("list_tasks"), task_context=task_context, timeout_s=_RPC_TIMEOUT_SHORT_REQUESTS, @@ -69,13 +69,13 @@ async def list_tasks( @log_decorator(_logger, level=logging.DEBUG) async def get_task_status( rabbitmq_rpc_client: RabbitMQRPCClient, - namespace: RabbitNamespace, + namespace: LRTNamespace, *, task_context: TaskContext, task_id: TaskId, ) -> TaskStatus: result = await rabbitmq_rpc_client.request( - get_namespace(namespace), + get_rabbit_namespace(namespace), TypeAdapter(RPCMethodName).validate_python("get_task_status"), task_context=task_context, task_id=task_id, @@ -88,13 +88,13 @@ async def get_task_status( @log_decorator(_logger, level=logging.DEBUG) async def get_task_result( rabbitmq_rpc_client: RabbitMQRPCClient, - namespace: RabbitNamespace, + namespace: LRTNamespace, *, task_context: TaskContext, task_id: TaskId, ) -> Any: serialized_result = await rabbitmq_rpc_client.request( - get_namespace(namespace), + get_rabbit_namespace(namespace), TypeAdapter(RPCMethodName).validate_python("get_task_result"), task_context=task_context, task_id=task_id, @@ -117,14 +117,14 @@ async def get_task_result( @log_decorator(_logger, level=logging.DEBUG) async def remove_task( rabbitmq_rpc_client: RabbitMQRPCClient, - namespace: RabbitNamespace, + namespace: LRTNamespace, *, task_context: TaskContext, task_id: TaskId, reraise_errors: bool = True, ) -> None: result = await rabbitmq_rpc_client.request( - get_namespace(namespace), + get_rabbit_namespace(namespace), TypeAdapter(RPCMethodName).validate_python("remove_task"), task_context=task_context, task_id=task_id, diff --git a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/namespace.py b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/namespace.py index fd3a9dc638c2..baf5198fdc80 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/namespace.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/namespace.py @@ -1,8 +1,8 @@ from models_library.rabbitmq_basic_types import RPCNamespace from pydantic import TypeAdapter -from ..models import RabbitNamespace +from ..models import LRTNamespace -def get_namespace(namespace: RabbitNamespace) -> RPCNamespace: +def get_rabbit_namespace(namespace: LRTNamespace) -> RPCNamespace: return TypeAdapter(RPCNamespace).validate_python(f"lrt-{namespace}") diff --git a/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py b/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py index 8fa7852895c3..0e52d6fbc7a5 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py +++ b/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py @@ -4,8 +4,8 @@ from settings_library.redis import RedisSettings from ..rabbitmq._client_rpc import RabbitMQRPCClient -from ._rabbit.namespace import get_namespace -from .models import RabbitNamespace, RedisNamespace +from ._rabbit.namespace import get_rabbit_namespace +from .models import LRTNamespace from .task import TasksManager @@ -20,16 +20,15 @@ def __init__( stale_task_detect_timeout: datetime.timedelta, redis_settings: RedisSettings, rabbit_settings: RabbitSettings, - redis_namespace: RedisNamespace, - rabbit_namespace: RabbitNamespace, + lrt_namespace: LRTNamespace, ): self._tasks_manager = TasksManager( stale_task_check_interval=stale_task_check_interval, stale_task_detect_timeout=stale_task_detect_timeout, redis_settings=redis_settings, - redis_namespace=redis_namespace, + lrt_namespace=lrt_namespace, ) - self._rabbit_namespace = rabbit_namespace + self._lrt_namespace = lrt_namespace self.rabbit_settings = rabbit_settings self._rpc_server: RabbitMQRPCClient | None = None self._rpc_client: RabbitMQRPCClient | None = None @@ -49,17 +48,17 @@ def rpc_client(self) -> RabbitMQRPCClient: return self._rpc_client @property - def rabbit_namespace(self) -> RabbitNamespace: - return self._rabbit_namespace + def lrt_namespace(self) -> LRTNamespace: + return self._lrt_namespace async def setup(self) -> None: await self._tasks_manager.setup() self._rpc_server = await RabbitMQRPCClient.create( - client_name=f"lrt-server-{self.rabbit_namespace}", + client_name=f"lrt-server-{self.lrt_namespace}", settings=self.rabbit_settings, ) self._rpc_client = await RabbitMQRPCClient.create( - client_name=f"lrt-client-{self.rabbit_namespace}", + client_name=f"lrt-client-{self.lrt_namespace}", settings=self.rabbit_settings, ) @@ -67,7 +66,7 @@ async def setup(self) -> None: await self.rpc_server.register_router( router, - get_namespace(self.rabbit_namespace), + get_rabbit_namespace(self.lrt_namespace), self, ) diff --git a/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py b/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py index 4d4a2c8b4f9e..c551f879b0b7 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py +++ b/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py @@ -45,7 +45,7 @@ async def start_task( return await lrt_client.start_task( long_running_manager.rpc_client, - long_running_manager.rabbit_namespace, + long_running_manager.lrt_namespace, registered_task_name=registered_task_name, unique=unique, task_context=task_context, @@ -62,7 +62,7 @@ async def list_tasks( ) -> list[TaskBase]: return await lrt_client.list_tasks( rabbitmq_rpc_client, - long_running_manager.rabbit_namespace, + long_running_manager.lrt_namespace, task_context=task_context, ) @@ -76,7 +76,7 @@ async def get_task_status( """returns the status of a task""" return await lrt_client.get_task_status( rabbitmq_rpc_client, - long_running_manager.rabbit_namespace, + long_running_manager.lrt_namespace, task_id=task_id, task_context=task_context, ) @@ -90,7 +90,7 @@ async def get_task_result( ) -> Any: return await lrt_client.get_task_result( rabbitmq_rpc_client, - long_running_manager.rabbit_namespace, + long_running_manager.lrt_namespace, task_context=task_context, task_id=task_id, ) @@ -105,7 +105,7 @@ async def remove_task( """cancels and removes the task""" await lrt_client.remove_task( rabbitmq_rpc_client, - long_running_manager.rabbit_namespace, + long_running_manager.lrt_namespace, task_id=task_id, task_context=task_context, ) diff --git a/packages/service-library/src/servicelib/long_running_tasks/models.py b/packages/service-library/src/servicelib/long_running_tasks/models.py index ab196bde2688..5dedb46291c4 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/models.py +++ b/packages/service-library/src/servicelib/long_running_tasks/models.py @@ -28,8 +28,7 @@ RequestBody: TypeAlias = Any TaskContext: TypeAlias = dict[str, Any] -RabbitNamespace: TypeAlias = str -RedisNamespace: TypeAlias = str +LRTNamespace: TypeAlias = str class ResultField(BaseModel): diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index 5be6555d31e3..d9c3195bc540 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -34,7 +34,7 @@ TaskNotRegisteredError, ) from .models import ( - RedisNamespace, + LRTNamespace, ResultField, TaskBase, TaskContext, @@ -119,17 +119,17 @@ def __init__( redis_settings: RedisSettings, stale_task_check_interval: datetime.timedelta, stale_task_detect_timeout: datetime.timedelta, - redis_namespace: RedisNamespace, + lrt_namespace: LRTNamespace, ): # Task groups: Every taskname maps to multiple asyncio.Task within TrackedTask model - self._tasks_data: BaseStore = RedisStore(redis_settings, redis_namespace) + self._tasks_data: BaseStore = RedisStore(redis_settings, lrt_namespace) self._created_tasks: dict[TaskId, asyncio.Task] = {} self.stale_task_check_interval = stale_task_check_interval self.stale_task_detect_timeout_s: PositiveFloat = ( stale_task_detect_timeout.total_seconds() ) - self.redis_namespace = redis_namespace + self.lrt_namespace = lrt_namespace self.redis_settings = redis_settings self.locks_redis_client_sdk: RedisClientSDK | None = None @@ -151,7 +151,7 @@ async def setup(self) -> None: self.locks_redis_client_sdk = RedisClientSDK( self.redis_settings.build_redis_dsn(RedisDatabase.LOCKS), - client_name=f"long_running_tasks_store_{self.redis_namespace}_lock", + client_name=f"{__name__}_{self.lrt_namespace}_lock", ) await self.locks_redis_client_sdk.setup() @@ -159,7 +159,7 @@ async def setup(self) -> None: self._task_stale_tasks_monitor = create_periodic_task( task=exclusive( self.locks_redis_client_sdk, - lock_key=f"{__name__}_{self.redis_namespace}_stale_tasks_monitor", + lock_key=f"{__name__}_{self.lrt_namespace}_stale_tasks_monitor", )(self._stale_tasks_monitor), interval=self.stale_task_check_interval, task_name=f"{__name__}.{self._stale_tasks_monitor.__name__}", @@ -437,8 +437,8 @@ async def remove_task( pass def _get_task_id(self, task_name: str, *, is_unique: bool) -> TaskId: - unique_part = "unique" if is_unique else f"{uuid4()}" - return f"{self.redis_namespace}.{task_name}.{unique_part}" + id_part = "unique" if is_unique else f"{uuid4()}" + return f"{self.lrt_namespace}.{task_name}.{id_part}" async def _update_progress( self, diff --git a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py index 3157bc6d4259..549fe2031abf 100644 --- a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py +++ b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py @@ -48,9 +48,8 @@ def app( long_running_tasks.server.setup( app, redis_settings=use_in_memory_redis, - redis_namespace="test", rabbit_settings=rabbit_service, - rabbit_namespace="test", + lrt_namespace="test", router_prefix="/futures", ) diff --git a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_client.py b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_client.py index 6310b679ac8b..9e8c9204acef 100644 --- a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_client.py +++ b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_client.py @@ -37,9 +37,8 @@ def app( long_running_tasks.server.setup( app, redis_settings=use_in_memory_redis, - redis_namespace="test", rabbit_settings=rabbit_service, - rabbit_namespace="test", + lrt_namespace="test", router_prefix="/futures", ) diff --git a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py index 925735540e4b..cef4a845ab8d 100644 --- a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py +++ b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py @@ -78,9 +78,8 @@ def app_with_task_context( long_running_tasks.server.setup( app, redis_settings=use_in_memory_redis, - redis_namespace="test", rabbit_settings=rabbit_service, - rabbit_namespace="test", + lrt_namespace="test", router_prefix="/futures_with_task_context", task_request_context_decorator=task_context_decorator, ) diff --git a/packages/service-library/tests/fastapi/long_running_tasks/conftest.py b/packages/service-library/tests/fastapi/long_running_tasks/conftest.py index c2bdbb70c166..f10a27c322ac 100644 --- a/packages/service-library/tests/fastapi/long_running_tasks/conftest.py +++ b/packages/service-library/tests/fastapi/long_running_tasks/conftest.py @@ -23,9 +23,8 @@ async def bg_task_app( long_running_tasks.server.setup( app, redis_settings=redis_service, - redis_namespace="test", rabbit_settings=rabbit_service, - rabbit_namespace="test", + lrt_namespace="test", router_prefix=router_prefix, ) return app diff --git a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py index dce021286aff..6af4b252e2f5 100644 --- a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py +++ b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py @@ -108,9 +108,8 @@ async def app( setup_server( app, redis_settings=use_in_memory_redis, - redis_namespace="test", rabbit_settings=rabbit_service, - rabbit_namespace="test", + lrt_namespace="test", ) setup_client(app) async with LifespanManager(app, startup_timeout=30, shutdown_timeout=30): diff --git a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py index 880d8fe65c25..481acdf61bd5 100644 --- a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py +++ b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py @@ -106,9 +106,8 @@ async def bg_task_app( app, router_prefix=router_prefix, redis_settings=use_in_memory_redis, - redis_namespace="test", rabbit_settings=rabbit_service, - rabbit_namespace="test", + lrt_namespace="test", ) setup_client(app, router_prefix=router_prefix) diff --git a/packages/service-library/tests/long_running_tasks/conftest.py b/packages/service-library/tests/long_running_tasks/conftest.py index 83b62d4075f7..9894a1e0c780 100644 --- a/packages/service-library/tests/long_running_tasks/conftest.py +++ b/packages/service-library/tests/long_running_tasks/conftest.py @@ -10,11 +10,10 @@ from faker import Faker from pytest_mock import MockerFixture from servicelib.logging_utils import log_catch -from servicelib.long_running_tasks._rabbit.lrt_client import RabbitNamespace from servicelib.long_running_tasks.base_long_running_manager import ( BaseLongRunningManager, ) -from servicelib.long_running_tasks.models import RedisNamespace +from servicelib.long_running_tasks.models import LRTNamespace from servicelib.long_running_tasks.task import TasksManager from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient from settings_library.rabbit import RabbitSettings @@ -29,7 +28,7 @@ async def get_long_running_manager( fast_long_running_tasks_cancellation: None, faker: Faker ) -> AsyncIterator[ Callable[ - [RedisSettings, RedisNamespace | None, RabbitSettings, RabbitNamespace], + [RedisSettings, RabbitSettings, LRTNamespace | None], Awaitable[BaseLongRunningManager], ] ]: @@ -37,17 +36,15 @@ async def get_long_running_manager( async def _( redis_settings: RedisSettings, - namespace: RedisNamespace | None, rabbit_settings: RabbitSettings, - rabbit_namespace: RabbitNamespace, + lrt_namespace: LRTNamespace | None, ) -> BaseLongRunningManager: manager = BaseLongRunningManager( stale_task_check_interval=timedelta(seconds=TEST_CHECK_STALE_INTERVAL_S), stale_task_detect_timeout=timedelta(seconds=TEST_CHECK_STALE_INTERVAL_S), - redis_namespace=namespace or f"test{faker.uuid4()}", redis_settings=redis_settings, - rabbit_namespace=rabbit_namespace, rabbit_settings=rabbit_settings, + lrt_namespace=lrt_namespace or f"test{faker.uuid4()}", ) await manager.setup() managers.append(manager) diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py index 4f0a21167954..22f0fd43559c 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py @@ -10,12 +10,11 @@ from models_library.api_schemas_long_running_tasks.base import TaskProgress from pydantic import NonNegativeInt from servicelib.long_running_tasks import lrt_api -from servicelib.long_running_tasks._rabbit.lrt_client import RabbitNamespace from servicelib.long_running_tasks.base_long_running_manager import ( BaseLongRunningManager, ) from servicelib.long_running_tasks.errors import TaskNotFoundError -from servicelib.long_running_tasks.models import RedisNamespace, TaskContext +from servicelib.long_running_tasks.models import LRTNamespace, TaskContext from servicelib.long_running_tasks.task import TaskId, TaskRegistry from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient from settings_library.rabbit import RabbitSettings @@ -71,14 +70,14 @@ async def long_running_managers( use_in_memory_redis: RedisSettings, rabbit_service: RabbitSettings, get_long_running_manager: Callable[ - [RedisSettings, RedisNamespace | None, RabbitSettings, RabbitNamespace], + [RedisSettings, RabbitSettings, LRTNamespace | None], Awaitable[BaseLongRunningManager], ], ) -> list[BaseLongRunningManager]: maanagers: list[BaseLongRunningManager] = [] for _ in range(managers_count): long_running_manager = await get_long_running_manager( - use_in_memory_redis, "same-service", rabbit_service, "some-service" + use_in_memory_redis, rabbit_service, "some-service" ) maanagers.append(long_running_manager) diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py index 0db0a8e9f268..f8698c3fa97a 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py @@ -14,7 +14,6 @@ from faker import Faker from models_library.api_schemas_long_running_tasks.base import ProgressMessage from servicelib.long_running_tasks import lrt_api -from servicelib.long_running_tasks._rabbit.lrt_client import RabbitNamespace from servicelib.long_running_tasks.base_long_running_manager import ( BaseLongRunningManager, ) @@ -25,7 +24,7 @@ TaskNotRegisteredError, ) from servicelib.long_running_tasks.models import ( - RedisNamespace, + LRTNamespace, TaskContext, TaskProgress, TaskStatus, @@ -95,12 +94,12 @@ async def long_running_manager( use_in_memory_redis: RedisSettings, rabbit_service: RabbitSettings, get_long_running_manager: Callable[ - [RedisSettings, RedisNamespace | None, RabbitSettings, RabbitNamespace], + [RedisSettings, RabbitSettings, LRTNamespace | None], Awaitable[BaseLongRunningManager], ], ) -> BaseLongRunningManager: return await get_long_running_manager( - use_in_memory_redis, None, rabbit_service, "rabbit-namespace" + use_in_memory_redis, rabbit_service, "rabbit-namespace" ) @@ -359,19 +358,19 @@ async def test_cancel_task_from_different_manager( rabbit_service: RabbitSettings, use_in_memory_redis: RedisSettings, get_long_running_manager: Callable[ - [RedisSettings, RedisNamespace | None, RabbitSettings, RabbitNamespace], + [RedisSettings, RabbitSettings, LRTNamespace | None], Awaitable[BaseLongRunningManager], ], empty_context: TaskContext, ): manager_1 = await get_long_running_manager( - use_in_memory_redis, "test-namespace", rabbit_service, "test-namespace" + use_in_memory_redis, rabbit_service, "test-namespace" ) manager_2 = await get_long_running_manager( - use_in_memory_redis, "test-namespace", rabbit_service, "test-namespace" + use_in_memory_redis, rabbit_service, "test-namespace" ) manager_3 = await get_long_running_manager( - use_in_memory_redis, "test-namespace", rabbit_service, "test-namespace" + use_in_memory_redis, rabbit_service, "test-namespace" ) task_id = await lrt_api.start_task( diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/module_setup.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/module_setup.py index aa2f1bf7b81d..5381566045c4 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/module_setup.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/module_setup.py @@ -1,13 +1,10 @@ from fastapi import FastAPI from servicelib.fastapi import long_running_tasks -from servicelib.long_running_tasks.models import RabbitNamespace, RedisNamespace +from ..._meta import APP_NAME from ...core.settings import AppSettings from . import api_client, scheduler -_LRT_REDIS_NAMESPACE: RedisNamespace = "director-v2" -_LRT_RABBIT_NAMESPACE: RabbitNamespace = "director-v2" - def setup(app: FastAPI) -> None: settings: AppSettings = app.state.settings @@ -16,9 +13,8 @@ def setup(app: FastAPI) -> None: long_running_tasks.server.setup( app, redis_settings=settings.REDIS, - redis_namespace=_LRT_REDIS_NAMESPACE, rabbit_settings=settings.DIRECTOR_V2_RABBITMQ, - rabbit_namespace=_LRT_RABBIT_NAMESPACE, + lrt_namespace=APP_NAME, ) async def on_startup() -> None: diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py index 37427459df1b..5b758c641bec 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py @@ -635,9 +635,8 @@ def setup_long_running_tasks(app: FastAPI) -> None: long_running_tasks.server.setup( app, redis_settings=app_settings.REDIS_SETTINGS, - redis_namespace=f"dy_sidecar-{app_settings.DY_SIDECAR_RUN_ID}", rabbit_settings=app_settings.RABBIT_SETTINGS, - rabbit_namespace=f"dy_sidecar-{app_settings.DY_SIDECAR_RUN_ID}", + lrt_namespace=f"{APP_NAME}-{app_settings.DY_SIDECAR_RUN_ID}", ) async def on_startup() -> None: diff --git a/services/storage/src/simcore_service_storage/modules/long_running_tasks.py b/services/storage/src/simcore_service_storage/modules/long_running_tasks.py index be8a5ead85be..27085b49d9f9 100644 --- a/services/storage/src/simcore_service_storage/modules/long_running_tasks.py +++ b/services/storage/src/simcore_service_storage/modules/long_running_tasks.py @@ -1,15 +1,9 @@ -from typing import Final - from fastapi import FastAPI from servicelib.fastapi.long_running_tasks._server import setup -from servicelib.long_running_tasks.models import RabbitNamespace, RedisNamespace -from .._meta import API_VTAG +from .._meta import API_VTAG, APP_NAME from ..core.settings import get_application_settings -_LRT_REDIS_NAMESPACE: Final[RedisNamespace] = "storage" -_LRT_RABBIT_NAMESPACE: Final[RabbitNamespace] = "storage" - def setup_rest_api_long_running_tasks_for_uploads(app: FastAPI) -> None: settings = get_application_settings(app) @@ -17,7 +11,6 @@ def setup_rest_api_long_running_tasks_for_uploads(app: FastAPI) -> None: app, router_prefix=f"/{API_VTAG}/futures", redis_settings=settings.STORAGE_REDIS, - redis_namespace=_LRT_REDIS_NAMESPACE, rabbit_settings=settings.STORAGE_RABBITMQ, - rabbit_namespace=_LRT_RABBIT_NAMESPACE, + lrt_namespace=APP_NAME, ) diff --git a/services/web/server/src/simcore_service_webserver/long_running_tasks/plugin.py b/services/web/server/src/simcore_service_webserver/long_running_tasks/plugin.py index 0ff8a6906438..dee65b67aa4c 100644 --- a/services/web/server/src/simcore_service_webserver/long_running_tasks/plugin.py +++ b/services/web/server/src/simcore_service_webserver/long_running_tasks/plugin.py @@ -1,6 +1,5 @@ import logging from functools import wraps -from typing import Final from aiohttp import web from models_library.utils.fastapi_encoders import jsonable_encoder @@ -12,7 +11,7 @@ from servicelib.aiohttp.typing_extension import Handler from .. import rabbitmq_settings, redis -from .._meta import API_VTAG +from .._meta import API_VTAG, APP_NAME from ..login.decorators import login_required from ..models import AuthenticatedRequestContext from ..projects.plugin import register_projects_long_running_tasks @@ -21,11 +20,8 @@ _logger = logging.getLogger(__name__) -_LRT_NAMESPACE_PREFIX: Final[str] = "webserver-legacy" - - -def _get_namespace(suffix: str) -> str: - return f"{_LRT_NAMESPACE_PREFIX}-{suffix}" +def _get_lrt_namespace(suffix: str) -> str: + return f"{APP_NAME}-{suffix}" def webserver_request_context_decorator(handler: Handler): @@ -57,8 +53,7 @@ def setup_long_running_tasks(app: web.Application) -> None: app, redis_settings=redis.get_plugin_settings(app), rabbit_settings=rabbitmq_settings.get_plugin_settings(app), - redis_namespace=_get_namespace(settings.LONG_RUNNING_TASKS_NAMESPACE_SUFFIX), - rabbit_namespace=_get_namespace(settings.LONG_RUNNING_TASKS_NAMESPACE_SUFFIX), + lrt_namespace=_get_lrt_namespace(settings.LONG_RUNNING_TASKS_NAMESPACE_SUFFIX), router_prefix=f"/{API_VTAG}/tasks-legacy", handler_check_decorator=login_required, task_request_context_decorator=webserver_request_context_decorator, From 8ce988a9b6f63a6722ae2e4b997fc3b5dec16f04 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 18 Aug 2025 10:38:48 +0200 Subject: [PATCH 061/119] removed unsued module --- .../{_store/redis.py => _redis_store.py} | 11 ++-- .../long_running_tasks/_store/__init__.py | 0 .../long_running_tasks/_store/base.py | 51 ------------------- .../src/servicelib/long_running_tasks/task.py | 7 ++- .../test_long_running_tasks__store.py | 7 ++- 5 files changed, 11 insertions(+), 65 deletions(-) rename packages/service-library/src/servicelib/long_running_tasks/{_store/redis.py => _redis_store.py} (94%) delete mode 100644 packages/service-library/src/servicelib/long_running_tasks/_store/__init__.py delete mode 100644 packages/service-library/src/servicelib/long_running_tasks/_store/base.py diff --git a/packages/service-library/src/servicelib/long_running_tasks/_store/redis.py b/packages/service-library/src/servicelib/long_running_tasks/_redis_store.py similarity index 94% rename from packages/service-library/src/servicelib/long_running_tasks/_store/redis.py rename to packages/service-library/src/servicelib/long_running_tasks/_redis_store.py index 6ec443619318..3fc80eaa4223 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_store/redis.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_redis_store.py @@ -6,11 +6,10 @@ from pydantic import TypeAdapter from settings_library.redis import RedisDatabase, RedisSettings -from ...redis._client import RedisClientSDK -from ...redis._utils import handle_redis_returns_union_types -from ...utils import limited_gather -from ..models import TaskContext, TaskData, TaskId -from .base import BaseStore +from ..redis._client import RedisClientSDK +from ..redis._utils import handle_redis_returns_union_types +from ..utils import limited_gather +from .models import TaskContext, TaskData, TaskId _STORE_TYPE_TASK_DATA: Final[str] = "TD" _STORE_TYPE_CANCELLED_TASKS: Final[str] = "CT" @@ -27,7 +26,7 @@ def _decode_dict(data: dict[str, str]) -> dict[str, Any]: return {k: json.loads(v) for k, v in data.items()} -class RedisStore(BaseStore): +class RedisStore: def __init__(self, redis_settings: RedisSettings, namespace: str): self.redis_settings = redis_settings self.namespace = namespace.upper() diff --git a/packages/service-library/src/servicelib/long_running_tasks/_store/__init__.py b/packages/service-library/src/servicelib/long_running_tasks/_store/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/packages/service-library/src/servicelib/long_running_tasks/_store/base.py b/packages/service-library/src/servicelib/long_running_tasks/_store/base.py deleted file mode 100644 index 6fc5df31f489..000000000000 --- a/packages/service-library/src/servicelib/long_running_tasks/_store/base.py +++ /dev/null @@ -1,51 +0,0 @@ -from abc import abstractmethod -from typing import Any - -from ..models import TaskContext, TaskData, TaskId - - -class BaseStore: - - @abstractmethod - async def get_task_data(self, task_id: TaskId) -> TaskData | None: - """Retrieve a tracked task""" - - @abstractmethod - async def add_task_data(self, task_id: TaskId, value: TaskData) -> None: - """Set a tracked task's data""" - - @abstractmethod - async def update_task_data( - self, task_id: TaskId, *, updates: dict[str, Any] - ) -> None: - """Update a tracked task's data by specifying each single field to update""" - - @abstractmethod - async def list_tasks_data(self) -> list[TaskData]: - """List all tracked tasks.""" - - @abstractmethod - async def delete_task_data(self, task_id: TaskId) -> None: - """Delete a tracked task.""" - - @abstractmethod - async def set_as_cancelled( - self, task_id: TaskId, with_task_context: TaskContext - ) -> None: - """Mark a tracked task as cancelled.""" - - @abstractmethod - async def delete_set_as_cancelled(self, task_id: TaskId) -> None: - """Remove a task from the cancelled tasks.""" - - @abstractmethod - async def get_cancelled(self) -> dict[TaskId, TaskContext]: - """Get cancelled tasks.""" - - @abstractmethod - async def setup(self) -> None: - """Setup the store, if needed.""" - - @abstractmethod - async def shutdown(self) -> None: - """Shutdown the store, if needed.""" diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index d9c3195bc540..93300f99d76d 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -23,9 +23,8 @@ from ..background_task import create_periodic_task from ..logging_utils import log_catch from ..redis import RedisClientSDK, exclusive +from ._redis_store import RedisStore from ._serialization import object_to_string, string_to_object -from ._store.base import BaseStore -from ._store.redis import RedisStore from .errors import ( TaskAlreadyRunningError, TaskCancelledError, @@ -81,7 +80,7 @@ def unregister(cls, task: TaskProtocol) -> None: async def _get_tasks_to_remove( - tracked_tasks: BaseStore, + tracked_tasks: RedisStore, stale_task_detect_timeout_s: PositiveFloat, ) -> list[tuple[TaskId, TaskContext]]: utc_now = datetime.datetime.now(tz=datetime.UTC) @@ -122,7 +121,7 @@ def __init__( lrt_namespace: LRTNamespace, ): # Task groups: Every taskname maps to multiple asyncio.Task within TrackedTask model - self._tasks_data: BaseStore = RedisStore(redis_settings, lrt_namespace) + self._tasks_data = RedisStore(redis_settings, lrt_namespace) self._created_tasks: dict[TaskId, asyncio.Task] = {} self.stale_task_check_interval = stale_task_check_interval diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks__store.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks__store.py index 41e499ac5b86..232140417492 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks__store.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks__store.py @@ -5,8 +5,7 @@ import pytest from pydantic import TypeAdapter -from servicelib.long_running_tasks._store.base import BaseStore -from servicelib.long_running_tasks._store.redis import RedisStore +from servicelib.long_running_tasks._redis_store import RedisStore from servicelib.long_running_tasks.models import TaskData from servicelib.redis._client import RedisClientSDK from settings_library.redis import RedisDatabase, RedisSettings @@ -25,7 +24,7 @@ async def store( get_redis_client_sdk: Callable[ [RedisDatabase], AbstractAsyncContextManager[RedisClientSDK] ], -) -> AsyncIterable[BaseStore]: +) -> AsyncIterable[RedisStore]: store = RedisStore(redis_settings=use_in_memory_redis, namespace="test") await store.setup() @@ -37,7 +36,7 @@ async def store( pass -async def test_workflow(store: BaseStore, task_data: TaskData) -> None: +async def test_workflow(store: RedisStore, task_data: TaskData) -> None: # task data assert await store.list_tasks_data() == [] assert await store.get_task_data("missing") is None From 5e31621a76bf84d8ba0ad0febda5e8a9695312ee Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 18 Aug 2025 10:46:51 +0200 Subject: [PATCH 062/119] rename interface --- .../servicelib/long_running_tasks/_redis_store.py | 11 +++++++---- .../src/servicelib/long_running_tasks/task.py | 6 +++--- .../test_long_running_tasks__store.py | 14 ++++++++------ .../test_long_running_tasks_task.py | 2 +- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/_redis_store.py b/packages/service-library/src/servicelib/long_running_tasks/_redis_store.py index 3fc80eaa4223..cc9733743b00 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_redis_store.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_redis_store.py @@ -118,23 +118,26 @@ async def delete_task_data(self, task_id: TaskId) -> None: self._redis.delete(self._get_redis_key_task_data_hash(task_id)) ) - # cancelled + # to cancel - async def set_as_cancelled( + async def set_to_cancel( self, task_id: TaskId, with_task_context: TaskContext ) -> None: + """marks a task_id to be cancelled""" await handle_redis_returns_union_types( self._redis.hset( self._get_key_cancelled_tasks(), task_id, json_dumps(with_task_context) ) ) - async def delete_set_as_cancelled(self, task_id: TaskId) -> None: + async def remove_to_cancel(self, task_id: TaskId) -> None: + """removes a task_id from the ones to be cancelled""" await handle_redis_returns_union_types( self._redis.hdel(self._get_key_cancelled_tasks(), task_id) ) - async def get_cancelled(self) -> dict[TaskId, TaskContext]: + async def get_all_to_cancel(self) -> dict[TaskId, TaskContext]: + """returns all task_ids that are to be cancelled""" result: dict[str, str | None] = await handle_redis_returns_union_types( self._redis.hgetall(self._get_key_cancelled_tasks()) ) diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index 93300f99d76d..8a2a8dc0dc10 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -275,7 +275,7 @@ async def _cancelled_tasks_removal(self) -> None: """ self._started_event_task_cancelled_tasks_removal.set() - cancelled_tasks = await self._tasks_data.get_cancelled() + cancelled_tasks = await self._tasks_data.get_all_to_cancel() for task_id in cancelled_tasks: await self._cancel_and_remove_local_task(task_id) @@ -397,7 +397,7 @@ async def _cancel_and_remove_local_task(self, task_id: TaskId) -> None: task_to_cancel = self._created_tasks.pop(task_id, None) if task_to_cancel is not None: await cancel_wait_task(task_to_cancel) - await self._tasks_data.delete_set_as_cancelled(task_id) + await self._tasks_data.remove_to_cancel(task_id) await self._tasks_data.delete_task_data(task_id) async def remove_task( @@ -415,7 +415,7 @@ async def remove_task( raise return - await self._tasks_data.set_as_cancelled( + await self._tasks_data.set_to_cancel( tracked_task.task_id, tracked_task.task_context ) diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks__store.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks__store.py index 232140417492..231ac4fad823 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks__store.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks__store.py @@ -50,11 +50,13 @@ async def test_workflow(store: RedisStore, task_data: TaskData) -> None: assert await store.list_tasks_data() == [] # cancelled tasks - assert await store.get_cancelled() == {} + assert await store.get_all_to_cancel() == {} - await store.set_as_cancelled(task_data.task_id, task_data.task_context) + await store.set_to_cancel(task_data.task_id, task_data.task_context) - assert await store.get_cancelled() == {task_data.task_id: task_data.task_context} + assert await store.get_all_to_cancel() == { + task_data.task_id: task_data.task_context + } @pytest.fixture @@ -87,15 +89,15 @@ async def test_workflow_multiple_redis_stores_with_different_namespaces( for store in redis_stores: assert await store.list_tasks_data() == [] - assert await store.get_cancelled() == {} + assert await store.get_all_to_cancel() == {} for store in redis_stores: await store.add_task_data(task_data.task_id, task_data) - await store.set_as_cancelled(task_data.task_id, None) + await store.set_to_cancel(task_data.task_id, {}) for store in redis_stores: assert await store.list_tasks_data() == [task_data] - assert await store.get_cancelled() == {task_data.task_id: None} + assert await store.get_all_to_cancel() == {task_data.task_id: {}} for store in redis_stores: await store.delete_task_data(task_data.task_id) diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py index f8698c3fa97a..c67c8c2a7d95 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py @@ -480,7 +480,7 @@ async def test__cancelled_tasks_worker_equivalent_of_cancellation_from_a_differe total_sleep=10, task_context=empty_context, ) - await long_running_manager.tasks_manager._tasks_data.set_as_cancelled( # noqa: SLF001 + await long_running_manager.tasks_manager._tasks_data.set_to_cancel( # noqa: SLF001 task_id, with_task_context=empty_context ) From 9e246d68a0835d6cdeda20bc755f08d044878c71 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 18 Aug 2025 11:12:10 +0200 Subject: [PATCH 063/119] fixed error formatting --- .../long_running_tasks/_rabbit/lrt_client.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py index 94184e6af050..1f1145690619 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py @@ -5,6 +5,7 @@ from models_library.rabbitmq_basic_types import RPCMethodName from pydantic import PositiveInt, TypeAdapter +from ...logging_errors import create_troubleshootting_log_kwargs from ...logging_utils import log_decorator from ...long_running_tasks.task import RegisteredTaskName from ...rabbitmq._client_rpc import RabbitMQRPCClient @@ -104,10 +105,16 @@ async def get_task_result( if isinstance(serialized_result, RPCErrorResponse): error = string_to_object(serialized_result.error_object) _logger.warning( - "Remote task finished with error: '%s: %s'\n%s", - error.__class__.__name__, - error, - serialized_result.str_traceback, + **create_troubleshootting_log_kwargs( + f"Remote task finished with error '{error.__class__.__name__}: {error}'\n{serialized_result.str_traceback}", + error=error, + error_context={ + "task_id": task_id, + "task_context": task_context, + "namespace": namespace, + }, + tip=f"Raised where the lrt_server was running, you can figure this out via {namespace=}", + ) ) raise error From ef3b6872655af9034c7cdfa29af9193234f9bef6 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 18 Aug 2025 11:26:42 +0200 Subject: [PATCH 064/119] rephrased --- .../src/servicelib/long_running_tasks/task.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index 8a2a8dc0dc10..504c6a72c84b 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -21,7 +21,6 @@ ) from ..background_task import create_periodic_task -from ..logging_utils import log_catch from ..redis import RedisClientSDK, exclusive from ._redis_store import RedisStore from ._serialization import object_to_string, string_to_object @@ -203,11 +202,11 @@ async def teardown(self) -> None: # since the task is using a redis lock if the lock could not be acquired # trying to cancel the task will hang, this avoids hanging # there are no sideeffects in timing out this cancellation - with log_catch(_logger, reraise=False): - await cancel_wait_task( - self._task_stale_tasks_monitor, - max_delay=_MAX_EXCLUSIVE_TASK_CANCEL_TIMEOUT, - ) + # with log_catch(_logger, reraise=False): + await cancel_wait_task( + self._task_stale_tasks_monitor, + max_delay=_MAX_EXCLUSIVE_TASK_CANCEL_TIMEOUT, + ) # cancelled_tasks_removal if self._task_cancelled_tasks_removal: @@ -277,7 +276,7 @@ async def _cancelled_tasks_removal(self) -> None: cancelled_tasks = await self._tasks_data.get_all_to_cancel() for task_id in cancelled_tasks: - await self._cancel_and_remove_local_task(task_id) + await self._attempt_cancel_and_remove_local_task(task_id) async def _status_update(self) -> None: """ @@ -391,8 +390,8 @@ async def get_task_result( return string_to_object(tracked_task.result_field.result) - async def _cancel_and_remove_local_task(self, task_id: TaskId) -> None: - """cancels task and removes if from local tracker if this is the worke that started it""" + async def _attempt_cancel_and_remove_local_task(self, task_id: TaskId) -> None: + """if task is running in the local process, cancel it and remove it""" task_to_cancel = self._created_tasks.pop(task_id, None) if task_to_cancel is not None: From bb928911b357dbfd735e694535b0214b4a2a0006 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 18 Aug 2025 11:30:52 +0200 Subject: [PATCH 065/119] refactor naming --- .../long_running_tasks/_redis_store.py | 20 +++++++++---------- .../src/servicelib/long_running_tasks/task.py | 12 +++++------ .../test_long_running_tasks__store.py | 12 +++++------ .../test_long_running_tasks_task.py | 2 +- 4 files changed, 22 insertions(+), 24 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/_redis_store.py b/packages/service-library/src/servicelib/long_running_tasks/_redis_store.py index cc9733743b00..7c9e5fc9a205 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_redis_store.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_redis_store.py @@ -55,7 +55,7 @@ def _get_redis_key_task_data_match(self) -> str: def _get_redis_key_task_data_hash(self, task_id: TaskId) -> str: return f"{self.namespace}:{_STORE_TYPE_TASK_DATA}:{task_id}" - def _get_key_cancelled_tasks(self) -> str: + def _get_key_to_remove(self) -> str: return f"{self.namespace}:{_STORE_TYPE_CANCELLED_TASKS}" # TaskData @@ -120,25 +120,25 @@ async def delete_task_data(self, task_id: TaskId) -> None: # to cancel - async def set_to_cancel( + async def set_to_remove( self, task_id: TaskId, with_task_context: TaskContext ) -> None: - """marks a task_id to be cancelled""" + """marks a task_id to be removed""" await handle_redis_returns_union_types( self._redis.hset( - self._get_key_cancelled_tasks(), task_id, json_dumps(with_task_context) + self._get_key_to_remove(), task_id, json_dumps(with_task_context) ) ) - async def remove_to_cancel(self, task_id: TaskId) -> None: - """removes a task_id from the ones to be cancelled""" + async def delete_to_remove(self, task_id: TaskId) -> None: + """deletes a task_id from the ones to be removed""" await handle_redis_returns_union_types( - self._redis.hdel(self._get_key_cancelled_tasks(), task_id) + self._redis.hdel(self._get_key_to_remove(), task_id) ) - async def get_all_to_cancel(self) -> dict[TaskId, TaskContext]: - """returns all task_ids that are to be cancelled""" + async def get_all_to_remove(self) -> dict[TaskId, TaskContext]: + """returns all task_ids that are to be removed""" result: dict[str, str | None] = await handle_redis_returns_union_types( - self._redis.hgetall(self._get_key_cancelled_tasks()) + self._redis.hgetall(self._get_key_to_remove()) ) return {task_id: json_loads(context) for task_id, context in result.items()} diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index 504c6a72c84b..42195010a33c 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -267,14 +267,12 @@ async def _stale_tasks_monitor(self) -> None: async def _cancelled_tasks_removal(self) -> None: """ - A task can be cancelled by the client, which implies it does not for sure - run in the same process as the one processing the request. - - This is a periodic task that ensures the cancellation occurred. + Periodicallu checks which tasks are maked for removal and attempts to remove the + task if it's handled by this process. """ self._started_event_task_cancelled_tasks_removal.set() - cancelled_tasks = await self._tasks_data.get_all_to_cancel() + cancelled_tasks = await self._tasks_data.get_all_to_remove() for task_id in cancelled_tasks: await self._attempt_cancel_and_remove_local_task(task_id) @@ -396,7 +394,7 @@ async def _attempt_cancel_and_remove_local_task(self, task_id: TaskId) -> None: task_to_cancel = self._created_tasks.pop(task_id, None) if task_to_cancel is not None: await cancel_wait_task(task_to_cancel) - await self._tasks_data.remove_to_cancel(task_id) + await self._tasks_data.delete_to_remove(task_id) await self._tasks_data.delete_task_data(task_id) async def remove_task( @@ -414,7 +412,7 @@ async def remove_task( raise return - await self._tasks_data.set_to_cancel( + await self._tasks_data.set_to_remove( tracked_task.task_id, tracked_task.task_context ) diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks__store.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks__store.py index 231ac4fad823..47a5222d310f 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks__store.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks__store.py @@ -50,11 +50,11 @@ async def test_workflow(store: RedisStore, task_data: TaskData) -> None: assert await store.list_tasks_data() == [] # cancelled tasks - assert await store.get_all_to_cancel() == {} + assert await store.get_all_to_remove() == {} - await store.set_to_cancel(task_data.task_id, task_data.task_context) + await store.set_to_remove(task_data.task_id, task_data.task_context) - assert await store.get_all_to_cancel() == { + assert await store.get_all_to_remove() == { task_data.task_id: task_data.task_context } @@ -89,15 +89,15 @@ async def test_workflow_multiple_redis_stores_with_different_namespaces( for store in redis_stores: assert await store.list_tasks_data() == [] - assert await store.get_all_to_cancel() == {} + assert await store.get_all_to_remove() == {} for store in redis_stores: await store.add_task_data(task_data.task_id, task_data) - await store.set_to_cancel(task_data.task_id, {}) + await store.set_to_remove(task_data.task_id, {}) for store in redis_stores: assert await store.list_tasks_data() == [task_data] - assert await store.get_all_to_cancel() == {task_data.task_id: {}} + assert await store.get_all_to_remove() == {task_data.task_id: {}} for store in redis_stores: await store.delete_task_data(task_data.task_id) diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py index c67c8c2a7d95..fe6e7dd749a6 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py @@ -480,7 +480,7 @@ async def test__cancelled_tasks_worker_equivalent_of_cancellation_from_a_differe total_sleep=10, task_context=empty_context, ) - await long_running_manager.tasks_manager._tasks_data.set_to_cancel( # noqa: SLF001 + await long_running_manager.tasks_manager._tasks_data.set_to_remove( # noqa: SLF001 task_id, with_task_context=empty_context ) From 2a342599e3783cdc997967768a16a1f72975e5d7 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 18 Aug 2025 11:31:45 +0200 Subject: [PATCH 066/119] docstring rename --- .../src/servicelib/long_running_tasks/task.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index 42195010a33c..92a154a9726a 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -272,9 +272,9 @@ async def _cancelled_tasks_removal(self) -> None: """ self._started_event_task_cancelled_tasks_removal.set() - cancelled_tasks = await self._tasks_data.get_all_to_remove() - for task_id in cancelled_tasks: - await self._attempt_cancel_and_remove_local_task(task_id) + to_remove = await self._tasks_data.get_all_to_remove() + for task_id in to_remove: + await self._attempt_to_remove_local_task(task_id) async def _status_update(self) -> None: """ @@ -388,8 +388,8 @@ async def get_task_result( return string_to_object(tracked_task.result_field.result) - async def _attempt_cancel_and_remove_local_task(self, task_id: TaskId) -> None: - """if task is running in the local process, cancel it and remove it""" + async def _attempt_to_remove_local_task(self, task_id: TaskId) -> None: + """if task is running in the local process, try to remove it""" task_to_cancel = self._created_tasks.pop(task_id, None) if task_to_cancel is not None: From a10f0658167293064ea71097a06772b013a3a491 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 18 Aug 2025 11:32:18 +0200 Subject: [PATCH 067/119] removed --- .../service-library/src/servicelib/long_running_tasks/task.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index 92a154a9726a..f3e36cd79fef 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -199,10 +199,6 @@ async def teardown(self) -> None: # stale_tasks_monitor if self._task_stale_tasks_monitor: - # since the task is using a redis lock if the lock could not be acquired - # trying to cancel the task will hang, this avoids hanging - # there are no sideeffects in timing out this cancellation - # with log_catch(_logger, reraise=False): await cancel_wait_task( self._task_stale_tasks_monitor, max_delay=_MAX_EXCLUSIVE_TASK_CANCEL_TIMEOUT, From a3ed827055b189fdebf4c3c546aa4e28037511e3 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 18 Aug 2025 11:56:16 +0200 Subject: [PATCH 068/119] rename --- .../src/servicelib/long_running_tasks/_rabbit/lrt_client.py | 4 ++-- .../src/servicelib/long_running_tasks/task.py | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py index 1f1145690619..e2a1789de203 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py @@ -16,7 +16,7 @@ _logger = logging.getLogger(__name__) -_RPC_TIMEOUT_VERY_LONG_REQUEST: Final[PositiveInt] = int( +_RPC_TIMEOUT_TASK_REMOVAL: Final[PositiveInt] = int( timedelta(minutes=60).total_seconds() ) _RPC_TIMEOUT_SHORT_REQUESTS: Final[PositiveInt] = int( @@ -136,6 +136,6 @@ async def remove_task( task_context=task_context, task_id=task_id, reraise_errors=reraise_errors, - timeout_s=_RPC_TIMEOUT_VERY_LONG_REQUEST, + timeout_s=_RPC_TIMEOUT_TASK_REMOVAL, ) assert result is None # nosec diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index f3e36cd79fef..0c2a199703de 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -419,14 +419,12 @@ async def remove_task( stop=stop_after_delay(_TASK_REMOVAL_MAX_WAIT), retry=retry_if_exception_type(TryAgain), ): - with attempt: - try: + with attempt: # noqa: SIM117 + with suppress(TaskNotFoundError): await self._get_tracked_task( tracked_task.task_id, tracked_task.task_context ) raise TryAgain - except TaskNotFoundError: - pass def _get_task_id(self, task_name: str, *, is_unique: bool) -> TaskId: id_part = "unique" if is_unique else f"{uuid4()}" From 308682972e556116327736af6cf8fa542ba0ade3 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 18 Aug 2025 12:00:02 +0200 Subject: [PATCH 069/119] fix pylint? --- .../service-library/tests/aiohttp/long_running_tasks/__init__.py | 1 + .../test_long_running_tasks_context_manager.py | 1 + 2 files changed, 2 insertions(+) diff --git a/packages/service-library/tests/aiohttp/long_running_tasks/__init__.py b/packages/service-library/tests/aiohttp/long_running_tasks/__init__.py index 2ab8f88ac10a..b4464b6ed7cb 100644 --- a/packages/service-library/tests/aiohttp/long_running_tasks/__init__.py +++ b/packages/service-library/tests/aiohttp/long_running_tasks/__init__.py @@ -1,2 +1,3 @@ +# pylint: disable=cyclic-import # DO NOT REMOVE # since there are tests with same name for aiohttp and fastapi diff --git a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py index 481acdf61bd5..a68f22461006 100644 --- a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py +++ b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py @@ -1,3 +1,4 @@ +# pylint: disable=cyclic-import # pylint: disable=redefined-outer-name # pylint: disable=unused-argument From 187d78f063e0552515fd5c968a00363171032d03 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 18 Aug 2025 13:23:48 +0200 Subject: [PATCH 070/119] revert change --- .../service-library/tests/aiohttp/long_running_tasks/__init__.py | 1 - .../test_long_running_tasks_context_manager.py | 1 - 2 files changed, 2 deletions(-) diff --git a/packages/service-library/tests/aiohttp/long_running_tasks/__init__.py b/packages/service-library/tests/aiohttp/long_running_tasks/__init__.py index b4464b6ed7cb..2ab8f88ac10a 100644 --- a/packages/service-library/tests/aiohttp/long_running_tasks/__init__.py +++ b/packages/service-library/tests/aiohttp/long_running_tasks/__init__.py @@ -1,3 +1,2 @@ -# pylint: disable=cyclic-import # DO NOT REMOVE # since there are tests with same name for aiohttp and fastapi diff --git a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py index a68f22461006..481acdf61bd5 100644 --- a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py +++ b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py @@ -1,4 +1,3 @@ -# pylint: disable=cyclic-import # pylint: disable=redefined-outer-name # pylint: disable=unused-argument From 95cd9d5a23596c0f89dee1c1bd5e83c36c39d555 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 18 Aug 2025 13:25:40 +0200 Subject: [PATCH 071/119] made modules flat --- .../{_rabbit/lrt_client.py => _lrt_client.py} | 14 +++++++------- .../{_rabbit/lrt_server.py => _lrt_server.py} | 12 ++++++------ .../long_running_tasks/{_rabbit => }/_models.py | 0 .../long_running_tasks/_rabbit/__init__.py | 0 .../{_rabbit/namespace.py => _rabbit_namespace.py} | 2 +- .../long_running_tasks/_serialization.py | 4 ---- .../base_long_running_manager.py | 5 ++--- .../src/servicelib/long_running_tasks/lrt_api.py | 12 ++++++------ 8 files changed, 22 insertions(+), 27 deletions(-) rename packages/service-library/src/servicelib/long_running_tasks/{_rabbit/lrt_client.py => _lrt_client.py} (91%) rename packages/service-library/src/servicelib/long_running_tasks/{_rabbit/lrt_server.py => _lrt_server.py} (90%) rename packages/service-library/src/servicelib/long_running_tasks/{_rabbit => }/_models.py (100%) delete mode 100644 packages/service-library/src/servicelib/long_running_tasks/_rabbit/__init__.py rename packages/service-library/src/servicelib/long_running_tasks/{_rabbit/namespace.py => _rabbit_namespace.py} (87%) diff --git a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py b/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py similarity index 91% rename from packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py rename to packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py index e2a1789de203..77f7109d5dfe 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_client.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py @@ -5,14 +5,14 @@ from models_library.rabbitmq_basic_types import RPCMethodName from pydantic import PositiveInt, TypeAdapter -from ...logging_errors import create_troubleshootting_log_kwargs -from ...logging_utils import log_decorator -from ...long_running_tasks.task import RegisteredTaskName -from ...rabbitmq._client_rpc import RabbitMQRPCClient -from .._serialization import string_to_object -from ..models import LRTNamespace, TaskBase, TaskContext, TaskId, TaskStatus +from ..logging_errors import create_troubleshootting_log_kwargs +from ..logging_utils import log_decorator +from ..rabbitmq._client_rpc import RabbitMQRPCClient from ._models import RPCErrorResponse -from .namespace import get_rabbit_namespace +from ._rabbit_namespace import get_rabbit_namespace +from ._serialization import string_to_object +from .models import LRTNamespace, TaskBase, TaskContext, TaskId, TaskStatus +from .task import RegisteredTaskName _logger = logging.getLogger(__name__) diff --git a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_server.py b/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py similarity index 90% rename from packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_server.py rename to packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py index 451c74bd75f9..542c90219015 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/lrt_server.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py @@ -1,13 +1,13 @@ import traceback from typing import Any -from ...rabbitmq import RPCRouter -from .._serialization import object_to_string -from ..base_long_running_manager import BaseLongRunningManager -from ..errors import BaseLongRunningError, TaskNotCompletedError, TaskNotFoundError -from ..models import TaskBase, TaskContext, TaskId, TaskStatus -from ..task import RegisteredTaskName +from ..rabbitmq import RPCRouter from ._models import RPCErrorResponse +from ._serialization import object_to_string +from .base_long_running_manager import BaseLongRunningManager +from .errors import BaseLongRunningError, TaskNotCompletedError, TaskNotFoundError +from .models import TaskBase, TaskContext, TaskId, TaskStatus +from .task import RegisteredTaskName router = RPCRouter() diff --git a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/_models.py b/packages/service-library/src/servicelib/long_running_tasks/_models.py similarity index 100% rename from packages/service-library/src/servicelib/long_running_tasks/_rabbit/_models.py rename to packages/service-library/src/servicelib/long_running_tasks/_models.py diff --git a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/__init__.py b/packages/service-library/src/servicelib/long_running_tasks/_rabbit/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/namespace.py b/packages/service-library/src/servicelib/long_running_tasks/_rabbit_namespace.py similarity index 87% rename from packages/service-library/src/servicelib/long_running_tasks/_rabbit/namespace.py rename to packages/service-library/src/servicelib/long_running_tasks/_rabbit_namespace.py index baf5198fdc80..7ace2e53a3dd 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_rabbit/namespace.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_rabbit_namespace.py @@ -1,7 +1,7 @@ from models_library.rabbitmq_basic_types import RPCNamespace from pydantic import TypeAdapter -from ..models import LRTNamespace +from .models import LRTNamespace def get_rabbit_namespace(namespace: LRTNamespace) -> RPCNamespace: diff --git a/packages/service-library/src/servicelib/long_running_tasks/_serialization.py b/packages/service-library/src/servicelib/long_running_tasks/_serialization.py index ae7125147a1f..37e5d1581164 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_serialization.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_serialization.py @@ -1,12 +1,8 @@ import base64 -import logging import pickle from abc import ABC, abstractmethod from typing import Any, Final, Generic, TypeVar -_logger = logging.getLogger(__name__) - - T = TypeVar("T") diff --git a/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py b/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py index 0e52d6fbc7a5..95791b518d65 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py +++ b/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py @@ -4,7 +4,7 @@ from settings_library.redis import RedisSettings from ..rabbitmq._client_rpc import RabbitMQRPCClient -from ._rabbit.namespace import get_rabbit_namespace +from ._rabbit_namespace import get_rabbit_namespace from .models import LRTNamespace from .task import TasksManager @@ -61,8 +61,7 @@ async def setup(self) -> None: client_name=f"lrt-client-{self.lrt_namespace}", settings=self.rabbit_settings, ) - - from ._rabbit.lrt_server import router + from ._lrt_server import router await self.rpc_server.register_router( router, diff --git a/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py b/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py index c551f879b0b7..c1b3c270d10b 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py +++ b/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py @@ -1,7 +1,7 @@ from typing import Any from ..rabbitmq._client_rpc import RabbitMQRPCClient -from ._rabbit import lrt_client +from . import _lrt_client from .base_long_running_manager import BaseLongRunningManager from .models import TaskBase, TaskContext, TaskId, TaskStatus from .task import RegisteredTaskName @@ -43,7 +43,7 @@ async def start_task( TaskId: the task unique identifier """ - return await lrt_client.start_task( + return await _lrt_client.start_task( long_running_manager.rpc_client, long_running_manager.lrt_namespace, registered_task_name=registered_task_name, @@ -60,7 +60,7 @@ async def list_tasks( long_running_manager: BaseLongRunningManager, task_context: TaskContext, ) -> list[TaskBase]: - return await lrt_client.list_tasks( + return await _lrt_client.list_tasks( rabbitmq_rpc_client, long_running_manager.lrt_namespace, task_context=task_context, @@ -74,7 +74,7 @@ async def get_task_status( task_id: TaskId, ) -> TaskStatus: """returns the status of a task""" - return await lrt_client.get_task_status( + return await _lrt_client.get_task_status( rabbitmq_rpc_client, long_running_manager.lrt_namespace, task_id=task_id, @@ -88,7 +88,7 @@ async def get_task_result( task_context: TaskContext, task_id: TaskId, ) -> Any: - return await lrt_client.get_task_result( + return await _lrt_client.get_task_result( rabbitmq_rpc_client, long_running_manager.lrt_namespace, task_context=task_context, @@ -103,7 +103,7 @@ async def remove_task( task_id: TaskId, ) -> None: """cancels and removes the task""" - await lrt_client.remove_task( + await _lrt_client.remove_task( rabbitmq_rpc_client, long_running_manager.lrt_namespace, task_id=task_id, From 7d6d885dafaaed6b8f5327f952a16cf9c9d232e2 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 18 Aug 2025 13:27:15 +0200 Subject: [PATCH 072/119] removed _models --- .../src/servicelib/long_running_tasks/_lrt_client.py | 10 ++++++++-- .../src/servicelib/long_running_tasks/_lrt_server.py | 3 +-- .../src/servicelib/long_running_tasks/_models.py | 6 ------ .../src/servicelib/long_running_tasks/models.py | 5 +++++ 4 files changed, 14 insertions(+), 10 deletions(-) delete mode 100644 packages/service-library/src/servicelib/long_running_tasks/_models.py diff --git a/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py b/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py index 77f7109d5dfe..bd61e63cf929 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py @@ -8,10 +8,16 @@ from ..logging_errors import create_troubleshootting_log_kwargs from ..logging_utils import log_decorator from ..rabbitmq._client_rpc import RabbitMQRPCClient -from ._models import RPCErrorResponse from ._rabbit_namespace import get_rabbit_namespace from ._serialization import string_to_object -from .models import LRTNamespace, TaskBase, TaskContext, TaskId, TaskStatus +from .models import ( + LRTNamespace, + RPCErrorResponse, + TaskBase, + TaskContext, + TaskId, + TaskStatus, +) from .task import RegisteredTaskName _logger = logging.getLogger(__name__) diff --git a/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py b/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py index 542c90219015..32c7d8ad775a 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py @@ -2,11 +2,10 @@ from typing import Any from ..rabbitmq import RPCRouter -from ._models import RPCErrorResponse from ._serialization import object_to_string from .base_long_running_manager import BaseLongRunningManager from .errors import BaseLongRunningError, TaskNotCompletedError, TaskNotFoundError -from .models import TaskBase, TaskContext, TaskId, TaskStatus +from .models import RPCErrorResponse, TaskBase, TaskContext, TaskId, TaskStatus from .task import RegisteredTaskName router = RPCRouter() diff --git a/packages/service-library/src/servicelib/long_running_tasks/_models.py b/packages/service-library/src/servicelib/long_running_tasks/_models.py deleted file mode 100644 index 5b3fa9f3a994..000000000000 --- a/packages/service-library/src/servicelib/long_running_tasks/_models.py +++ /dev/null @@ -1,6 +0,0 @@ -from pydantic import BaseModel - - -class RPCErrorResponse(BaseModel): - str_traceback: str - error_object: str diff --git a/packages/service-library/src/servicelib/long_running_tasks/models.py b/packages/service-library/src/servicelib/long_running_tasks/models.py index 5dedb46291c4..8440e0f43f97 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/models.py +++ b/packages/service-library/src/servicelib/long_running_tasks/models.py @@ -100,6 +100,11 @@ class ClientConfiguration(BaseModel): default_timeout: PositiveFloat +class RPCErrorResponse(BaseModel): + str_traceback: str + error_object: str + + @dataclass(frozen=True) class LRTask: progress: TaskProgress From 984846e52d4c0d2d1836b5790b8ad3b1609f7e5c Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 18 Aug 2025 13:39:04 +0200 Subject: [PATCH 073/119] fixed ciruclar dependency --- .../long_running_tasks/_lrt_server.py | 19 +++++++++++-------- .../base_long_running_manager.py | 2 +- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py b/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py index 32c7d8ad775a..893997d2ee2c 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py @@ -1,19 +1,22 @@ import traceback -from typing import Any +from typing import TYPE_CHECKING, Any from ..rabbitmq import RPCRouter from ._serialization import object_to_string -from .base_long_running_manager import BaseLongRunningManager from .errors import BaseLongRunningError, TaskNotCompletedError, TaskNotFoundError from .models import RPCErrorResponse, TaskBase, TaskContext, TaskId, TaskStatus from .task import RegisteredTaskName +if TYPE_CHECKING: + from .base_long_running_manager import BaseLongRunningManager + + router = RPCRouter() @router.expose(reraise_if_error_type=(BaseLongRunningError,)) async def start_task( - long_running_manager: BaseLongRunningManager, + long_running_manager: "BaseLongRunningManager", *, registered_task_name: RegisteredTaskName, unique: bool = False, @@ -34,7 +37,7 @@ async def start_task( @router.expose(reraise_if_error_type=(BaseLongRunningError,)) async def list_tasks( - long_running_manager: BaseLongRunningManager, *, task_context: TaskContext + long_running_manager: "BaseLongRunningManager", *, task_context: TaskContext ) -> list[TaskBase]: return await long_running_manager.tasks_manager.list_tasks( with_task_context=task_context @@ -43,7 +46,7 @@ async def list_tasks( @router.expose(reraise_if_error_type=(BaseLongRunningError,)) async def get_task_status( - long_running_manager: BaseLongRunningManager, + long_running_manager: "BaseLongRunningManager", *, task_context: TaskContext, task_id: TaskId, @@ -54,7 +57,7 @@ async def get_task_status( async def _get_transferarble_task_result( - long_running_manager: BaseLongRunningManager, + long_running_manager: "BaseLongRunningManager", *, task_context: TaskContext, task_id: TaskId, @@ -79,7 +82,7 @@ async def _get_transferarble_task_result( @router.expose(reraise_if_error_type=(BaseLongRunningError, Exception)) async def get_task_result( - long_running_manager: BaseLongRunningManager, + long_running_manager: "BaseLongRunningManager", *, task_context: TaskContext, task_id: TaskId, @@ -99,7 +102,7 @@ async def get_task_result( @router.expose(reraise_if_error_type=(BaseLongRunningError,)) async def remove_task( - long_running_manager: BaseLongRunningManager, + long_running_manager: "BaseLongRunningManager", *, task_context: TaskContext, task_id: TaskId, diff --git a/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py b/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py index 95791b518d65..3c58705b55c3 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py +++ b/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py @@ -4,6 +4,7 @@ from settings_library.redis import RedisSettings from ..rabbitmq._client_rpc import RabbitMQRPCClient +from ._lrt_server import router from ._rabbit_namespace import get_rabbit_namespace from .models import LRTNamespace from .task import TasksManager @@ -61,7 +62,6 @@ async def setup(self) -> None: client_name=f"lrt-client-{self.lrt_namespace}", settings=self.rabbit_settings, ) - from ._lrt_server import router await self.rpc_server.register_router( router, From 3863e9c10040f531de085ef8a293c27849fc3ad7 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 19 Aug 2025 10:49:12 +0200 Subject: [PATCH 074/119] renamed --- .../long_running_tasks/_redis_store.py | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/_redis_store.py b/packages/service-library/src/servicelib/long_running_tasks/_redis_store.py index 7c9e5fc9a205..f74855a444e0 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_redis_store.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_redis_store.py @@ -1,4 +1,3 @@ -import json from typing import Any, Final import redis.asyncio as aioredis @@ -16,14 +15,14 @@ _LIST_CONCURRENCY: Final[int] = 2 -def _encode_dict(data: dict[str, Any]) -> dict[str, str]: - """replaces dict with a JSON-serializable dict""" +def _to_redis(data: dict[str, Any]) -> dict[str, str]: + """convert values to redis compatible data""" return {k: json_dumps(v) for k, v in data.items()} -def _decode_dict(data: dict[str, str]) -> dict[str, Any]: - """replaces dict with a JSON-deserialized dict""" - return {k: json.loads(v) for k, v in data.items()} +def _from_redis(data: dict[str, str]) -> dict[str, Any]: + """converrt back from redis compatible data to python types""" + return {k: json_loads(v) for k, v in data.items()} class RedisStore: @@ -67,7 +66,7 @@ async def get_task_data(self, task_id: TaskId) -> TaskData | None: ) ) return ( - TypeAdapter(TaskData).validate_python(_decode_dict(result)) + TypeAdapter(TaskData).validate_python(_from_redis(result)) if result and len(result) else None ) @@ -76,7 +75,7 @@ async def add_task_data(self, task_id: TaskId, value: TaskData) -> None: await handle_redis_returns_union_types( self._redis.hset( self._get_redis_key_task_data_hash(task_id), - mapping=_encode_dict(value.model_dump()), + mapping=_to_redis(value.model_dump()), ) ) @@ -89,7 +88,7 @@ async def update_task_data( await handle_redis_returns_union_types( self._redis.hset( self._get_redis_key_task_data_hash(task_id), - mapping=_encode_dict(updates), + mapping=_to_redis(updates), ) ) @@ -108,7 +107,7 @@ async def list_tasks_data(self) -> list[TaskData]: ) return [ - TypeAdapter(TaskData).validate_python(_decode_dict(item)) + TypeAdapter(TaskData).validate_python(_from_redis(item)) for item in result if item ] @@ -120,24 +119,21 @@ async def delete_task_data(self, task_id: TaskId) -> None: # to cancel - async def set_to_remove( + async def mark_task_for_removal( self, task_id: TaskId, with_task_context: TaskContext ) -> None: - """marks a task_id to be removed""" await handle_redis_returns_union_types( self._redis.hset( self._get_key_to_remove(), task_id, json_dumps(with_task_context) ) ) - async def delete_to_remove(self, task_id: TaskId) -> None: - """deletes a task_id from the ones to be removed""" + async def completed_task_removal(self, task_id: TaskId) -> None: await handle_redis_returns_union_types( self._redis.hdel(self._get_key_to_remove(), task_id) ) - async def get_all_to_remove(self) -> dict[TaskId, TaskContext]: - """returns all task_ids that are to be removed""" + async def list_tasks_to_remove(self) -> dict[TaskId, TaskContext]: result: dict[str, str | None] = await handle_redis_returns_union_types( self._redis.hgetall(self._get_key_to_remove()) ) From 5559e6df367b9ed48b59b4141a5338df6771a74c Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 19 Aug 2025 10:58:15 +0200 Subject: [PATCH 075/119] renamed --- .../fastapi/long_running_tasks/_manager.py | 8 +++++++- .../long_running_tasks/base_long_running_manager.py | 10 ++++++++-- .../src/servicelib/long_running_tasks/task.py | 6 +++--- .../tests/long_running_tasks/conftest.py | 11 +++++++++-- .../test_long_running_tasks__store.py | 12 ++++++------ .../test_long_running_tasks_task.py | 2 +- 6 files changed, 34 insertions(+), 15 deletions(-) diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_manager.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_manager.py index 54215c0fe015..8f04f828705e 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_manager.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_manager.py @@ -1,5 +1,11 @@ +from fastapi import Request + from ...long_running_tasks.base_long_running_manager import BaseLongRunningManager +from ...long_running_tasks.models import TaskContext class FastAPILongRunningManager(BaseLongRunningManager): - pass + @staticmethod + def get_task_context(request: Request) -> TaskContext: + _ = request + return {} diff --git a/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py b/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py index 3c58705b55c3..d79f60133582 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py +++ b/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py @@ -1,4 +1,5 @@ import datetime +from abc import ABC, abstractmethod from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings @@ -6,11 +7,11 @@ from ..rabbitmq._client_rpc import RabbitMQRPCClient from ._lrt_server import router from ._rabbit_namespace import get_rabbit_namespace -from .models import LRTNamespace +from .models import LRTNamespace, TaskContext from .task import TasksManager -class BaseLongRunningManager: +class BaseLongRunningManager(ABC): """ Provides a commond inteface for aiohttp and fastapi services """ @@ -79,3 +80,8 @@ async def teardown(self) -> None: if self._rpc_client is not None: await self._rpc_client.close() self._rpc_client = None + + @abstractmethod + @staticmethod + def get_task_context(request) -> TaskContext: + """return the task context based on the current request""" diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index 0c2a199703de..853a90f39424 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -268,7 +268,7 @@ async def _cancelled_tasks_removal(self) -> None: """ self._started_event_task_cancelled_tasks_removal.set() - to_remove = await self._tasks_data.get_all_to_remove() + to_remove = await self._tasks_data.list_tasks_to_remove() for task_id in to_remove: await self._attempt_to_remove_local_task(task_id) @@ -390,7 +390,7 @@ async def _attempt_to_remove_local_task(self, task_id: TaskId) -> None: task_to_cancel = self._created_tasks.pop(task_id, None) if task_to_cancel is not None: await cancel_wait_task(task_to_cancel) - await self._tasks_data.delete_to_remove(task_id) + await self._tasks_data.completed_task_removal(task_id) await self._tasks_data.delete_task_data(task_id) async def remove_task( @@ -408,7 +408,7 @@ async def remove_task( raise return - await self._tasks_data.set_to_remove( + await self._tasks_data.mark_task_for_removal( tracked_task.task_id, tracked_task.task_context ) diff --git a/packages/service-library/tests/long_running_tasks/conftest.py b/packages/service-library/tests/long_running_tasks/conftest.py index 9894a1e0c780..ef3339786f27 100644 --- a/packages/service-library/tests/long_running_tasks/conftest.py +++ b/packages/service-library/tests/long_running_tasks/conftest.py @@ -13,7 +13,7 @@ from servicelib.long_running_tasks.base_long_running_manager import ( BaseLongRunningManager, ) -from servicelib.long_running_tasks.models import LRTNamespace +from servicelib.long_running_tasks.models import LRTNamespace, TaskContext from servicelib.long_running_tasks.task import TasksManager from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient from settings_library.rabbit import RabbitSettings @@ -23,6 +23,13 @@ _logger = logging.getLogger(__name__) +class _TestingLongRunningManager(BaseLongRunningManager): + @staticmethod + def get_task_context(request) -> TaskContext: + _ = request + return {} + + @pytest.fixture async def get_long_running_manager( fast_long_running_tasks_cancellation: None, faker: Faker @@ -39,7 +46,7 @@ async def _( rabbit_settings: RabbitSettings, lrt_namespace: LRTNamespace | None, ) -> BaseLongRunningManager: - manager = BaseLongRunningManager( + manager = _TestingLongRunningManager( stale_task_check_interval=timedelta(seconds=TEST_CHECK_STALE_INTERVAL_S), stale_task_detect_timeout=timedelta(seconds=TEST_CHECK_STALE_INTERVAL_S), redis_settings=redis_settings, diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks__store.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks__store.py index 47a5222d310f..218af7a9aaae 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks__store.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks__store.py @@ -50,11 +50,11 @@ async def test_workflow(store: RedisStore, task_data: TaskData) -> None: assert await store.list_tasks_data() == [] # cancelled tasks - assert await store.get_all_to_remove() == {} + assert await store.list_tasks_to_remove() == {} - await store.set_to_remove(task_data.task_id, task_data.task_context) + await store.mark_task_for_removal(task_data.task_id, task_data.task_context) - assert await store.get_all_to_remove() == { + assert await store.list_tasks_to_remove() == { task_data.task_id: task_data.task_context } @@ -89,15 +89,15 @@ async def test_workflow_multiple_redis_stores_with_different_namespaces( for store in redis_stores: assert await store.list_tasks_data() == [] - assert await store.get_all_to_remove() == {} + assert await store.list_tasks_to_remove() == {} for store in redis_stores: await store.add_task_data(task_data.task_id, task_data) - await store.set_to_remove(task_data.task_id, {}) + await store.mark_task_for_removal(task_data.task_id, {}) for store in redis_stores: assert await store.list_tasks_data() == [task_data] - assert await store.get_all_to_remove() == {task_data.task_id: {}} + assert await store.list_tasks_to_remove() == {task_data.task_id: {}} for store in redis_stores: await store.delete_task_data(task_data.task_id) diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py index fe6e7dd749a6..cc9820631464 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py @@ -480,7 +480,7 @@ async def test__cancelled_tasks_worker_equivalent_of_cancellation_from_a_differe total_sleep=10, task_context=empty_context, ) - await long_running_manager.tasks_manager._tasks_data.set_to_remove( # noqa: SLF001 + await long_running_manager.tasks_manager._tasks_data.mark_task_for_removal( # noqa: SLF001 task_id, with_task_context=empty_context ) From eb1826cd4f5030f066c20b4bef598a1d9e28b372 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 19 Aug 2025 11:04:53 +0200 Subject: [PATCH 076/119] rename --- .../projects/_controller/projects_rest.py | 8 ++++---- .../projects/_crud_api_create.py | 8 +++++--- .../projects/_permalink_service.py | 18 ++++++++++++------ .../studies_dispatcher/_projects_permalinks.py | 15 +++++++++------ .../simcore_service_webserver/utils_aiohttp.py | 6 +++--- ...t_studies_dispatcher_projects_permalinks.py | 16 ++++++++-------- 6 files changed, 41 insertions(+), 30 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py index 2b0b4a6f3eee..4168ec608def 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py @@ -101,8 +101,8 @@ async def create_project(request: web.Request): fire_and_forget=True, task_context=jsonable_encoder(req_ctx), # arguments - url=request.url, - headers=dict(request.headers), + request_url=request.url, + request_headers=dict(request.headers), new_project_was_hidden_before_data_was_copied=query_params.hidden, from_study=query_params.from_study, as_template=query_params.as_template, @@ -424,8 +424,8 @@ async def clone_project(request: web.Request): fire_and_forget=True, task_context=jsonable_encoder(req_ctx), # arguments - url=request.url, - headers=dict(request.headers), + request_url=request.url, + request_headers=dict(request.headers), new_project_was_hidden_before_data_was_copied=False, from_study=path_params.project_id, as_template=False, diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py index ef0efa1262e7..5f09adae36dd 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py @@ -251,8 +251,8 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche progress: TaskProgress, *, app: web.Application, - url: URL, - headers: dict[str, str], + request_url: URL, + request_headers: dict[str, str], new_project_was_hidden_before_data_was_copied: bool, from_study: ProjectID | None, as_template: bool, @@ -439,7 +439,9 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche await progress.update() # Adds permalink - await update_or_pop_permalink_in_project(app, url, headers, new_project) + await update_or_pop_permalink_in_project( + app, request_url, request_headers, new_project + ) # Adds folderId user_specific_project_data_db = ( diff --git a/services/web/server/src/simcore_service_webserver/projects/_permalink_service.py b/services/web/server/src/simcore_service_webserver/projects/_permalink_service.py index bb1a470e9546..d3c52a985012 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_permalink_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_permalink_service.py @@ -18,8 +18,8 @@ class CreateLinkCoroutine(Protocol): async def __call__( self, app: web.Application, - url: URL, - headers: dict[str, str], + request_url: URL, + request_headers: dict[str, str], project_uuid: ProjectID, ) -> ProjectPermalink: ... @@ -43,13 +43,16 @@ def _get_factory(app: web.Application) -> CreateLinkCoroutine: async def _create_permalink( - app: web.Application, url: URL, headers: dict[str, str], project_uuid: ProjectID + app: web.Application, + request_url: URL, + request_headers: dict[str, str], + project_uuid: ProjectID, ) -> ProjectPermalink: create_coro: CreateLinkCoroutine = _get_factory(app) try: permalink: ProjectPermalink = await asyncio.wait_for( - create_coro(app, url, headers, project_uuid), + create_coro(app, request_url, request_headers, project_uuid), timeout=_PERMALINK_CREATE_TIMEOUT_S, ) return permalink @@ -59,7 +62,10 @@ async def _create_permalink( async def update_or_pop_permalink_in_project( - app: web.Application, url: URL, headers: dict[str, str], project: ProjectDict + app: web.Application, + request_url: URL, + request_headers: dict[str, str], + project: ProjectDict, ) -> ProjectPermalink | None: """Updates permalink entry in project @@ -69,7 +75,7 @@ async def update_or_pop_permalink_in_project( """ try: permalink = await _create_permalink( - app, url, headers, project_uuid=project["uuid"] + app, request_url, request_headers, project_uuid=project["uuid"] ) assert permalink # nosec diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects_permalinks.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects_permalinks.py index 37ee33ba34a8..f63fc3cf1c0f 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects_permalinks.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects_permalinks.py @@ -36,8 +36,8 @@ class _GroupAccessRightsDict(TypedDict): def create_permalink_for_study( app: web.Application, *, - url: URL, - headers: dict[str, str], + request_url: URL, + request_headers: dict[str, str], project_uuid: ProjectID | ProjectIDStr, project_type: ProjectType, project_access_rights: dict[_GroupID, _GroupAccessRightsDict], @@ -68,7 +68,7 @@ def create_permalink_for_study( raise PermalinkNotAllowedError(msg) # create - url_for = create_url_for_function(app, url, headers) + url_for = create_url_for_function(app, request_url, request_headers) permalink = TypeAdapter(HttpUrl).validate_python( url_for(route_name="get_redirection_to_study_page", id=f"{project_uuid}"), ) @@ -80,7 +80,10 @@ def create_permalink_for_study( async def permalink_factory( - app: web.Application, url: URL, headers: dict[str, str], project_uuid: ProjectID + app: web.Application, + request_url: URL, + request_headers: dict[str, str], + project_uuid: ProjectID, ) -> ProjectPermalink: """ - Assumes project_id is up-to-date in the database @@ -125,8 +128,8 @@ async def permalink_factory( return create_permalink_for_study( app, - url=url, - headers=headers, + request_url=request_url, + request_headers=request_headers, project_uuid=row.uuid, project_type=row.type, project_access_rights=row.access_rights, diff --git a/services/web/server/src/simcore_service_webserver/utils_aiohttp.py b/services/web/server/src/simcore_service_webserver/utils_aiohttp.py index 0458897f12a8..e0b753e8aee9 100644 --- a/services/web/server/src/simcore_service_webserver/utils_aiohttp.py +++ b/services/web/server/src/simcore_service_webserver/utils_aiohttp.py @@ -36,7 +36,7 @@ def get_routes_view(routes: RouteTableDef) -> str: def create_url_for_function( - app: web.Application, url: URL, headers: dict[str, str] + app: web.Application, request_url: URL, request_headers: dict[str, str] ) -> Callable: def _url_for(route_name: str, **params: dict[str, Any]) -> str: @@ -46,11 +46,11 @@ def _url_for(route_name: str, **params: dict[str, Any]) -> str: **{k: f"{v}" for k, v in params.items()} ) _url: URL = ( - url.origin() + request_url.origin() .with_scheme( # Custom header by traefik. See labels in docker-compose as: # - traefik.http.middlewares.${SWARM_STACK_NAME_NO_HYPHEN}_sslheader.headers.customrequestheaders.X-Forwarded-Proto=http - headers.get(X_FORWARDED_PROTO, url.scheme) + request_headers.get(X_FORWARDED_PROTO, request_url.scheme) ) .with_path(str(rel_url)) ) diff --git a/services/web/server/tests/unit/isolated/test_studies_dispatcher_projects_permalinks.py b/services/web/server/tests/unit/isolated/test_studies_dispatcher_projects_permalinks.py index b448e806a681..384041df0bd9 100644 --- a/services/web/server/tests/unit/isolated/test_studies_dispatcher_projects_permalinks.py +++ b/services/web/server/tests/unit/isolated/test_studies_dispatcher_projects_permalinks.py @@ -92,8 +92,8 @@ def test_create_permalink(fake_get_project_request: web.Request, is_public: bool permalink = create_permalink_for_study( fake_get_project_request.app, - url=fake_get_project_request.url, - headers=dict(fake_get_project_request.headers), + request_url=fake_get_project_request.url, + request_headers=dict(fake_get_project_request.headers), project_uuid=project_uuid, project_type=ProjectType.TEMPLATE, project_access_rights={"1": {"read": True, "write": False, "delete": False}}, @@ -122,8 +122,8 @@ def test_permalink_only_for_template_projects( with pytest.raises(PermalinkNotAllowedError): create_permalink_for_study( fake_get_project_request.app, - url=fake_get_project_request.url, - headers=dict(fake_get_project_request.headers), + request_url=fake_get_project_request.url, + request_headers=dict(fake_get_project_request.headers), **{**valid_project_kwargs, "project_type": ProjectType.STANDARD} ) @@ -134,8 +134,8 @@ def test_permalink_only_when_read_access_to_everyone( with pytest.raises(PermalinkNotAllowedError): create_permalink_for_study( fake_get_project_request.app, - url=fake_get_project_request.url, - headers=dict(fake_get_project_request.headers), + request_url=fake_get_project_request.url, + request_headers=dict(fake_get_project_request.headers), **{ **valid_project_kwargs, "project_access_rights": { @@ -147,8 +147,8 @@ def test_permalink_only_when_read_access_to_everyone( with pytest.raises(PermalinkNotAllowedError): create_permalink_for_study( fake_get_project_request.app, - url=fake_get_project_request.url, - headers=dict(fake_get_project_request.headers), + request_url=fake_get_project_request.url, + request_headers=dict(fake_get_project_request.headers), **{ **valid_project_kwargs, "project_access_rights": { From 18be3a4e169c569c4976f6fa2c37241ab1ecdaa9 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 19 Aug 2025 11:05:56 +0200 Subject: [PATCH 077/119] correct decorator order --- .../servicelib/long_running_tasks/base_long_running_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py b/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py index d79f60133582..1a2417e04099 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py +++ b/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py @@ -81,7 +81,7 @@ async def teardown(self) -> None: await self._rpc_client.close() self._rpc_client = None - @abstractmethod @staticmethod + @abstractmethod def get_task_context(request) -> TaskContext: """return the task context based on the current request""" From 9200ba15c58a1b48e7f780685df8ab84acc3702a Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 19 Aug 2025 14:12:31 +0200 Subject: [PATCH 078/119] fixed flaky test removing tasks tha have not been started --- .../src/servicelib/long_running_tasks/task.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index 853a90f39424..54299e855db3 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -92,14 +92,16 @@ async def _get_tasks_to_remove( if tracked_task.last_status_check is None: # the task just added or never received a poll request - elapsed_from_start = (utc_now - tracked_task.started).seconds + elapsed_from_start = (utc_now - tracked_task.started).total_seconds() if elapsed_from_start > stale_task_detect_timeout_s: tasks_to_remove.append( (tracked_task.task_id, tracked_task.task_context) ) else: # the task status was already queried by the client - elapsed_from_last_poll = (utc_now - tracked_task.last_status_check).seconds + elapsed_from_last_poll = ( + utc_now - tracked_task.last_status_check + ).total_seconds() if elapsed_from_last_poll > stale_task_detect_timeout_s: tasks_to_remove.append( (tracked_task.task_id, tracked_task.task_context) From d510c8ec7521d6d7e1760fd1fe5facb552255fe8 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 19 Aug 2025 14:13:05 +0200 Subject: [PATCH 079/119] fixed tests --- .../long_running_tasks/test_long_running_tasks_task.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py index cc9820631464..60676583bcd7 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py @@ -147,14 +147,17 @@ async def test_task_is_auto_removed( ) +@pytest.mark.parametrize("wait_multiplier", [1, 2, 3, 4, 5, 6]) async def test_checked_task_is_not_auto_removed( - long_running_manager: BaseLongRunningManager, empty_context: TaskContext + long_running_manager: BaseLongRunningManager, + empty_context: TaskContext, + wait_multiplier: int, ): task_id = await lrt_api.start_task( long_running_manager, a_background_task.__name__, raise_when_finished=False, - total_sleep=5 * TEST_CHECK_STALE_INTERVAL_S, + total_sleep=wait_multiplier * TEST_CHECK_STALE_INTERVAL_S, task_context=empty_context, ) async for attempt in AsyncRetrying(**_RETRY_PARAMS): From 6a0d40ccedcb59f908101d2ab0a36f831092369c Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 19 Aug 2025 14:18:25 +0200 Subject: [PATCH 080/119] fixed wrong usage --- services/web/server/tests/unit/with_dbs/01/test_api_keys.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/01/test_api_keys.py b/services/web/server/tests/unit/with_dbs/01/test_api_keys.py index 38966e1fa827..902942840f54 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_api_keys.py +++ b/services/web/server/tests/unit/with_dbs/01/test_api_keys.py @@ -248,7 +248,7 @@ async def test_create_api_key_with_expiration( "/v0/auth/api-keys", json={ "displayName": expected_api_key, - "expiration": expiration_interval.seconds, + "expiration": expiration_interval.total_seconds(), }, ) @@ -264,7 +264,9 @@ async def test_create_api_key_with_expiration( assert [d["displayName"] for d in data] == [expected_api_key] # wait for api-key for it to expire and force-run scheduled task - await asyncio.sleep(EXPIRATION_WAIT_FACTOR * expiration_interval.seconds) + await asyncio.sleep( + EXPIRATION_WAIT_FACTOR * expiration_interval.total_seconds() + ) deleted = await api_keys_service.prune_expired_api_keys(client.app) assert deleted == [expected_api_key] From 3e62d90035e574ca8247d96ed85292ae843f5ac5 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 19 Aug 2025 14:37:53 +0200 Subject: [PATCH 081/119] corrected tests --- .../test_long_running_tasks_client.py | 26 ++++++++++++++----- .../test_long_running_tasks.py | 10 +------ 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_client.py b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_client.py index 9e8c9204acef..8ee6c6a082aa 100644 --- a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_client.py +++ b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_client.py @@ -17,6 +17,12 @@ from servicelib.aiohttp.rest_middlewares import append_rest_middlewares from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings +from tenacity import ( + AsyncRetrying, + retry_if_exception_type, + stop_after_delay, + wait_fixed, +) from yarl import URL pytest_simcore_core_services_selection = [ @@ -112,12 +118,20 @@ async def test_long_running_task_request_timeout( ): print(f"<-- received {task=}") - # check the task was properly aborted by the client - list_url = client.app.router["list_tasks"].url_for() - result = await client.get(f"{list_url}") - data, error = await assert_status(result, status.HTTP_200_OK) - assert not error - assert data == [] + # does not wait for removal any longer + async for attempt in AsyncRetrying( + wait=wait_fixed(0.1), + stop=stop_after_delay(5), + reraise=True, + retry=retry_if_exception_type(AssertionError), + ): + with attempt: + # check the task was properly aborted by the client + list_url = client.app.router["list_tasks"].url_for() + result = await client.get(f"{list_url}") + data, error = await assert_status(result, status.HTTP_200_OK) + assert not error + assert data == [] async def test_long_running_task_request_error( diff --git a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py index 6af4b252e2f5..c8727e123028 100644 --- a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py +++ b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py @@ -200,15 +200,7 @@ async def test_workflow( ("generated item", 0.8), ("finished", 1.0), ] - - async for attempt in AsyncRetrying( - wait=wait_fixed(0.1), - stop=stop_after_delay(10), - reraise=True, - retry=retry_if_exception_type(AssertionError), - ): - with attempt: - assert all(x in progress_updates for x in EXPECTED_MESSAGES) + assert all(x in progress_updates for x in EXPECTED_MESSAGES) # now check the result result_url = app.url_path_for("get_task_result", task_id=task_id) result = await client.get(f"{result_url}") From fbf229d4318ef1ea18ceaea59647b157a791e703 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 19 Aug 2025 14:38:14 +0200 Subject: [PATCH 082/119] corrected tests --- .../test_long_running_tasks.py | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py index 549fe2031abf..c9246d1f22d6 100644 --- a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py +++ b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py @@ -109,14 +109,7 @@ async def test_workflow( ("generated item", 0.8), ("finished", 1.0), ] - async for attempt in AsyncRetrying( - wait=wait_fixed(0.1), - stop=stop_after_delay(10), - reraise=True, - retry=retry_if_exception_type(AssertionError), - ): - with attempt: - assert all(x in progress_updates for x in EXPECTED_MESSAGES) + assert all(x in progress_updates for x in EXPECTED_MESSAGES) # now get the result result_url = client.app.router["get_task_result"].url_for(task_id=task_id) result = await client.get(f"{result_url}") @@ -201,10 +194,18 @@ async def test_cancel_task( assert not data assert not error - # it should be gone, so no status - status_url = client.app.router["get_task_status"].url_for(task_id=task_id) - result = await client.get(f"{status_url}") - await assert_status(result, status.HTTP_404_NOT_FOUND) + # it should eventually go away, so no status + # does not wait for removal any longer + async for attempt in AsyncRetrying( + wait=wait_fixed(0.1), + stop=stop_after_delay(5), + reraise=True, + retry=retry_if_exception_type(AssertionError), + ): + with attempt: + status_url = client.app.router["get_task_status"].url_for(task_id=task_id) + result = await client.get(f"{status_url}") + await assert_status(result, status.HTTP_404_NOT_FOUND) # and also no results result_url = client.app.router["get_task_result"].url_for(task_id=task_id) result = await client.get(f"{result_url}") From b432be3de89606f33554a46105553d160953d7b5 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 19 Aug 2025 14:38:29 +0200 Subject: [PATCH 083/119] let app initialize --- .../test_long_running_tasks_context_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py index 481acdf61bd5..94feb6d231e3 100644 --- a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py +++ b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py @@ -111,7 +111,7 @@ async def bg_task_app( ) setup_client(app, router_prefix=router_prefix) - async with LifespanManager(app): + async with LifespanManager(app, startup_timeout=30, shutdown_timeout=30): yield app From 3f764114a66229e6b0838e4a282960338caaaef4 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 19 Aug 2025 14:38:46 +0200 Subject: [PATCH 084/119] explicit wait for removal --- .../test_long_running_tasks_lrt_api.py | 1 + .../test_long_running_tasks_task.py | 15 +++++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py index 22f0fd43559c..39be66212e2c 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py @@ -310,6 +310,7 @@ async def test_remove_task( _get_task_manager(long_running_managers), saved_context, task_id, + wait_for_removal=True, ) await _assert_task_is_no_longer_present( diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py index 60676583bcd7..acfd7df39a83 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py @@ -419,7 +419,7 @@ async def test_remove_task( task_id, with_task_context=empty_context ) await long_running_manager.tasks_manager.remove_task( - task_id, with_task_context=empty_context + task_id, with_task_context=empty_context, wait_for_removal=True ) with pytest.raises(TaskNotFoundError): await long_running_manager.tasks_manager.get_task_status( @@ -453,10 +453,10 @@ async def test_remove_task_with_task_context( # removing task fails if wrong task context given with pytest.raises(TaskNotFoundError): await long_running_manager.tasks_manager.remove_task( - task_id, with_task_context={"wrong_task_context": 12} + task_id, with_task_context={"wrong_task_context": 12}, wait_for_removal=True ) await long_running_manager.tasks_manager.remove_task( - task_id, with_task_context=empty_context + task_id, with_task_context=empty_context, wait_for_removal=True ) @@ -465,11 +465,14 @@ async def test_remove_unknown_task( ): with pytest.raises(TaskNotFoundError): await long_running_manager.tasks_manager.remove_task( - "invalid_id", with_task_context=empty_context + "invalid_id", with_task_context=empty_context, wait_for_removal=True ) await long_running_manager.tasks_manager.remove_task( - "invalid_id", with_task_context=empty_context, reraise_errors=False + "invalid_id", + with_task_context=empty_context, + wait_for_removal=True, + reraise_errors=False, ) @@ -532,7 +535,7 @@ async def test_list_tasks( ) for task_index, task_id in enumerate(task_ids): await long_running_manager.tasks_manager.remove_task( - task_id, with_task_context=empty_context + task_id, with_task_context=empty_context, wait_for_removal=True ) assert len( await long_running_manager.tasks_manager.list_tasks( From f160ddb553a2a005d33b2849153fc86dafe0bdd5 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 19 Aug 2025 14:52:14 +0200 Subject: [PATCH 085/119] removal does not wait for task to be removed and supports a timeout --- .../aiohttp/long_running_tasks/_routes.py | 1 + .../aiohttp/long_running_tasks/_server.py | 1 + .../fastapi/long_running_tasks/_routes.py | 1 + .../long_running_tasks/_lrt_client.py | 20 ++++++++++-- .../long_running_tasks/_lrt_server.py | 6 +++- .../servicelib/long_running_tasks/lrt_api.py | 4 +++ .../src/servicelib/long_running_tasks/task.py | 10 +++++- ...st_long_running_tasks_with_task_context.py | 32 +++++++++++++++---- 8 files changed, 64 insertions(+), 11 deletions(-) diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py index 5c4ea1ca9ce0..65412a852bd3 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py @@ -75,5 +75,6 @@ async def remove_task(request: web.Request) -> web.Response: long_running_manager, long_running_manager.get_task_context(request), path_params.task_id, + wait_for_removal=False, # frontend does not care about waiting for this ) return web.json_response(status=status.HTTP_204_NO_CONTENT) diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py index ff9bffbf6f4d..60ac1b32994b 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py @@ -103,6 +103,7 @@ async def start_long_running_task( long_running_manager, task_context, task_id, + wait_for_removal=True, ) raise diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py index e4caf671cb76..3ec4f62e9a01 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py @@ -106,4 +106,5 @@ async def remove_task( long_running_manager, task_context={}, task_id=task_id, + wait_for_removal=True, # only used by internal services, they will wait as before ) diff --git a/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py b/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py index bd61e63cf929..30e0c62131fa 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py @@ -22,7 +22,7 @@ _logger = logging.getLogger(__name__) -_RPC_TIMEOUT_TASK_REMOVAL: Final[PositiveInt] = int( +_RPC_MAX_CANCELLATION_TIMEOUT: Final[PositiveInt] = int( timedelta(minutes=60).total_seconds() ) _RPC_TIMEOUT_SHORT_REQUESTS: Final[PositiveInt] = int( @@ -134,14 +134,28 @@ async def remove_task( *, task_context: TaskContext, task_id: TaskId, - reraise_errors: bool = True, + wait_for_removal: bool, + reraise_errors: bool, + cancellation_timeout: timedelta | None = None, ) -> None: + timeout_s = ( + _RPC_MAX_CANCELLATION_TIMEOUT + if cancellation_timeout is None + else int(cancellation_timeout.total_seconds()) + ) + + # NOTE: task always gets cancelled even if not waiting for it + # request will return immediatlye, no need to wait so much + if not wait_for_removal: + timeout_s = _RPC_TIMEOUT_SHORT_REQUESTS + result = await rabbitmq_rpc_client.request( get_rabbit_namespace(namespace), TypeAdapter(RPCMethodName).validate_python("remove_task"), task_context=task_context, task_id=task_id, + wait_for_removal=wait_for_removal, reraise_errors=reraise_errors, - timeout_s=_RPC_TIMEOUT_TASK_REMOVAL, + timeout_s=timeout_s, ) assert result is None # nosec diff --git a/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py b/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py index 893997d2ee2c..55ea64311c98 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py @@ -106,8 +106,12 @@ async def remove_task( *, task_context: TaskContext, task_id: TaskId, + wait_for_removal: bool, reraise_errors: bool, ) -> None: await long_running_manager.tasks_manager.remove_task( - task_id, with_task_context=task_context, reraise_errors=reraise_errors + task_id, + with_task_context=task_context, + wait_for_removal=wait_for_removal, + reraise_errors=reraise_errors, ) diff --git a/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py b/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py index c1b3c270d10b..f4c5cd59de53 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py +++ b/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py @@ -101,6 +101,8 @@ async def remove_task( long_running_manager: BaseLongRunningManager, task_context: TaskContext, task_id: TaskId, + *, + wait_for_removal: bool, ) -> None: """cancels and removes the task""" await _lrt_client.remove_task( @@ -108,4 +110,6 @@ async def remove_task( long_running_manager.lrt_namespace, task_id=task_id, task_context=task_context, + wait_for_removal=wait_for_removal, + reraise_errors=True, ) diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index 54299e855db3..909c0e23d700 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -189,6 +189,7 @@ async def teardown(self) -> None: tracked_task.task_id, tracked_task.task_context, # when closing we do not care about pending errors + wait_for_removal=True, reraise_errors=False, ) @@ -260,7 +261,10 @@ async def _stale_tasks_monitor(self) -> None: ).model_dump_json(), ) await self.remove_task( - task_id, with_task_context=task_context, reraise_errors=False + task_id, + with_task_context=task_context, + wait_for_removal=True, + reraise_errors=False, ) async def _cancelled_tasks_removal(self) -> None: @@ -400,6 +404,7 @@ async def remove_task( task_id: TaskId, with_task_context: TaskContext, *, + wait_for_removal: bool, reraise_errors: bool = True, ) -> None: """cancels and removes task""" @@ -414,6 +419,9 @@ async def remove_task( tracked_task.task_id, tracked_task.task_context ) + if not wait_for_removal: + return + # wait for task to be removed since it might not have been running # in this process async for attempt in AsyncRetrying( diff --git a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py index cef4a845ab8d..ffed3531911e 100644 --- a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py +++ b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py @@ -29,6 +29,10 @@ from servicelib.long_running_tasks.task import TaskContext from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings +from tenacity.asyncio import AsyncRetrying +from tenacity.retry import retry_if_exception_type +from tenacity.stop import stop_after_delay +from tenacity.wait import wait_fixed pytest_simcore_core_services_selection = [ "rabbit", @@ -176,15 +180,31 @@ async def test_cancel_task( task_id=task_id ) # calling cancel without task context should find nothing - resp = await client_with_task_context.delete(f"{cancel_url}") - await assert_status(resp, status.HTTP_404_NOT_FOUND) + # no longer waits for removal to end + async for attempt in AsyncRetrying( + wait=wait_fixed(0.1), + stop=stop_after_delay(5), + reraise=True, + retry=retry_if_exception_type(AssertionError), + ): + with attempt: + resp = await client_with_task_context.delete(f"{cancel_url}") + await assert_status(resp, status.HTTP_404_NOT_FOUND) # calling with context should find and delete the task resp = await client_with_task_context.delete( f"{cancel_url.update_query(task_context)}" ) await assert_status(resp, status.HTTP_204_NO_CONTENT) # calling with context a second time should find nothing - resp = await client_with_task_context.delete( - f"{cancel_url.update_query(task_context)}" - ) - await assert_status(resp, status.HTTP_404_NOT_FOUND) + # no longer waits for removal to end + async for attempt in AsyncRetrying( + wait=wait_fixed(0.1), + stop=stop_after_delay(5), + reraise=True, + retry=retry_if_exception_type(AssertionError), + ): + with attempt: + resp = await client_with_task_context.delete( + f"{cancel_url.update_query(task_context)}" + ) + await assert_status(resp, status.HTTP_404_NOT_FOUND) From 888efc89d3268953c3dfe3098985f640bf259c70 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 19 Aug 2025 15:01:57 +0200 Subject: [PATCH 086/119] added missing --- .../src/servicelib/long_running_tasks/_lrt_server.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py b/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py index 55ea64311c98..96de43b79f6d 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py @@ -67,7 +67,10 @@ async def _get_transferarble_task_result( task_id, with_task_context=task_context ) await long_running_manager.tasks_manager.remove_task( - task_id, with_task_context=task_context, reraise_errors=False + task_id, + with_task_context=task_context, + wait_for_removal=True, + reraise_errors=False, ) return task_result except (TaskNotFoundError, TaskNotCompletedError): @@ -75,7 +78,10 @@ async def _get_transferarble_task_result( except Exception: # the task shall be removed in this case await long_running_manager.tasks_manager.remove_task( - task_id, with_task_context=task_context, reraise_errors=False + task_id, + with_task_context=task_context, + wait_for_removal=True, + reraise_errors=False, ) raise From afde03d1d2598ff1b93164069736df20e0f9754c Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 20 Aug 2025 10:49:52 +0200 Subject: [PATCH 087/119] refactor error translaton layer to accepet specific error classes only --- .../fastapi/long_running_tasks/_routes.py | 11 ++- .../long_running_tasks/_lrt_client.py | 24 ++++-- .../long_running_tasks/_lrt_server.py | 83 ++++++++++--------- .../servicelib/long_running_tasks/lrt_api.py | 2 + .../servicelib/long_running_tasks/models.py | 11 ++- .../src/servicelib/long_running_tasks/task.py | 63 +++++++------- .../servicelib/long_running_tasks/utils.py | 23 +++++ .../src/servicelib/rabbitmq/_rpc_router.py | 2 +- .../aiohttp/long_running_tasks/conftest.py | 6 +- .../test_long_running_tasks.py | 14 +++- ...test_long_running_tasks_context_manager.py | 6 +- .../test_long_running_tasks_lrt_api.py | 9 +- .../test_long_running_tasks_task.py | 41 ++++++--- 13 files changed, 194 insertions(+), 101 deletions(-) create mode 100644 packages/service-library/src/servicelib/long_running_tasks/utils.py diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py index 3ec4f62e9a01..0bba29e780f9 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py @@ -1,9 +1,10 @@ from typing import Annotated, Any -from fastapi import APIRouter, Depends, Request, status +from fastapi import APIRouter, Depends, Query, Request, status from ...long_running_tasks import lrt_api from ...long_running_tasks.models import TaskGet, TaskId, TaskResult, TaskStatus +from ...long_running_tasks.utils import decode_error_types from ..requests_decorators import cancel_on_disconnect from ._dependencies import get_long_running_manager from ._manager import FastAPILongRunningManager @@ -73,6 +74,13 @@ async def get_task_result( long_running_manager: Annotated[ FastAPILongRunningManager, Depends(get_long_running_manager) ], + allowed_errors: Annotated[ + str, + Query( + description="list of json encoded tuples of allowed errors", + example='["tests.fastapi.long_running_tasks.test_long_running_tasks", "_TestingError"]', + ), + ] = "", ) -> TaskResult | Any: assert request # nosec return await lrt_api.get_task_result( @@ -80,6 +88,7 @@ async def get_task_result( long_running_manager, task_context={}, task_id=task_id, + allowed_errors=decode_error_types(allowed_errors), ) diff --git a/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py b/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py index 30e0c62131fa..2d86a26af468 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py @@ -9,10 +9,10 @@ from ..logging_utils import log_decorator from ..rabbitmq._client_rpc import RabbitMQRPCClient from ._rabbit_namespace import get_rabbit_namespace -from ._serialization import string_to_object +from ._serialization import object_to_string, string_to_object from .models import ( + ErrorResponse, LRTNamespace, - RPCErrorResponse, TaskBase, TaskContext, TaskId, @@ -99,27 +99,33 @@ async def get_task_result( *, task_context: TaskContext, task_id: TaskId, + allowed_errors: tuple[type[BaseException], ...], ) -> Any: serialized_result = await rabbitmq_rpc_client.request( get_rabbit_namespace(namespace), TypeAdapter(RPCMethodName).validate_python("get_task_result"), task_context=task_context, task_id=task_id, + allowed_errors_str=object_to_string(allowed_errors), timeout_s=_RPC_TIMEOUT_SHORT_REQUESTS, ) - assert isinstance(serialized_result, RPCErrorResponse | str) # nosec - if isinstance(serialized_result, RPCErrorResponse): - error = string_to_object(serialized_result.error_object) - _logger.warning( + assert isinstance(serialized_result, ErrorResponse | str) # nosec + if isinstance(serialized_result, ErrorResponse): + error = string_to_object(serialized_result.str_error_object) + _logger.info( **create_troubleshootting_log_kwargs( - f"Remote task finished with error '{error.__class__.__name__}: {error}'\n{serialized_result.str_traceback}", + f"Task '{task_id}' raised the following error:\n{serialized_result.str_traceback}", error=error, error_context={ "task_id": task_id, - "task_context": task_context, "namespace": namespace, + "task_context": task_context, + "allowed_errors": allowed_errors, }, - tip=f"Raised where the lrt_server was running, you can figure this out via {namespace=}", + tip=( + f"The caller of this function should handle the exception. " + f"To figure out where it was running check {namespace=}" + ), ) ) raise error diff --git a/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py b/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py index 96de43b79f6d..018a95d9a8ab 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py @@ -1,12 +1,15 @@ -import traceback +import logging from typing import TYPE_CHECKING, Any +from ..logging_errors import create_troubleshootting_log_kwargs from ..rabbitmq import RPCRouter -from ._serialization import object_to_string -from .errors import BaseLongRunningError, TaskNotCompletedError, TaskNotFoundError -from .models import RPCErrorResponse, TaskBase, TaskContext, TaskId, TaskStatus +from ._serialization import string_to_object +from .errors import BaseLongRunningError +from .models import ErrorResponse, TaskBase, TaskContext, TaskId, TaskStatus from .task import RegisteredTaskName +_logger = logging.getLogger(__name__) + if TYPE_CHECKING: from .base_long_running_manager import BaseLongRunningManager @@ -56,54 +59,56 @@ async def get_task_status( ) -async def _get_transferarble_task_result( +@router.expose(reraise_if_error_type=(BaseLongRunningError,)) +async def get_task_result( long_running_manager: "BaseLongRunningManager", *, task_context: TaskContext, task_id: TaskId, -) -> Any: + allowed_errors_str: str, +) -> ErrorResponse | str: + allowed_errors: tuple[type[BaseException], ...] = string_to_object( + allowed_errors_str + ) try: - task_result = await long_running_manager.tasks_manager.get_task_result( + result_field = await long_running_manager.tasks_manager.get_task_result( task_id, with_task_context=task_context ) + if result_field.error_response is not None: + task_raised_error_traceback = result_field.error_response.str_traceback + task_raised_error = string_to_object( + result_field.error_response.str_error_object + ) + _logger.info( + **create_troubleshootting_log_kwargs( + f"Execution of {task_id=} finished with error:\n{task_raised_error_traceback}", + error=task_raised_error, + error_context={ + "task_id": task_id, + "task_context": task_context, + "namespace": long_running_manager.lrt_namespace, + }, + tip="This exception is logged for debugging purposes, the client side will handle it", + ) + ) + if type(task_raised_error) in allowed_errors: + return result_field.error_response + + raise task_raised_error + + if result_field.str_result is not None: + return result_field.str_result + + msg = f"Please check {result_field=}, both fields should never be None" + raise ValueError(msg) + finally: + # Ensure the task is removed regardless of the result await long_running_manager.tasks_manager.remove_task( task_id, with_task_context=task_context, wait_for_removal=True, reraise_errors=False, ) - return task_result - except (TaskNotFoundError, TaskNotCompletedError): - raise - except Exception: - # the task shall be removed in this case - await long_running_manager.tasks_manager.remove_task( - task_id, - with_task_context=task_context, - wait_for_removal=True, - reraise_errors=False, - ) - raise - - -@router.expose(reraise_if_error_type=(BaseLongRunningError, Exception)) -async def get_task_result( - long_running_manager: "BaseLongRunningManager", - *, - task_context: TaskContext, - task_id: TaskId, -) -> RPCErrorResponse | str: - try: - return object_to_string( - await _get_transferarble_task_result( - long_running_manager, task_context=task_context, task_id=task_id - ) - ) - except Exception as exc: # pylint:disable=broad-exception-caught - return RPCErrorResponse( - str_traceback="".join(traceback.format_tb(exc.__traceback__)), - error_object=object_to_string(exc), - ) @router.expose(reraise_if_error_type=(BaseLongRunningError,)) diff --git a/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py b/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py index f4c5cd59de53..93fd26bef065 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py +++ b/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py @@ -87,12 +87,14 @@ async def get_task_result( long_running_manager: BaseLongRunningManager, task_context: TaskContext, task_id: TaskId, + allowed_errors: tuple[type[BaseException], ...] = (), ) -> Any: return await _lrt_client.get_task_result( rabbitmq_rpc_client, long_running_manager.lrt_namespace, task_context=task_context, task_id=task_id, + allowed_errors=allowed_errors, ) diff --git a/packages/service-library/src/servicelib/long_running_tasks/models.py b/packages/service-library/src/servicelib/long_running_tasks/models.py index 8440e0f43f97..2ab72274d990 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/models.py +++ b/packages/service-library/src/servicelib/long_running_tasks/models.py @@ -31,13 +31,18 @@ LRTNamespace: TypeAlias = str +class ErrorResponse(BaseModel): + str_error_object: str + str_traceback: str + + class ResultField(BaseModel): - result: str | None = None - error: str | None = None + str_result: str | None = None + error_response: ErrorResponse | None = None @model_validator(mode="after") def validate_mutually_exclusive(self) -> "ResultField": - if self.result is not None and self.error is not None: + if self.str_result is not None and self.error_response is not None: msg = "Cannot set both 'result' and 'error' - they are mutually exclusive" raise ValueError(msg) return self diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index 909c0e23d700..d2bde2b468df 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -3,6 +3,7 @@ import functools import inspect import logging +import traceback import urllib.parse from contextlib import suppress from typing import Any, ClassVar, Final, Protocol, TypeAlias @@ -23,7 +24,7 @@ from ..background_task import create_periodic_task from ..redis import RedisClientSDK, exclusive from ._redis_store import RedisStore -from ._serialization import object_to_string, string_to_object +from ._serialization import object_to_string from .errors import ( TaskAlreadyRunningError, TaskCancelledError, @@ -32,6 +33,7 @@ TaskNotRegisteredError, ) from .models import ( + ErrorResponse, LRTNamespace, ResultField, TaskBase, @@ -142,9 +144,9 @@ def __init__( self._task_cancelled_tasks_removal: asyncio.Task | None = None self._started_event_task_cancelled_tasks_removal = asyncio.Event() - # status_update - self._task_status_update: asyncio.Task | None = None - self._started_event_task_status_update = asyncio.Event() + # tasks_monitor + self._task_tasks_monitor: asyncio.Task | None = None + self._started_event_task_tasks_monitor = asyncio.Event() async def setup(self) -> None: await self._tasks_data.setup() @@ -174,13 +176,13 @@ async def setup(self) -> None: ) await self._started_event_task_cancelled_tasks_removal.wait() - # status_update - self._task_status_update = create_periodic_task( - task=self._status_update, + # tasks_monitor + self._task_tasks_monitor = create_periodic_task( + task=self._tasks_monitor, interval=_STATUS_UPDATE_CHECK_INTERNAL, - task_name=f"{__name__}.{self._status_update.__name__}", + task_name=f"{__name__}.{self._tasks_monitor.__name__}", ) - await self._started_event_task_status_update.wait() + await self._started_event_task_tasks_monitor.wait() async def teardown(self) -> None: # ensure all created tasks are cancelled @@ -211,9 +213,9 @@ async def teardown(self) -> None: if self._task_cancelled_tasks_removal: await cancel_wait_task(self._task_cancelled_tasks_removal) - # status_update - if self._task_status_update: - await cancel_wait_task(self._task_status_update) + # tasks_monitor + if self._task_tasks_monitor: + await cancel_wait_task(self._task_tasks_monitor) if self.locks_redis_client_sdk is not None: await self.locks_redis_client_sdk.shutdown() @@ -278,12 +280,12 @@ async def _cancelled_tasks_removal(self) -> None: for task_id in to_remove: await self._attempt_to_remove_local_task(task_id) - async def _status_update(self) -> None: + async def _tasks_monitor(self) -> None: """ A task which monitors locally running tasks and updates their status in the Redis store when they are done. """ - self._started_event_task_status_update.set() + self._started_event_task_tasks_monitor.set() task_id: TaskId for task_id in set(self._created_tasks.keys()): if task := self._created_tasks.get(task_id, None): @@ -301,16 +303,28 @@ async def _status_update(self) -> None: result_field: ResultField | None = None # get task result try: - result_field = ResultField(result=object_to_string(task.result())) + result_field = ResultField( + str_result=object_to_string(task.result()) + ) except asyncio.InvalidStateError: # task was not completed try again next time and see if it is done continue - except asyncio.CancelledError: + except asyncio.CancelledError as e: result_field = ResultField( - error=object_to_string(TaskCancelledError(task_id=task_id)) + error_response=ErrorResponse( + str_error_object=object_to_string( + TaskCancelledError(task_id=task_id) + ), + str_traceback="".join(traceback.format_tb(e.__traceback__)), + ) ) except Exception as e: # pylint:disable=broad-except - result_field = ResultField(error=object_to_string(e)) + result_field = ResultField( + error_response=ErrorResponse( + str_error_object=object_to_string(e), + str_traceback="".join(traceback.format_tb(e.__traceback__)), + ) + ) # update and store in Redis updates = {"is_done": is_done, "result_field": task_data.result_field} @@ -369,12 +383,11 @@ async def get_task_status( async def get_task_result( self, task_id: TaskId, with_task_context: TaskContext - ) -> Any: + ) -> ResultField: """ - returns: the result of the task + returns: the result of the task wrapped in ResultField raises TaskNotFoundError if the task cannot be found - raises TaskCancelledError if the task was cancelled raises TaskNotCompletedError if the task is not completed """ tracked_task = await self._get_tracked_task(task_id, with_task_context) @@ -382,13 +395,7 @@ async def get_task_result( if not tracked_task.is_done or tracked_task.result_field is None: raise TaskNotCompletedError(task_id=task_id) - if tracked_task.result_field.error is not None: - raise string_to_object(tracked_task.result_field.error) - - if tracked_task.result_field.result is None: - return None - - return string_to_object(tracked_task.result_field.result) + return tracked_task.result_field async def _attempt_to_remove_local_task(self, task_id: TaskId) -> None: """if task is running in the local process, try to remove it""" diff --git a/packages/service-library/src/servicelib/long_running_tasks/utils.py b/packages/service-library/src/servicelib/long_running_tasks/utils.py new file mode 100644 index 000000000000..1d948fa51627 --- /dev/null +++ b/packages/service-library/src/servicelib/long_running_tasks/utils.py @@ -0,0 +1,23 @@ +import importlib +import json + + +def encode_error_types( + error_types: tuple[type[BaseException], ...], +) -> str: + """Encode a tuple of error types into a JSON string.""" + return json.dumps( + [[error_type.__module__, error_type.__name__] for error_type in error_types] + ) + + +def decode_error_types(encoded_errors: str) -> tuple[type[BaseException], ...]: + """Decode a JSON string into a tuple of error types.""" + if not encoded_errors: + return () + + error_types_list = json.loads(encoded_errors) + return tuple( + getattr(importlib.import_module(module), name) + for module, name in error_types_list + ) diff --git a/packages/service-library/src/servicelib/rabbitmq/_rpc_router.py b/packages/service-library/src/servicelib/rabbitmq/_rpc_router.py index 49cab08f79b2..b4a45ab4a08b 100644 --- a/packages/service-library/src/servicelib/rabbitmq/_rpc_router.py +++ b/packages/service-library/src/servicelib/rabbitmq/_rpc_router.py @@ -66,7 +66,7 @@ async def _wrapper(*args, **kwargs): _logger.exception( "Unhandled exception on the rpc-server side." - " Re-raising as RPCServerError." + f" Re-raising as {RPCServerError.__name__}." ) # NOTE: we do not return internal exceptions over RPC formatted_traceback = "\n".join( diff --git a/packages/service-library/tests/aiohttp/long_running_tasks/conftest.py b/packages/service-library/tests/aiohttp/long_running_tasks/conftest.py index 917cd335c65c..13ab8fbd51e7 100644 --- a/packages/service-library/tests/aiohttp/long_running_tasks/conftest.py +++ b/packages/service-library/tests/aiohttp/long_running_tasks/conftest.py @@ -26,6 +26,10 @@ from tenacity.wait import wait_fixed +class _TestingError(Exception): + pass + + async def _string_list_task( progress: TaskProgress, num_strings: int, @@ -39,7 +43,7 @@ async def _string_list_task( await progress.update(message="generated item", percent=index / num_strings) if fail: msg = "We were asked to fail!!" - raise RuntimeError(msg) + raise _TestingError(msg) # NOTE: this code is used just for the sake of not returning the default 200 return web.json_response( diff --git a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py index c8727e123028..34a4cf729165 100644 --- a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py +++ b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py @@ -33,6 +33,7 @@ TaskStatus, ) from servicelib.long_running_tasks.task import TaskContext, TaskRegistry +from servicelib.long_running_tasks.utils import encode_error_types from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings from tenacity.asyncio import AsyncRetrying @@ -48,6 +49,10 @@ ITEM_PUBLISH_SLEEP: Final[float] = 0.1 +class _TestingError(Exception): + pass + + async def _string_list_task( progress: TaskProgress, num_strings: int, @@ -61,7 +66,7 @@ async def _string_list_task( await progress.update(message="generated item", percent=index / num_strings) if fail: msg = "We were asked to fail!!" - raise RuntimeError(msg) + raise _TestingError(msg) return generated_strings @@ -241,8 +246,11 @@ async def test_failing_task_returns_error( await wait_for_task(app, client, task_id, {}) # get the result result_url = app.url_path_for("get_task_result", task_id=task_id) - with pytest.raises(RuntimeError) as exec_info: - await client.get(f"{result_url}") + + encoded_errors = encode_error_types((_TestingError,)) + url = f"{result_url}?allowed_errors={encoded_errors}" + with pytest.raises(_TestingError) as exec_info: + await client.get(url) assert f"{exec_info.value}" == "We were asked to fail!!" diff --git a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py index 94feb6d231e3..75e73531e954 100644 --- a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py +++ b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py @@ -56,11 +56,15 @@ async def a_test_task(progress: TaskProgress) -> int: TaskRegistry.register(a_test_task) +class _TestingError(Exception): + pass + + async def a_failing_test_task(progress: TaskProgress) -> None: _ = progress await asyncio.sleep(TASK_SLEEP_INTERVAL) msg = "I am failing as requested" - raise RuntimeError(msg) + raise _TestingError(msg) TaskRegistry.register(a_failing_test_task) diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py index 39be66212e2c..9ba27ee476d7 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py @@ -43,9 +43,13 @@ async def _task_echo_input(progress: TaskProgress, to_return: Any) -> Any: return to_return +class _TestingError(Exception): + pass + + async def _task_always_raise(progress: TaskProgress) -> None: msg = "This task always raises an error" - raise RuntimeError(msg) + raise _TestingError(msg) async def _task_takes_too_long(progress: TaskProgress) -> None: @@ -270,12 +274,13 @@ async def test_workflow_raises_error( ) for task_id in task_ids: - with pytest.raises(RuntimeError, match="This task always raises an error"): + with pytest.raises(_TestingError, match="This task always raises an error"): await lrt_api.get_task_result( rabbitmq_rpc_client, _get_task_manager(long_running_managers), saved_context, task_id, + allowed_errors=(_TestingError,), ) await _assert_task_is_no_longer_present( diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py index acfd7df39a83..52b7231a2a4f 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py @@ -14,6 +14,9 @@ from faker import Faker from models_library.api_schemas_long_running_tasks.base import ProgressMessage from servicelib.long_running_tasks import lrt_api +from servicelib.long_running_tasks._serialization import ( + string_to_object, +) from servicelib.long_running_tasks.base_long_running_manager import ( BaseLongRunningManager, ) @@ -25,6 +28,7 @@ ) from servicelib.long_running_tasks.models import ( LRTNamespace, + ResultField, TaskContext, TaskProgress, TaskStatus, @@ -52,6 +56,10 @@ } +class _TetingError(Exception): + pass + + async def a_background_task( progress: TaskProgress, raise_when_finished: bool, @@ -63,7 +71,7 @@ async def a_background_task( await progress.update(percent=(i + 1) / total_sleep) if raise_when_finished: msg = "raised this error as instructed" - raise RuntimeError(msg) + raise _TetingError(msg) return 42 @@ -76,7 +84,7 @@ async def fast_background_task(progress: TaskProgress) -> int: async def failing_background_task(progress: TaskProgress): """this task does nothing and returns a constant""" msg = "failing asap" - raise RuntimeError(msg) + raise _TetingError(msg) TaskRegistry.register(a_background_task) @@ -172,6 +180,10 @@ async def test_checked_task_is_not_auto_removed( assert result +def _get_resutlt(result_field: ResultField) -> Any: + return string_to_object(result_field.str_result) + + async def test_fire_and_forget_task_is_not_auto_removed( long_running_manager: BaseLongRunningManager, empty_context: TaskContext ): @@ -195,7 +207,7 @@ async def test_fire_and_forget_task_is_not_auto_removed( task_result = await long_running_manager.tasks_manager.get_task_result( task_id, with_task_context=empty_context ) - assert task_result == 42 + assert _get_resutlt(task_result) == 42 async def test_get_result_of_unfinished_task_raises( @@ -265,12 +277,12 @@ async def not_unique_task(progress: TaskProgress): async def test_get_task_id( long_running_manager: BaseLongRunningManager, faker: Faker, is_unique: bool ): - obj1 = long_running_manager.tasks_manager._get_task_id( + obj1 = long_running_manager.tasks_manager._get_task_id( # noqa: SLF001 faker.word(), is_unique=is_unique - ) # noqa: SLF001 - obj2 = long_running_manager.tasks_manager._get_task_id( + ) + obj2 = long_running_manager.tasks_manager._get_task_id( # noqa: SLF001 faker.word(), is_unique=is_unique - ) # noqa: SLF001 + ) assert obj1 != obj2 @@ -321,7 +333,7 @@ async def test_get_result( result = await long_running_manager.tasks_manager.get_task_result( task_id, with_task_context=empty_context ) - assert result == 42 + assert _get_resutlt(result) == 42 async def test_get_result_missing( @@ -351,10 +363,13 @@ async def test_get_result_finished_with_error( ) ).done - with pytest.raises(RuntimeError, match="failing asap"): - await long_running_manager.tasks_manager.get_task_result( - task_id, with_task_context=empty_context - ) + result = await long_running_manager.tasks_manager.get_task_result( + task_id, with_task_context=empty_context + ) + assert result.error_response is not None # nosec + error = string_to_object(result.error_response.str_error_object) + with pytest.raises(_TetingError, match="failing asap"): + raise error async def test_cancel_task_from_different_manager( @@ -402,7 +417,7 @@ async def test_cancel_task_from_different_manager( task_result = await manager.tasks_manager.get_task_result( task_id, empty_context ) - assert task_result == 42 + assert _get_resutlt(task_result) == 42 async def test_remove_task( From ba0cafd4eedf848e4b185ca61cccebb2e60941bc Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 20 Aug 2025 10:54:29 +0200 Subject: [PATCH 088/119] pylint --- .../service-library/src/servicelib/rabbitmq/_rpc_router.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/service-library/src/servicelib/rabbitmq/_rpc_router.py b/packages/service-library/src/servicelib/rabbitmq/_rpc_router.py index b4a45ab4a08b..10dbf26a4497 100644 --- a/packages/service-library/src/servicelib/rabbitmq/_rpc_router.py +++ b/packages/service-library/src/servicelib/rabbitmq/_rpc_router.py @@ -65,8 +65,8 @@ async def _wrapper(*args, **kwargs): raise _logger.exception( - "Unhandled exception on the rpc-server side." - f" Re-raising as {RPCServerError.__name__}." + "Unhandled exception on the rpc-server side. Re-raising as %s.", + RPCServerError.__name__, ) # NOTE: we do not return internal exceptions over RPC formatted_traceback = "\n".join( From d878b17555c2929a9ee8ff6b280e0d646b7eb1ea Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 20 Aug 2025 13:43:51 +0200 Subject: [PATCH 089/119] refactored error registration and detection --- .../aiohttp/long_running_tasks/_server.py | 8 +++- .../fastapi/long_running_tasks/_routes.py | 11 +----- .../long_running_tasks/_lrt_client.py | 7 +--- .../long_running_tasks/_lrt_server.py | 19 ++++++--- .../servicelib/long_running_tasks/lrt_api.py | 5 +-- .../servicelib/long_running_tasks/models.py | 4 ++ .../src/servicelib/long_running_tasks/task.py | 39 ++++++++++++++++--- .../servicelib/long_running_tasks/utils.py | 23 ----------- .../aiohttp/long_running_tasks/conftest.py | 2 +- .../test_long_running_tasks.py | 2 +- .../test_long_running_tasks.py | 7 +--- .../test_long_running_tasks_lrt_api.py | 3 +- 12 files changed, 65 insertions(+), 65 deletions(-) delete mode 100644 packages/service-library/src/servicelib/long_running_tasks/utils.py diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py index 60ac1b32994b..dacf2e2e648a 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py @@ -21,8 +21,12 @@ DEFAULT_STALE_TASK_CHECK_INTERVAL, DEFAULT_STALE_TASK_DETECT_TIMEOUT, ) -from ...long_running_tasks.models import LRTNamespace, TaskContext, TaskGet -from ...long_running_tasks.task import RegisteredTaskName +from ...long_running_tasks.models import ( + LRTNamespace, + RegisteredTaskName, + TaskContext, + TaskGet, +) from ..typing_extension import Handler from . import _routes from ._constants import ( diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py index 0bba29e780f9..3ec4f62e9a01 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py @@ -1,10 +1,9 @@ from typing import Annotated, Any -from fastapi import APIRouter, Depends, Query, Request, status +from fastapi import APIRouter, Depends, Request, status from ...long_running_tasks import lrt_api from ...long_running_tasks.models import TaskGet, TaskId, TaskResult, TaskStatus -from ...long_running_tasks.utils import decode_error_types from ..requests_decorators import cancel_on_disconnect from ._dependencies import get_long_running_manager from ._manager import FastAPILongRunningManager @@ -74,13 +73,6 @@ async def get_task_result( long_running_manager: Annotated[ FastAPILongRunningManager, Depends(get_long_running_manager) ], - allowed_errors: Annotated[ - str, - Query( - description="list of json encoded tuples of allowed errors", - example='["tests.fastapi.long_running_tasks.test_long_running_tasks", "_TestingError"]', - ), - ] = "", ) -> TaskResult | Any: assert request # nosec return await lrt_api.get_task_result( @@ -88,7 +80,6 @@ async def get_task_result( long_running_manager, task_context={}, task_id=task_id, - allowed_errors=decode_error_types(allowed_errors), ) diff --git a/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py b/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py index 2d86a26af468..c64da0cd1ae4 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py @@ -9,16 +9,16 @@ from ..logging_utils import log_decorator from ..rabbitmq._client_rpc import RabbitMQRPCClient from ._rabbit_namespace import get_rabbit_namespace -from ._serialization import object_to_string, string_to_object +from ._serialization import string_to_object from .models import ( ErrorResponse, LRTNamespace, + RegisteredTaskName, TaskBase, TaskContext, TaskId, TaskStatus, ) -from .task import RegisteredTaskName _logger = logging.getLogger(__name__) @@ -99,14 +99,12 @@ async def get_task_result( *, task_context: TaskContext, task_id: TaskId, - allowed_errors: tuple[type[BaseException], ...], ) -> Any: serialized_result = await rabbitmq_rpc_client.request( get_rabbit_namespace(namespace), TypeAdapter(RPCMethodName).validate_python("get_task_result"), task_context=task_context, task_id=task_id, - allowed_errors_str=object_to_string(allowed_errors), timeout_s=_RPC_TIMEOUT_SHORT_REQUESTS, ) assert isinstance(serialized_result, ErrorResponse | str) # nosec @@ -120,7 +118,6 @@ async def get_task_result( "task_id": task_id, "namespace": namespace, "task_context": task_context, - "allowed_errors": allowed_errors, }, tip=( f"The caller of this function should handle the exception. " diff --git a/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py b/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py index 018a95d9a8ab..58f726c519a7 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py @@ -5,8 +5,14 @@ from ..rabbitmq import RPCRouter from ._serialization import string_to_object from .errors import BaseLongRunningError -from .models import ErrorResponse, TaskBase, TaskContext, TaskId, TaskStatus -from .task import RegisteredTaskName +from .models import ( + ErrorResponse, + RegisteredTaskName, + TaskBase, + TaskContext, + TaskId, + TaskStatus, +) _logger = logging.getLogger(__name__) @@ -65,11 +71,7 @@ async def get_task_result( *, task_context: TaskContext, task_id: TaskId, - allowed_errors_str: str, ) -> ErrorResponse | str: - allowed_errors: tuple[type[BaseException], ...] = string_to_object( - allowed_errors_str - ) try: result_field = await long_running_manager.tasks_manager.get_task_result( task_id, with_task_context=task_context @@ -91,6 +93,11 @@ async def get_task_result( tip="This exception is logged for debugging purposes, the client side will handle it", ) ) + allowed_errors = ( + await long_running_manager.tasks_manager.get_allowed_errors( + task_id, with_task_context=task_context + ) + ) if type(task_raised_error) in allowed_errors: return result_field.error_response diff --git a/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py b/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py index 93fd26bef065..05ac4fcaadca 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py +++ b/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py @@ -3,8 +3,7 @@ from ..rabbitmq._client_rpc import RabbitMQRPCClient from . import _lrt_client from .base_long_running_manager import BaseLongRunningManager -from .models import TaskBase, TaskContext, TaskId, TaskStatus -from .task import RegisteredTaskName +from .models import RegisteredTaskName, TaskBase, TaskContext, TaskId, TaskStatus async def start_task( @@ -87,14 +86,12 @@ async def get_task_result( long_running_manager: BaseLongRunningManager, task_context: TaskContext, task_id: TaskId, - allowed_errors: tuple[type[BaseException], ...] = (), ) -> Any: return await _lrt_client.get_task_result( rabbitmq_rpc_client, long_running_manager.lrt_namespace, task_context=task_context, task_id=task_id, - allowed_errors=allowed_errors, ) diff --git a/packages/service-library/src/servicelib/long_running_tasks/models.py b/packages/service-library/src/servicelib/long_running_tasks/models.py index 2ab72274d990..5bad6a06f1a4 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/models.py +++ b/packages/service-library/src/servicelib/long_running_tasks/models.py @@ -30,6 +30,8 @@ LRTNamespace: TypeAlias = str +RegisteredTaskName: TypeAlias = str + class ErrorResponse(BaseModel): str_error_object: str @@ -49,6 +51,7 @@ def validate_mutually_exclusive(self) -> "ResultField": class TaskData(BaseModel): + registered_task_name: RegisteredTaskName task_id: str task_progress: TaskProgress # NOTE: this context lifetime is with the tracked task (similar to aiohttp storage concept) @@ -86,6 +89,7 @@ class TaskData(BaseModel): json_schema_extra={ "examples": [ { + "registered_task_name": "a-task-name", "task_id": "1a119618-7186-4bc1-b8de-7e3ff314cb7e", "task_name": "running-task", "task_status": "running", diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index d2bde2b468df..e4d1efbde74e 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -35,6 +35,7 @@ from .models import ( ErrorResponse, LRTNamespace, + RegisteredTaskName, ResultField, TaskBase, TaskContext, @@ -51,8 +52,7 @@ _MAX_EXCLUSIVE_TASK_CANCEL_TIMEOUT: Final[NonNegativeFloat] = 5 _TASK_REMOVAL_MAX_WAIT: Final[NonNegativeFloat] = 60 - -RegisteredTaskName: TypeAlias = str +AllowedErrrors: TypeAlias = tuple[type[BaseException], ...] class TaskProtocol(Protocol): @@ -65,14 +65,29 @@ def __name__(self) -> str: ... class TaskRegistry: - REGISTERED_TASKS: ClassVar[dict[RegisteredTaskName, TaskProtocol]] = {} + REGISTERED_TASKS: ClassVar[ + dict[RegisteredTaskName, tuple[AllowedErrrors, TaskProtocol]] + ] = {} @classmethod - def register(cls, task: TaskProtocol, **partial_kwargs) -> None: + def register( + cls, + task: TaskProtocol, + allowed_errors: AllowedErrrors = (), + **partial_kwargs, + ) -> None: partial_task = functools.partial(task, **partial_kwargs) # allows to call the partial via it's original name partial_task.__name__ = task.__name__ # type: ignore[attr-defined] - cls.REGISTERED_TASKS[task.__name__] = partial_task # type: ignore[assignment] + cls.REGISTERED_TASKS[task.__name__] = [allowed_errors, partial_task] # type: ignore[assignment] + + @classmethod + def get_task(cls, task_name: RegisteredTaskName) -> TaskProtocol: + return cls.REGISTERED_TASKS[task_name][1] + + @classmethod + def get_allowed_errors(cls, task_name: RegisteredTaskName) -> AllowedErrrors: + return cls.REGISTERED_TASKS[task_name][0] @classmethod def unregister(cls, task: TaskProtocol) -> None: @@ -381,6 +396,17 @@ async def get_task_status( } ) + async def get_allowed_errors( + self, task_id: TaskId, with_task_context: TaskContext + ) -> AllowedErrrors: + """ + returns: the allowed errors for the task + + raises TaskNotFoundError if the task cannot be found + """ + task_data = await self._get_tracked_task(task_id, with_task_context) + return TaskRegistry.get_allowed_errors(task_data.registered_task_name) + async def get_task_result( self, task_id: TaskId, with_task_context: TaskContext ) -> ResultField: @@ -483,7 +509,7 @@ async def start_task( task_name=registered_task_name, tasks=TaskRegistry.REGISTERED_TASKS ) - task = TaskRegistry.REGISTERED_TASKS[registered_task_name] + task = TaskRegistry.get_task(registered_task_name) # NOTE: If not task name is given, it will be composed of the handler's module and it's name # to keep the urls shorter and more meaningful. @@ -521,6 +547,7 @@ async def _task_with_progress(progress: TaskProgress, handler: TaskProtocol): ) tracked_task = TaskData( + registered_task_name=registered_task_name, task_id=task_id, task_progress=task_progress, task_context=context_to_use, diff --git a/packages/service-library/src/servicelib/long_running_tasks/utils.py b/packages/service-library/src/servicelib/long_running_tasks/utils.py deleted file mode 100644 index 1d948fa51627..000000000000 --- a/packages/service-library/src/servicelib/long_running_tasks/utils.py +++ /dev/null @@ -1,23 +0,0 @@ -import importlib -import json - - -def encode_error_types( - error_types: tuple[type[BaseException], ...], -) -> str: - """Encode a tuple of error types into a JSON string.""" - return json.dumps( - [[error_type.__module__, error_type.__name__] for error_type in error_types] - ) - - -def decode_error_types(encoded_errors: str) -> tuple[type[BaseException], ...]: - """Decode a JSON string into a tuple of error types.""" - if not encoded_errors: - return () - - error_types_list = json.loads(encoded_errors) - return tuple( - getattr(importlib.import_module(module), name) - for module, name in error_types_list - ) diff --git a/packages/service-library/tests/aiohttp/long_running_tasks/conftest.py b/packages/service-library/tests/aiohttp/long_running_tasks/conftest.py index 13ab8fbd51e7..3bf527ab2c82 100644 --- a/packages/service-library/tests/aiohttp/long_running_tasks/conftest.py +++ b/packages/service-library/tests/aiohttp/long_running_tasks/conftest.py @@ -51,7 +51,7 @@ async def _string_list_task( ) -TaskRegistry.register(_string_list_task) +TaskRegistry.register(_string_list_task, allowed_errors=(_TestingError,)) @pytest.fixture diff --git a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py index c9246d1f22d6..5683385da079 100644 --- a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py +++ b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py @@ -164,7 +164,7 @@ async def test_failing_task_returns_error( # The actual error details should be logged, not returned in response log_messages = caplog.text assert "OEC" in log_messages - assert "RuntimeError" in log_messages + assert "_TestingError" in log_messages assert "We were asked to fail!!" in log_messages diff --git a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py index 34a4cf729165..e5cc58e68816 100644 --- a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py +++ b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py @@ -33,7 +33,6 @@ TaskStatus, ) from servicelib.long_running_tasks.task import TaskContext, TaskRegistry -from servicelib.long_running_tasks.utils import encode_error_types from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings from tenacity.asyncio import AsyncRetrying @@ -71,7 +70,7 @@ async def _string_list_task( return generated_strings -TaskRegistry.register(_string_list_task) +TaskRegistry.register(_string_list_task, allowed_errors=(_TestingError,)) @pytest.fixture @@ -247,10 +246,8 @@ async def test_failing_task_returns_error( # get the result result_url = app.url_path_for("get_task_result", task_id=task_id) - encoded_errors = encode_error_types((_TestingError,)) - url = f"{result_url}?allowed_errors={encoded_errors}" with pytest.raises(_TestingError) as exec_info: - await client.get(url) + await client.get(f"{result_url}") assert f"{exec_info.value}" == "We were asked to fail!!" diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py index 9ba27ee476d7..2f7f22ab57ed 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py @@ -58,7 +58,7 @@ async def _task_takes_too_long(progress: TaskProgress) -> None: TaskRegistry.register(_task_echo_input) -TaskRegistry.register(_task_always_raise) +TaskRegistry.register(_task_always_raise, allowed_errors=(_TestingError,)) TaskRegistry.register(_task_takes_too_long) @@ -280,7 +280,6 @@ async def test_workflow_raises_error( _get_task_manager(long_running_managers), saved_context, task_id, - allowed_errors=(_TestingError,), ) await _assert_task_is_no_longer_present( From 46cd3b387f6be0456e5fcf3a2dc1721619d77bdd Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 20 Aug 2025 14:08:13 +0200 Subject: [PATCH 090/119] refactor --- .../modules/long_running_tasks.py | 73 ++++++++++--------- 1 file changed, 39 insertions(+), 34 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py index 5b758c641bec..7e8a4d5fe916 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py @@ -639,42 +639,47 @@ def setup_long_running_tasks(app: FastAPI) -> None: lrt_namespace=f"{APP_NAME}-{app_settings.DY_SIDECAR_RUN_ID}", ) - async def on_startup() -> None: - shared_store: SharedStore = app.state.shared_store - mounted_volumes: MountedVolumes = app.state.mounted_volumes - outputs_manager: OutputsManager = app.state.outputs_manager - - context_app_store: dict[str, Any] = { - "app": app, - "shared_store": shared_store, - } - context_app_store_volumes: dict[str, Any] = { - "app": app, - "shared_store": shared_store, - "mounted_volumes": mounted_volumes, - } - context_app_volumes: dict[str, Any] = { - "app": app, - "mounted_volumes": mounted_volumes, - } - context_app_outputs: dict[str, Any] = { - "app": app, - "outputs_manager": outputs_manager, - } - - task_context: dict[TaskProtocol, dict[str, Any]] = { - task_pull_user_servcices_docker_images: context_app_store, - task_create_service_containers: context_app_store, - task_runs_docker_compose_down: context_app_store_volumes, - task_restore_state: context_app_volumes, - task_save_state: context_app_volumes, - task_ports_inputs_pull: context_app_volumes, - task_ports_outputs_pull: context_app_volumes, - task_ports_outputs_push: context_app_outputs, - task_containers_restart: context_app_store, - } + shared_store: SharedStore = app.state.shared_store + mounted_volumes: MountedVolumes = app.state.mounted_volumes + outputs_manager: OutputsManager = app.state.outputs_manager + + context_app_store: dict[str, Any] = { + "app": app, + "shared_store": shared_store, + } + context_app_store_volumes: dict[str, Any] = { + "app": app, + "shared_store": shared_store, + "mounted_volumes": mounted_volumes, + } + context_app_volumes: dict[str, Any] = { + "app": app, + "mounted_volumes": mounted_volumes, + } + context_app_outputs: dict[str, Any] = { + "app": app, + "outputs_manager": outputs_manager, + } + + task_context: dict[TaskProtocol, dict[str, Any]] = { + task_pull_user_servcices_docker_images: context_app_store, + task_create_service_containers: context_app_store, + task_runs_docker_compose_down: context_app_store_volumes, + task_restore_state: context_app_volumes, + task_save_state: context_app_volumes, + task_ports_inputs_pull: context_app_volumes, + task_ports_outputs_pull: context_app_volumes, + task_ports_outputs_push: context_app_outputs, + task_containers_restart: context_app_store, + } + async def on_startup() -> None: for handler, context in task_context.items(): TaskRegistry.register(handler, **context) + async def _on_shutdown() -> None: + for handler in task_context: + TaskRegistry.unregister(handler) + app.add_event_handler("startup", on_startup) + app.add_event_handler("shutdown", _on_shutdown) From 4d8d93d98265eb540785526de7841e00d4e98ec3 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 20 Aug 2025 14:21:00 +0200 Subject: [PATCH 091/119] fixed tests --- .../projects/_controller/nodes_rest.py | 4 +++- .../projects/_crud_api_create.py | 11 ++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/nodes_rest.py b/services/web/server/src/simcore_service_webserver/projects/_controller/nodes_rest.py index b2341339ab02..8208df41a0ed 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/nodes_rest.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/nodes_rest.py @@ -332,7 +332,9 @@ async def _stop_dynamic_service_task( def register_stop_dynamic_service_task(app: web.Application) -> None: - TaskRegistry.register(_stop_dynamic_service_task, app=app) + TaskRegistry.register( + _stop_dynamic_service_task, allowed_errors=(web.HTTPNotFound,), app=app + ) @routes.post( diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py index 5f09adae36dd..bad787ca1136 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py @@ -523,4 +523,13 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche def register_create_project_task(app: web.Application) -> None: - TaskRegistry.register(create_project, app=app) + TaskRegistry.register( + create_project, + allowed_errors=( + web.HTTPUnprocessableEntity, + web.HTTPBadRequest, + web.HTTPNotFound, + web.HTTPForbidden, + ), + app=app, + ) From e11088320eb05472da6408ea639c32a7d8ae6bd0 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 20 Aug 2025 14:29:16 +0200 Subject: [PATCH 092/119] added note --- services/docker-compose.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/services/docker-compose.yml b/services/docker-compose.yml index dd9130d467c0..7e263ed2139e 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -699,6 +699,7 @@ services: WEBSERVER_LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} WEBSERVER_LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} + # NOTE: keep in sync with the prefix form the hostname LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: wb # WEBSERVER_SERVER_HOST @@ -931,6 +932,7 @@ services: WEBSERVER_STATICWEB: "null" WEBSERVER_FUNCTIONS: ${WEBSERVER_FUNCTIONS} # needed for api-server + # NOTE: keep in sync with the prefix form the hostname LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: api networks: *webserver_networks @@ -942,6 +944,7 @@ services: environment: WEBSERVER_LOGLEVEL: ${WB_DB_EL_LOGLEVEL} + # NOTE: keep in sync with the prefix form the hostname LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: db WEBSERVER_HOST: ${WEBSERVER_HOST} @@ -1040,6 +1043,7 @@ services: LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + # NOTE: keep in sync with the prefix form the hostname LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: gc # WEBSERVER_DB @@ -1134,6 +1138,7 @@ services: WEBSERVER_APP_FACTORY_NAME: WEBSERVER_AUTHZ_APP_FACTORY WEBSERVER_LOGLEVEL: ${WB_AUTH_LOGLEVEL} + # NOTE: keep in sync with the prefix form the hostname LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: auth GUNICORN_CMD_ARGS: ${WEBSERVER_GUNICORN_CMD_ARGS} From a48bd025cf80b5ffdbffab42c0f1e9350ac83e25 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 20 Aug 2025 14:30:03 +0200 Subject: [PATCH 093/119] removed unused --- .../src/servicelib/long_running_tasks/models.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/models.py b/packages/service-library/src/servicelib/long_running_tasks/models.py index 5bad6a06f1a4..1d8e1cfc53bb 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/models.py +++ b/packages/service-library/src/servicelib/long_running_tasks/models.py @@ -109,11 +109,6 @@ class ClientConfiguration(BaseModel): default_timeout: PositiveFloat -class RPCErrorResponse(BaseModel): - str_traceback: str - error_object: str - - @dataclass(frozen=True) class LRTask: progress: TaskProgress From 4a7341a75b97bb10159a2313b45e2a7da369db62 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 20 Aug 2025 14:45:22 +0200 Subject: [PATCH 094/119] rename --- .../service-library/src/servicelib/long_running_tasks/task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index e4d1efbde74e..69d6f71f3fe5 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -286,7 +286,7 @@ async def _stale_tasks_monitor(self) -> None: async def _cancelled_tasks_removal(self) -> None: """ - Periodicallu checks which tasks are maked for removal and attempts to remove the + Periodically checks which tasks are marked for removal and attempts to remove the task if it's handled by this process. """ self._started_event_task_cancelled_tasks_removal.set() From 6552ddfc0189f5027eb83e8289a2e2d408b412c6 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 20 Aug 2025 14:57:23 +0200 Subject: [PATCH 095/119] fixed public lrt_api interface --- .../aiohttp/long_running_tasks/_routes.py | 8 +-- .../aiohttp/long_running_tasks/_server.py | 5 +- .../fastapi/long_running_tasks/_routes.py | 10 +-- .../servicelib/long_running_tasks/lrt_api.py | 38 ++++++----- .../test_long_running_tasks.py | 3 +- ...test_long_running_tasks_context_manager.py | 10 ++- .../test_long_running_tasks_lrt_api.py | 29 +++++---- .../test_long_running_tasks_task.py | 65 +++++++++++++------ .../api/routes/dynamic_scheduler.py | 12 ++-- .../api/rest/containers_long_running_tasks.py | 27 +++++--- .../simcore_service_webserver/tasks/_rest.py | 2 +- 11 files changed, 132 insertions(+), 77 deletions(-) diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py index 65412a852bd3..1443b7090db4 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py @@ -30,7 +30,7 @@ async def list_tasks(request: web.Request) -> web.Response: ) for t in await lrt_api.list_tasks( long_running_manager.rpc_client, - long_running_manager, + long_running_manager.lrt_namespace, long_running_manager.get_task_context(request), ) ] @@ -44,7 +44,7 @@ async def get_task_status(request: web.Request) -> web.Response: task_status = await lrt_api.get_task_status( long_running_manager.rpc_client, - long_running_manager, + long_running_manager.lrt_namespace, long_running_manager.get_task_context(request), path_params.task_id, ) @@ -59,7 +59,7 @@ async def get_task_result(request: web.Request) -> web.Response | Any: # NOTE: this might raise an exception that will be catched by the _error_handlers return await lrt_api.get_task_result( long_running_manager.rpc_client, - long_running_manager, + long_running_manager.lrt_namespace, long_running_manager.get_task_context(request), path_params.task_id, ) @@ -72,7 +72,7 @@ async def remove_task(request: web.Request) -> web.Response: await lrt_api.remove_task( long_running_manager.rpc_client, - long_running_manager, + long_running_manager.lrt_namespace, long_running_manager.get_task_context(request), path_params.task_id, wait_for_removal=False, # frontend does not care about waiting for this diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py index dacf2e2e648a..495e03498f97 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py @@ -68,7 +68,8 @@ async def start_long_running_task( task_id = None try: task_id = await lrt_api.start_task( - long_running_manager, + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, registerd_task_name, fire_and_forget=fire_and_forget, task_context=task_context, @@ -104,7 +105,7 @@ async def start_long_running_task( if task_id: await lrt_api.remove_task( long_running_manager.rpc_client, - long_running_manager, + long_running_manager.lrt_namespace, task_context, task_id, wait_for_removal=True, diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py index 3ec4f62e9a01..bbbba4abcd53 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py @@ -28,7 +28,9 @@ async def list_tasks( abort_href=str(request.url_for("remove_task", task_id=t.task_id)), ) for t in await lrt_api.list_tasks( - long_running_manager.rpc_client, long_running_manager, task_context={} + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, + task_context={}, ) ] @@ -51,7 +53,7 @@ async def get_task_status( assert request # nosec return await lrt_api.get_task_status( long_running_manager.rpc_client, - long_running_manager, + long_running_manager.lrt_namespace, task_context={}, task_id=task_id, ) @@ -77,7 +79,7 @@ async def get_task_result( assert request # nosec return await lrt_api.get_task_result( long_running_manager.rpc_client, - long_running_manager, + long_running_manager.lrt_namespace, task_context={}, task_id=task_id, ) @@ -103,7 +105,7 @@ async def remove_task( assert request # nosec await lrt_api.remove_task( long_running_manager.rpc_client, - long_running_manager, + long_running_manager.lrt_namespace, task_context={}, task_id=task_id, wait_for_removal=True, # only used by internal services, they will wait as before diff --git a/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py b/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py index 05ac4fcaadca..c7dc733fa89e 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py +++ b/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py @@ -2,12 +2,19 @@ from ..rabbitmq._client_rpc import RabbitMQRPCClient from . import _lrt_client -from .base_long_running_manager import BaseLongRunningManager -from .models import RegisteredTaskName, TaskBase, TaskContext, TaskId, TaskStatus +from .models import ( + LRTNamespace, + RegisteredTaskName, + TaskBase, + TaskContext, + TaskId, + TaskStatus, +) async def start_task( - long_running_manager: BaseLongRunningManager, + rabbitmq_rpc_client: RabbitMQRPCClient, + lrt_namespace: LRTNamespace, registered_task_name: RegisteredTaskName, *, unique: bool = False, @@ -43,8 +50,8 @@ async def start_task( """ return await _lrt_client.start_task( - long_running_manager.rpc_client, - long_running_manager.lrt_namespace, + rabbitmq_rpc_client, + lrt_namespace, registered_task_name=registered_task_name, unique=unique, task_context=task_context, @@ -56,40 +63,35 @@ async def start_task( async def list_tasks( rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_manager: BaseLongRunningManager, + lrt_namespace: LRTNamespace, task_context: TaskContext, ) -> list[TaskBase]: return await _lrt_client.list_tasks( - rabbitmq_rpc_client, - long_running_manager.lrt_namespace, - task_context=task_context, + rabbitmq_rpc_client, lrt_namespace, task_context=task_context ) async def get_task_status( rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_manager: BaseLongRunningManager, + lrt_namespace: LRTNamespace, task_context: TaskContext, task_id: TaskId, ) -> TaskStatus: """returns the status of a task""" return await _lrt_client.get_task_status( - rabbitmq_rpc_client, - long_running_manager.lrt_namespace, - task_id=task_id, - task_context=task_context, + rabbitmq_rpc_client, lrt_namespace, task_id=task_id, task_context=task_context ) async def get_task_result( rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_manager: BaseLongRunningManager, + lrt_namespace: LRTNamespace, task_context: TaskContext, task_id: TaskId, ) -> Any: return await _lrt_client.get_task_result( rabbitmq_rpc_client, - long_running_manager.lrt_namespace, + lrt_namespace, task_context=task_context, task_id=task_id, ) @@ -97,7 +99,7 @@ async def get_task_result( async def remove_task( rabbitmq_rpc_client: RabbitMQRPCClient, - long_running_manager: BaseLongRunningManager, + lrt_namespace: LRTNamespace, task_context: TaskContext, task_id: TaskId, *, @@ -106,7 +108,7 @@ async def remove_task( """cancels and removes the task""" await _lrt_client.remove_task( rabbitmq_rpc_client, - long_running_manager.lrt_namespace, + lrt_namespace, task_id=task_id, task_context=task_context, wait_for_removal=wait_for_removal, diff --git a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py index e5cc58e68816..1b72713dbd5c 100644 --- a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py +++ b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks.py @@ -90,7 +90,8 @@ async def create_string_list_task( fail: bool = False, ) -> TaskId: return await lrt_api.start_task( - long_running_manager, + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, _string_list_task.__name__, num_strings=num_strings, sleep_time=sleep_time, diff --git a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py index 75e73531e954..30418fd922a3 100644 --- a/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py +++ b/packages/service-library/tests/fastapi/long_running_tasks/test_long_running_tasks_context_manager.py @@ -80,7 +80,11 @@ async def create_task_user_defined_route( FastAPILongRunningManager, Depends(get_long_running_manager) ], ) -> TaskId: - return await lrt_api.start_task(long_running_manager, a_test_task.__name__) + return await lrt_api.start_task( + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, + a_test_task.__name__, + ) @router.get("/api/failing", status_code=status.HTTP_200_OK) async def create_task_which_fails( @@ -89,7 +93,9 @@ async def create_task_which_fails( ], ) -> TaskId: return await lrt_api.start_task( - long_running_manager, a_failing_test_task.__name__ + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, + a_failing_test_task.__name__, ) return router diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py index 2f7f22ab57ed..e1742a17013b 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py @@ -88,7 +88,7 @@ async def long_running_managers( return maanagers -def _get_task_manager( +def _get_long_running_manager( long_running_managers: list[BaseLongRunningManager], ) -> BaseLongRunningManager: return secrets.choice(long_running_managers) @@ -102,7 +102,7 @@ async def _assert_task_status( is_done: bool ) -> None: result = await lrt_api.get_task_status( - rabbitmq_rpc_client, long_running_manager, TaskContext(), task_id + rabbitmq_rpc_client, long_running_manager.lrt_namespace, TaskContext(), task_id ) assert result.done is is_done @@ -117,7 +117,7 @@ async def _assert_task_status_on_random_manager( for task_id in task_ids: result = await lrt_api.get_task_status( rabbitmq_rpc_client, - _get_task_manager(long_running_managers), + _get_long_running_manager(long_running_managers).lrt_namespace, TaskContext(), task_id, ) @@ -135,7 +135,7 @@ async def _assert_task_status_done_on_all_managers( with attempt: await _assert_task_status( rabbitmq_rpc_client, - _get_task_manager(long_running_managers), + _get_long_running_manager(long_running_managers), task_id, is_done=is_done, ) @@ -154,7 +154,9 @@ async def _assert_list_tasks_from_all_managers( task_count: int, ) -> None: for manager in long_running_managers: - tasks = await lrt_api.list_tasks(rabbitmq_rpc_client, manager, task_context) + tasks = await lrt_api.list_tasks( + rabbitmq_rpc_client, manager.lrt_namespace, task_context + ) assert len(tasks) == task_count @@ -167,7 +169,7 @@ async def _assert_task_is_no_longer_present( with pytest.raises(TaskNotFoundError): await lrt_api.get_task_status( rabbitmq_rpc_client, - _get_task_manager(long_running_managers), + _get_long_running_manager(long_running_managers).lrt_namespace, task_context, task_id, ) @@ -196,7 +198,8 @@ async def test_workflow_with_result( task_ids: list[TaskId] = [] for _ in range(task_count): task_id = await lrt_api.start_task( - _get_task_manager(long_running_managers), + _get_long_running_manager(long_running_managers).rpc_client, + _get_long_running_manager(long_running_managers).lrt_namespace, _task_echo_input.__name__, unique=is_unique, task_name=None, @@ -223,7 +226,7 @@ async def test_workflow_with_result( for task_id in task_ids: result = await lrt_api.get_task_result( rabbitmq_rpc_client, - _get_task_manager(long_running_managers), + _get_long_running_manager(long_running_managers).lrt_namespace, saved_context, task_id, ) @@ -250,7 +253,8 @@ async def test_workflow_raises_error( task_ids: list[TaskId] = [] for _ in range(task_count): task_id = await lrt_api.start_task( - _get_task_manager(long_running_managers), + _get_long_running_manager(long_running_managers).rpc_client, + _get_long_running_manager(long_running_managers).lrt_namespace, _task_always_raise.__name__, unique=is_unique, task_name=None, @@ -277,7 +281,7 @@ async def test_workflow_raises_error( with pytest.raises(_TestingError, match="This task always raises an error"): await lrt_api.get_task_result( rabbitmq_rpc_client, - _get_task_manager(long_running_managers), + _get_long_running_manager(long_running_managers).lrt_namespace, saved_context, task_id, ) @@ -296,7 +300,8 @@ async def test_remove_task( task_context: TaskContext | None, ): task_id = await lrt_api.start_task( - _get_task_manager(long_running_managers), + _get_long_running_manager(long_running_managers).rpc_client, + _get_long_running_manager(long_running_managers).lrt_namespace, _task_takes_too_long.__name__, unique=is_unique, task_name=None, @@ -311,7 +316,7 @@ async def test_remove_task( await lrt_api.remove_task( rabbitmq_rpc_client, - _get_task_manager(long_running_managers), + _get_long_running_manager(long_running_managers).lrt_namespace, saved_context, task_id, wait_for_removal=True, diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py index 52b7231a2a4f..b0a9ac46bc3c 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py @@ -118,7 +118,8 @@ async def test_task_is_auto_removed( empty_context: TaskContext, ): task_id = await lrt_api.start_task( - long_running_manager, + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, a_background_task.__name__, raise_when_finished=False, total_sleep=10 * TEST_CHECK_STALE_INTERVAL_S, @@ -162,7 +163,8 @@ async def test_checked_task_is_not_auto_removed( wait_multiplier: int, ): task_id = await lrt_api.start_task( - long_running_manager, + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, a_background_task.__name__, raise_when_finished=False, total_sleep=wait_multiplier * TEST_CHECK_STALE_INTERVAL_S, @@ -188,7 +190,8 @@ async def test_fire_and_forget_task_is_not_auto_removed( long_running_manager: BaseLongRunningManager, empty_context: TaskContext ): task_id = await lrt_api.start_task( - long_running_manager, + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, a_background_task.__name__, raise_when_finished=False, total_sleep=5 * TEST_CHECK_STALE_INTERVAL_S, @@ -214,7 +217,8 @@ async def test_get_result_of_unfinished_task_raises( long_running_manager: BaseLongRunningManager, empty_context: TaskContext ): task_id = await lrt_api.start_task( - long_running_manager, + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, a_background_task.__name__, raise_when_finished=False, total_sleep=5 * TEST_CHECK_STALE_INTERVAL_S, @@ -236,7 +240,8 @@ async def unique_task(progress: TaskProgress): TaskRegistry.register(unique_task) await lrt_api.start_task( - long_running_manager, + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, unique_task.__name__, unique=True, task_context=empty_context, @@ -245,7 +250,8 @@ async def unique_task(progress: TaskProgress): # ensure unique running task regardless of how many times it gets started with pytest.raises(TaskAlreadyRunningError) as exec_info: await lrt_api.start_task( - long_running_manager, + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, unique_task.__name__, unique=True, task_context=empty_context, @@ -265,7 +271,8 @@ async def not_unique_task(progress: TaskProgress): for _ in range(5): await lrt_api.start_task( - long_running_manager, + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, not_unique_task.__name__, task_context=empty_context, ) @@ -290,7 +297,8 @@ async def test_get_status( long_running_manager: BaseLongRunningManager, empty_context: TaskContext ): task_id = await lrt_api.start_task( - long_running_manager, + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, a_background_task.__name__, raise_when_finished=False, total_sleep=10, @@ -320,7 +328,10 @@ async def test_get_result( long_running_manager: BaseLongRunningManager, empty_context: TaskContext ): task_id = await lrt_api.start_task( - long_running_manager, fast_background_task.__name__, task_context=empty_context + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, + fast_background_task.__name__, + task_context=empty_context, ) async for attempt in AsyncRetrying(**_RETRY_PARAMS): @@ -350,7 +361,8 @@ async def test_get_result_finished_with_error( long_running_manager: BaseLongRunningManager, empty_context: TaskContext ): task_id = await lrt_api.start_task( - long_running_manager, + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, failing_background_task.__name__, task_context=empty_context, ) @@ -392,7 +404,8 @@ async def test_cancel_task_from_different_manager( ) task_id = await lrt_api.start_task( - manager_1, + manager_1.rpc_client, + manager_1.lrt_namespace, a_background_task.__name__, raise_when_finished=False, total_sleep=1, @@ -424,7 +437,8 @@ async def test_remove_task( long_running_manager: BaseLongRunningManager, empty_context: TaskContext ): task_id = await lrt_api.start_task( - long_running_manager, + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, a_background_task.__name__, raise_when_finished=False, total_sleep=10, @@ -450,7 +464,8 @@ async def test_remove_task_with_task_context( long_running_manager: BaseLongRunningManager, empty_context: TaskContext ): task_id = await lrt_api.start_task( - long_running_manager, + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, a_background_task.__name__, raise_when_finished=False, total_sleep=10, @@ -495,7 +510,8 @@ async def test__cancelled_tasks_worker_equivalent_of_cancellation_from_a_differe long_running_manager: BaseLongRunningManager, empty_context: TaskContext ): task_id = await lrt_api.start_task( - long_running_manager, + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, a_background_task.__name__, raise_when_finished=False, total_sleep=10, @@ -533,7 +549,8 @@ async def test_list_tasks( for _ in range(NUM_TASKS): task_ids.append( # noqa: PERF401 await lrt_api.start_task( - long_running_manager, + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, a_background_task.__name__, raise_when_finished=False, total_sleep=10, @@ -563,21 +580,24 @@ async def test_list_tasks_filtering( long_running_manager: BaseLongRunningManager, empty_context: TaskContext ): await lrt_api.start_task( - long_running_manager, + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, a_background_task.__name__, raise_when_finished=False, total_sleep=10, task_context=empty_context, ) await lrt_api.start_task( - long_running_manager, + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, a_background_task.__name__, raise_when_finished=False, total_sleep=10, task_context={"user_id": 213}, ) await lrt_api.start_task( - long_running_manager, + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, a_background_task.__name__, raise_when_finished=False, total_sleep=10, @@ -622,7 +642,8 @@ async def test_define_task_name( ): task_name = faker.name() task_id = await lrt_api.start_task( - long_running_manager, + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, a_background_task.__name__, raise_when_finished=False, total_sleep=10, @@ -636,4 +657,8 @@ async def test_start_not_registered_task( long_running_manager: BaseLongRunningManager, ): with pytest.raises(TaskNotRegisteredError): - await lrt_api.start_task(long_running_manager, "not_registered_task") + await lrt_api.start_task( + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, + "not_registered_task", + ) diff --git a/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_scheduler.py b/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_scheduler.py index 6e1c2a09acaa..53aaac235044 100644 --- a/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_scheduler.py @@ -116,7 +116,8 @@ async def _progress_callback( try: return await lrt_api.start_task( - long_running_manager, + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, _task_remove_service_containers.__name__, unique=True, node_uuid=node_uuid, @@ -181,7 +182,8 @@ async def _progress_callback( try: return await lrt_api.start_task( - long_running_manager, + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, _task_save_service_state.__name__, unique=True, node_uuid=node_uuid, @@ -228,7 +230,8 @@ async def _progress_callback( try: return await lrt_api.start_task( - long_running_manager, + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, _task_push_service_outputs.__name__, unique=True, node_uuid=node_uuid, @@ -270,7 +273,8 @@ async def _task_cleanup_service_docker_resources( try: return await lrt_api.start_task( - long_running_manager, + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, _task_cleanup_service_docker_resources.__name__, unique=True, node_uuid=node_uuid, diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_long_running_tasks.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_long_running_tasks.py index 91900a0a0e41..a8f7c3e69ccc 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_long_running_tasks.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/rest/containers_long_running_tasks.py @@ -50,7 +50,8 @@ async def pull_user_servcices_docker_images( try: return await lrt_api.start_task( - long_running_manager, + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, task_pull_user_servcices_docker_images.__name__, unique=True, ) @@ -87,7 +88,8 @@ async def create_service_containers_task( # pylint: disable=too-many-arguments try: return await lrt_api.start_task( - long_running_manager, + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, task_create_service_containers.__name__, unique=True, settings=settings, @@ -116,7 +118,8 @@ async def runs_docker_compose_down_task( try: return await lrt_api.start_task( - long_running_manager, + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, task_runs_docker_compose_down.__name__, unique=True, settings=settings, @@ -143,7 +146,8 @@ async def state_restore_task( try: return await lrt_api.start_task( - long_running_manager, + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, task_restore_state.__name__, unique=True, settings=settings, @@ -170,7 +174,8 @@ async def state_save_task( try: return await lrt_api.start_task( - long_running_manager, + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, task_save_state.__name__, unique=True, settings=settings, @@ -199,7 +204,8 @@ async def ports_inputs_pull_task( try: return await lrt_api.start_task( - long_running_manager, + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, task_ports_inputs_pull.__name__, unique=True, port_keys=port_keys, @@ -228,7 +234,8 @@ async def ports_outputs_pull_task( try: return await lrt_api.start_task( - long_running_manager, + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, task_ports_outputs_pull.__name__, unique=True, port_keys=port_keys, @@ -254,7 +261,8 @@ async def ports_outputs_push_task( try: return await lrt_api.start_task( - long_running_manager, + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, task_ports_outputs_push.__name__, unique=True, ) @@ -280,7 +288,8 @@ async def containers_restart_task( try: return await lrt_api.start_task( - long_running_manager, + long_running_manager.rpc_client, + long_running_manager.lrt_namespace, task_containers_restart.__name__, unique=True, settings=settings, diff --git a/services/web/server/src/simcore_service_webserver/tasks/_rest.py b/services/web/server/src/simcore_service_webserver/tasks/_rest.py index 288945d0ea6a..45c6457bc582 100644 --- a/services/web/server/src/simcore_service_webserver/tasks/_rest.py +++ b/services/web/server/src/simcore_service_webserver/tasks/_rest.py @@ -60,7 +60,7 @@ async def get_async_jobs(request: web.Request) -> web.Response: inprocess_long_running_manager = get_long_running_manager(request.app) inprocess_tracked_tasks = await lrt_api.list_tasks( inprocess_long_running_manager.rpc_client, - inprocess_long_running_manager, + inprocess_long_running_manager.lrt_namespace, inprocess_long_running_manager.get_task_context(request), ) From 2d7e5872de184414a76c92716085bb465646b893 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 20 Aug 2025 15:00:58 +0200 Subject: [PATCH 096/119] updated docstrings --- .../servicelib/aiohttp/long_running_tasks/_server.py | 12 +++++++----- .../servicelib/fastapi/long_running_tasks/_server.py | 12 +++++++----- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py index 495e03498f97..b5ae54cb07f9 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_server.py @@ -155,19 +155,21 @@ def setup( redis_settings: RedisSettings, rabbit_settings: RabbitSettings, lrt_namespace: LRTNamespace, - handler_check_decorator: Callable = _no_ops_decorator, - task_request_context_decorator: Callable = _no_task_context_decorator, stale_task_check_interval: datetime.timedelta = DEFAULT_STALE_TASK_CHECK_INTERVAL, stale_task_detect_timeout: datetime.timedelta = DEFAULT_STALE_TASK_DETECT_TIMEOUT, + handler_check_decorator: Callable = _no_ops_decorator, + task_request_context_decorator: Callable = _no_task_context_decorator, ) -> None: """ - `router_prefix` APIs are mounted on `/...`, this will change them to be mounted as `{router_prefix}/...` - - `stale_task_check_interval_s` interval at which the + - `redis_settings` settings for Redis connection + - `rabbit_settings` settings for RabbitMQ connection + - `lrt_namespace` namespace for the long-running tasks + - `stale_task_check_interval` interval at which the TaskManager checks for tasks which are no longer being actively monitored by a client - - `stale_task_detect_timeout_s` interval after which a - task is considered stale + - `stale_task_detect_timeout` interval after which atask is considered stale """ async def on_cleanup_ctx(app: web.Application) -> AsyncGenerator[None, None]: diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py index 81e2bb745f80..9cf4c526acee 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_server.py @@ -26,13 +26,15 @@ def setup( stale_task_detect_timeout: datetime.timedelta = DEFAULT_STALE_TASK_DETECT_TIMEOUT, ) -> None: """ - - `router_prefix` APIs are mounted on `/task/...`, this - will change them to be mounted as `{router_prefix}/task/...` - - `stale_task_check_interval_s` interval at which the + - `router_prefix` APIs are mounted on `/...`, this + will change them to be mounted as `{router_prefix}/...` + - `redis_settings` settings for Redis connection + - `rabbit_settings` settings for RabbitMQ connection + - `lrt_namespace` namespace for the long-running tasks + - `stale_task_check_interval` interval at which the TaskManager checks for tasks which are no longer being actively monitored by a client - - `stale_task_detect_timeout_s` interval after which a - task is considered stale + - `stale_task_detect_timeout` interval after which atask is considered stale """ async def on_startup() -> None: From 7b26893e66c357eb5b71db27e60b05caeaa1e605 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 20 Aug 2025 15:14:49 +0200 Subject: [PATCH 097/119] updated docstrings --- .../long_running_tasks/_lrt_client.py | 6 +++--- .../servicelib/long_running_tasks/lrt_api.py | 19 +++++++++++++++++-- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py b/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py index c64da0cd1ae4..db86c81b504f 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py @@ -23,7 +23,7 @@ _logger = logging.getLogger(__name__) _RPC_MAX_CANCELLATION_TIMEOUT: Final[PositiveInt] = int( - timedelta(minutes=60).total_seconds() + timedelta(hours=1).total_seconds() ) _RPC_TIMEOUT_SHORT_REQUESTS: Final[PositiveInt] = int( timedelta(seconds=20).total_seconds() @@ -137,9 +137,9 @@ async def remove_task( *, task_context: TaskContext, task_id: TaskId, - wait_for_removal: bool, reraise_errors: bool, - cancellation_timeout: timedelta | None = None, + wait_for_removal: bool, + cancellation_timeout: timedelta | None, ) -> None: timeout_s = ( _RPC_MAX_CANCELLATION_TIMEOUT diff --git a/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py b/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py index c7dc733fa89e..16da285ebfa0 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py +++ b/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py @@ -1,3 +1,4 @@ +from datetime import timedelta from typing import Any from ..rabbitmq._client_rpc import RabbitMQRPCClient @@ -104,13 +105,27 @@ async def remove_task( task_id: TaskId, *, wait_for_removal: bool, + cancellation_timeout: timedelta | None = None, ) -> None: - """cancels and removes the task""" + """cancels and removes a task + + Arguments: + wait_for_removal -- if True, then it will wait for the task to be removed + before returning otherwise returns immediately + + Keyword Arguments: + cancellation_timeout (default: {None}) -- if specified it's the amount of + time to wait before cancellation is timedout + if not specified and: + - wait_for_removal is True, it's set to _RPC_TIMEOUT_SHORT_REQUESTS + - wait_for_removal is False it's set to _RPC_MAX_CANCELLATION_TIMEOUT + """ await _lrt_client.remove_task( rabbitmq_rpc_client, lrt_namespace, task_id=task_id, task_context=task_context, - wait_for_removal=wait_for_removal, reraise_errors=True, + wait_for_removal=wait_for_removal, + cancellation_timeout=cancellation_timeout, ) From 8cfffb46e5d8fff4a4c7df05c5cc7869e6786031 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 20 Aug 2025 15:21:25 +0200 Subject: [PATCH 098/119] fixed namesapce --- .../src/servicelib/long_running_tasks/_redis_store.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/_redis_store.py b/packages/service-library/src/servicelib/long_running_tasks/_redis_store.py index f74855a444e0..bd2bc79e0dc0 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_redis_store.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_redis_store.py @@ -8,7 +8,7 @@ from ..redis._client import RedisClientSDK from ..redis._utils import handle_redis_returns_union_types from ..utils import limited_gather -from .models import TaskContext, TaskData, TaskId +from .models import LRTNamespace, TaskContext, TaskData, TaskId _STORE_TYPE_TASK_DATA: Final[str] = "TD" _STORE_TYPE_CANCELLED_TASKS: Final[str] = "CT" @@ -26,9 +26,9 @@ def _from_redis(data: dict[str, str]) -> dict[str, Any]: class RedisStore: - def __init__(self, redis_settings: RedisSettings, namespace: str): + def __init__(self, redis_settings: RedisSettings, namespace: LRTNamespace): self.redis_settings = redis_settings - self.namespace = namespace.upper() + self.namespace: LRTNamespace = namespace.upper() self._client: RedisClientSDK | None = None From d1fad6f239d7e39755bdf8ff9a3007288e6fe1be Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 20 Aug 2025 15:38:58 +0200 Subject: [PATCH 099/119] fixed broken sidecar --- .../modules/long_running_tasks.py | 70 ++++++++++--------- 1 file changed, 37 insertions(+), 33 deletions(-) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py index 7e8a4d5fe916..94fb33ea5c7a 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py @@ -639,41 +639,45 @@ def setup_long_running_tasks(app: FastAPI) -> None: lrt_namespace=f"{APP_NAME}-{app_settings.DY_SIDECAR_RUN_ID}", ) - shared_store: SharedStore = app.state.shared_store - mounted_volumes: MountedVolumes = app.state.mounted_volumes - outputs_manager: OutputsManager = app.state.outputs_manager - - context_app_store: dict[str, Any] = { - "app": app, - "shared_store": shared_store, - } - context_app_store_volumes: dict[str, Any] = { - "app": app, - "shared_store": shared_store, - "mounted_volumes": mounted_volumes, - } - context_app_volumes: dict[str, Any] = { - "app": app, - "mounted_volumes": mounted_volumes, - } - context_app_outputs: dict[str, Any] = { - "app": app, - "outputs_manager": outputs_manager, - } - - task_context: dict[TaskProtocol, dict[str, Any]] = { - task_pull_user_servcices_docker_images: context_app_store, - task_create_service_containers: context_app_store, - task_runs_docker_compose_down: context_app_store_volumes, - task_restore_state: context_app_volumes, - task_save_state: context_app_volumes, - task_ports_inputs_pull: context_app_volumes, - task_ports_outputs_pull: context_app_volumes, - task_ports_outputs_push: context_app_outputs, - task_containers_restart: context_app_store, - } + task_context: dict[TaskProtocol, dict[str, Any]] = {} async def on_startup() -> None: + shared_store: SharedStore = app.state.shared_store + mounted_volumes: MountedVolumes = app.state.mounted_volumes + outputs_manager: OutputsManager = app.state.outputs_manager + + context_app_store: dict[str, Any] = { + "app": app, + "shared_store": shared_store, + } + context_app_store_volumes: dict[str, Any] = { + "app": app, + "shared_store": shared_store, + "mounted_volumes": mounted_volumes, + } + context_app_volumes: dict[str, Any] = { + "app": app, + "mounted_volumes": mounted_volumes, + } + context_app_outputs: dict[str, Any] = { + "app": app, + "outputs_manager": outputs_manager, + } + + task_context.update( + { + task_pull_user_servcices_docker_images: context_app_store, + task_create_service_containers: context_app_store, + task_runs_docker_compose_down: context_app_store_volumes, + task_restore_state: context_app_volumes, + task_save_state: context_app_volumes, + task_ports_inputs_pull: context_app_volumes, + task_ports_outputs_pull: context_app_volumes, + task_ports_outputs_push: context_app_outputs, + task_containers_restart: context_app_store, + } + ) + for handler, context in task_context.items(): TaskRegistry.register(handler, **context) From 4dd8535c8bf94b5934b9a025313961befd764e40 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 20 Aug 2025 16:17:55 +0200 Subject: [PATCH 100/119] avoid tests form hanging --- packages/service-library/tests/long_running_tasks/conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/service-library/tests/long_running_tasks/conftest.py b/packages/service-library/tests/long_running_tasks/conftest.py index ef3339786f27..2d1d6d2d6377 100644 --- a/packages/service-library/tests/long_running_tasks/conftest.py +++ b/packages/service-library/tests/long_running_tasks/conftest.py @@ -2,6 +2,7 @@ # pylint: disable=redefined-outer-name # pylint: disable=unused-argument +import asyncio import logging from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable from datetime import timedelta @@ -61,7 +62,7 @@ async def _( for manager in managers: with log_catch(_logger, reraise=False): - await manager.teardown() + await asyncio.wait_for(manager.teardown(), timeout=5) @pytest.fixture From 0d3208ee4f2155ec23eb66e320e5f9d624bca89c Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 21 Aug 2025 11:46:26 +0200 Subject: [PATCH 101/119] renamed --- .../long_running_tasks/_redis_store.py | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/_redis_store.py b/packages/service-library/src/servicelib/long_running_tasks/_redis_store.py index bd2bc79e0dc0..acf70bb87e48 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_redis_store.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_redis_store.py @@ -15,13 +15,11 @@ _LIST_CONCURRENCY: Final[int] = 2 -def _to_redis(data: dict[str, Any]) -> dict[str, str]: - """convert values to redis compatible data""" +def _to_redis_hash_mapping(data: dict[str, Any]) -> dict[str, str]: return {k: json_dumps(v) for k, v in data.items()} -def _from_redis(data: dict[str, str]) -> dict[str, Any]: - """converrt back from redis compatible data to python types""" +def _load_from_redis_hash(data: dict[str, str]) -> dict[str, Any]: return {k: json_loads(v) for k, v in data.items()} @@ -51,7 +49,7 @@ def _redis(self) -> aioredis.Redis: def _get_redis_key_task_data_match(self) -> str: return f"{self.namespace}:{_STORE_TYPE_TASK_DATA}*" - def _get_redis_key_task_data_hash(self, task_id: TaskId) -> str: + def _get_redis_task_data_key(self, task_id: TaskId) -> str: return f"{self.namespace}:{_STORE_TYPE_TASK_DATA}:{task_id}" def _get_key_to_remove(self) -> str: @@ -62,11 +60,11 @@ def _get_key_to_remove(self) -> str: async def get_task_data(self, task_id: TaskId) -> TaskData | None: result: dict[str, Any] = await handle_redis_returns_union_types( self._redis.hgetall( - self._get_redis_key_task_data_hash(task_id), + self._get_redis_task_data_key(task_id), ) ) return ( - TypeAdapter(TaskData).validate_python(_from_redis(result)) + TypeAdapter(TaskData).validate_python(_load_from_redis_hash(result)) if result and len(result) else None ) @@ -74,8 +72,8 @@ async def get_task_data(self, task_id: TaskId) -> TaskData | None: async def add_task_data(self, task_id: TaskId, value: TaskData) -> None: await handle_redis_returns_union_types( self._redis.hset( - self._get_redis_key_task_data_hash(task_id), - mapping=_to_redis(value.model_dump()), + self._get_redis_task_data_key(task_id), + mapping=_to_redis_hash_mapping(value.model_dump()), ) ) @@ -87,8 +85,8 @@ async def update_task_data( ) -> None: await handle_redis_returns_union_types( self._redis.hset( - self._get_redis_key_task_data_hash(task_id), - mapping=_to_redis(updates), + self._get_redis_task_data_key(task_id), + mapping=_to_redis_hash_mapping(updates), ) ) @@ -107,14 +105,14 @@ async def list_tasks_data(self) -> list[TaskData]: ) return [ - TypeAdapter(TaskData).validate_python(_from_redis(item)) + TypeAdapter(TaskData).validate_python(_load_from_redis_hash(item)) for item in result if item ] async def delete_task_data(self, task_id: TaskId) -> None: await handle_redis_returns_union_types( - self._redis.delete(self._get_redis_key_task_data_hash(task_id)) + self._redis.delete(self._get_redis_task_data_key(task_id)) ) # to cancel From 4bf6d6379c0fd3bc44b8fb3ad0fa35b16b064f21 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 21 Aug 2025 12:50:15 +0200 Subject: [PATCH 102/119] renamed --- .../service-library/src/servicelib/long_running_tasks/task.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index 69d6f71f3fe5..36a74bb03ec6 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -470,8 +470,8 @@ async def remove_task( raise TryAgain def _get_task_id(self, task_name: str, *, is_unique: bool) -> TaskId: - id_part = "unique" if is_unique else f"{uuid4()}" - return f"{self.lrt_namespace}.{task_name}.{id_part}" + suffix = "unique" if is_unique else f"{uuid4()}" + return f"{self.lrt_namespace}.{task_name}.{suffix}" async def _update_progress( self, From 407fa83eeaf540d57bf65b6420bdaa65f9c97ce2 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 21 Aug 2025 13:17:47 +0200 Subject: [PATCH 103/119] fixed interfaces --- .../aiohttp/long_running_tasks/_routes.py | 25 ++++++++++++--- .../fastapi/long_running_tasks/_routes.py | 28 ++++++++++------ .../long_running_tasks/_lrt_client.py | 2 -- .../long_running_tasks/_lrt_server.py | 17 +++++----- .../servicelib/long_running_tasks/lrt_api.py | 1 - .../src/servicelib/long_running_tasks/task.py | 32 ++++++++----------- .../test_long_running_tasks_task.py | 7 ---- 7 files changed, 61 insertions(+), 51 deletions(-) diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py index 1443b7090db4..3a041491672b 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py @@ -1,12 +1,16 @@ -from typing import Any +from typing import Annotated, Any from aiohttp import web -from pydantic import BaseModel +from models_library.rest_base import RequestParameters +from pydantic import BaseModel, Field from ...aiohttp import status from ...long_running_tasks import lrt_api from ...long_running_tasks.models import TaskGet, TaskId -from ..requests_validation import parse_request_path_parameters_as +from ..requests_validation import ( + parse_request_path_parameters_as, + parse_request_query_parameters_as, +) from ..rest_responses import create_data_response from ._manager import get_long_running_manager @@ -65,9 +69,22 @@ async def get_task_result(request: web.Request) -> web.Response | Any: ) +class MyRequestQueryParams(RequestParameters): + wait_for_removal: Annotated[ + bool, + Field( + description=( + "when True waits for the task to be removed " + "completly instead of returning immediately" + ) + ), + ] = True + + @routes.delete("/{task_id}", name="remove_task") async def remove_task(request: web.Request) -> web.Response: path_params = parse_request_path_parameters_as(_PathParam, request) + query_params = parse_request_query_parameters_as(MyRequestQueryParams, request) long_running_manager = get_long_running_manager(request.app) await lrt_api.remove_task( @@ -75,6 +92,6 @@ async def remove_task(request: web.Request) -> web.Response: long_running_manager.lrt_namespace, long_running_manager.get_task_context(request), path_params.task_id, - wait_for_removal=False, # frontend does not care about waiting for this + wait_for_removal=query_params.wait_for_removal, ) return web.json_response(status=status.HTTP_204_NO_CONTENT) diff --git a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py index bbbba4abcd53..bf347ba0d0aa 100644 --- a/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py +++ b/packages/service-library/src/servicelib/fastapi/long_running_tasks/_routes.py @@ -1,6 +1,6 @@ from typing import Annotated, Any -from fastapi import APIRouter, Depends, Request, status +from fastapi import APIRouter, Depends, Query, Request, status from ...long_running_tasks import lrt_api from ...long_running_tasks.models import TaskGet, TaskId, TaskResult, TaskStatus @@ -30,7 +30,7 @@ async def list_tasks( for t in await lrt_api.list_tasks( long_running_manager.rpc_client, long_running_manager.lrt_namespace, - task_context={}, + long_running_manager.get_task_context(request), ) ] @@ -45,16 +45,16 @@ async def list_tasks( @cancel_on_disconnect async def get_task_status( request: Request, - task_id: TaskId, long_running_manager: Annotated[ FastAPILongRunningManager, Depends(get_long_running_manager) ], + task_id: TaskId, ) -> TaskStatus: assert request # nosec return await lrt_api.get_task_status( long_running_manager.rpc_client, long_running_manager.lrt_namespace, - task_context={}, + long_running_manager.get_task_context(request), task_id=task_id, ) @@ -71,16 +71,16 @@ async def get_task_status( @cancel_on_disconnect async def get_task_result( request: Request, - task_id: TaskId, long_running_manager: Annotated[ FastAPILongRunningManager, Depends(get_long_running_manager) ], + task_id: TaskId, ) -> TaskResult | Any: assert request # nosec return await lrt_api.get_task_result( long_running_manager.rpc_client, long_running_manager.lrt_namespace, - task_context={}, + long_running_manager.get_task_context(request), task_id=task_id, ) @@ -97,16 +97,26 @@ async def get_task_result( @cancel_on_disconnect async def remove_task( request: Request, - task_id: TaskId, long_running_manager: Annotated[ FastAPILongRunningManager, Depends(get_long_running_manager) ], + task_id: TaskId, + *, + wait_for_removal: Annotated[ + bool, + Query( + description=( + "when True waits for the task to be removed " + "completly instead of returning immediately" + ), + ), + ] = True, ) -> None: assert request # nosec await lrt_api.remove_task( long_running_manager.rpc_client, long_running_manager.lrt_namespace, - task_context={}, + long_running_manager.get_task_context(request), task_id=task_id, - wait_for_removal=True, # only used by internal services, they will wait as before + wait_for_removal=wait_for_removal, ) diff --git a/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py b/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py index db86c81b504f..ccbf53ee6156 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py @@ -137,7 +137,6 @@ async def remove_task( *, task_context: TaskContext, task_id: TaskId, - reraise_errors: bool, wait_for_removal: bool, cancellation_timeout: timedelta | None, ) -> None: @@ -158,7 +157,6 @@ async def remove_task( task_context=task_context, task_id=task_id, wait_for_removal=wait_for_removal, - reraise_errors=reraise_errors, timeout_s=timeout_s, ) assert result is None # nosec diff --git a/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py b/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py index 58f726c519a7..7c7c0628fa01 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py @@ -1,10 +1,11 @@ import logging +from contextlib import suppress from typing import TYPE_CHECKING, Any from ..logging_errors import create_troubleshootting_log_kwargs from ..rabbitmq import RPCRouter from ._serialization import string_to_object -from .errors import BaseLongRunningError +from .errors import BaseLongRunningError, TaskNotFoundError from .models import ( ErrorResponse, RegisteredTaskName, @@ -110,12 +111,12 @@ async def get_task_result( raise ValueError(msg) finally: # Ensure the task is removed regardless of the result - await long_running_manager.tasks_manager.remove_task( - task_id, - with_task_context=task_context, - wait_for_removal=True, - reraise_errors=False, - ) + with suppress(TaskNotFoundError): + await long_running_manager.tasks_manager.remove_task( + task_id, + with_task_context=task_context, + wait_for_removal=True, + ) @router.expose(reraise_if_error_type=(BaseLongRunningError,)) @@ -125,11 +126,9 @@ async def remove_task( task_context: TaskContext, task_id: TaskId, wait_for_removal: bool, - reraise_errors: bool, ) -> None: await long_running_manager.tasks_manager.remove_task( task_id, with_task_context=task_context, wait_for_removal=wait_for_removal, - reraise_errors=reraise_errors, ) diff --git a/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py b/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py index 16da285ebfa0..b9be616b26db 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py +++ b/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py @@ -125,7 +125,6 @@ async def remove_task( lrt_namespace, task_id=task_id, task_context=task_context, - reraise_errors=True, wait_for_removal=wait_for_removal, cancellation_timeout=cancellation_timeout, ) diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index 36a74bb03ec6..98e92244bfd3 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -202,13 +202,13 @@ async def setup(self) -> None: async def teardown(self) -> None: # ensure all created tasks are cancelled for tracked_task in await self._tasks_data.list_tasks_data(): - await self.remove_task( - tracked_task.task_id, - tracked_task.task_context, - # when closing we do not care about pending errors - wait_for_removal=True, - reraise_errors=False, - ) + # when closing we do not care about pending errors + with suppress(TaskNotFoundError): + await self.remove_task( + tracked_task.task_id, + tracked_task.task_context, + wait_for_removal=True, + ) for task in self._created_tasks.values(): _logger.warning( @@ -278,10 +278,7 @@ async def _stale_tasks_monitor(self) -> None: ).model_dump_json(), ) await self.remove_task( - task_id, - with_task_context=task_context, - wait_for_removal=True, - reraise_errors=False, + task_id, with_task_context=task_context, wait_for_removal=True ) async def _cancelled_tasks_removal(self) -> None: @@ -438,15 +435,12 @@ async def remove_task( with_task_context: TaskContext, *, wait_for_removal: bool, - reraise_errors: bool = True, ) -> None: - """cancels and removes task""" - try: - tracked_task = await self._get_tracked_task(task_id, with_task_context) - except TaskNotFoundError: - if reraise_errors: - raise - return + """ + cancels and removes task + raises TaskNotFoundError if the task cannot be found + """ + tracked_task = await self._get_tracked_task(task_id, with_task_context) await self._tasks_data.mark_task_for_removal( tracked_task.task_id, tracked_task.task_context diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py index b0a9ac46bc3c..401f8dd88b0e 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py @@ -498,13 +498,6 @@ async def test_remove_unknown_task( "invalid_id", with_task_context=empty_context, wait_for_removal=True ) - await long_running_manager.tasks_manager.remove_task( - "invalid_id", - with_task_context=empty_context, - wait_for_removal=True, - reraise_errors=False, - ) - async def test__cancelled_tasks_worker_equivalent_of_cancellation_from_a_different_process( long_running_manager: BaseLongRunningManager, empty_context: TaskContext From 3095ba6cb3baa47670fe8690336f1ae6b1076f1d Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 21 Aug 2025 13:18:21 +0200 Subject: [PATCH 104/119] rename --- .../src/servicelib/aiohttp/long_running_tasks/_routes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py index 3a041491672b..9e8ac646c330 100644 --- a/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py +++ b/packages/service-library/src/servicelib/aiohttp/long_running_tasks/_routes.py @@ -69,7 +69,7 @@ async def get_task_result(request: web.Request) -> web.Response | Any: ) -class MyRequestQueryParams(RequestParameters): +class _RemoveTaskQueryParams(RequestParameters): wait_for_removal: Annotated[ bool, Field( @@ -84,7 +84,7 @@ class MyRequestQueryParams(RequestParameters): @routes.delete("/{task_id}", name="remove_task") async def remove_task(request: web.Request) -> web.Response: path_params = parse_request_path_parameters_as(_PathParam, request) - query_params = parse_request_query_parameters_as(MyRequestQueryParams, request) + query_params = parse_request_query_parameters_as(_RemoveTaskQueryParams, request) long_running_manager = get_long_running_manager(request.app) await lrt_api.remove_task( From 67471338fc24efc25b5332e79511e60e6e1fbe17 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Thu, 21 Aug 2025 13:41:35 +0200 Subject: [PATCH 105/119] reverted changes --- .../long_running_tasks/_lrt_client.py | 7 ++-- .../servicelib/long_running_tasks/lrt_api.py | 11 +------ .../test_long_running_tasks.py | 16 +++------- .../test_long_running_tasks_client.py | 26 ++++----------- ...st_long_running_tasks_with_task_context.py | 32 ++++--------------- 5 files changed, 19 insertions(+), 73 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py b/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py index ccbf53ee6156..eada93610023 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py @@ -22,9 +22,6 @@ _logger = logging.getLogger(__name__) -_RPC_MAX_CANCELLATION_TIMEOUT: Final[PositiveInt] = int( - timedelta(hours=1).total_seconds() -) _RPC_TIMEOUT_SHORT_REQUESTS: Final[PositiveInt] = int( timedelta(seconds=20).total_seconds() ) @@ -141,14 +138,14 @@ async def remove_task( cancellation_timeout: timedelta | None, ) -> None: timeout_s = ( - _RPC_MAX_CANCELLATION_TIMEOUT + None if cancellation_timeout is None else int(cancellation_timeout.total_seconds()) ) # NOTE: task always gets cancelled even if not waiting for it # request will return immediatlye, no need to wait so much - if not wait_for_removal: + if wait_for_removal is False: timeout_s = _RPC_TIMEOUT_SHORT_REQUESTS result = await rabbitmq_rpc_client.request( diff --git a/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py b/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py index b9be616b26db..e066b0cff2d6 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py +++ b/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py @@ -109,16 +109,7 @@ async def remove_task( ) -> None: """cancels and removes a task - Arguments: - wait_for_removal -- if True, then it will wait for the task to be removed - before returning otherwise returns immediately - - Keyword Arguments: - cancellation_timeout (default: {None}) -- if specified it's the amount of - time to wait before cancellation is timedout - if not specified and: - - wait_for_removal is True, it's set to _RPC_TIMEOUT_SHORT_REQUESTS - - wait_for_removal is False it's set to _RPC_MAX_CANCELLATION_TIMEOUT + When `wait_for_removal` is True, `cancellationt_timeout` is set to _RPC_TIMEOUT_SHORT_REQUESTS """ await _lrt_client.remove_task( rabbitmq_rpc_client, diff --git a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py index 5683385da079..49604fd3a15e 100644 --- a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py +++ b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks.py @@ -194,18 +194,10 @@ async def test_cancel_task( assert not data assert not error - # it should eventually go away, so no status - # does not wait for removal any longer - async for attempt in AsyncRetrying( - wait=wait_fixed(0.1), - stop=stop_after_delay(5), - reraise=True, - retry=retry_if_exception_type(AssertionError), - ): - with attempt: - status_url = client.app.router["get_task_status"].url_for(task_id=task_id) - result = await client.get(f"{status_url}") - await assert_status(result, status.HTTP_404_NOT_FOUND) + # it should be gone, so no status + status_url = client.app.router["get_task_status"].url_for(task_id=task_id) + result = await client.get(f"{status_url}") + await assert_status(result, status.HTTP_404_NOT_FOUND) # and also no results result_url = client.app.router["get_task_result"].url_for(task_id=task_id) result = await client.get(f"{result_url}") diff --git a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_client.py b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_client.py index 8ee6c6a082aa..9e8c9204acef 100644 --- a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_client.py +++ b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_client.py @@ -17,12 +17,6 @@ from servicelib.aiohttp.rest_middlewares import append_rest_middlewares from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings -from tenacity import ( - AsyncRetrying, - retry_if_exception_type, - stop_after_delay, - wait_fixed, -) from yarl import URL pytest_simcore_core_services_selection = [ @@ -118,20 +112,12 @@ async def test_long_running_task_request_timeout( ): print(f"<-- received {task=}") - # does not wait for removal any longer - async for attempt in AsyncRetrying( - wait=wait_fixed(0.1), - stop=stop_after_delay(5), - reraise=True, - retry=retry_if_exception_type(AssertionError), - ): - with attempt: - # check the task was properly aborted by the client - list_url = client.app.router["list_tasks"].url_for() - result = await client.get(f"{list_url}") - data, error = await assert_status(result, status.HTTP_200_OK) - assert not error - assert data == [] + # check the task was properly aborted by the client + list_url = client.app.router["list_tasks"].url_for() + result = await client.get(f"{list_url}") + data, error = await assert_status(result, status.HTTP_200_OK) + assert not error + assert data == [] async def test_long_running_task_request_error( diff --git a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py index ffed3531911e..cef4a845ab8d 100644 --- a/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py +++ b/packages/service-library/tests/aiohttp/long_running_tasks/test_long_running_tasks_with_task_context.py @@ -29,10 +29,6 @@ from servicelib.long_running_tasks.task import TaskContext from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings -from tenacity.asyncio import AsyncRetrying -from tenacity.retry import retry_if_exception_type -from tenacity.stop import stop_after_delay -from tenacity.wait import wait_fixed pytest_simcore_core_services_selection = [ "rabbit", @@ -180,31 +176,15 @@ async def test_cancel_task( task_id=task_id ) # calling cancel without task context should find nothing - # no longer waits for removal to end - async for attempt in AsyncRetrying( - wait=wait_fixed(0.1), - stop=stop_after_delay(5), - reraise=True, - retry=retry_if_exception_type(AssertionError), - ): - with attempt: - resp = await client_with_task_context.delete(f"{cancel_url}") - await assert_status(resp, status.HTTP_404_NOT_FOUND) + resp = await client_with_task_context.delete(f"{cancel_url}") + await assert_status(resp, status.HTTP_404_NOT_FOUND) # calling with context should find and delete the task resp = await client_with_task_context.delete( f"{cancel_url.update_query(task_context)}" ) await assert_status(resp, status.HTTP_204_NO_CONTENT) # calling with context a second time should find nothing - # no longer waits for removal to end - async for attempt in AsyncRetrying( - wait=wait_fixed(0.1), - stop=stop_after_delay(5), - reraise=True, - retry=retry_if_exception_type(AssertionError), - ): - with attempt: - resp = await client_with_task_context.delete( - f"{cancel_url.update_query(task_context)}" - ) - await assert_status(resp, status.HTTP_404_NOT_FOUND) + resp = await client_with_task_context.delete( + f"{cancel_url.update_query(task_context)}" + ) + await assert_status(resp, status.HTTP_404_NOT_FOUND) From 1add852bff0ec1429c630f084ebc3373d1a1d980 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 22 Aug 2025 05:47:48 +0200 Subject: [PATCH 106/119] remove unused --- .../simcore_service_storage/core/application.py | 3 +-- .../modules/long_running_tasks.py | 16 ---------------- 2 files changed, 1 insertion(+), 18 deletions(-) delete mode 100644 services/storage/src/simcore_service_storage/modules/long_running_tasks.py diff --git a/services/storage/src/simcore_service_storage/core/application.py b/services/storage/src/simcore_service_storage/core/application.py index ebe84c5643ff..305e13bb3eae 100644 --- a/services/storage/src/simcore_service_storage/core/application.py +++ b/services/storage/src/simcore_service_storage/core/application.py @@ -38,7 +38,6 @@ from ..exceptions.handlers import set_exception_handlers from ..modules.celery import setup_task_manager from ..modules.db import setup_db -from ..modules.long_running_tasks import setup_rest_api_long_running_tasks_for_uploads from ..modules.rabbitmq import setup as setup_rabbitmq from ..modules.redis import setup as setup_redis from ..modules.s3 import setup_s3 @@ -77,7 +76,7 @@ def create_app(settings: ApplicationSettings) -> FastAPI: # noqa: C901 setup_task_manager(app, celery_settings=settings.STORAGE_CELERY) setup_rpc_routes(app) - setup_rest_api_long_running_tasks_for_uploads(app) + setup_rest_api_routes(app, API_VTAG) set_exception_handlers(app) diff --git a/services/storage/src/simcore_service_storage/modules/long_running_tasks.py b/services/storage/src/simcore_service_storage/modules/long_running_tasks.py deleted file mode 100644 index 27085b49d9f9..000000000000 --- a/services/storage/src/simcore_service_storage/modules/long_running_tasks.py +++ /dev/null @@ -1,16 +0,0 @@ -from fastapi import FastAPI -from servicelib.fastapi.long_running_tasks._server import setup - -from .._meta import API_VTAG, APP_NAME -from ..core.settings import get_application_settings - - -def setup_rest_api_long_running_tasks_for_uploads(app: FastAPI) -> None: - settings = get_application_settings(app) - setup( - app, - router_prefix=f"/{API_VTAG}/futures", - redis_settings=settings.STORAGE_REDIS, - rabbit_settings=settings.STORAGE_RABBITMQ, - lrt_namespace=APP_NAME, - ) From 501a672409a3be51622982a1f8a3df238059b455 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 22 Aug 2025 06:22:33 +0200 Subject: [PATCH 107/119] changed how error is transferred --- .../long_running_tasks/_lrt_client.py | 42 ++++++------------- .../long_running_tasks/_lrt_server.py | 37 +++------------- .../servicelib/long_running_tasks/errors.py | 7 ++++ .../servicelib/long_running_tasks/models.py | 9 +--- .../src/servicelib/long_running_tasks/task.py | 35 +++++++++------- .../test_long_running_tasks_task.py | 4 +- 6 files changed, 50 insertions(+), 84 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py b/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py index eada93610023..b8a1c8afff60 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py @@ -5,13 +5,12 @@ from models_library.rabbitmq_basic_types import RPCMethodName from pydantic import PositiveInt, TypeAdapter -from ..logging_errors import create_troubleshootting_log_kwargs from ..logging_utils import log_decorator from ..rabbitmq._client_rpc import RabbitMQRPCClient from ._rabbit_namespace import get_rabbit_namespace from ._serialization import string_to_object +from .errors import RPCTransferrableTaskError from .models import ( - ErrorResponse, LRTNamespace, RegisteredTaskName, TaskBase, @@ -97,34 +96,19 @@ async def get_task_result( task_context: TaskContext, task_id: TaskId, ) -> Any: - serialized_result = await rabbitmq_rpc_client.request( - get_rabbit_namespace(namespace), - TypeAdapter(RPCMethodName).validate_python("get_task_result"), - task_context=task_context, - task_id=task_id, - timeout_s=_RPC_TIMEOUT_SHORT_REQUESTS, - ) - assert isinstance(serialized_result, ErrorResponse | str) # nosec - if isinstance(serialized_result, ErrorResponse): - error = string_to_object(serialized_result.str_error_object) - _logger.info( - **create_troubleshootting_log_kwargs( - f"Task '{task_id}' raised the following error:\n{serialized_result.str_traceback}", - error=error, - error_context={ - "task_id": task_id, - "namespace": namespace, - "task_context": task_context, - }, - tip=( - f"The caller of this function should handle the exception. " - f"To figure out where it was running check {namespace=}" - ), - ) + try: + serialized_result = await rabbitmq_rpc_client.request( + get_rabbit_namespace(namespace), + TypeAdapter(RPCMethodName).validate_python("get_task_result"), + task_context=task_context, + task_id=task_id, + timeout_s=_RPC_TIMEOUT_SHORT_REQUESTS, ) - raise error - - return string_to_object(serialized_result) + assert isinstance(serialized_result, str) # nosec + return string_to_object(serialized_result) + except RPCTransferrableTaskError as e: + decoded_error = string_to_object(f"{e}") + raise decoded_error from None @log_decorator(_logger, level=logging.DEBUG) diff --git a/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py b/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py index 7c7c0628fa01..d63c5d370ce4 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py @@ -2,12 +2,9 @@ from contextlib import suppress from typing import TYPE_CHECKING, Any -from ..logging_errors import create_troubleshootting_log_kwargs from ..rabbitmq import RPCRouter -from ._serialization import string_to_object -from .errors import BaseLongRunningError, TaskNotFoundError +from .errors import BaseLongRunningError, RPCTransferrableTaskError, TaskNotFoundError from .models import ( - ErrorResponse, RegisteredTaskName, TaskBase, TaskContext, @@ -66,43 +63,19 @@ async def get_task_status( ) -@router.expose(reraise_if_error_type=(BaseLongRunningError,)) +@router.expose(reraise_if_error_type=(BaseLongRunningError, RPCTransferrableTaskError)) async def get_task_result( long_running_manager: "BaseLongRunningManager", *, task_context: TaskContext, task_id: TaskId, -) -> ErrorResponse | str: +) -> str: try: result_field = await long_running_manager.tasks_manager.get_task_result( task_id, with_task_context=task_context ) - if result_field.error_response is not None: - task_raised_error_traceback = result_field.error_response.str_traceback - task_raised_error = string_to_object( - result_field.error_response.str_error_object - ) - _logger.info( - **create_troubleshootting_log_kwargs( - f"Execution of {task_id=} finished with error:\n{task_raised_error_traceback}", - error=task_raised_error, - error_context={ - "task_id": task_id, - "task_context": task_context, - "namespace": long_running_manager.lrt_namespace, - }, - tip="This exception is logged for debugging purposes, the client side will handle it", - ) - ) - allowed_errors = ( - await long_running_manager.tasks_manager.get_allowed_errors( - task_id, with_task_context=task_context - ) - ) - if type(task_raised_error) in allowed_errors: - return result_field.error_response - - raise task_raised_error + if result_field.str_error is not None: + raise RPCTransferrableTaskError(result_field.str_error) if result_field.str_result is not None: return result_field.str_result diff --git a/packages/service-library/src/servicelib/long_running_tasks/errors.py b/packages/service-library/src/servicelib/long_running_tasks/errors.py index 01aad0c81569..c95a228720b7 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/errors.py +++ b/packages/service-library/src/servicelib/long_running_tasks/errors.py @@ -44,3 +44,10 @@ class GenericClientError(BaseLongRunningError): msg_template: str = ( "Unexpected error while '{action}' for '{task_id}': status={status} body={body}" ) + + +class RPCTransferrableTaskError(Exception): + """ + The message contains the task's exception serialized as string. + Decode it and raise to obtain the task's original exception. + """ diff --git a/packages/service-library/src/servicelib/long_running_tasks/models.py b/packages/service-library/src/servicelib/long_running_tasks/models.py index 1d8e1cfc53bb..7a99f9a5cf34 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/models.py +++ b/packages/service-library/src/servicelib/long_running_tasks/models.py @@ -33,18 +33,13 @@ RegisteredTaskName: TypeAlias = str -class ErrorResponse(BaseModel): - str_error_object: str - str_traceback: str - - class ResultField(BaseModel): str_result: str | None = None - error_response: ErrorResponse | None = None + str_error: str | None = None @model_validator(mode="after") def validate_mutually_exclusive(self) -> "ResultField": - if self.str_result is not None and self.error_response is not None: + if self.str_result is not None and self.str_error is not None: msg = "Cannot set both 'result' and 'error' - they are mutually exclusive" raise ValueError(msg) return self diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index 98e92244bfd3..efd83f736574 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -3,7 +3,6 @@ import functools import inspect import logging -import traceback import urllib.parse from contextlib import suppress from typing import Any, ClassVar, Final, Protocol, TypeAlias @@ -22,6 +21,7 @@ ) from ..background_task import create_periodic_task +from ..logging_errors import create_troubleshootting_log_kwargs from ..redis import RedisClientSDK, exclusive from ._redis_store import RedisStore from ._serialization import object_to_string @@ -33,7 +33,6 @@ TaskNotRegisteredError, ) from .models import ( - ErrorResponse, LRTNamespace, RegisteredTaskName, ResultField, @@ -321,22 +320,30 @@ async def _tasks_monitor(self) -> None: except asyncio.InvalidStateError: # task was not completed try again next time and see if it is done continue - except asyncio.CancelledError as e: + except asyncio.CancelledError: result_field = ResultField( - error_response=ErrorResponse( - str_error_object=object_to_string( - TaskCancelledError(task_id=task_id) - ), - str_traceback="".join(traceback.format_tb(e.__traceback__)), - ) + str_error=object_to_string(TaskCancelledError(task_id=task_id)) ) except Exception as e: # pylint:disable=broad-except - result_field = ResultField( - error_response=ErrorResponse( - str_error_object=object_to_string(e), - str_traceback="".join(traceback.format_tb(e.__traceback__)), - ) + allowed_errors = TaskRegistry.get_allowed_errors( + task_data.registered_task_name ) + if type(e) not in allowed_errors: + _logger.exception( + **create_troubleshootting_log_kwargs( + ( + f"Execution of {task_id=} finished with unexpected error, " + f"only the following are {allowed_errors=} are permitted" + ), + error=e, + error_context={ + "task_id": task_id, + "task_data": task_data, + "namespace": self.lrt_namespace, + }, + ), + ) + result_field = ResultField(str_error=object_to_string(e)) # update and store in Redis updates = {"is_done": is_done, "result_field": task_data.result_field} diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py index 401f8dd88b0e..c20ae746bc03 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py @@ -378,8 +378,8 @@ async def test_get_result_finished_with_error( result = await long_running_manager.tasks_manager.get_task_result( task_id, with_task_context=empty_context ) - assert result.error_response is not None # nosec - error = string_to_object(result.error_response.str_error_object) + assert result.str_error is not None # nosec + error = string_to_object(result.str_error) with pytest.raises(_TetingError, match="failing asap"): raise error From 01be053fee1217e3964f98ab03137700f5e86da2 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 22 Aug 2025 06:29:54 +0200 Subject: [PATCH 108/119] refactor --- .../src/servicelib/long_running_tasks/task.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index efd83f736574..45dbf9f61008 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -14,8 +14,7 @@ from settings_library.redis import RedisDatabase, RedisSettings from tenacity import ( AsyncRetrying, - TryAgain, - retry_if_exception_type, + retry_unless_exception_type, stop_after_delay, wait_exponential, ) @@ -458,17 +457,16 @@ async def remove_task( # wait for task to be removed since it might not have been running # in this process - async for attempt in AsyncRetrying( - wait=wait_exponential(max=1), - stop=stop_after_delay(_TASK_REMOVAL_MAX_WAIT), - retry=retry_if_exception_type(TryAgain), - ): - with attempt: # noqa: SIM117 - with suppress(TaskNotFoundError): + with suppress(TaskNotFoundError): + async for attempt in AsyncRetrying( + wait=wait_exponential(max=1), + stop=stop_after_delay(_TASK_REMOVAL_MAX_WAIT), + retry=retry_unless_exception_type(TaskNotFoundError), + ): + with attempt: await self._get_tracked_task( tracked_task.task_id, tracked_task.task_context ) - raise TryAgain def _get_task_id(self, task_name: str, *, is_unique: bool) -> TaskId: suffix = "unique" if is_unique else f"{uuid4()}" From 2dfe2fc55f67d519eb6b9ee04754dfd5704fe54b Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 22 Aug 2025 08:48:14 +0200 Subject: [PATCH 109/119] renamed --- .../src/servicelib/long_running_tasks/_lrt_client.py | 6 +++--- .../servicelib/long_running_tasks/_serialization.py | 4 ++-- .../src/servicelib/long_running_tasks/task.py | 10 ++++------ .../test_long_running_tasks__redis_serialization.py | 8 ++++---- .../test_long_running_tasks__serialization.py | 8 ++++---- .../long_running_tasks/test_long_running_tasks_task.py | 7 ++++--- 6 files changed, 21 insertions(+), 22 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py b/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py index b8a1c8afff60..519f6495cf76 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py @@ -8,7 +8,7 @@ from ..logging_utils import log_decorator from ..rabbitmq._client_rpc import RabbitMQRPCClient from ._rabbit_namespace import get_rabbit_namespace -from ._serialization import string_to_object +from ._serialization import loads from .errors import RPCTransferrableTaskError from .models import ( LRTNamespace, @@ -105,9 +105,9 @@ async def get_task_result( timeout_s=_RPC_TIMEOUT_SHORT_REQUESTS, ) assert isinstance(serialized_result, str) # nosec - return string_to_object(serialized_result) + return loads(serialized_result) except RPCTransferrableTaskError as e: - decoded_error = string_to_object(f"{e}") + decoded_error = loads(f"{e}") raise decoded_error from None diff --git a/packages/service-library/src/servicelib/long_running_tasks/_serialization.py b/packages/service-library/src/servicelib/long_running_tasks/_serialization.py index 37e5d1581164..8212c3518d87 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_serialization.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_serialization.py @@ -38,7 +38,7 @@ def register_custom_serialization( _MODULE_FIELD: Final[str] = "__pickle__module__field__" -def object_to_string(e: Any) -> str: +def dumps(e: Any) -> str: """Serialize object to base64-encoded string.""" to_serialize: Any | dict = e object_class = type(e) @@ -55,7 +55,7 @@ def object_to_string(e: Any) -> str: return base64.b85encode(pickle.dumps(to_serialize)).decode("utf-8") -def string_to_object(error_str: str) -> Any: +def loads(error_str: str) -> Any: """Deserialize object from base64-encoded string.""" data = pickle.loads(base64.b85decode(error_str)) # noqa: S301 diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index 45dbf9f61008..1364fb64a491 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -23,7 +23,7 @@ from ..logging_errors import create_troubleshootting_log_kwargs from ..redis import RedisClientSDK, exclusive from ._redis_store import RedisStore -from ._serialization import object_to_string +from ._serialization import dumps from .errors import ( TaskAlreadyRunningError, TaskCancelledError, @@ -313,15 +313,13 @@ async def _tasks_monitor(self) -> None: result_field: ResultField | None = None # get task result try: - result_field = ResultField( - str_result=object_to_string(task.result()) - ) + result_field = ResultField(str_result=dumps(task.result())) except asyncio.InvalidStateError: # task was not completed try again next time and see if it is done continue except asyncio.CancelledError: result_field = ResultField( - str_error=object_to_string(TaskCancelledError(task_id=task_id)) + str_error=dumps(TaskCancelledError(task_id=task_id)) ) except Exception as e: # pylint:disable=broad-except allowed_errors = TaskRegistry.get_allowed_errors( @@ -342,7 +340,7 @@ async def _tasks_monitor(self) -> None: }, ), ) - result_field = ResultField(str_error=object_to_string(e)) + result_field = ResultField(str_error=dumps(e)) # update and store in Redis updates = {"is_done": is_done, "result_field": task_data.result_field} diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks__redis_serialization.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks__redis_serialization.py index a4d84f8873e7..83e978756ac8 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks__redis_serialization.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks__redis_serialization.py @@ -4,9 +4,9 @@ from aiohttp.web import HTTPException, HTTPInternalServerError from servicelib.aiohttp.long_running_tasks._server import AiohttpHTTPExceptionSerializer from servicelib.long_running_tasks._serialization import ( - object_to_string, + dumps, + loads, register_custom_serialization, - string_to_object, ) register_custom_serialization(HTTPException, AiohttpHTTPExceptionSerializer) @@ -38,9 +38,9 @@ def __init__(self, arg1, arg2, kwarg1=None, kwarg2=None): ], ) def test_serialization(obj: Any): - str_data = object_to_string(obj) + str_data = dumps(obj) - reconstructed_obj = string_to_object(str_data) + reconstructed_obj = loads(str_data) assert type(reconstructed_obj) is type(obj) if hasattr(obj, "__dict__"): diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks__serialization.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks__serialization.py index a4d84f8873e7..83e978756ac8 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks__serialization.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks__serialization.py @@ -4,9 +4,9 @@ from aiohttp.web import HTTPException, HTTPInternalServerError from servicelib.aiohttp.long_running_tasks._server import AiohttpHTTPExceptionSerializer from servicelib.long_running_tasks._serialization import ( - object_to_string, + dumps, + loads, register_custom_serialization, - string_to_object, ) register_custom_serialization(HTTPException, AiohttpHTTPExceptionSerializer) @@ -38,9 +38,9 @@ def __init__(self, arg1, arg2, kwarg1=None, kwarg2=None): ], ) def test_serialization(obj: Any): - str_data = object_to_string(obj) + str_data = dumps(obj) - reconstructed_obj = string_to_object(str_data) + reconstructed_obj = loads(str_data) assert type(reconstructed_obj) is type(obj) if hasattr(obj, "__dict__"): diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py index c20ae746bc03..78b3ac74a0b6 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks_task.py @@ -15,7 +15,7 @@ from models_library.api_schemas_long_running_tasks.base import ProgressMessage from servicelib.long_running_tasks import lrt_api from servicelib.long_running_tasks._serialization import ( - string_to_object, + loads, ) from servicelib.long_running_tasks.base_long_running_manager import ( BaseLongRunningManager, @@ -183,7 +183,8 @@ async def test_checked_task_is_not_auto_removed( def _get_resutlt(result_field: ResultField) -> Any: - return string_to_object(result_field.str_result) + assert result_field.str_result + return loads(result_field.str_result) async def test_fire_and_forget_task_is_not_auto_removed( @@ -379,7 +380,7 @@ async def test_get_result_finished_with_error( task_id, with_task_context=empty_context ) assert result.str_error is not None # nosec - error = string_to_object(result.str_error) + error = loads(result.str_error) with pytest.raises(_TetingError, match="failing asap"): raise error From e3f4bcd8fba1d0c6d5d06d4cc145af8863d469e8 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 22 Aug 2025 09:15:00 +0200 Subject: [PATCH 110/119] changed error raising --- .../src/servicelib/long_running_tasks/_lrt_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py b/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py index 519f6495cf76..3bc3caf5e804 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py @@ -108,7 +108,7 @@ async def get_task_result( return loads(serialized_result) except RPCTransferrableTaskError as e: decoded_error = loads(f"{e}") - raise decoded_error from None + raise decoded_error from e @log_decorator(_logger, level=logging.DEBUG) From f8785db61278c45b6a22983deb8c029acbed87f8 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 22 Aug 2025 09:27:53 +0200 Subject: [PATCH 111/119] renamed --- .../long_running_tasks/_serialization.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/_serialization.py b/packages/service-library/src/servicelib/long_running_tasks/_serialization.py index 8212c3518d87..472b7d80f840 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/_serialization.py +++ b/packages/service-library/src/servicelib/long_running_tasks/_serialization.py @@ -38,26 +38,26 @@ def register_custom_serialization( _MODULE_FIELD: Final[str] = "__pickle__module__field__" -def dumps(e: Any) -> str: +def dumps(obj: Any) -> str: """Serialize object to base64-encoded string.""" - to_serialize: Any | dict = e - object_class = type(e) + to_serialize: Any | dict = obj + object_class = type(obj) for registered_class, object_serializer in _SERIALIZERS.items(): if issubclass(object_class, registered_class): to_serialize = { - _TYPE_FIELD: type(e).__name__, - _MODULE_FIELD: type(e).__module__, - **object_serializer.get_init_kwargs_from_object(e), + _TYPE_FIELD: type(obj).__name__, + _MODULE_FIELD: type(obj).__module__, + **object_serializer.get_init_kwargs_from_object(obj), } break return base64.b85encode(pickle.dumps(to_serialize)).decode("utf-8") -def loads(error_str: str) -> Any: +def loads(obj_str: str) -> Any: """Deserialize object from base64-encoded string.""" - data = pickle.loads(base64.b85decode(error_str)) # noqa: S301 + data = pickle.loads(base64.b85decode(obj_str)) # noqa: S301 if isinstance(data, dict) and _TYPE_FIELD in data and _MODULE_FIELD in data: try: @@ -71,7 +71,7 @@ def loads(error_str: str) -> Any: data.pop(_TYPE_FIELD) data.pop(_MODULE_FIELD) - return exception_class( + raise exception_class( **object_serializer.prepare_object_init_kwargs(data) ) except (ImportError, AttributeError, TypeError) as e: From 2c77471aee26c2dc0deb09a23b5fabbce4598f31 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 22 Aug 2025 09:42:07 +0200 Subject: [PATCH 112/119] fixed broken tests --- ...long_running_tasks__redis_serialization.py | 47 ------------------- .../test_long_running_tasks__serialization.py | 5 +- 2 files changed, 4 insertions(+), 48 deletions(-) delete mode 100644 packages/service-library/tests/long_running_tasks/test_long_running_tasks__redis_serialization.py diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks__redis_serialization.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks__redis_serialization.py deleted file mode 100644 index 83e978756ac8..000000000000 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks__redis_serialization.py +++ /dev/null @@ -1,47 +0,0 @@ -from typing import Any - -import pytest -from aiohttp.web import HTTPException, HTTPInternalServerError -from servicelib.aiohttp.long_running_tasks._server import AiohttpHTTPExceptionSerializer -from servicelib.long_running_tasks._serialization import ( - dumps, - loads, - register_custom_serialization, -) - -register_custom_serialization(HTTPException, AiohttpHTTPExceptionSerializer) - - -class PositionalArguments: - def __init__(self, arg1, arg2, *args): - self.arg1 = arg1 - self.arg2 = arg2 - self.args = args - - -class MixedArguments: - def __init__(self, arg1, arg2, kwarg1=None, kwarg2=None): - self.arg1 = arg1 - self.arg2 = arg2 - self.kwarg1 = kwarg1 - self.kwarg2 = kwarg2 - - -@pytest.mark.parametrize( - "obj", - [ - HTTPInternalServerError(reason="Uh-oh!", text="Failure!"), - PositionalArguments("arg1", "arg2", "arg3", "arg4"), - MixedArguments("arg1", "arg2", kwarg1="kwarg1", kwarg2="kwarg2"), - "a_string", - 1, - ], -) -def test_serialization(obj: Any): - str_data = dumps(obj) - - reconstructed_obj = loads(str_data) - - assert type(reconstructed_obj) is type(obj) - if hasattr(obj, "__dict__"): - assert reconstructed_obj.__dict__ == obj.__dict__ diff --git a/packages/service-library/tests/long_running_tasks/test_long_running_tasks__serialization.py b/packages/service-library/tests/long_running_tasks/test_long_running_tasks__serialization.py index 83e978756ac8..3b7562e55503 100644 --- a/packages/service-library/tests/long_running_tasks/test_long_running_tasks__serialization.py +++ b/packages/service-library/tests/long_running_tasks/test_long_running_tasks__serialization.py @@ -40,7 +40,10 @@ def __init__(self, arg1, arg2, kwarg1=None, kwarg2=None): def test_serialization(obj: Any): str_data = dumps(obj) - reconstructed_obj = loads(str_data) + try: + reconstructed_obj = loads(str_data) + except Exception as exc: # pylint:disable=broad-exception-caught + reconstructed_obj = exc assert type(reconstructed_obj) is type(obj) if hasattr(obj, "__dict__"): From 1a9d1eb992034616f19b7603e98dd8180398f6df Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 22 Aug 2025 11:21:47 +0200 Subject: [PATCH 113/119] updated specs --- api/specs/web-server/_long_running_tasks.py | 26 ++++++++----------- .../api/v0/openapi.yaml | 10 +++---- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/api/specs/web-server/_long_running_tasks.py b/api/specs/web-server/_long_running_tasks.py index 4183f3c5839a..71a1906549f3 100644 --- a/api/specs/web-server/_long_running_tasks.py +++ b/api/specs/web-server/_long_running_tasks.py @@ -32,44 +32,40 @@ @router.get( "/tasks", response_model=Envelope[list[TaskGet]], - name="list_tasks", - description="Lists all long running tasks", responses=_export_data_responses, ) -def get_async_jobs(): ... +def list_tasks(): + """Lists all long running tasks""" @router.get( "/tasks/{task_id}", response_model=Envelope[TaskStatus], - name="get_task_status", - description="Retrieves the status of a task", responses=_export_data_responses, ) -def get_async_job_status( +def get_task_status( _path_params: Annotated[_PathParam, Depends()], -): ... +): + """Retrieves the status of a task""" @router.delete( "/tasks/{task_id}", - name="remove_task", - description="Cancels and removes a task", responses=_export_data_responses, status_code=status.HTTP_204_NO_CONTENT, ) -def cancel_async_job( +def remove_task( _path_params: Annotated[_PathParam, Depends()], -): ... +): + """Cancels and removes a task""" @router.get( "/tasks/{task_id}/result", response_model=Any, - name="get_task_result", - description="Retrieves the result of a task", responses=_export_data_responses, ) -def get_async_job_result( +def get_task_result( _path_params: Annotated[_PathParam, Depends()], -): ... +): + """Retrieves the result of a task""" diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index afe021328ee6..67af72232504 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -3874,7 +3874,7 @@ paths: - long-running-tasks summary: List Tasks description: Lists all long running tasks - operationId: get_async_jobs + operationId: list_tasks responses: '200': description: Successful Response @@ -3912,7 +3912,7 @@ paths: - long-running-tasks summary: Get Task Status description: Retrieves the status of a task - operationId: get_async_job_status + operationId: get_task_status parameters: - name: task_id in: path @@ -3956,7 +3956,7 @@ paths: - long-running-tasks summary: Remove Task description: Cancels and removes a task - operationId: cancel_async_job + operationId: remove_task parameters: - name: task_id in: path @@ -3997,7 +3997,7 @@ paths: - long-running-tasks summary: Get Task Result description: Retrieves the result of a task - operationId: get_async_job_result + operationId: get_task_result parameters: - name: task_id in: path @@ -4011,7 +4011,7 @@ paths: content: application/json: schema: - title: Response Get Async Job Result + title: Response Get Task Result '404': content: application/json: From 89fbc259bf6990c38c6119a4cf8178324ce9ef21 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 22 Aug 2025 11:22:34 +0200 Subject: [PATCH 114/119] renamed modules --- .../{_lrt_client.py => _rpc_client.py} | 0 .../{_lrt_server.py => _rpc_server.py} | 0 .../long_running_tasks/base_long_running_manager.py | 2 +- .../src/servicelib/long_running_tasks/lrt_api.py | 12 ++++++------ 4 files changed, 7 insertions(+), 7 deletions(-) rename packages/service-library/src/servicelib/long_running_tasks/{_lrt_client.py => _rpc_client.py} (100%) rename packages/service-library/src/servicelib/long_running_tasks/{_lrt_server.py => _rpc_server.py} (100%) diff --git a/packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py b/packages/service-library/src/servicelib/long_running_tasks/_rpc_client.py similarity index 100% rename from packages/service-library/src/servicelib/long_running_tasks/_lrt_client.py rename to packages/service-library/src/servicelib/long_running_tasks/_rpc_client.py diff --git a/packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py b/packages/service-library/src/servicelib/long_running_tasks/_rpc_server.py similarity index 100% rename from packages/service-library/src/servicelib/long_running_tasks/_lrt_server.py rename to packages/service-library/src/servicelib/long_running_tasks/_rpc_server.py diff --git a/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py b/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py index 1a2417e04099..2090df4e1adb 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py +++ b/packages/service-library/src/servicelib/long_running_tasks/base_long_running_manager.py @@ -5,8 +5,8 @@ from settings_library.redis import RedisSettings from ..rabbitmq._client_rpc import RabbitMQRPCClient -from ._lrt_server import router from ._rabbit_namespace import get_rabbit_namespace +from ._rpc_server import router from .models import LRTNamespace, TaskContext from .task import TasksManager diff --git a/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py b/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py index e066b0cff2d6..73fdebb4cfa9 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py +++ b/packages/service-library/src/servicelib/long_running_tasks/lrt_api.py @@ -2,7 +2,7 @@ from typing import Any from ..rabbitmq._client_rpc import RabbitMQRPCClient -from . import _lrt_client +from . import _rpc_client from .models import ( LRTNamespace, RegisteredTaskName, @@ -50,7 +50,7 @@ async def start_task( TaskId: the task unique identifier """ - return await _lrt_client.start_task( + return await _rpc_client.start_task( rabbitmq_rpc_client, lrt_namespace, registered_task_name=registered_task_name, @@ -67,7 +67,7 @@ async def list_tasks( lrt_namespace: LRTNamespace, task_context: TaskContext, ) -> list[TaskBase]: - return await _lrt_client.list_tasks( + return await _rpc_client.list_tasks( rabbitmq_rpc_client, lrt_namespace, task_context=task_context ) @@ -79,7 +79,7 @@ async def get_task_status( task_id: TaskId, ) -> TaskStatus: """returns the status of a task""" - return await _lrt_client.get_task_status( + return await _rpc_client.get_task_status( rabbitmq_rpc_client, lrt_namespace, task_id=task_id, task_context=task_context ) @@ -90,7 +90,7 @@ async def get_task_result( task_context: TaskContext, task_id: TaskId, ) -> Any: - return await _lrt_client.get_task_result( + return await _rpc_client.get_task_result( rabbitmq_rpc_client, lrt_namespace, task_context=task_context, @@ -111,7 +111,7 @@ async def remove_task( When `wait_for_removal` is True, `cancellationt_timeout` is set to _RPC_TIMEOUT_SHORT_REQUESTS """ - await _lrt_client.remove_task( + await _rpc_client.remove_task( rabbitmq_rpc_client, lrt_namespace, task_id=task_id, From 66c5dde11d2a0b0b882d09d04388c82b793ef894 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 22 Aug 2025 11:31:43 +0200 Subject: [PATCH 115/119] removed relative imports --- .../service-library/src/servicelib/fastapi/client_session.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/service-library/src/servicelib/fastapi/client_session.py b/packages/service-library/src/servicelib/fastapi/client_session.py index c77a6ea339ea..f9c126272eec 100644 --- a/packages/service-library/src/servicelib/fastapi/client_session.py +++ b/packages/service-library/src/servicelib/fastapi/client_session.py @@ -2,9 +2,10 @@ import httpx from fastapi import FastAPI -from servicelib.fastapi.tracing import setup_httpx_client_tracing from settings_library.tracing import TracingSettings +from .tracing import setup_httpx_client_tracing + def setup_client_session( app: FastAPI, From 5ca6a589621adf5574bd360ebc94d0d894d9109c Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 22 Aug 2025 11:31:55 +0200 Subject: [PATCH 116/119] feedback --- .../src/servicelib/long_running_tasks/task.py | 41 +++++++++---------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index 1364fb64a491..486ec7d5d3bb 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -21,6 +21,7 @@ from ..background_task import create_periodic_task from ..logging_errors import create_troubleshootting_log_kwargs +from ..logging_utils import log_context from ..redis import RedisClientSDK, exclusive from ._redis_store import RedisStore from ._serialization import dumps @@ -63,7 +64,7 @@ def __name__(self) -> str: ... class TaskRegistry: - REGISTERED_TASKS: ClassVar[ + _REGISTERED_TASKS: ClassVar[ dict[RegisteredTaskName, tuple[AllowedErrrors, TaskProtocol]] ] = {} @@ -77,20 +78,20 @@ def register( partial_task = functools.partial(task, **partial_kwargs) # allows to call the partial via it's original name partial_task.__name__ = task.__name__ # type: ignore[attr-defined] - cls.REGISTERED_TASKS[task.__name__] = [allowed_errors, partial_task] # type: ignore[assignment] + cls._REGISTERED_TASKS[task.__name__] = [allowed_errors, partial_task] # type: ignore[assignment] @classmethod def get_task(cls, task_name: RegisteredTaskName) -> TaskProtocol: - return cls.REGISTERED_TASKS[task_name][1] + return cls._REGISTERED_TASKS[task_name][1] @classmethod def get_allowed_errors(cls, task_name: RegisteredTaskName) -> AllowedErrrors: - return cls.REGISTERED_TASKS[task_name][0] + return cls._REGISTERED_TASKS[task_name][0] @classmethod def unregister(cls, task: TaskProtocol) -> None: - if task.__name__ in cls.REGISTERED_TASKS: - del cls.REGISTERED_TASKS[task.__name__] + if task.__name__ in cls._REGISTERED_TASKS: + del cls._REGISTERED_TASKS[task.__name__] async def _get_tasks_to_remove( @@ -200,7 +201,6 @@ async def setup(self) -> None: async def teardown(self) -> None: # ensure all created tasks are cancelled for tracked_task in await self._tasks_data.list_tasks_data(): - # when closing we do not care about pending errors with suppress(TaskNotFoundError): await self.remove_task( tracked_task.task_id, @@ -266,18 +266,17 @@ async def _stale_tasks_monitor(self) -> None: # - finished with errors # we just print the status from where one can infer the above with suppress(TaskNotFoundError): - _logger.warning( - "Removing stale task '%s' with status '%s'", - task_id, - ( - await self.get_task_status( - task_id, with_task_context=task_context - ) - ).model_dump_json(), - ) - await self.remove_task( - task_id, with_task_context=task_context, wait_for_removal=True + task_status = await self.get_task_status( + task_id, with_task_context=task_context ) + with log_context( + _logger, + logging.WARNING, + f"Removing stale task '{task_id}' with status '{task_status.model_dump_json()}'", + ): + await self.remove_task( + task_id, with_task_context=task_context, wait_for_removal=True + ) async def _cancelled_tasks_removal(self) -> None: """ @@ -330,7 +329,7 @@ async def _tasks_monitor(self) -> None: **create_troubleshootting_log_kwargs( ( f"Execution of {task_id=} finished with unexpected error, " - f"only the following are {allowed_errors=} are permitted" + f"only the following {allowed_errors=} are permitted" ), error=e, error_context={ @@ -501,9 +500,9 @@ async def start_task( fire_and_forget: bool, **task_kwargs: Any, ) -> TaskId: - if registered_task_name not in TaskRegistry.REGISTERED_TASKS: + if registered_task_name not in TaskRegistry._REGISTERED_TASKS: raise TaskNotRegisteredError( - task_name=registered_task_name, tasks=TaskRegistry.REGISTERED_TASKS + task_name=registered_task_name, tasks=TaskRegistry._REGISTERED_TASKS ) task = TaskRegistry.get_task(registered_task_name) From ddac38fb4309fd7898704c3c15e31ad6c0b3ddb4 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 22 Aug 2025 12:22:11 +0200 Subject: [PATCH 117/119] fixed test --- .../src/servicelib/long_running_tasks/task.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/service-library/src/servicelib/long_running_tasks/task.py b/packages/service-library/src/servicelib/long_running_tasks/task.py index 486ec7d5d3bb..e76b486954cc 100644 --- a/packages/service-library/src/servicelib/long_running_tasks/task.py +++ b/packages/service-library/src/servicelib/long_running_tasks/task.py @@ -80,6 +80,12 @@ def register( partial_task.__name__ = task.__name__ # type: ignore[attr-defined] cls._REGISTERED_TASKS[task.__name__] = [allowed_errors, partial_task] # type: ignore[assignment] + @classmethod + def get_registered_tasks( + cls, + ) -> dict[RegisteredTaskName, tuple[AllowedErrrors, TaskProtocol]]: + return cls._REGISTERED_TASKS + @classmethod def get_task(cls, task_name: RegisteredTaskName) -> TaskProtocol: return cls._REGISTERED_TASKS[task_name][1] @@ -500,9 +506,10 @@ async def start_task( fire_and_forget: bool, **task_kwargs: Any, ) -> TaskId: - if registered_task_name not in TaskRegistry._REGISTERED_TASKS: + registered_tasks = TaskRegistry.get_registered_tasks() + if registered_task_name not in registered_tasks: raise TaskNotRegisteredError( - task_name=registered_task_name, tasks=TaskRegistry._REGISTERED_TASKS + task_name=registered_task_name, tasks=registered_tasks ) task = TaskRegistry.get_task(registered_task_name) From 5e0efab8987032b452633bfd6c584c2178f4d4e1 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 22 Aug 2025 12:30:52 +0200 Subject: [PATCH 118/119] fixed issue --- api/specs/web-server/_long_running_tasks.py | 8 ++++---- .../api/v0/openapi.yaml | 18 +++++++++--------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/api/specs/web-server/_long_running_tasks.py b/api/specs/web-server/_long_running_tasks.py index 71a1906549f3..fdea7ef40286 100644 --- a/api/specs/web-server/_long_running_tasks.py +++ b/api/specs/web-server/_long_running_tasks.py @@ -34,7 +34,7 @@ response_model=Envelope[list[TaskGet]], responses=_export_data_responses, ) -def list_tasks(): +def list_tasks_(): """Lists all long running tasks""" @@ -43,7 +43,7 @@ def list_tasks(): response_model=Envelope[TaskStatus], responses=_export_data_responses, ) -def get_task_status( +def get_task_status_( _path_params: Annotated[_PathParam, Depends()], ): """Retrieves the status of a task""" @@ -54,7 +54,7 @@ def get_task_status( responses=_export_data_responses, status_code=status.HTTP_204_NO_CONTENT, ) -def remove_task( +def remove_task_( _path_params: Annotated[_PathParam, Depends()], ): """Cancels and removes a task""" @@ -65,7 +65,7 @@ def remove_task( response_model=Any, responses=_export_data_responses, ) -def get_task_result( +def get_task_result_( _path_params: Annotated[_PathParam, Depends()], ): """Retrieves the result of a task""" diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 78211db082d1..df4b1579c674 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -3851,9 +3851,9 @@ paths: get: tags: - long-running-tasks - summary: List Tasks + summary: 'List Tasks ' description: Lists all long running tasks - operationId: list_tasks + operationId: list_tasks_ responses: '200': description: Successful Response @@ -3889,9 +3889,9 @@ paths: get: tags: - long-running-tasks - summary: Get Task Status + summary: 'Get Task Status ' description: Retrieves the status of a task - operationId: get_task_status + operationId: get_task_status_ parameters: - name: task_id in: path @@ -3933,9 +3933,9 @@ paths: delete: tags: - long-running-tasks - summary: Remove Task + summary: 'Remove Task ' description: Cancels and removes a task - operationId: remove_task + operationId: remove_task_ parameters: - name: task_id in: path @@ -3974,9 +3974,9 @@ paths: get: tags: - long-running-tasks - summary: Get Task Result + summary: 'Get Task Result ' description: Retrieves the result of a task - operationId: get_task_result + operationId: get_task_result_ parameters: - name: task_id in: path @@ -3990,7 +3990,7 @@ paths: content: application/json: schema: - title: Response Get Task Result + title: 'Response Get Task Result ' '404': content: application/json: From 9fc652ae058ff698ddf006ce5b0782d7c4f86cc8 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 22 Aug 2025 13:23:48 +0200 Subject: [PATCH 119/119] fixed borken tests --- api/specs/web-server/_long_running_tasks.py | 8 ++++---- .../api/v0/openapi.yaml | 18 +++++++++--------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/api/specs/web-server/_long_running_tasks.py b/api/specs/web-server/_long_running_tasks.py index fdea7ef40286..77b979b2e3e8 100644 --- a/api/specs/web-server/_long_running_tasks.py +++ b/api/specs/web-server/_long_running_tasks.py @@ -34,7 +34,7 @@ response_model=Envelope[list[TaskGet]], responses=_export_data_responses, ) -def list_tasks_(): +def get_async_jobs(): """Lists all long running tasks""" @@ -43,7 +43,7 @@ def list_tasks_(): response_model=Envelope[TaskStatus], responses=_export_data_responses, ) -def get_task_status_( +def get_async_job_status( _path_params: Annotated[_PathParam, Depends()], ): """Retrieves the status of a task""" @@ -54,7 +54,7 @@ def get_task_status_( responses=_export_data_responses, status_code=status.HTTP_204_NO_CONTENT, ) -def remove_task_( +def cancel_async_job( _path_params: Annotated[_PathParam, Depends()], ): """Cancels and removes a task""" @@ -65,7 +65,7 @@ def remove_task_( response_model=Any, responses=_export_data_responses, ) -def get_task_result_( +def get_async_job_result( _path_params: Annotated[_PathParam, Depends()], ): """Retrieves the result of a task""" diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index df4b1579c674..2e40aa33250c 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -3851,9 +3851,9 @@ paths: get: tags: - long-running-tasks - summary: 'List Tasks ' + summary: Get Async Jobs description: Lists all long running tasks - operationId: list_tasks_ + operationId: get_async_jobs responses: '200': description: Successful Response @@ -3889,9 +3889,9 @@ paths: get: tags: - long-running-tasks - summary: 'Get Task Status ' + summary: Get Async Job Status description: Retrieves the status of a task - operationId: get_task_status_ + operationId: get_async_job_status parameters: - name: task_id in: path @@ -3933,9 +3933,9 @@ paths: delete: tags: - long-running-tasks - summary: 'Remove Task ' + summary: Cancel Async Job description: Cancels and removes a task - operationId: remove_task_ + operationId: cancel_async_job parameters: - name: task_id in: path @@ -3974,9 +3974,9 @@ paths: get: tags: - long-running-tasks - summary: 'Get Task Result ' + summary: Get Async Job Result description: Retrieves the result of a task - operationId: get_task_result_ + operationId: get_async_job_result parameters: - name: task_id in: path @@ -3990,7 +3990,7 @@ paths: content: application/json: schema: - title: 'Response Get Task Result ' + title: Response Get Async Job Result '404': content: application/json: