fixed tests

Author: Andrei Neagu

packages/pytest-simcore/src/pytest_simcore/helpers/long_running_tasks.py

@@ -0,0 +1,41 @@
+# pylint: disable=protected-access
+
+import pytest
+from fastapi import FastAPI
+from servicelib.long_running_tasks.errors import TaskNotFoundError
+from servicelib.long_running_tasks.manager import (
+    LongRunningManager,
+)
+from servicelib.long_running_tasks.models import TaskContext
+from servicelib.long_running_tasks.task import TaskId
+from tenacity import (
+    AsyncRetrying,
+    TryAgain,
+    retry_if_exception_type,
+    stop_after_delay,
+    wait_fixed,
+)
+
+
+def get_fastapi_long_running_manager(app: FastAPI) -> LongRunningManager:
+    manager = app.state.long_running_manager
+    assert isinstance(manager, LongRunningManager)
+    return manager
+
+
+async def assert_task_is_no_longer_present(
+    manager: LongRunningManager, task_id: TaskId, task_context: TaskContext
+) -> None:
+    async for attempt in AsyncRetrying(
+        reraise=True,
+        wait=wait_fixed(0.1),
+        stop=stop_after_delay(60),
+        retry=retry_if_exception_type((AssertionError, TryAgain)),
+    ):
+        with attempt:  # noqa: SIM117
+            with pytest.raises(TaskNotFoundError):  # noqa: PT012
+                # use internals to detirmine when it's no longer here
+                await manager._tasks_manager._get_tracked_task(  # noqa: SLF001
+                    task_id, task_context
+                )
+                raise TryAgain

packages/service-library/tests/long_running_tasks/test_long_running_tasks_lrt_api.py

@@ -10,11 +10,9 @@
 import pytest
 from models_library.api_schemas_long_running_tasks.base import TaskProgress
 from pydantic import NonNegativeInt
+from pytest_simcore.helpers.long_running_tasks import assert_task_is_no_longer_present
 from servicelib.long_running_tasks import lrt_api
-from servicelib.long_running_tasks.errors import TaskNotFoundError
-from servicelib.long_running_tasks.manager import (
-    LongRunningManager,
-)
+from servicelib.long_running_tasks.manager import LongRunningManager
 from servicelib.long_running_tasks.models import LRTNamespace, TaskContext
 from servicelib.long_running_tasks.task import TaskId, TaskRegistry
 from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient
@@ -161,22 +159,6 @@ async def _assert_list_tasks_from_all_managers(
         assert len(tasks) == task_count
 
 
-async def _assert_task_is_no_longer_present(
-    long_running_managers: list[LongRunningManager],
-    task_context: TaskContext,
-    task_id: TaskId,
-) -> None:
-    manager = _get_long_running_manager(long_running_managers)
-    async for attempt in AsyncRetrying(**_RETRY_PARAMS):
-        with attempt:  # noqa: SIM117
-            with pytest.raises(TaskNotFoundError):  # noqa: PT012
-                # use internals to detirmine when it's no longer here
-                await manager._tasks_manager._get_tracked_task(  # noqa: SLF001
-                    task_id, task_context
-                )
-                raise TryAgain
-
-
 _TASK_CONTEXT: Final[list[TaskContext | None]] = [{"a": "context"}, None]
 _IS_UNIQUE: Final[list[bool]] = [False, True]
 _TASK_COUNT: Final[list[int]] = [5]
@@ -188,6 +170,7 @@ async def _assert_task_is_no_longer_present(
 @pytest.mark.parametrize("to_return", [{"key": "value"}])
 async def test_workflow_with_result(
     disable_stale_tasks_monitor: None,
+    fast_long_running_tasks_cancellation: None,
     long_running_managers: list[LongRunningManager],
     rabbitmq_rpc_client: RabbitMQRPCClient,
     task_count: int,
@@ -235,8 +218,8 @@ async def test_workflow_with_result(
         )
         assert result == to_return
 
-        await _assert_task_is_no_longer_present(
-            long_running_managers, saved_context, task_id
+        await assert_task_is_no_longer_present(
+            _get_long_running_manager(long_running_managers), task_id, saved_context
         )
 
 
@@ -245,6 +228,7 @@ async def test_workflow_with_result(
 @pytest.mark.parametrize("is_unique", _IS_UNIQUE)
 async def test_workflow_raises_error(
     disable_stale_tasks_monitor: None,
+    fast_long_running_tasks_cancellation: None,
     long_running_managers: list[LongRunningManager],
     rabbitmq_rpc_client: RabbitMQRPCClient,
     task_count: int,
@@ -290,15 +274,16 @@ async def test_workflow_raises_error(
                 task_id,
             )
 
-        await _assert_task_is_no_longer_present(
-            long_running_managers, saved_context, task_id
+        await assert_task_is_no_longer_present(
+            _get_long_running_manager(long_running_managers), task_id, saved_context
         )
 
 
 @pytest.mark.parametrize("task_context", _TASK_CONTEXT)
 @pytest.mark.parametrize("is_unique", _IS_UNIQUE)
 async def test_remove_task(
     disable_stale_tasks_monitor: None,
+    fast_long_running_tasks_cancellation: None,
     long_running_managers: list[LongRunningManager],
     rabbitmq_rpc_client: RabbitMQRPCClient,
     is_unique: bool,
@@ -326,6 +311,6 @@ async def test_remove_task(
         task_id,
     )
 
-    await _assert_task_is_no_longer_present(
-        long_running_managers, saved_context, task_id
+    await assert_task_is_no_longer_present(
+        _get_long_running_manager(long_running_managers), task_id, saved_context
     )

services/dynamic-sidecar/tests/unit/test_api_rest_workflow_service_metrics.py

@@ -36,6 +36,10 @@
 from models_library.services_creation import CreateServiceMetricsAdditionalParams
 from pydantic import AnyHttpUrl, TypeAdapter
 from pytest_mock import MockerFixture
+from pytest_simcore.helpers.long_running_tasks import (
+    assert_task_is_no_longer_present,
+    get_fastapi_long_running_manager,
+)
 from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict
 from servicelib.fastapi.long_running_tasks.client import (
     HttpClient,
@@ -61,6 +65,11 @@
 _CREATE_SERVICE_CONTAINERS_TIMEOUT: Final[float] = 60
 _BASE_HEART_BEAT_INTERVAL: Final[float] = 0.1
 
+_RETRY_PRAMS: Final[dict[str, Any]] = {
+    "wait": wait_fixed(0.1),
+    "stop": stop_after_delay(5),
+}
+
 
 @pytest.fixture(params=[1, 2])
 def container_names(request: pytest.FixtureRequest) -> list[str]:
@@ -92,6 +101,7 @@ def backend_url() -> AnyHttpUrl:
 
 @pytest.fixture
 async def mock_environment(
+    fast_long_running_tasks_cancellation: None,
     mock_postgres_check: None,
     mock_registry_service: AsyncMock,
     mock_environment: EnvVarsDict,
@@ -351,14 +361,18 @@ async def test_user_services_fail_to_stop_or_save_data(
     # in case of manual intervention multiple stops will be sent
     _EXPECTED_STOP_MESSAGES = 4
     for _ in range(_EXPECTED_STOP_MESSAGES):
+        task_id = await _get_task_id_docker_compose_down(httpx_async_client)
         with pytest.raises(TaskExceptionError):
             async with periodic_task_result(
                 client=http_client,
-                task_id=await _get_task_id_docker_compose_down(httpx_async_client),
+                task_id=task_id,
                 task_timeout=_CREATE_SERVICE_CONTAINERS_TIMEOUT,
                 status_poll_interval=_FAST_STATUS_POLL,
             ):
                 ...
+        await assert_task_is_no_longer_present(
+            get_fastapi_long_running_manager(app), task_id, {}
+        )
 
     # Ensure messages arrive in the expected order
     resource_tracking_messages = _get_resource_tracking_messages(
@@ -482,13 +496,17 @@ async def test_user_services_crash_when_running(
     # will be sent due to manual intervention
     _EXPECTED_STOP_MESSAGES = 4
     for _ in range(_EXPECTED_STOP_MESSAGES):
+        task_id = await _get_task_id_docker_compose_down(httpx_async_client)
         async with periodic_task_result(
             client=http_client,
-            task_id=await _get_task_id_docker_compose_down(httpx_async_client),
+            task_id=task_id,
             task_timeout=_CREATE_SERVICE_CONTAINERS_TIMEOUT,
             status_poll_interval=_FAST_STATUS_POLL,
         ) as result:
             assert result is None
+        await assert_task_is_no_longer_present(
+            get_fastapi_long_running_manager(app), task_id, {}
+        )
 
     resource_tracking_messages = _get_resource_tracking_messages(
         mock_post_rabbit_message