fixed flaky test

Author: Andrei Neagu

services/dynamic-sidecar/tests/unit/test_api_rest_containers_long_running_tasks.py

@@ -45,6 +45,12 @@
 from simcore_service_dynamic_sidecar.modules.inputs import enable_inputs_pulling
 from simcore_service_dynamic_sidecar.modules.outputs._context import OutputsContext
 from simcore_service_dynamic_sidecar.modules.outputs._manager import OutputsManager
+from tenacity import (
+    AsyncRetrying,
+    retry_if_exception_type,
+    stop_after_delay,
+    wait_fixed,
+)
 
 FAST_STATUS_POLL: Final[float] = 0.1
 CREATE_SERVICE_CONTAINERS_TIMEOUT: Final[float] = 60
@@ -384,6 +390,19 @@ async def _debug_progress(
     print(f"{task_id} {percent} {message}")
 
 
+async def _assert_progress_finished(
+    last_progress_message: tuple[ProgressMessage, ProgressPercent] | None,
+) -> None:
+    async for attempt in AsyncRetrying(
+        stop=stop_after_delay(5),
+        wait=wait_fixed(0.1),
+        retry=retry_if_exception_type(AssertionError),
+        reraise=True,
+    ):
+        with attempt:
+            assert last_progress_message == ("finished", 1.0)
+
+
 async def test_create_containers_task(
     httpx_async_client: AsyncClient,
     client: Client,
@@ -392,10 +411,13 @@ async def test_create_containers_task(
     mock_metrics_params: CreateServiceMetricsAdditionalParams,
     shared_store: SharedStore,
 ) -> None:
-    last_progress_message: tuple[str, float] | None = None
+    last_progress_message: tuple[ProgressMessage, ProgressPercent] | None = None
 
-    async def create_progress(message: str, percent: float, _: TaskId) -> None:
+    async def create_progress(
+        message: ProgressMessage, percent: ProgressPercent | None, _: TaskId
+    ) -> None:
         nonlocal last_progress_message
+        assert percent is not None
         last_progress_message = (message, percent)
         print(message, percent)
 
@@ -410,7 +432,7 @@ async def create_progress(message: str, percent: float, _: TaskId) -> None:
     ) as result:
         assert shared_store.container_names == result
 
-    assert last_progress_message == ("finished", 1.0)
+    await _assert_progress_finished(last_progress_message)
 
 
 async def test_pull_user_servcices_docker_images(
@@ -442,7 +464,7 @@ async def create_progress(
     ) as result:
         assert shared_store.container_names == result
 
-    assert last_progress_message == ("finished", 1.0)
+    await _assert_progress_finished(last_progress_message)
 
     async with periodic_task_result(
         client=client,
@@ -454,7 +476,7 @@ async def create_progress(
         progress_callback=_debug_progress,
     ) as result:
         assert result is None
-    assert last_progress_message == ("finished", 1.0)
+    await _assert_progress_finished(last_progress_message)
 
 
 async def test_create_containers_task_invalid_yaml_spec(