🐛added retry when creating services (ITISFoundation#3297)

Co-authored-by: Andrei Neagu <[email protected]>

Author: GitHK

services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/docker_compose_utils.py

@@ -103,10 +103,7 @@ async def docker_compose_create(
     compose_spec_yaml: str, settings: ApplicationSettings
 ) -> CommandResult:
     """
-    (Re)creates, starts, and attaches to containers for a service
-
-    - does NOT build images
-    - runs in DETACHED mode, i.e. runs containers in the background, prints new container names
+    Creates containers required by the service.
 
     [SEE docker-compose](https://docs.docker.com/engine/reference/commandline/compose_up/)
     """

services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/docker_utils.py

@@ -5,6 +5,7 @@
 
 import aiodocker
 from aiodocker.utils import clean_filters
+from pydantic import PositiveInt
 
 from .errors import UnexpectedDockerError, VolumeNotFoundError
 
@@ -40,3 +41,15 @@ async def get_volume_by_label(label: str, run_id: UUID) -> dict[str, Any]:
             raise VolumeNotFoundError(label, run_id, volumes)
         volume_details = volumes[0]
         return volume_details  # type: ignore
+
+
+async def get_running_containers_count_from_names(
+    container_names: list[str],
+) -> PositiveInt:
+    if len(container_names) == 0:
+        return 0
+
+    async with docker_client() as docker:
+        filters = clean_filters({"name": container_names})
+        containers = await docker.containers.list(all=True, filters=filters)
+        return len(containers)

services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/long_running_tasks.py

@@ -21,10 +21,11 @@
     docker_compose_start,
 )
 from ..core.docker_logs import start_log_fetching, stop_log_fetching
+from ..core.docker_utils import get_running_containers_count_from_names
 from ..core.rabbitmq import RabbitMQ, send_message
 from ..core.settings import ApplicationSettings
 from ..core.utils import CommandResult, assemble_container_names
-from ..core.validation import validate_compose_spec
+from ..core.validation import parse_compose_spec, validate_compose_spec
 from ..models.schemas.application_health import ApplicationHealth
 from ..models.schemas.containers import ContainersCreate
 from ..models.shared_store import SharedStore
@@ -43,7 +44,7 @@
 
 
 @retry(
-    wait=wait_random_exponential(),
+    wait=wait_random_exponential(max=30),
     stop=stop_after_delay(5 * _MINUTE),
     retry=retry_if_result(lambda result: result.success is False),
     reraise=False,
@@ -57,6 +58,28 @@ async def _retry_docker_compose_start(
     return await docker_compose_start(compose_spec, settings)
 
 
+@retry(
+    wait=wait_random_exponential(max=30),
+    stop=stop_after_delay(5 * _MINUTE),
+    retry=retry_if_result(lambda result: result is False),
+    reraise=True,
+)
+async def _retry_docker_compose_create(
+    compose_spec: str, settings: ApplicationSettings
+) -> bool:
+    await docker_compose_create(compose_spec, settings)
+
+    compose_spec_dict = parse_compose_spec(compose_spec)
+    container_names = list(compose_spec_dict["services"].keys())
+
+    expected_num_containers = len(container_names)
+    actual_num_containers = await get_running_containers_count_from_names(
+        container_names
+    )
+
+    return expected_num_containers == actual_num_containers
+
+
 async def task_create_service_containers(
     progress: TaskProgress,
     settings: ApplicationSettings,
@@ -90,7 +113,7 @@ async def task_create_service_containers(
         await docker_compose_pull(shared_store.compose_spec, settings)
 
         progress.update(message="creating and starting containers", percent=0.90)
-        await docker_compose_create(shared_store.compose_spec, settings)
+        await _retry_docker_compose_create(shared_store.compose_spec, settings)
 
         progress.update(message="ensure containers are started", percent=0.95)
         r = await _retry_docker_compose_start(shared_store.compose_spec, settings)

services/dynamic-sidecar/tests/unit/test_core_docker_utils.py

@@ -1,15 +1,22 @@
 # pylint: disable=redefined-outer-name
 # pylint: disable=unused-argument
 # pylint: disable=unused-variable
-from typing import AsyncIterable
+from typing import AsyncIterable, AsyncIterator
 from uuid import UUID
 
 import aiodocker
 import pytest
+from _pytest.fixtures import FixtureRequest
 from faker import Faker
-from simcore_service_dynamic_sidecar.core.docker_utils import get_volume_by_label
+from pydantic import PositiveInt
+from simcore_service_dynamic_sidecar.core.docker_utils import (
+    get_running_containers_count_from_names,
+    get_volume_by_label,
+)
 from simcore_service_dynamic_sidecar.core.errors import VolumeNotFoundError
 
+# FIXTURES
+
 
 @pytest.fixture(scope="session")
 def volume_name() -> str:
@@ -39,16 +46,54 @@ async def volume_with_label(volume_name: str, run_id: str) -> AsyncIterable[None
         await volume.delete()
 
 
+@pytest.fixture(params=[0, 1, 2, 3])
+def container_count(request: FixtureRequest) -> PositiveInt:
+    return request.param
+
+
+@pytest.fixture
+def container_names(container_count: PositiveInt) -> list[str]:
+    return [f"container_test_{i}" for i in range(container_count)]
+
+
+@pytest.fixture
+async def started_services(container_names: list[str]) -> AsyncIterator[None]:
+    async with aiodocker.Docker() as docker_client:
+        started_containers = []
+        for container_name in container_names:
+            container = await docker_client.containers.create(
+                config={"Image": "busybox:latest"},
+                name=container_name,
+            )
+            started_containers.append(container)
+
+        yield
+
+        for container in started_containers:
+            await container.stop()
+            await container.delete()
+
+
+# TESTS
+
+
 async def test_volume_with_label(
     volume_with_label: None, volume_name: str, run_id: UUID
-) -> None:
+):
     assert await get_volume_by_label(volume_name, run_id)
 
 
-async def test_volume_label_missing(run_id: UUID) -> None:
+async def test_volume_label_missing(run_id: UUID):
     with pytest.raises(VolumeNotFoundError) as exc_info:
         await get_volume_by_label("not_exist", run_id)
 
     error_msg = f"{exc_info.value}"
     assert f"{run_id}" in error_msg
     assert "not_exist" in error_msg
+
+
+async def test_get_running_containers_count_from_names(
+    started_services: None, container_names: list[str], container_count: PositiveInt
+):
+    found_containers = await get_running_containers_count_from_names(container_names)
+    assert found_containers == container_count