🐛 Is638/ fixes dy-sidecar docker-compose process down command and timeout (ITISFoundation#3182)

- closes Make docker_compose_down reliable ITISFoundation#3172
- fixes ``async_command`` timeout feature

Author: pcrespov

services/dynamic-sidecar/VERSION

@@ -1 +1 @@
-1.1.0
+1.1.1

services/dynamic-sidecar/openapi.json

@@ -3,7 +3,7 @@
   "info": {
     "title": "simcore-service-dynamic-sidecar",
     "description": " Implements a sidecar service to manage user's dynamic/interactive services",
-    "version": "1.1.0"
+    "version": "1.1.1"
   },
   "servers": [
     {
@@ -103,9 +103,9 @@
             "required": false,
             "schema": {
               "title": "Command Timeout",
-              "type": "number",
+              "type": "integer",
               "description": "docker-compose up command timeout run as a background",
-              "default": 3600.0
+              "default": 3600
             },
             "name": "command_timeout",
             "in": "query"
@@ -115,9 +115,9 @@
             "required": false,
             "schema": {
               "title": "Validation Timeout",
-              "type": "number",
+              "type": "integer",
               "description": "docker-compose config timeout (EXPERIMENTAL)",
-              "default": 60.0
+              "default": 60
             },
             "name": "validation_timeout",
             "in": "query"
@@ -168,9 +168,9 @@
             "required": false,
             "schema": {
               "title": "Command Timeout",
-              "type": "number",
+              "type": "integer",
               "description": "docker-compose down command timeout default  (EXPERIMENTAL)",
-              "default": 10.0
+              "default": 10
             },
             "name": "command_timeout",
             "in": "query"
@@ -600,9 +600,9 @@
             "required": false,
             "schema": {
               "title": "Command Timeout",
-              "type": "number",
+              "type": "integer",
               "description": "docker-compose stop command timeout default",
-              "default": 10.0
+              "default": 10
             },
             "name": "command_timeout",
             "in": "query"

services/dynamic-sidecar/requirements/_base.in

@@ -30,6 +30,7 @@ aiofiles
 docker-compose
 fastapi
 httpx
+psutil
 pydantic
 python-magic # file type identification library. See 'magic.from_file(...)'
 PyYAML

services/dynamic-sidecar/requirements/_base.txt

@@ -209,6 +209,8 @@ paramiko==2.11.0
     #   docker
 pint==0.19.2
     # via -r requirements/../../../packages/simcore-sdk/requirements/_base.in
+psutil==5.9.1
+    # via -r requirements/_base.in
 psycopg2-binary==2.9.3
     # via
     #   aiopg

services/dynamic-sidecar/setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.1.0
+current_version = 1.1.1
 commit = True
 message = services/dynamic-sidecar version: {current_version} → {new_version}
 tag = False

services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py

@@ -12,7 +12,11 @@
 from pydantic import BaseModel
 from servicelib.fastapi.requests_decorators import cancel_on_disconnect
 
-from ..core.docker_compose_utils import docker_compose_down, docker_compose_up
+from ..core.docker_compose_utils import (
+    docker_compose_down,
+    docker_compose_rm,
+    docker_compose_up,
+)
 from ..core.docker_logs import start_log_fetching, stop_log_fetching
 from ..core.docker_utils import docker_client
 from ..core.rabbitmq import RabbitMQ
@@ -50,18 +54,21 @@ async def _task_docker_compose_up_and_send_message(
     app: FastAPI,
     application_health: ApplicationHealth,
     rabbitmq: RabbitMQ,
-    command_timeout: float,
+    command_timeout: int,
 ) -> None:
     # building is a security risk hence is disabled via "--no-build" parameter
     await send_message(rabbitmq, "starting service containers")
     assert shared_store.compose_spec  # nosec
 
     with directory_watcher_disabled(app):
+        # prunes first stopped containers
+        await docker_compose_rm(shared_store.compose_spec, settings)
+
         r = await docker_compose_up(
-            shared_store, settings, command_timeout=command_timeout
+            shared_store.compose_spec, settings, timeout=command_timeout
         )
 
-    message = f"Finished docker-compose up with output\n{r.decoded_stdout}"
+    message = f"Finished docker-compose up with output\n{r.message}"
 
     if r.success:
         await send_message(rabbitmq, "service containers started")
@@ -117,11 +124,11 @@ async def create_containers(
     request: Request,
     containers_create: ContainersCreate,
     background_tasks: BackgroundTasks,
-    command_timeout: float = Query(
-        3600.0, description="docker-compose up command timeout run as a background"
+    command_timeout: int = Query(
+        3600, description="docker-compose up command timeout run as a background"
     ),
-    validation_timeout: float = Query(
-        60.0, description="docker-compose config timeout (EXPERIMENTAL)"
+    validation_timeout: int = Query(
+        60, description="docker-compose config timeout (EXPERIMENTAL)"
     ),
     settings: DynamicSidecarSettings = Depends(get_settings),
     shared_store: SharedStore = Depends(get_shared_store),
@@ -176,8 +183,8 @@ async def create_containers(
     },
 )
 async def runs_docker_compose_down(
-    command_timeout: float = Query(
-        10.0, description="docker-compose down command timeout default  (EXPERIMENTAL)"
+    command_timeout: int = Query(
+        10, description="docker-compose down command timeout default  (EXPERIMENTAL)"
     ),
     settings: DynamicSidecarSettings = Depends(get_settings),
     shared_store: SharedStore = Depends(get_shared_store),
@@ -193,24 +200,28 @@ async def runs_docker_compose_down(
         )
 
     result = await docker_compose_down(
-        shared_store=shared_store,
-        settings=settings,
-        command_timeout=command_timeout,
+        shared_store.compose_spec,
+        settings,
+        timeout=min(command_timeout, settings.DYNAMIC_SIDECAR_STOP_AND_REMOVE_TIMEOUT),
     )
 
     for container_name in shared_store.container_names:
         await stop_log_fetching(app, container_name)
 
+    await docker_compose_rm(shared_store.compose_spec, settings)
+
     if not result.success:
         logger.warning(
             "docker-compose down command finished with errors\n%s",
-            result.decoded_stdout,
-        )
-        raise HTTPException(
-            status.HTTP_422_UNPROCESSABLE_ENTITY, detail=result.decoded_stdout
+            result.message,
         )
+        raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, detail=result.message)
+
+    # removing compose-file spec
+    assert result.success  # nosec
+    shared_store.clear()
 
-    return result.decoded_stdout
+    return result.message
 
 
 @containers_router.get(

services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers_extension.py

@@ -270,8 +270,8 @@ async def push_output_ports(
 @cancel_on_disconnect
 async def restarts_containers(
     request: Request,
-    command_timeout: float = Query(
-        10.0, description="docker-compose stop command timeout default"
+    command_timeout: int = Query(
+        10, description="docker-compose stop command timeout default"
     ),
     app: FastAPI = Depends(get_application),
     settings: DynamicSidecarSettings = Depends(get_settings),
@@ -293,16 +293,16 @@ async def restarts_containers(
         await stop_log_fetching(app, container_name)
 
     result = await docker_compose_restart(
-        shared_store.compose_spec, settings, command_timeout=command_timeout
+        shared_store.compose_spec,
+        settings,
+        timeout=min(command_timeout, settings.DYNAMIC_SIDECAR_STOP_AND_REMOVE_TIMEOUT),
     )
 
     if not result.success:
         logger.warning(
-            "docker-compose restart finished with errors\n%s", result.decoded_stdout
-        )
-        raise HTTPException(
-            status.HTTP_422_UNPROCESSABLE_ENTITY, detail=result.decoded_stdout
+            "docker-compose restart finished with errors\n%s", result.message
         )
+        raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, detail=result.message)
 
     for container_name in shared_store.container_names:
         await start_log_fetching(app, container_name)

services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py

@@ -60,7 +60,7 @@ class AppState:
     }
 
     def __init__(self, initialized_app: FastAPI):
-        # guarantees app states are in place
+        # Ensures states are initialized upon construction
         errors = [
             "app.state.{name}"
             for name, type_ in AppState._STATES.items()
@@ -84,13 +84,13 @@ def mounted_volumes(self) -> MountedVolumes:
         return self._app.state.mounted_volumes
 
     @property
-    def shared_store(self) -> SharedStore:
+    def _shared_store(self) -> SharedStore:
         assert isinstance(self._app.state.shared_store, SharedStore)  # nosec
         return self._app.state.shared_store
 
     @property
     def compose_spec(self) -> Optional[str]:
-        return self.shared_store.compose_spec
+        return self._shared_store.compose_spec
 
 
 def setup_logger(settings: DynamicSidecarSettings):
@@ -161,18 +161,19 @@ async def _on_startup() -> None:
 
     async def _on_shutdown() -> None:
         if docker_compose_yaml := app_state.compose_spec:
-            logger.info("Removing spawned containers%s", docker_compose_yaml)
+            logger.info("Removing spawned containers")
 
             result = await docker_compose_down(
-                app.state.shared_store,
+                docker_compose_yaml,
                 app.state.settings,
-                command_timeout=app.state.settings.DYNAMIC_SIDECAR_DOCKER_COMPOSE_DOWN_TIMEOUT,
+                # NOTE: in the event of a SIGTERM, there is a limited time to cleanup
+                timeout=app.state.settings.DYNAMIC_SIDECAR_DOCKER_COMPOSE_DOWN_TIMEOUT,
             )
 
             logger.log(
                 logging.INFO if result.success else logging.ERROR,
                 "Removed spawned containers:\n%s",
-                result.decoded_stdout,
+                result.message,
             )
         # FINISHED
         print(APP_FINISHED_BANNER_MSG, flush=True)