Is1747/remove unused volumes (#1750)

sidecar shows on which host it runs
deletes mounted volumes after use
gracefully stops

Author: sanderegg

services/docker-compose.devel.yml

@@ -48,6 +48,7 @@ services:
       - ./sidecar:/devel/services/sidecar
       - ./storage/client-sdk:/devel/services/storage/client-sdk
       - ../packages:/devel/packages
+      - /etc/hostname:/home/scu/hostname:ro
     environment:
       # force to start as cpu mode otherwise it will boot in gpu mode in development
       - START_AS_MODE_CPU=1
@@ -81,6 +82,7 @@ services:
       - ./sidecar:/devel/services/sidecar
       - ./storage/client-sdk:/devel/services/storage/client-sdk
       - ../packages:/devel/packages
+      - /etc/hostname:/home/scu/hostname:ro
     ports:
       - "3007:3000"
     environment:
@@ -135,6 +137,7 @@ services:
       - ./sidecar:/devel/services/sidecar
       - ./storage/client-sdk:/devel/services/storage/client-sdk
       - ../packages:/devel/packages
+      - /etc/hostname:/home/scu/hostname:ro
     ports:
       - "3008:3000"
     environment:

services/docker-compose.yml

@@ -149,6 +149,7 @@ services:
       - output:/home/scu/output
       - log:/home/scu/log
       - /var/run/docker.sock:/var/run/docker.sock
+      - /etc/hostname:/home/scu/hostname:ro
     environment:
       - RABBIT_HOST=${RABBIT_HOST}
       - RABBIT_PORT=${RABBIT_PORT}

services/sidecar/Dockerfile

@@ -10,37 +10,44 @@ FROM python:${PYTHON_VERSION}-slim-buster as base
 
 LABEL maintainer=mguidon
 
+RUN set -eux; \
+  apt-get update; \
+  apt-get install -y gosu; \
+  rm -rf /var/lib/apt/lists/*; \
+  # verify that the binary works
+  gosu nobody true
+
 # simcore-user uid=8004(scu) gid=8004(scu) groups=8004(scu)
 ENV SC_USER_ID=8004 \
-    SC_USER_NAME=scu \
-    SC_BUILD_TARGET=base \
-    SC_BOOT_MODE=default
+  SC_USER_NAME=scu \
+  SC_BUILD_TARGET=base \
+  SC_BOOT_MODE=default
 
 RUN adduser \
-    --uid ${SC_USER_ID} \
-    --disabled-password \
-    --gecos "" \
-    --shell /bin/sh \
-    --home /home/${SC_USER_NAME} \
-    ${SC_USER_NAME}
+  --uid ${SC_USER_ID} \
+  --disabled-password \
+  --gecos "" \
+  --shell /bin/sh \
+  --home /home/${SC_USER_NAME} \
+  ${SC_USER_NAME}
 
 # Sets utf-8 encoding for Python et al
 ENV LANG=C.UTF-8
 # Turns off writing .pyc files; superfluous on an ephemeral container.
 ENV PYTHONDONTWRITEBYTECODE=1 \
-    VIRTUAL_ENV=/home/scu/.venv
+  VIRTUAL_ENV=/home/scu/.venv
 # Ensures that the python and pip executables used
 # in the image will be those from our virtualenv.
 ENV PATH="${VIRTUAL_ENV}/bin:$PATH"
 
 # environment variables
 ENV SWARM_STACK_NAME=""\
-    SIDECAR_SERVICES_MAX_NANO_CPUS=4000000000 \
-    SIDECAR_SERVICES_MAX_MEMORY_BYTES=2147483648 \
-    SIDECAR_SERVICES_TIMEOUT_SECONDS=1200 \
-    SIDECAR_INPUT_FOLDER=/home/scu/input \
-    SIDECAR_OUTPUT_FOLDER=/home/scu/output \
-    SIDECAR_LOG_FOLDER=/home/scu/log
+  SIDECAR_SERVICES_MAX_NANO_CPUS=4000000000 \
+  SIDECAR_SERVICES_MAX_MEMORY_BYTES=2147483648 \
+  SIDECAR_SERVICES_TIMEOUT_SECONDS=1200 \
+  SIDECAR_INPUT_FOLDER=/home/scu/input \
+  SIDECAR_OUTPUT_FOLDER=/home/scu/output \
+  SIDECAR_LOG_FOLDER=/home/scu/log
 
 EXPOSE 8080
 
@@ -54,16 +61,16 @@ FROM base as build
 ENV SC_BUILD_TARGET=build
 
 RUN apt-get update &&\
-    apt-get install -y --no-install-recommends \
-    build-essential
+  apt-get install -y --no-install-recommends \
+  build-essential
 
 # NOTE: python virtualenv is used here such that installed packages may be moved to production image easily by copying the venv
 RUN python -m venv ${VIRTUAL_ENV}
 
 RUN pip --no-cache-dir install --upgrade \
-    pip~=20.2.2  \
-    wheel \
-    setuptools
+  pip~=20.2.2  \
+  wheel \
+  setuptools
 
 # copy sidecar and dependencies
 COPY --chown=scu:scu packages /build/packages
@@ -95,7 +102,7 @@ RUN pip --no-cache-dir install -r requirements/prod.txt
 FROM base as production
 
 ENV SC_BUILD_TARGET=production \
-    SC_BOOT_MODE=production
+  SC_BOOT_MODE=production
 ENV PYTHONOPTIMIZE=TRUE
 
 WORKDIR /home/scu

services/sidecar/docker/boot.sh

@@ -22,21 +22,20 @@ then
   pip install --no-cache-dir -r requirements/dev.txt
   cd - || exit 1
   echo "$INFO" "PIP :"
-  pip list | sed 's/^/    /'  
+  pip list | sed 's/^/    /'
 fi
 
 # RUNNING application ----------------------------------------
 
 # default
 CONCURRENCY=1
 POOL=prefork
-
 if [ "${SC_BOOT_MODE}" = "debug-ptvsd" ]
 then
   # NOTE: in this case, remote debugging is only available in development mode!
   # FIXME: workaround since PTVSD does not support prefork subprocess debugging: https://github.com/microsoft/ptvsd/issues/943
   POOL=solo
-  watchmedo auto-restart --recursive --pattern="*.py" -- \
+  exec watchmedo auto-restart --recursive --pattern="*.py" -- \
     celery worker \
       --app simcore_service_sidecar.celery:app \
       --concurrency ${CONCURRENCY} \
@@ -49,5 +48,3 @@ else
       --loglevel="${SIDECAR_LOGLEVEL-WARNING}" \
       --pool=${POOL}
 fi
-
-

services/sidecar/docker/entrypoint.sh

@@ -56,7 +56,7 @@ then
 
         echo "changing $SC_USER_NAME:$SC_USER_NAME ($SC_USER_ID:$SC_USER_ID) to $SC_USER_NAME:$CONT_GROUPNAME ($HOST_USERID:$HOST_GROUPID)"
         usermod --uid "$HOST_USERID" --gid "$HOST_GROUPID" "$SC_USER_NAME"
-        
+
         echo "Changing group properties of files around from $SC_USER_ID to group $CONT_GROUPNAME"
         find / -path /proc -prune -o -group "$SC_USER_ID" -exec chgrp --no-dereference "$CONT_GROUPNAME" {} \;
         # change user property of files already around
@@ -100,8 +100,8 @@ chown -R $USERNAME:"$GROUPNAME" "${SIDECAR_LOG_FOLDER}"
 echo "$INFO Starting $* ..."
 echo "  $SC_USER_NAME rights    : $(id "$SC_USER_NAME")"
 echo "  local dir : $(ls -al)"
-echo "  input dir : $(ls -al "${SIDECAR_INPUT_FOLDER}")"    
+echo "  input dir : $(ls -al "${SIDECAR_INPUT_FOLDER}")"
 echo "  output dir : $(ls -al "${SIDECAR_OUTPUT_FOLDER}")"
 echo "  log dir : $(ls -al "${SIDECAR_LOG_FOLDER}")"
 
-su --command "$*" "$SC_USER_NAME"
+exec gosu "$SC_USER_NAME" "$@"

services/sidecar/src/simcore_service_sidecar/celery.py

@@ -1,8 +1,35 @@
-from .remote_debug import setup_remote_debugging
+import asyncio
+
+from celery.signals import worker_shutting_down
+
 from .celery_configurator import get_rabbitmq_config_and_celery_app
+from .celery_log_setup import get_task_logger
+from .cli import run_sidecar
+from .remote_debug import setup_remote_debugging
 
 setup_remote_debugging()
 
 rabbit_config, app = get_rabbitmq_config_and_celery_app()
 
+log = get_task_logger(__name__)
+
+
+@worker_shutting_down.connect
+def worker_shutting_down_handler(
+    # pylint: disable=unused-argument
+    sig,
+    how,
+    exitcode,
+    **kwargs
+):
+    # NOTE: this function shall be adapted when we switch to python 3.7+
+    log.info("detected worker_shutting_down signal(%s, %s, %s)", sig, how, exitcode)
+    tasks = asyncio.Task.all_tasks()
+    for task in tasks:
+        # pylint: disable=protected-access
+        if task._coro.__name__ == run_sidecar.__name__:
+            log.warning("canceling task....................")
+            task.cancel()
+
+
 __all__ = ["rabbit_config", "app"]

services/sidecar/src/simcore_service_sidecar/cli.py

@@ -1,9 +1,10 @@
+import asyncio
 import logging
 from typing import List, Optional, Tuple
 
 import click
 
-from .config import RABBIT_CONFIG
+from .config import RABBIT_CONFIG, SIDECAR_LOGLEVEL
 from .core import inspect
 from .db import DBContextManager
 from .rabbitmq import RabbitMQ
@@ -58,6 +59,11 @@ async def run_sidecar(
                     node_id,
                 )
                 return next_task_nodes, None
+    except asyncio.CancelledError as e:
+        log.warning(
+            "Run cancelled", exc_info=True, stack_info=SIDECAR_LOGLEVEL == logging.DEBUG
+        )
+        return None, f"Run cancelled: {str(e)}"
     except Exception as e:  # pylint: disable=broad-except
         error_message = f"run_sidecar caught the following exception: {str(e)}"
         log.exception(error_message)

services/sidecar/src/simcore_service_sidecar/config.py

@@ -37,6 +37,11 @@
 SIDECAR_DOCKER_VOLUME_LOG: str = os.environ.get(
     "SIDECAR_DOCKER_VOLUME_LOG", f"{SWARM_STACK_NAME}_log"
 )
+
+SIDECAR_HOST_HOSTNAME_PATH: Path = Path(
+    os.environ.get("SIDECAR_HOST_HOSTNAME_PATH", Path.home() / "hostname")
+)
+
 SIDECAR_LOGLEVEL: str = getattr(
     logging, os.environ.get("SIDECAR_LOGLEVEL", "WARNING").upper(), logging.WARNING
 )

services/sidecar/src/simcore_service_sidecar/core.py

@@ -2,7 +2,7 @@
 from datetime import datetime
 from typing import Dict, List, Optional, Union
 
-import aiodocker
+import asyncio
 import networkx as nx
 from aiopg.sa import Engine, SAConnection
 from aiopg.sa.result import RowProxy
@@ -97,7 +97,8 @@ async def _try_get_task_from_db(
         comp_tasks.update()
         .where(
             and_(
-                comp_tasks.c.node_id == node_id, comp_tasks.c.project_id == project_id,
+                comp_tasks.c.node_id == node_id,
+                comp_tasks.c.project_id == project_id,
             )
         )
         .values(job_id=job_request_id, state=RUNNING, start=datetime.utcnow())
@@ -114,7 +115,8 @@ async def _try_get_task_from_db(
 
 
 async def _get_pipeline_from_db(
-    db_connection: SAConnection, project_id: str,
+    db_connection: SAConnection,
+    project_id: str,
 ) -> RowProxy:
     # get the pipeline
     result = await db_connection.execute(
@@ -174,12 +176,19 @@ async def inspect(
     run_result = SUCCESS
     next_task_nodes = []
     try:
-        sidecar = Executor(
-            db_engine=db_engine, rabbit_mq=rabbit_mq, task=task, user_id=user_id,
+        executor = Executor(
+            db_engine=db_engine,
+            rabbit_mq=rabbit_mq,
+            task=task,
+            user_id=user_id,
         )
-        await sidecar.run()
+        await executor.run()
         next_task_nodes = list(graph.successors(node_id))
-    except (aiodocker.exceptions.DockerError, exceptions.SidecarException):
+    except asyncio.CancelledError:
+        run_result = FAILED
+        log.warning("Task has been cancelled")
+        raise
+    except exceptions.SidecarException:
         run_result = FAILED
         log.exception("Error during execution")