Enhancement 2121: overwrite computational services limits (#2129)

* disable sidecar timeout
* read limitations from docker labels
* add service settings model
* add testing of model

Author: sanderegg

packages/models-library/src/models_library/service_settings.py

@@ -0,0 +1,69 @@
+from typing import Any, List
+
+from pydantic import BaseModel, Extra, Field
+
+
+class ServiceSetting(BaseModel):
+    name: str = Field(..., description="The name of the service setting")
+    setting_type: str = Field(
+        ...,
+        description="The type of the service setting (follows Docker REST API naming scheme)",
+        alias="type",
+    )
+    value: Any = Field(
+        ...,
+        description="The value of the service setting (shall follow Docker REST API scheme for services",
+    )
+
+    class Config:
+        extra = Extra.forbid
+        schema_extra = {
+            "examples": [
+                # constraints
+                {
+                    "name": "constraints",
+                    "type": "string",
+                    "value": ["node.platform.os == linux"],
+                },
+                # resources
+                {
+                    "name": "Resources",
+                    "type": "Resources",
+                    "value": {
+                        "Limits": {"NanoCPUs": 4000000000, "MemoryBytes": 17179869184},
+                        "Reservations": {
+                            "NanoCPUs": 100000000,
+                            "MemoryBytes": 536870912,
+                            "GenericResources": [
+                                {"DiscreteResourceSpec": {"Kind": "VRAM", "Value": 1}}
+                            ],
+                        },
+                    },
+                },
+                # mounts
+                {
+                    "name": "mount",
+                    "type": "object",
+                    "value": [
+                        {
+                            "ReadOnly": True,
+                            "Source": "/tmp/.X11-unix",  # nosec
+                            "Target": "/tmp/.X11-unix",  # nosec
+                            "Type": "bind",
+                        }
+                    ],
+                },
+                # environments
+                {"name": "env", "type": "string", "value": ["DISPLAY=:0"]},
+            ]
+        }
+
+
+class ServiceSettings(BaseModel):
+    __root__: List[ServiceSetting]
+
+    def __iter__(self):
+        return iter(self.__root__)
+
+    def __getitem__(self, item):
+        return self.__root__[item]

packages/models-library/tests/test_service_settings.py

@@ -0,0 +1,21 @@
+# pylint:disable=unused-variable
+# pylint:disable=unused-argument
+# pylint:disable=redefined-outer-name
+
+from typing import Any, Dict
+
+import pytest
+from models_library.service_settings import ServiceSetting, ServiceSettings
+
+
+@pytest.mark.parametrize("example", ServiceSetting.Config.schema_extra["examples"])
+def test_service_setting(example: Dict[str, Any]):
+    service_setting_instance = ServiceSetting.parse_obj(example)
+    assert service_setting_instance
+
+
+def test_service_settings():
+    service_settings_instance = ServiceSettings.parse_obj(
+        ServiceSetting.Config.schema_extra["examples"]
+    )
+    assert service_settings_instance

services/sidecar/Dockerfile

@@ -42,9 +42,6 @@ ENV PATH="${VIRTUAL_ENV}/bin:$PATH"
 
 # environment variables
 ENV SWARM_STACK_NAME=""\
-  SIDECAR_SERVICES_MAX_NANO_CPUS=4000000000 \
-  SIDECAR_SERVICES_MAX_MEMORY_BYTES=2147483648 \
-  SIDECAR_SERVICES_TIMEOUT_SECONDS=1200 \
   SIDECAR_INPUT_FOLDER=/home/scu/input \
   SIDECAR_OUTPUT_FOLDER=/home/scu/output \
   SIDECAR_LOG_FOLDER=/home/scu/log
@@ -62,7 +59,7 @@ ENV SC_BUILD_TARGET=build
 
 RUN apt-get update \
   && apt-get install -y --no-install-recommends \
-    build-essential \
+  build-essential \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/*
 

services/sidecar/src/simcore_service_sidecar/config.py

@@ -14,7 +14,7 @@
     os.environ.get("SIDECAR_SERVICES_MAX_MEMORY_BYTES", 2 * pow(1024, 3))
 )
 SERVICES_TIMEOUT_SECONDS: int = int(
-    os.environ.get("SIDECAR_SERVICES_TIMEOUT_SECONDS", 20 * 60)
+    os.environ.get("SIDECAR_SERVICES_TIMEOUT_SECONDS", 0)
 )
 SWARM_STACK_NAME: str = os.environ.get("SWARM_STACK_NAME", "simcore")
 

services/sidecar/src/simcore_service_sidecar/executor.py

@@ -6,14 +6,17 @@
 import time
 import zipfile
 from pathlib import Path
+from pprint import pformat
 from typing import Dict, Optional
 
 import aiopg
 import attr
 from aiodocker import Docker
 from aiodocker.containers import DockerContainer
 from aiodocker.exceptions import DockerContainerError, DockerError
+from models_library.service_settings import ServiceSettings
 from packaging import version
+from pydantic import BaseModel
 from servicelib.logging_utils import log_decorator
 from servicelib.utils import fire_and_forget_task, logged_gather
 from simcore_sdk import node_data, node_ports_v2
@@ -70,6 +73,13 @@ def log_error(_, path, excinfo):
                 shutil.rmtree(str(folder), onerror=log_error)
 
 
+class ServiceResources(BaseModel):
+    memory_reservation: int = 0
+    memory_limit: int = config.SERVICES_MAX_MEMORY_BYTES
+    nano_cpus_reservation: int = 0
+    nano_cpus_limit: int = config.SERVICES_MAX_NANO_CPUS
+
+
 @attr.s(auto_attribs=True)
 class Executor:
     db_engine: aiopg.sa.Engine = None
@@ -80,6 +90,7 @@ class Executor:
     stack_name: str = config.SWARM_STACK_NAME
     shared_folders: TaskSharedVolumes = None
     integration_version: version.Version = version.parse("0.0.0")
+    service_settings: ServiceSettings = []
 
     @log_decorator(logger=log)
     async def run(self):
@@ -187,6 +198,10 @@ async def _process_task_inputs(self) -> Dict:
                 for port in (await PORTS.inputs).values()
             ]
         )
+        await self._post_messages(
+            LogType.LOG,
+            "[sidecar]Downloaded inputs.",
+        )
         return input_ports
 
     @log_decorator(logger=log)
@@ -225,6 +240,17 @@ async def _pull_image(self):
                         image_cfg["Config"]["Labels"]["io.simcore.integration-version"]
                     )["integration-version"]
                 )
+            # get service settings
+            self.service_settings = ServiceSettings.parse_raw(
+                image_cfg["Config"]["Labels"].get("simcore.service.settings", "[]")
+            )
+            log.debug(
+                "found following service settings: %s", pformat(self.service_settings)
+            )
+            await self._post_messages(
+                LogType.LOG,
+                f"[sidecar]Pulled {self.task.image['name']}:{self.task.image['tag']}",
+            )
 
     @log_decorator(logger=log)
     async def _create_container_config(self, docker_image: str) -> Dict:
@@ -247,6 +273,32 @@ async def _create_container_config(self, docker_image: str) -> Dict:
         )
         host_log_path = await get_volume_mount_point(config.SIDECAR_DOCKER_VOLUME_LOG)
 
+        # get user-defined IT limitations
+        async def _get_resource_limitations() -> Dict[str, int]:
+            resource_limitations = {
+                "Memory": config.SERVICES_MAX_MEMORY_BYTES,
+                "NanoCPUs": config.SERVICES_MAX_NANO_CPUS,
+            }
+            for setting in self.service_settings:
+                if not setting.name == "Resources":
+                    continue
+                if not isinstance(setting.value, dict):
+                    continue
+
+                limits = setting.value.get("Limits", {})
+                resource_limitations["Memory"] = limits.get(
+                    "MemoryBytes", config.SERVICES_MAX_MEMORY_BYTES
+                )
+                resource_limitations["NanoCPUs"] = limits.get(
+                    "NanoCPUs", config.SERVICES_MAX_NANO_CPUS
+                )
+            log.debug(
+                "Current resource limitations are %s", pformat(resource_limitations)
+            )
+            return resource_limitations
+
+        resource_limitations = await _get_resource_limitations()
+
         docker_container_config = {
             "Env": env_vars,
             "Cmd": "run",
@@ -255,12 +307,12 @@ async def _create_container_config(self, docker_image: str) -> Dict:
                 "user_id": str(self.user_id),
                 "study_id": str(self.task.project_id),
                 "node_id": str(self.task.node_id),
-                "nano_cpus_limit": str(config.SERVICES_MAX_NANO_CPUS),
-                "mem_limit": str(config.SERVICES_MAX_MEMORY_BYTES),
+                "nano_cpus_limit": str(resource_limitations["NanoCPUs"]),
+                "mem_limit": str(resource_limitations["Memory"]),
             },
             "HostConfig": {
-                "Memory": config.SERVICES_MAX_MEMORY_BYTES,
-                "NanoCPUs": config.SERVICES_MAX_NANO_CPUS,
+                "Memory": resource_limitations["Memory"],
+                "NanoCPUs": resource_limitations["NanoCPUs"],
                 "Init": True,
                 "AutoRemove": False,
                 "Binds": [
@@ -272,6 +324,7 @@ async def _create_container_config(self, docker_image: str) -> Dict:
                 ],
             },
         }
+
         return docker_container_config
 
     @log_decorator(logger=log)