(⚠️ devops) 🐛 Adding missing custom constraints (ITISFoundation#3017)

* adding new placement constraints

* no longer optional

* added example

* if data cannot be saved, dynamic-sidecars will not be removed

* added optional

* added constraints

* fixed regex

* adding commands to help with service state recovery

* refactor

* using default_factory

* pylint

* fixed failing test

* fix test

Co-authored-by: Andrei Neagu <[email protected]>

Author: GitHK

services/director-v2/src/simcore_service_director_v2/core/settings.py

@@ -5,7 +5,7 @@
 from enum import Enum
 from functools import cached_property
 from pathlib import Path
-from typing import Dict, Optional, Set
+from typing import Dict, List, Optional, Set
 
 from models_library.basic_types import (
     BootModeEnum,
@@ -21,7 +21,15 @@
     NoAuthentication,
 )
 from models_library.projects_networks import SERVICE_NETWORK_RE
-from pydantic import AnyHttpUrl, AnyUrl, Field, PositiveFloat, PositiveInt, validator
+from pydantic import (
+    AnyHttpUrl,
+    AnyUrl,
+    Field,
+    PositiveFloat,
+    PositiveInt,
+    constr,
+    validator,
+)
 from settings_library.base import BaseCustomSettings
 from settings_library.docker_registry import RegistrySettings
 from settings_library.http_client_request import ClientRequestSettings
@@ -51,6 +59,10 @@
 
 SUPPORTED_TRAEFIK_LOG_LEVELS: Set[str] = {"info", "debug", "warn", "error"}
 
+PlacementConstraint = constr(
+    strip_whitespace=True, regex=r"^[a-zA-Z0-9. ]*(!=|==){1}[a-zA-Z0-9. ]*$"
+)
+
 
 class S3Provider(str, Enum):
     AWS = "AWS"
@@ -430,6 +442,13 @@ class AppSettings(BaseCustomSettings, MixinLoggingSettings):
 
     DIRECTOR_V2_DOCKER_REGISTRY: RegistrySettings = Field(auto_default_from_env=True)
 
+    # This is just a service placement constraint, see
+    # https://docs.docker.com/engine/swarm/services/#control-service-placement.
+    DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS: List[PlacementConstraint] = Field(
+        default_factory=list,
+        example='["node.labels.region==east", "one!=yes"]',
+    )
+
     @validator("LOG_LEVEL", pre=True)
     @classmethod
     def _validate_loglevel(cls, value) -> str:

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py

@@ -179,8 +179,6 @@ def get_dynamic_sidecar_spec(
         "labels": {
             # TODO: let's use a pydantic model with descriptions
             "io.simcore.zone": scheduler_data.simcore_traefik_zone,
-            "port": f"{dynamic_sidecar_settings.DYNAMIC_SIDECAR_PORT}",
-            "study_id": f"{scheduler_data.project_id}",
             "traefik.docker.network": scheduler_data.dynamic_sidecar_network_name,  # also used for scheduling
             "traefik.enable": "true",
             f"traefik.http.routers.{scheduler_data.service_name}.entrypoints": "http",
@@ -189,10 +187,13 @@ def get_dynamic_sidecar_spec(
             f"traefik.http.services.{scheduler_data.service_name}.loadbalancer.server.port": f"{dynamic_sidecar_settings.DYNAMIC_SIDECAR_PORT}",
             "type": ServiceType.MAIN.value,  # required to be listed as an interactive service and be properly cleaned up
             "user_id": f"{scheduler_data.user_id}",
+            "port": f"{dynamic_sidecar_settings.DYNAMIC_SIDECAR_PORT}",
+            "study_id": f"{scheduler_data.project_id}",
             # the following are used for scheduling
             "uuid": f"{scheduler_data.node_uuid}",  # also needed for removal when project is closed
             "swarm_stack_name": dynamic_sidecar_settings.SWARM_STACK_NAME,  # required for listing services with uuid
             DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL: scheduler_data.as_label_data(),
+            "service_image": dynamic_sidecar_settings.DYNAMIC_SIDECAR_IMAGE,
         },
         "name": scheduler_data.service_name,
         "networks": [swarm_network_id, dynamic_sidecar_network_id],
@@ -207,7 +208,9 @@ def get_dynamic_sidecar_spec(
                 "Labels": {},
                 "Mounts": mounts,
             },
-            "Placement": {"Constraints": []},
+            "Placement": {
+                "Constraints": app_settings.DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS
+            },
             "RestartPolicy": {
                 "Condition": "on-failure",
                 "Delay": 5000000,

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/events.py

@@ -467,6 +467,7 @@ async def action(cls, app: FastAPI, scheduler_data: SchedulerData) -> None:
             await dynamic_sidecar_client.begin_service_destruction(
                 dynamic_sidecar_endpoint=scheduler_data.dynamic_sidecar.endpoint
             )
+        # NOTE: ANE: need to use more specific exception here
         except Exception as e:  # pylint: disable=broad-except
             logger.warning(
                 "Could not contact dynamic-sidecar to begin destruction of %s\n%s",
@@ -502,6 +503,7 @@ async def action(cls, app: FastAPI, scheduler_data: SchedulerData) -> None:
                     )
                 await logged_gather(*tasks)
                 logger.info("Ports data pushed by dynamic-sidecar")
+            # NOTE: ANE: need to use more specific exception here
             except Exception as e:  # pylint: disable=broad-except
                 logger.warning(
                     (
@@ -511,6 +513,10 @@ async def action(cls, app: FastAPI, scheduler_data: SchedulerData) -> None:
                     scheduler_data.service_name,
                     str(e),
                 )
+                # ensure dynamic-sidecar does not get removed
+                # user data can be manually saved and manual
+                # cleanup of the dynamic-sidecar is required
+                raise e
 
         # remove the 2 services
         await remove_dynamic_sidecar_stack(

services/director-v2/tests/integration/02/test_dynamic_services_routes.py

@@ -184,6 +184,7 @@ def mock_dynamic_sidecar_api_calls(mocker: MockerFixture) -> None:
         ("service_restore_state", None),
         ("service_pull_output_ports", 42),
         ("service_outputs_create_dirs", None),
+        ("service_push_output_ports", None),
     ]:
         mocker.patch(
             f"{class_path}.{function_name}",

services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py

@@ -203,6 +203,7 @@ def mock_dynamic_sidecar_client(mocker: MockerFixture) -> None:
     for method_name, return_value in [
         ("service_restore_state", None),
         ("service_pull_output_ports", 42),
+        ("service_push_output_ports", None),
     ]:
         mocker.patch(
             f"simcore_service_director_v2.modules.dynamic_sidecar.client_api.DynamicSidecarClient.{method_name}",

services/director-v2/tests/unit/test_core_settings.py

@@ -2,7 +2,7 @@
 # pylint:disable=unused-argument
 # pylint:disable=redefined-outer-name
 
-from typing import Any, Dict, Set
+from typing import Any, Dict, List, Set
 
 import pytest
 from _pytest.monkeypatch import MonkeyPatch
@@ -133,3 +133,32 @@ def test_expected_failure_dynamic_sidecar_settings(
 ) -> None:
     with pytest.raises(ValidationError) as exc_info:
         DynamicSidecarSettings()
+
+
+@pytest.mark.parametrize(
+    "custom_constraints, expected",
+    (
+        ("[]", []),
+        ('["one==yes"]', ["one==yes"]),
+        ('["two!=no"]', ["two!=no"]),
+        ('["one==yes", "two!=no"]', ["one==yes", "two!=no"]),
+        ('["     strips.white.spaces   ==  ok "]', ["strips.white.spaces   ==  ok"]),
+    ),
+)
+def test_services_custom_constraints(
+    custom_constraints: str,
+    expected: List[str],
+    project_env_devel_environment,
+    monkeypatch: MonkeyPatch,
+) -> None:
+    monkeypatch.setenv("DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS", custom_constraints)
+    settings = AppSettings()
+    assert type(settings.DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS) == list
+    assert settings.DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS == expected
+
+
+def test_services_custom_constraints_default_empty_list(
+    project_env_devel_environment,
+) -> None:
+    settings = AppSettings()
+    assert settings.DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS == []

services/dynamic-sidecar/Dockerfile

@@ -14,6 +14,7 @@ RUN set -eux && \
   apt-get update && \
   apt-get install -y \
   gosu \
+  curl \
   libmagic1 \
   && \
   rm -rf /var/lib/apt/lists/* && \
@@ -77,6 +78,10 @@ WORKDIR /build
 COPY --chown=scu:scu services/dynamic-sidecar/requirements/_base.txt .
 RUN pip --no-cache-dir install -r _base.txt
 
+# copy utility devops scripts
+COPY --chown=scu:scu services/dynamic-sidecar/extra/Makefile /home/scu
+COPY --chown=root:root services/dynamic-sidecar/extra/Makefile /root
+
 # --------------------------Prod-depends-only stage -------------------
 # This stage is for production only dependencies that get partially wiped out afterwards (final docker image concerns)
 #
@@ -95,7 +100,6 @@ WORKDIR /build/services/dynamic-sidecar
 RUN pip --no-cache-dir install -r requirements/prod.txt &&\
   pip --no-cache-dir list -v
 
-
 # --------------------------Production stage -------------------
 # Final cleanup up to reduce image size and startup setup
 # Runs as scu (non-root user)

services/dynamic-sidecar/extra/Makefile

@@ -0,0 +1,24 @@
+
+SHELL = /bin/bash
+.DEFAULT_GOAL := help
+
+BASE_ADDRESS = http://localhost:8000/v1/
+
+.PHONY: save-state
+save-state:  ## start saving the state of this service
+	@echo ">>>>> Expect a 204 reply if OK <<<<<"
+	curl -v -X POST ${BASE_ADDRESS}/containers/state:save
+
+
+.PHONY: push-outputs
+push-outputs:  ## push the outputs for this service
+	@echo ">>>>> Expect a 204 reply if OK <<<<<"
+	curl -v -X POST ${BASE_ADDRESS}/containers/ports/outputs:push
+
+
+.PHONY: help
+help:   ## this help
+	@grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) \
+	| sed -n 's/^\(.*\): \(.*\)##\(.*\)/\1\t\3/p'
+
+