✨Replacing dynamic sidecar proxy with Caddy (ITISFoundation#2597)

* from traefik to caddy proxy

* removed traefik rules proxy configuration

* update proxy config

* extended API to return the name of entrypoint container

* replaced traefik with caddy in dy-sidecar

* fixed imports

* moved to constant and clarified usage

* fix pylint

* fixed responses

* fixed API path name

* added missing response

* updated endpoint

* moved description to constnts

* made method private

* changed API signature

* update docstring

* fix codeclimate

* fixing resource allocation for the proxy

* no longer needed

* fixed http error types

* fixed logging for each retry

* using correct name in filter

* adjusted proxy limits and restart policy

* removed comment

* updated openapi spec

* added soem typing

* added proper regex validation for ids

* aded refactor note

* changed interface for fetching container name

* removed unused

Co-authored-by: Andrei Neagu <[email protected]>

Author: GitHK

services/director-v2/src/simcore_service_director_v2/core/settings.py

@@ -67,33 +67,12 @@ class CelerySettings(BaseCelerySettings):
     CELERY_PUBLICATION_TIMEOUT: int = 60
 
 
-class DynamicSidecarTraefikSettings(BaseCustomSettings):
-    DYNAMIC_SIDECAR_TRAEFIK_VERSION: str = Field(
-        "v2.4.13",
-        description="current version of the Traefik image to be pulled and used from dockerhub",
-    )
-    DYNAMIC_SIDECAR_TRAEFIK_LOGLEVEL: str = Field(
-        "warn", description="set Treafik's loglevel to be used"
-    )
-
-    DYNAMIC_SIDECAR_TRAEFIK_ACCESS_LOG: bool = Field(
-        False, description="enables or disables access log"
+class DynamicSidecarProxySettings(BaseCustomSettings):
+    DYNAMIC_SIDECAR_CADDY_VERSION: str = Field(
+        "2.4.5-alpine",
+        description="current version of the Caddy image to be pulled and used from dockerhub",
     )
 
-    @validator("DYNAMIC_SIDECAR_TRAEFIK_LOGLEVEL", pre=True)
-    @classmethod
-    def validate_log_level(cls, v) -> str:
-        if v not in SUPPORTED_TRAEFIK_LOG_LEVELS:
-            message = (
-                "Got log level '{v}', expected one of '{SUPPORTED_TRAEFIK_LOG_LEVELS}'"
-            )
-            raise ValueError(message)
-        return v
-
-    @cached_property
-    def access_log_as_string(self) -> str:
-        return str(self.DYNAMIC_SIDECAR_TRAEFIK_ACCESS_LOG).lower()
-
 
 class DynamicSidecarSettings(BaseCustomSettings):
     SC_BOOT_MODE: BootModeEnum = Field(
@@ -147,6 +126,14 @@ class DynamicSidecarSettings(BaseCustomSettings):
             "twards the dynamic-sidecar, as is the case with the above timeout field."
         ),
     )
+    DYNAMIC_SIDECAR_WAIT_FOR_CONTAINERS_TO_START: PositiveFloat = Field(
+        60.0 * 60.0,
+        description=(
+            "After running `docker-compose up`, images might need to be pulled "
+            "before everything is started. Using default 1hour timeout to let this "
+            "operation finish."
+        ),
+    )
 
     TRAEFIK_SIMCORE_ZONE: str = Field(
         ...,
@@ -158,7 +145,7 @@ class DynamicSidecarSettings(BaseCustomSettings):
         description="in case there are several deployments on the same docker swarm, it is attached as a label on all spawned services",
     )
 
-    DYNAMIC_SIDECAR_TRAEFIK_SETTINGS: DynamicSidecarTraefikSettings
+    DYNAMIC_SIDECAR_PROXY_SETTINGS: DynamicSidecarProxySettings
 
     REGISTRY: RegistrySettings
 

services/director-v2/src/simcore_service_director_v2/models/schemas/dynamic_services/scheduler.py

@@ -13,7 +13,7 @@
     PathMappingsLabel,
     SimcoreServiceLabels,
 )
-from pydantic import BaseModel, Extra, Field, PositiveInt, PrivateAttr
+from pydantic import BaseModel, Extra, Field, PositiveInt, PrivateAttr, constr
 
 from ..constants import (
     DYNAMIC_PROXY_SERVICE_PREFIX,
@@ -27,6 +27,9 @@
 
 MAX_ALLOWED_SERVICE_NAME_LENGTH: int = 63
 
+SHA256 = constr(max_length=64, regex=r"\b[A-Fa-f0-9]{64}\b")
+ServiceId = SHA256
+NetworkId = SHA256
 
 logger = logging.getLogger()
 
@@ -141,6 +144,7 @@ def compose_spec_submitted(self) -> bool:
         scription="docker inspect results from all the container ran at regular intervals",
     )
 
+    was_dynamic_sidecar_started: bool = False
     were_services_created: bool = Field(
         False,
         description=(
@@ -149,6 +153,21 @@ def compose_spec_submitted(self) -> bool:
         ),
     )
 
+    # below had already been validated and
+    # used only to start the proxy
+    dynamic_sidecar_id: Optional[ServiceId] = Field(
+        None, description="returned by the docker engine; used for starting the proxy"
+    )
+    dynamic_sidecar_network_id: Optional[NetworkId] = Field(
+        None, description="returned by the docker engine; used for starting the proxy"
+    )
+    swarm_network_id: Optional[NetworkId] = Field(
+        None, description="returned by the docker engine; used for starting the proxy"
+    )
+    swarm_network_name: Optional[str] = Field(
+        None, description="used for starting the proxy"
+    )
+
     @property
     def can_save_state(self) -> bool:
         """

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/client_api.py

@@ -1,13 +1,19 @@
+import json
 import logging
 import traceback
 from typing import Any, Dict
 
 import httpx
 from fastapi import FastAPI
+from starlette import status
 
 from ...core.settings import DynamicSidecarSettings
 from ...models.schemas.dynamic_services import SchedulerData
-from .errors import DynamicSchedulerException, DynamicSidecarNetworkError
+from .errors import (
+    DynamicSchedulerException,
+    DynamicSidecarNetworkError,
+    EntrypointContainerNotFoundError,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -41,6 +47,8 @@ class DynamicSidecarClient:
     # The previous implementation (with a shared client) raised
     # RuntimeErrors because resources were already locked.
 
+    API_VERSION = "v1"
+
     def __init__(self, app: FastAPI):
         dynamic_sidecar_settings: DynamicSidecarSettings = (
             app.state.settings.DYNAMIC_SERVICES.DYNAMIC_SIDECAR
@@ -73,11 +81,11 @@ async def containers_inspect(self, dynamic_sidecar_endpoint: str) -> Dict[str, A
         returns dict containing docker inspect result form
         all dynamic-sidecar started containers
         """
-        url = get_url(dynamic_sidecar_endpoint, "/v1/containers")
+        url = get_url(dynamic_sidecar_endpoint, f"/{self.API_VERSION}/containers")
         try:
             async with httpx.AsyncClient(timeout=self._base_timeout) as client:
                 response = await client.get(url=url)
-            if response.status_code != 200:
+            if response.status_code != status.HTTP_200_OK:
                 message = (
                     f"error during request status={response.status_code}, "
                     f"body={response.text}"
@@ -93,11 +101,11 @@ async def containers_inspect(self, dynamic_sidecar_endpoint: str) -> Dict[str, A
     async def containers_docker_status(
         self, dynamic_sidecar_endpoint: str
     ) -> Dict[str, Dict[str, str]]:
-        url = get_url(dynamic_sidecar_endpoint, "/v1/containers")
+        url = get_url(dynamic_sidecar_endpoint, f"/{self.API_VERSION}/containers")
         try:
             async with httpx.AsyncClient(timeout=self._base_timeout) as client:
                 response = await client.get(url=url, params=dict(only_status=True))
-            if response.status_code != 200:
+            if response.status_code != status.HTTP_200_OK:
                 logging.warning(
                     "error during request status=%s, body=%s",
                     response.status_code,
@@ -114,7 +122,7 @@ async def start_service_creation(
         self, dynamic_sidecar_endpoint: str, compose_spec: str
     ) -> None:
         """returns: True if the compose up was submitted correctly"""
-        url = get_url(dynamic_sidecar_endpoint, "/v1/containers")
+        url = get_url(dynamic_sidecar_endpoint, f"/{self.API_VERSION}/containers")
         try:
             async with httpx.AsyncClient(timeout=self._base_timeout) as client:
                 response = await client.post(url, data=compose_spec)
@@ -134,11 +142,11 @@ async def start_service_creation(
 
     async def begin_service_destruction(self, dynamic_sidecar_endpoint: str) -> None:
         """runs docker compose down on the started spec"""
-        url = get_url(dynamic_sidecar_endpoint, "/v1/containers:down")
+        url = get_url(dynamic_sidecar_endpoint, f"/{self.API_VERSION}/containers:down")
         try:
             async with httpx.AsyncClient(timeout=self._base_timeout) as client:
                 response = await client.post(url)
-            if response.status_code != 200:
+            if response.status_code != status.HTTP_200_OK:
                 message = (
                     f"ERROR during service destruction request: "
                     f"status={response.status_code}, body={response.text}"
@@ -151,6 +159,31 @@ async def begin_service_destruction(self, dynamic_sidecar_endpoint: str) -> None
             log_httpx_http_error(url, "POST", traceback.format_exc())
             raise e
 
+    async def get_entrypoint_container_name(
+        self, dynamic_sidecar_endpoint: str, dynamic_sidecar_network_name: str
+    ) -> str:
+        """
+        While this API raises EntrypointContainerNotFoundError
+        it should be called again, because in the menwhile the containers
+        might still be starting.
+        """
+        filters = json.dumps({"network": dynamic_sidecar_network_name})
+        url = get_url(
+            dynamic_sidecar_endpoint,
+            f"/{self.API_VERSION}/containers/name?filters={filters}",
+        )
+        try:
+            async with httpx.AsyncClient(timeout=self._base_timeout) as client:
+                response = await client.get(url=url)
+                if response.status_code == status.HTTP_404_NOT_FOUND:
+                    raise EntrypointContainerNotFoundError()
+                response.raise_for_status()
+
+                return response.json()
+        except httpx.HTTPError:
+            log_httpx_http_error(url, "GET", traceback.format_exc())
+            raise
+
 
 async def setup_api_client(app: FastAPI) -> None:
     logger.debug("dynamic-sidecar api client setup")

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api.py

@@ -250,7 +250,7 @@ def _make_pending() -> Tuple[ServiceState, str]:
     return service_state, message
 
 
-async def are_services_missing(
+async def is_dynamic_sidecar_missing(
     node_uuid: NodeID, dynamic_sidecar_settings: DynamicSidecarSettings
 ) -> bool:
     """Used to check if the service should be created"""

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_compose_specs.py

@@ -3,8 +3,7 @@
 
 import yaml
 from fastapi.applications import FastAPI
-from models_library.service_settings_labels import ComposeSpecLabel, PathMappingsLabel
-from pydantic import PositiveInt
+from models_library.service_settings_labels import ComposeSpecLabel
 
 from ...core.settings import DynamicSidecarSettings
 from .docker_service_specs import MATCH_SERVICE_VERSION, MATCH_SIMCORE_REGISTRY
@@ -16,14 +15,15 @@
 }
 
 
-def _inject_traefik_configuration(
+def _inject_proxy_network_configuration(
     service_spec: Dict[str, Any],
     target_container: str,
     dynamic_sidecar_network_name: str,
-    simcore_traefik_zone: str,
-    service_port: PositiveInt,
 ) -> None:
-    """Injects configuration to allow the service to be accessible on the uuid.services.SERVICE_DNS"""
+    """
+    Injects network configuration to allow the service
+    to be accessible on `uuid.services.SERVICE_DNS`
+    """
 
     # add external network to existing networks defined in the container
     service_spec["networks"] = {
@@ -41,21 +41,6 @@ def _inject_traefik_configuration(
     container_networks.append(dynamic_sidecar_network_name)
     target_container_spec["networks"] = container_networks
 
-    # expose spawned container to the internet
-    labels = target_container_spec.get("labels", [])
-    labels.extend(
-        [
-            f"io.simcore.zone={simcore_traefik_zone}",
-            "traefik.enable=true",
-            f"traefik.http.services.{target_container}.loadbalancer.server.port={service_port}",
-            f"traefik.http.routers.{target_container}.entrypoints=http",
-            f"traefik.http.routers.{target_container}.rule=PathPrefix(`/`)",
-        ]
-    )
-
-    # put back updated labels
-    target_container_spec["labels"] = labels
-
 
 def _assemble_from_service_key_and_tag(
     resolved_registry_url: str,
@@ -83,16 +68,12 @@ def _replace_env_vars_in_compose_spec(
 
 
 async def assemble_spec(
-    # pylint: disable=too-many-arguments
     app: FastAPI,
     service_key: str,
     service_tag: str,
-    paths_mapping: PathMappingsLabel,  # pylint: disable=unused-argument
     compose_spec: ComposeSpecLabel,
     container_http_entry: Optional[str],
     dynamic_sidecar_network_name: str,
-    simcore_traefik_zone: str,
-    service_port: PositiveInt,
 ) -> str:
     """
     returns a docker-compose spec used by
@@ -113,20 +94,13 @@ async def assemble_spec(
             service_tag=service_tag,
         )
         container_name = CONTAINER_NAME
-    else:
-        # TODO: need to be sorted out:
-        # - inject paths mapping
-        # - remove above # pylint: disable=unused-argument
-        pass
 
     assert container_name is not None  # nosec
 
-    _inject_traefik_configuration(
+    _inject_proxy_network_configuration(
         service_spec,
         target_container=container_name,
         dynamic_sidecar_network_name=dynamic_sidecar_network_name,
-        simcore_traefik_zone=simcore_traefik_zone,
-        service_port=service_port,
     )
 
     stringified_service_spec = yaml.safe_dump(service_spec)

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/__init__.py

@@ -0,0 +1,8 @@
+from .spec_dynamic_sidecar import (
+    MATCH_SERVICE_VERSION,
+    MATCH_SIMCORE_REGISTRY,
+    extract_service_port_from_compose_start_spec,
+    get_dynamic_sidecar_spec,
+    merge_settings_before_use,
+)
+from .spec_proxy import get_dynamic_proxy_spec