boot options injected to all containers (ITISFoundation#3102)

Co-authored-by: Andrei Neagu <[email protected]>

Author: GitHK

packages/models-library/src/models_library/service_settings_labels.py

@@ -4,7 +4,7 @@
 from enum import Enum
 from functools import cached_property
 from pathlib import Path
-from typing import Any, Dict, Literal, Optional
+from typing import Any, Literal, Optional
 
 from pydantic import BaseModel, Extra, Field, Json, PrivateAttr, validator
 
@@ -44,7 +44,7 @@ class SimcoreServiceSettingLabelEntry(BaseModel):
     Specifically the section under ``TaskTemplate``
     """
 
-    _destination_container: str = PrivateAttr()
+    _destination_containers: list[str] = PrivateAttr()
     name: str = Field(..., description="The name of the service setting")
     setting_type: Literal[
         "string",
@@ -162,7 +162,7 @@ class Config(_BaseConfig):
         }
 
 
-ComposeSpecLabel = Dict[str, Any]
+ComposeSpecLabel = dict[str, Any]
 
 
 class RestartPolicy(str, Enum):

packages/models-library/tests/test_service_settings_labels.py

@@ -6,7 +6,7 @@
 from collections import namedtuple
 from copy import deepcopy
 from pprint import pformat
-from typing import Any, Dict, Type
+from typing import Any, Type
 
 import pytest
 from models_library.service_settings_labels import (
@@ -50,7 +50,7 @@
     ids=[x.id for x in SIMCORE_SERVICE_EXAMPLES],
 )
 def test_simcore_service_labels(
-    example: Dict, items: int, uses_dynamic_sidecar: bool
+    example: dict, items: int, uses_dynamic_sidecar: bool
 ) -> None:
     simcore_service_labels = SimcoreServiceLabels.parse_obj(example)
 
@@ -72,7 +72,7 @@ def test_service_settings() -> None:
     # ensure private attribute assignment
     for service_setting in simcore_settings_settings_label:
         # pylint: disable=protected-access
-        service_setting._destination_container = "random_value"
+        service_setting._destination_containers = ["random_value1", "random_value2"]
 
 
 @pytest.mark.parametrize(
@@ -84,7 +84,7 @@ def test_service_settings() -> None:
     ),
 )
 def test_service_settings_model_examples(
-    model_cls: Type[BaseModel], model_cls_examples: Dict[str, Dict[str, Any]]
+    model_cls: Type[BaseModel], model_cls_examples: dict[str, dict[str, Any]]
 ) -> None:
     for name, example in model_cls_examples.items():
         print(name, ":", pformat(example))
@@ -97,7 +97,7 @@ def test_service_settings_model_examples(
     (SimcoreServiceLabels,),
 )
 def test_correctly_detect_dynamic_sidecar_boot(
-    model_cls: Type[BaseModel], model_cls_examples: Dict[str, Dict[str, Any]]
+    model_cls: Type[BaseModel], model_cls_examples: dict[str, dict[str, Any]]
 ) -> None:
     for name, example in model_cls_examples.items():
         print(name, ":", pformat(example))
@@ -108,7 +108,7 @@ def test_correctly_detect_dynamic_sidecar_boot(
 
 
 def test_raises_error_if_http_entrypoint_is_missing() -> None:
-    simcore_service_labels: Dict[str, Any] = deepcopy(
+    simcore_service_labels: dict[str, Any] = deepcopy(
         SimcoreServiceLabels.Config.schema_extra["examples"][2]
     )
     del simcore_service_labels["simcore.service.container-http-entrypoint"]
@@ -141,7 +141,7 @@ def test_simcore_services_labels_compose_spec_null_container_http_entry_provided
 
 
 def test_raises_error_wrong_restart_policy() -> None:
-    simcore_service_labels: Dict[str, Any] = deepcopy(
+    simcore_service_labels: dict[str, Any] = deepcopy(
         SimcoreServiceLabels.Config.schema_extra["examples"][2]
     )
     simcore_service_labels["simcore.service.restart-policy"] = "__not_a_valid_policy__"

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/settings.py

@@ -1,7 +1,7 @@
 import json
 import logging
 from collections import deque
-from typing import Any, Deque, Dict, List, Optional, cast
+from typing import Any, Optional, cast
 
 from models_library.boot_options import BootOption, EnvVarKey
 from models_library.service_settings_labels import (
@@ -33,7 +33,7 @@
 log = logging.getLogger(__name__)
 
 
-def _parse_mount_settings(settings: List[Dict]) -> List[Dict]:
+def _parse_mount_settings(settings: list[dict]) -> list[dict]:
     mounts = []
     for s in settings:
         log.debug("Retrieved mount settings %s", s)
@@ -57,7 +57,7 @@ def _parse_mount_settings(settings: List[Dict]) -> List[Dict]:
     return mounts
 
 
-def _parse_env_settings(settings: List[str]) -> Dict:
+def _parse_env_settings(settings: list[str]) -> dict:
     envs = {}
     for s in settings:
         log.debug("Retrieved env settings %s", s)
@@ -76,7 +76,7 @@ def _parse_env_settings(settings: List[str]) -> Dict:
 # TODO: PC->ANE: i tend to agree with pylint, perhaps we can refactor this together
 def update_service_params_from_settings(
     labels_service_settings: SimcoreServiceSettingsLabel,
-    create_service_params: Dict[str, Any],
+    create_service_params: dict[str, Any],
 ) -> None:
     for param in labels_service_settings:
         param: SimcoreServiceSettingLabelEntry = param
@@ -143,7 +143,7 @@ def update_service_params_from_settings(
                 )
         elif param.name == "mount":
             log.debug("Found mount parameter %s", param.value)
-            mount_settings: List[Dict] = _parse_mount_settings(param.value)
+            mount_settings: list[dict] = _parse_mount_settings(param.value)
             if mount_settings:
                 create_service_params["task_template"]["ContainerSpec"][
                     "Mounts"
@@ -175,7 +175,7 @@ async def _extract_osparc_involved_service_labels(
     service_key: str,
     service_tag: str,
     service_labels: SimcoreServiceLabels,
-) -> Dict[str, SimcoreServiceLabels]:
+) -> dict[str, SimcoreServiceLabels]:
     """
     Returns all the involved oSPARC services from the provided service labels.
 
@@ -188,13 +188,13 @@ async def _extract_osparc_involved_service_labels(
     # initialize with existing labels
     # stores labels mapped by image_name service:tag
     _default_key = _assemble_key(service_key=service_key, service_tag=service_tag)
-    docker_image_name_by_services: Dict[str, SimcoreServiceLabels] = {
+    docker_image_name_by_services: dict[str, SimcoreServiceLabels] = {
         _default_key: service_labels
     }
     # maps form image_name to compose_spec key
-    reverse_mapping: Dict[str, str] = {_default_key: DEFAULT_SINGLE_SERVICE_NAME}
+    reverse_mapping: dict[str, str] = {_default_key: DEFAULT_SINGLE_SERVICE_NAME}
 
-    def remap_to_compose_spec_key() -> Dict[str, str]:
+    def remap_to_compose_spec_key() -> dict[str, str]:
         # remaps from image_name as key to compose_spec key
         return {reverse_mapping[k]: v for k, v in docker_image_name_by_services.items()}
 
@@ -252,27 +252,27 @@ def remap_to_compose_spec_key() -> Dict[str, str]:
     return remap_to_compose_spec_key()
 
 
-def _add_compose_destination_container_to_settings_entries(
-    settings: SimcoreServiceSettingsLabel, destination_container: str
-) -> List[SimcoreServiceSettingLabelEntry]:
+def _add_compose_destination_containers_to_settings_entries(
+    settings: SimcoreServiceSettingsLabel, destination_containers: list[str]
+) -> list[SimcoreServiceSettingLabelEntry]:
     def _inject_destination_container(
         item: SimcoreServiceSettingLabelEntry,
     ) -> SimcoreServiceSettingLabelEntry:
         # pylint: disable=protected-access
-        item._destination_container = destination_container
+        item._destination_containers = destination_containers
         return item
 
     return [_inject_destination_container(x) for x in settings]
 
 
 def _merge_resources_in_settings(
-    settings: Deque[SimcoreServiceSettingLabelEntry],
+    settings: deque[SimcoreServiceSettingLabelEntry],
     service_resources: ServiceResourcesDict,
-) -> Deque[SimcoreServiceSettingLabelEntry]:
+) -> deque[SimcoreServiceSettingLabelEntry]:
     """All oSPARC services which have defined resource requirements will be added"""
     log.debug("MERGING\n%s\nAND\n%s", f"{settings=}", f"{service_resources}")
 
-    result: Deque[SimcoreServiceSettingLabelEntry] = deque()
+    result: deque[SimcoreServiceSettingLabelEntry] = deque()
 
     for entry in settings:
         entry: SimcoreServiceSettingLabelEntry = entry
@@ -343,14 +343,14 @@ def _merge_resources_in_settings(
 
 
 def _patch_target_service_into_env_vars(
-    settings: Deque[SimcoreServiceSettingLabelEntry],
-) -> Deque[SimcoreServiceSettingLabelEntry]:
+    settings: deque[SimcoreServiceSettingLabelEntry],
+) -> deque[SimcoreServiceSettingLabelEntry]:
     """NOTE: this method will modify settings in place"""
 
-    def _format_env_var(env_var: str, destination_container: str) -> str:
+    def _format_env_var(env_var: str, destination_container: list[str]) -> str:
         var_name, var_payload = env_var.split("=")
         json_encoded = json.dumps(
-            dict(destination_container=destination_container, env_var=var_payload)
+            dict(destination_containers=destination_container, env_var=var_payload)
         )
         return f"{var_name}={json_encoded}"
 
@@ -361,21 +361,21 @@ def _format_env_var(env_var: str, destination_container: str) -> str:
             list_of_env_vars = entry.value if entry.value else []
 
             # pylint: disable=protected-access
-            destination_container = entry._destination_container
+            destination_containers: list[str] = entry._destination_containers
 
             # transforms settings defined environment variables
             # from `ENV_VAR=PAYLOAD`
-            # to   `ENV_VAR={"destination_container": "destination_container", "env_var": "PAYLOAD"}`
+            # to   `ENV_VAR={"destination_container": ["destination_container"], "env_var": "PAYLOAD"}`
             entry.value = [
-                _format_env_var(x, destination_container) for x in list_of_env_vars
+                _format_env_var(x, destination_containers) for x in list_of_env_vars
             ]
 
     return settings
 
 
 def _get_boot_options(
     service_labels: SimcoreServiceLabels,
-) -> Optional[Dict[EnvVarKey, BootOption]]:
+) -> Optional[dict[EnvVarKey, BootOption]]:
     as_dict = service_labels.dict()
     boot_options_encoded = as_dict.get("io.simcore.boot-options", None)
     if boot_options_encoded is None:
@@ -387,11 +387,11 @@ def _get_boot_options(
 
 
 def _assemble_env_vars_for_boot_options(
-    boot_options: Dict[EnvVarKey, BootOption],
-    service_user_selection_boot_options: Dict[EnvVarKey, str],
+    boot_options: dict[EnvVarKey, BootOption],
+    service_user_selection_boot_options: dict[EnvVarKey, str],
 ) -> SimcoreServiceSettingsLabel:
 
-    env_vars: Deque[str] = deque()
+    env_vars: deque[str] = deque()
     for env_var_key, boot_option in boot_options.items():
         # fetch value selected by the user or use default if not present
         value = service_user_selection_boot_options.get(
@@ -411,7 +411,7 @@ def _assemble_env_vars_for_boot_options(
 
 async def get_labels_for_involved_services(
     director_v0_client: DirectorV0Client, service_key: str, service_tag: str
-) -> Dict[str, SimcoreServiceLabels]:
+) -> dict[str, SimcoreServiceLabels]:
     simcore_service_labels: SimcoreServiceLabels = (
         await director_v0_client.get_service_labels(
             service=ServiceKeyVersion(key=service_key, version=service_tag)
@@ -424,7 +424,7 @@ async def get_labels_for_involved_services(
     # paths_mapping express how to map dynamic-sidecar paths to the compose-spec volumes
     # where the service expects to find its certain folders
 
-    labels_for_involved_services: Dict[
+    labels_for_involved_services: dict[
         str, SimcoreServiceLabels
     ] = await _extract_osparc_involved_service_labels(
         director_v0_client=director_v0_client,
@@ -440,7 +440,7 @@ async def merge_settings_before_use(
     director_v0_client: DirectorV0Client,
     service_key: str,
     service_tag: str,
-    service_user_selection_boot_options: Dict[EnvVarKey, str],
+    service_user_selection_boot_options: dict[EnvVarKey, str],
     service_resources: ServiceResourcesDict,
 ) -> SimcoreServiceSettingsLabel:
     labels_for_involved_services = await get_labels_for_involved_services(
@@ -449,34 +449,36 @@ async def merge_settings_before_use(
         service_tag=service_tag,
     )
 
-    # merge the settings from the all the involved services
-    settings: Deque[SimcoreServiceSettingLabelEntry] = deque()  # TODO: fix typing here
-    for compose_spec_key, service_labels in labels_for_involved_services.items():
-        service_settings: SimcoreServiceSettingsLabel = cast(
-            SimcoreServiceSettingsLabel, service_labels.settings
-        )
+    settings: deque[SimcoreServiceSettingLabelEntry] = deque()  # TODO: fix typing here
 
-        settings.extend(
-            # inject compose spec key, used to target container specific services
-            _add_compose_destination_container_to_settings_entries(
-                settings=service_settings, destination_container=compose_spec_key
-            )
-        )
-
-        # inject boot options as env vars
+    boot_options_settings_env_vars: Optional[SimcoreServiceSettingsLabel] = None
+    # search for boot options first and inject to all containers
+    for compose_spec_key, service_labels in labels_for_involved_services.items():
         labels_boot_options = _get_boot_options(service_labels)
         if labels_boot_options:
-            # create a new setting from SimcoreServiceSettingsLabel as env var to pass to target container
+            # create a new setting from SimcoreServiceSettingsLabel as env var
             boot_options_settings_env_vars = _assemble_env_vars_for_boot_options(
                 labels_boot_options, service_user_selection_boot_options
             )
             settings.extend(
-                # inject compose spec key, used to target container specific services
-                _add_compose_destination_container_to_settings_entries(
+                _add_compose_destination_containers_to_settings_entries(
                     settings=boot_options_settings_env_vars,
-                    destination_container=compose_spec_key,
+                    destination_containers=list(labels_for_involved_services.keys()),
                 )
             )
+            break
+
+    # merge the settings from the all the involved services
+    for compose_spec_key, service_labels in labels_for_involved_services.items():
+        service_settings: SimcoreServiceSettingsLabel = cast(
+            SimcoreServiceSettingsLabel, service_labels.settings
+        )
+        settings.extend(
+            # inject compose spec key, used to target container specific services
+            _add_compose_destination_containers_to_settings_entries(
+                settings=service_settings, destination_containers=[compose_spec_key]
+            )
+        )
 
     settings = _merge_resources_in_settings(settings, service_resources)
     settings = _patch_target_service_into_env_vars(settings)

services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/validation.py

@@ -49,9 +49,9 @@ def _get_forwarded_env_vars(container_key: str) -> List[str]:
         if key.startswith("FORWARD_ENV_"):
             new_entry_key = key.replace("FORWARD_ENV_", "")
 
-            # parsing `VAR={"destination_container": "destination_container", "env_var": "PAYLOAD"}`
+            # parsing `VAR={"destination_containers": ["destination_container"], "env_var": "PAYLOAD"}`
             new_entry_payload = json.loads(os.environ[key])
-            if new_entry_payload["destination_container"] != container_key:
+            if container_key not in new_entry_payload["destination_containers"]:
                 continue
 
             new_entry_value = new_entry_payload["env_var"]