✨ Vendor secrets and session oenvs (part 3) 🗃️ (#3921)

Author: pcrespov

.pre-commit-config.yaml

@@ -29,7 +29,7 @@ repos:
           - "--py310-plus"
         name: upgrade code
   - repo: https://github.com/hadialqattan/pycln
-    rev: v1.2.5
+    rev: v2.1.4
     hooks:
       - id: pycln
         args: [--all, --expand-stars]

README.md

@@ -144,7 +144,7 @@ This project is licensed under the terms of the [MIT license](LICENSE).
 ---
 
 <p align="center">
-<image src="https://github.com/ITISFoundation/osparc-simcore-python-client/blob/4e8b18494f3191d55f6692a6a605818aeeb83f95/docs/_media/mwl.png" alt="Made with love at www.z43.swiss" width="20%" />
+<image src="https://raw.githubusercontent.com/ITISFoundation/osparc-simcore-python-client/4e8b18494f3191d55f6692a6a605818aeeb83f95/docs/_media/mwl.png" alt="Made with love (and lots of hard work) at www.z43.swiss" width="20%" />
 </p>
 
 <!-- ADD REFERENCES BELOW AND KEEP THEM IN ALPHABETICAL ORDER -->

packages/models-library/requirements/_test.in

@@ -19,4 +19,5 @@ pytest-instafail
 pytest-mock
 pytest-runner
 pytest-sugar
+python-dotenv
 pyyaml

packages/models-library/requirements/_test.txt

@@ -82,6 +82,8 @@ python-dateutil==2.8.2
     # via
     #   -c requirements/_base.txt
     #   faker
+python-dotenv==1.0.0
+    # via -r requirements/_test.in
 pyyaml==6.0
     # via -r requirements/_test.in
 six==1.16.0

packages/models-library/setup.cfg

@@ -15,5 +15,6 @@ test = pytest
 
 [tool:pytest]
 asyncio_mode = auto
-markers = 
+markers =
 	diagnostics: "can be used to run diagnostics against deployed data (e.g. database, registry etc)"
+	testit: "marks test to run during development"

packages/models-library/src/models_library/service_settings_labels.py

@@ -1,6 +1,7 @@
 # pylint: disable=unsubscriptable-object
 
 import json
+import re
 from enum import Enum
 from functools import cached_property
 from pathlib import Path
@@ -9,6 +10,7 @@
 from pydantic import (
     BaseModel,
     ByteSize,
+    ConstrainedStr,
     Extra,
     Field,
     Json,
@@ -22,10 +24,18 @@
 from .basic_types import PortInt
 from .generics import ListModel
 from .services_resources import DEFAULT_SINGLE_SERVICE_NAME
+from .utils.string_substitution import OSPARC_IDENTIFIER_PREFIX
 
 # Cloudflare DNS server address
 DEFAULT_DNS_SERVER_ADDRESS: Final[str] = "1.1.1.1"  # NOSONAR
-DEFAULT_DNS_SERVER_PORT: Final[PortInt] = 53
+DEFAULT_DNS_SERVER_PORT: Final[PortInt] = parse_obj_as(PortInt, 53)
+
+
+# NOTE: To allow parametrized value, set the type to Union[OEnvSubstitutionStr, ...]
+
+
+class OEnvSubstitutionStr(ConstrainedStr):
+    regex = re.compile(rf"^\${OSPARC_IDENTIFIER_PREFIX}\w+$")
 
 
 class _BaseConfig:
@@ -57,7 +67,9 @@ class Config(_BaseConfig):
 
 
 class SimcoreServiceSettingLabelEntry(BaseModel):
-    """These values are used to build the request body of https://docs.docker.com/engine/api/v1.41/#operation/ServiceCreate
+    """Content of "simcore.service.settings" label
+
+    These values are used to build the request body of https://docs.docker.com/engine/api/v1.41/#operation/ServiceCreate
     Specifically the section under ``TaskTemplate``
     """
 
@@ -151,6 +163,8 @@ class Config(_BaseConfig):
 
 
 class PathMappingsLabel(BaseModel):
+    """Content of "simcore.service.paths-mapping" label"""
+
     inputs_path: Path = Field(
         ..., description="folder path where the service expects all the inputs"
     )
@@ -240,10 +254,12 @@ class Config(_BaseConfig):
         }
 
 
-ComposeSpecLabel: TypeAlias = dict[str, Any]
+ComposeSpecLabelDict: TypeAlias = dict[str, Any]
 
 
 class RestartPolicy(str, Enum):
+    """Content of "simcore.service.restart-policy" label"""
+
     NO_RESTART = "no-restart"
     ON_INPUTS_DOWNLOADED = "on-inputs-downloaded"
 
@@ -281,6 +297,8 @@ class Config(_BaseConfig):
 
 
 class NATRule(BaseModel):
+    """Content of "simcore.service.containers-allowed-outgoing-permit-list" label"""
+
     hostname: str
     tcp_ports: list[_PortRange | PortInt]
     dns_resolver: DNSResolver = Field(
@@ -299,6 +317,8 @@ def iter_tcp_ports(self) -> Iterator[PortInt]:
 
 
 class DynamicSidecarServiceLabels(BaseModel):
+    """All "simcore.service.*" labels including keys"""
+
     paths_mapping: Json[PathMappingsLabel] | None = Field(
         None,
         alias="simcore.service.paths-mapping",
@@ -308,7 +328,7 @@ class DynamicSidecarServiceLabels(BaseModel):
         ),
     )
 
-    compose_spec: Json[ComposeSpecLabel] | None = Field(
+    compose_spec: Json[ComposeSpecLabelDict] | None = Field(
         None,
         alias="simcore.service.compose-spec",
         description=(

packages/models-library/src/models_library/utils/docker_compose.py

@@ -1,6 +1,6 @@
 import yaml
 
-from .string_substitution import SubstitutionsDict, TemplateText
+from .string_substitution import SubstitutionsDict, TextTemplate
 
 # Notes on below env var names:
 # - SIMCORE_REGISTRY will be replaced by the url of the simcore docker registry
@@ -17,7 +17,7 @@
 
 
 def replace_env_vars_in_compose_spec(
-    service_spec: "ComposeSpecLabel",
+    service_spec: "ComposeSpecLabelDict",
     *,
     replace_simcore_registry: str,
     replace_service_version: str,
@@ -29,7 +29,7 @@ def replace_env_vars_in_compose_spec(
 
     content: str = yaml.safe_dump(service_spec)
 
-    template = TemplateText(content)
+    template = TextTemplate(content)
     substitutions = SubstitutionsDict(
         {
             "SERVICE_VERSION": replace_service_version,

packages/models-library/src/models_library/utils/specs_substitution.py

@@ -0,0 +1,76 @@
+from typing import Any, TypeAlias
+
+import yaml
+from models_library.utils.string_substitution import (
+    SubstitutionsDict,
+    TextTemplate,
+    substitute_all_legacy_identifiers,
+)
+from pydantic import StrictBool, StrictFloat, StrictInt
+
+from .string_substitution import SubstitutionsDict, TextTemplate
+
+# This constraint on substitution values is to avoid
+# deserialization issues on the TextTemplate substitution!
+SubstitutionValue: TypeAlias = StrictBool | StrictInt | StrictFloat | str
+
+
+class SpecsSubstitutionsResolver:
+    """
+    Resolve specs dict by substituting identifiers
+
+    'specs' is defined here as dict[str, Any]. E.g. a docker-compose.yml loaded as a dict are 'specs'.
+
+    """
+
+    def __init__(self, specs: dict[str, Any], upgrade: bool):
+        self._template = self._create_text_template(specs, upgrade=upgrade)
+        self._substitutions: SubstitutionsDict = SubstitutionsDict()
+
+    @classmethod
+    def _create_text_template(
+        cls, specs: dict[str, Any], *, upgrade: bool
+    ) -> TextTemplate:
+        # convert to yaml (less symbols as in json)
+        service_spec_str: str = yaml.safe_dump(specs)
+
+        if upgrade:  # legacy
+            service_spec_str = substitute_all_legacy_identifiers(service_spec_str)
+
+        # template
+        template = TextTemplate(service_spec_str)
+        assert template.is_valid()  # nosec
+
+        return template
+
+    def get_identifiers(self) -> list[str]:
+        """lists identifiers in specs in order of apperance. Can have repetitions"""
+        return self._template.get_identifiers()
+
+    def get_replaced(self) -> set[str]:
+        return self._substitutions.used
+
+    @property
+    def substitutions(self):
+        return self._substitutions
+
+    def set_substitutions(
+        self, environs: dict[str, SubstitutionValue]
+    ) -> SubstitutionsDict:
+        """NOTE: ONLY targets identifiers declared in the specs"""
+        identifiers_needed = self.get_identifiers()
+
+        # picks only needed for substitution
+        self._substitutions = SubstitutionsDict(
+            {
+                identifier: environs[identifier]
+                for identifier in identifiers_needed
+                if identifier in environs
+            }
+        )
+        return self._substitutions
+
+    def run(self) -> dict[str, Any]:
+        new_specs_txt: str = self._template.safe_substitute(self._substitutions)
+        new_specs: dict = yaml.safe_load(new_specs_txt)
+        return new_specs

packages/models-library/src/models_library/utils/string_substitution.py

@@ -7,7 +7,7 @@
 from string import Template
 from typing import Any
 
-OSPARC_IDENTIFIER_PREFIX = "OSPARC_ENVIRONMENT"
+OSPARC_IDENTIFIER_PREFIX = "OSPARC_ENVIRONMENT_"
 
 
 def upgrade_identifier(identifier: str) -> str:
@@ -18,7 +18,8 @@ def upgrade_identifier(identifier: str) -> str:
     identifier = re.sub(r"[.-]", "_", identifier)
     identifier = identifier.upper()
     if not identifier.startswith(OSPARC_IDENTIFIER_PREFIX):
-        identifier = f"{OSPARC_IDENTIFIER_PREFIX}_{identifier}"
+        assert OSPARC_IDENTIFIER_PREFIX.endswith("_")  # nosec
+        identifier = OSPARC_IDENTIFIER_PREFIX + identifier
     return identifier
 
 
@@ -40,7 +41,7 @@ def _upgrade(match):
     return re.sub(_LEGACY_IDENTIFIER_RE_PATTERN, _upgrade, text)
 
 
-class TemplateText(Template):
+class TextTemplate(Template):
     """Template strings support `$`-based substitutions, using the following rules:
 
     - `$$` is an escape; it is replaced with a single `$`.