♻️ Enhances BaseCustomSettings in settings_library (ITISFoundation#2750)

Author: pcrespov

packages/pytest-simcore/src/pytest_simcore/docker_compose.py

@@ -26,8 +26,8 @@
     FIXTURE_CONFIG_OPS_SERVICES_SELECTION,
 )
 from .helpers.constants import HEADER_STR
+from .helpers.typing_env import EnvVarsDict
 from .helpers.utils_docker import get_ip, run_docker_compose_config, save_docker_infos
-from .helpers.utils_environs import EnvVarsDict
 
 
 @pytest.fixture(scope="session")

packages/pytest-simcore/src/pytest_simcore/docker_swarm.py

@@ -22,9 +22,9 @@
 from tenacity.wait import wait_fixed
 
 from .helpers.constants import HEADER_STR, MINUTE
+from .helpers.typing_env import EnvVarsDict
 from .helpers.utils_dict import copy_from_dict
 from .helpers.utils_docker import get_ip
-from .helpers.utils_environs import EnvVarsDict
 
 log = logging.getLogger(__name__)
 

packages/pytest-simcore/src/pytest_simcore/helpers/typing_env.py

@@ -0,0 +1,13 @@
+from typing import Dict, Optional
+
+EnvVarsDict = Dict[str, Optional[str]]
+#
+# NOTE: that this means that env vars do not require a value. If that happens a None is assigned
+#   For instance, a valid env file is
+#
+#   NAME=foo
+#   INDEX=33
+#   ONLY_NAME=
+#
+# will return env: EnvVarsDict = {"NAME": "foo", "INDEX": 33, "ONLY_NAME": None}
+#

packages/pytest-simcore/src/pytest_simcore/helpers/utils_environs.py

@@ -2,16 +2,22 @@
 
 """
 import re
+import warnings
 from copy import deepcopy
 from pathlib import Path
-from typing import Dict, List, Union
+from typing import Dict, List
 
 import yaml
 
-EnvVarsDict = Dict[str, Union[str, None]]
+from .typing_env import EnvVarsDict
 
 VARIABLE_SUBSTITUTION = re.compile(r"\$\{(\w+)(?:(:{0,1}[-?]{0,1})(.*))?\}$")
 
+warnings.warn(
+    f"{__name__} is deprecated, use instead pytest_simcore.helpers.utils_envs",
+    DeprecationWarning,
+)
+
 
 def _load_env(file_handler) -> Dict:
     """Deserializes an environment file like .env-devel and
@@ -36,7 +42,7 @@ def eval_environs_in_docker_compose(
     docker_compose_dir: Path,
     host_environ: Dict = None,
     *,
-    use_env_devel=True
+    use_env_devel=True,
 ):
     """Resolves environments in docker compose and sets them under 'environment' section
 
@@ -56,7 +62,7 @@ def replace_environs_in_docker_compose_service(
     docker_compose_dir: Path,
     host_environ: Dict = None,
     *,
-    use_env_devel=True
+    use_env_devel=True,
 ):
     """Resolves environments in docker-compose's service section,
     drops any reference to env_file and sets all
@@ -107,7 +113,7 @@ def eval_service_environ(
     host_environ: Dict = None,
     image_environ: Dict = None,
     *,
-    use_env_devel=True
+    use_env_devel=True,
 ) -> EnvVarsDict:
     """Deduces a service environment with it runs in a stack from confirmation
 

packages/pytest-simcore/src/pytest_simcore/helpers/utils_envs.py

@@ -0,0 +1,27 @@
+import os
+from io import StringIO
+
+from _pytest.monkeypatch import MonkeyPatch
+from dotenv import dotenv_values
+
+from .typing_env import EnvVarsDict
+
+
+def setenvs_as_envfile(monkeypatch: MonkeyPatch, envfile_text: str) -> EnvVarsDict:
+    envs = dotenv_values(stream=StringIO(envfile_text))
+    for key, value in envs.items():
+        monkeypatch.setenv(key, str(value))
+
+    assert all(env in os.environ for env in envs)
+    return envs
+
+
+def delenvs_as_envfile(
+    monkeypatch: MonkeyPatch, envfile_text: str, raising: bool
+) -> EnvVarsDict:
+    envs = dotenv_values(stream=StringIO(envfile_text))
+    for key in envs.keys():
+        monkeypatch.delenv(key, raising=raising)
+
+    assert all(env not in os.environ for env in envs)
+    return envs

packages/pytest-simcore/src/pytest_simcore/simcore_services.py

@@ -19,8 +19,8 @@
 from yarl import URL
 
 from .helpers.constants import MINUTE
+from .helpers.typing_env import EnvVarsDict
 from .helpers.utils_docker import get_ip, get_service_published_port
-from .helpers.utils_environs import EnvVarsDict
 
 log = logging.getLogger(__name__)
 

packages/settings-library/requirements/_base.in

@@ -3,7 +3,7 @@
 #
 --constraint ../../../requirements/constraints.txt
 
-pydantic
+pydantic>=1.9
 
 
 # extra

packages/settings-library/src/settings_library/base.py

@@ -1,74 +1,110 @@
-import logging
-import os
 from functools import cached_property
-from typing import List, Tuple, Type
+from typing import Sequence, get_args
 
-from pydantic import BaseSettings, Extra, SecretStr, ValidationError
+from pydantic import BaseConfig, BaseSettings, Extra, ValidationError, validator
+from pydantic.error_wrappers import ErrorList, ErrorWrapper
+from pydantic.fields import ModelField, Undefined
+from pydantic.types import SecretStr
 
-logger = logging.getLogger(__name__)
+
+class DefaultFromEnvFactoryError(ValidationError):
+    ...
+
+
+def create_settings_from_env(field: ModelField):
+    # NOTE: Cannot pass only field.type_ because @prepare_field (when this function is called)
+    #  this value is still not resolved (field.type_ at that moment has a weak_ref).
+    #  Therefore we keep the entire 'field' but MUST be treated here as read-only
+
+    def _default_factory():
+        """Creates default from sub-settings or None (if nullable)"""
+        field_settings_cls = field.type_
+        try:
+            return field_settings_cls()
+
+        except ValidationError as err:
+            if field.allow_none:
+                return None
+
+            def _prepend_field_name(ee: ErrorList):
+                if isinstance(ee, ErrorWrapper):
+                    return ErrorWrapper(ee.exc, (field.name,) + ee.loc_tuple())
+                assert isinstance(ee, Sequence)  # nosec
+                return [_prepend_field_name(e) for e in ee]
+
+            raise DefaultFromEnvFactoryError(
+                errors=_prepend_field_name(err.raw_errors),  # type: ignore
+                model=err.model,
+                # FIXME: model = shall be the parent settings?? but I dont find how retrieve it from the field
+            ) from err
+
+    return _default_factory
 
 
 class BaseCustomSettings(BaseSettings):
-    class Config:
-        # SEE https://pydantic-docs.helpmanual.io/usage/model_config/
-        case_sensitive = False
+    """
+    - Customized configuration for all settings
+    - If a field is a BaseCustomSettings subclass, it allows creating a default from env vars setting the Field
+      option 'auto_default_from_env=True'.
+
+    SEE tests for details.
+    """
+
+    @validator("*", pre=True)
+    @classmethod
+    def parse_none(cls, v, field: ModelField):
+        # WARNING: In nullable fields, envs equal to null or none are parsed as None !!
+        if field.allow_none:
+            if isinstance(v, str) and v.lower() in ("null", "none"):
+                return None
+        return v
+
+    class Config(BaseConfig):
+        case_sensitive = True  # All must be capitalized
         extra = Extra.forbid
         allow_mutation = False
         frozen = True
         validate_all = True
         json_encoders = {SecretStr: lambda v: v.get_secret_value()}
         keep_untouched = (cached_property,)
 
-    @classmethod
-    def _set_defaults_with_default_constructors(
-        cls, default_fields: List[Tuple[str, Type["BaseCustomSettings"]]]
-    ):
-        # This function can set defaults on fields that are BaseSettings as well
-        # It is used in control construction of defaults.
-        # Pydantic offers a defaults_factory but it is executed upon creation of the Settings **class**
-        # which is too early for our purpose. Instead, we want to create the defaults just
-        # before the settings instance is constructed
-
-        assert issubclass(cls, BaseCustomSettings)  # nosec
-
-        # Builds defaults at this point
-        for name, default_cls in default_fields:
-            try:
-                default = default_cls.create_from_envs()
-                field_obj = cls.__fields__[name]
-                field_obj.default = default
-                field_obj.field_info.default = default
-                field_obj.required = False
-            except ValidationError as e:
-                logger.error(
-                    (
-                        "Could not validate '%s', field '%s' "
-                        "contains errors, see below:\n%s"
-                        "\n======ENV_VARS=====\n%s"
-                        "\n==================="
-                    ),
-                    cls.__name__,
-                    default_cls.__name__,
-                    str(e),
-                    "\n".join(f"{k}={v}" for k, v in os.environ.items()),
+        @classmethod
+        def prepare_field(cls, field: ModelField) -> None:
+            super().prepare_field(field)
+
+            auto_default_from_env = field.field_info.extra.get(
+                "auto_default_from_env", False
+            )
+
+            field_type = field.type_
+            if args := get_args(field_type):
+                field_type = next(a for a in args if a != type(None))
+
+            if issubclass(field_type, BaseCustomSettings):
+
+                if auto_default_from_env:
+
+                    assert field.field_info.default is Undefined
+                    assert field.field_info.default_factory is None
+
+                    field.default_factory = create_settings_from_env(field)
+                    # Having a default value, makes this field automatically optional
+                    field.required = False
+
+            elif issubclass(field_type, BaseSettings):
+                raise ValueError(
+                    f"{cls}.{field.name} of type {field_type} must inherit from BaseCustomSettings"
                 )
-                raise e
 
-    @classmethod
-    def create_from_envs(cls):
-        """Constructs settings instance capturing envs (even for defaults) at this call moment"""
-
-        # captures envs here to build defaults for BaseCustomSettings sub-settings
-        default_fields = []
-        for name, field in cls.__fields__.items():
-            if issubclass(field.type_, BaseCustomSettings):
-                default_fields.append((name, field.type_))
-            elif issubclass(field.type_, BaseSettings):
+            elif auto_default_from_env:
                 raise ValueError(
-                    f"{name} field class {field.type_} must inherit from BaseCustomSettings"
+                    "auto_default_from_env=True can only be used in BaseCustomSettings subclasses"
+                    f"but field {cls}.{field.name} is {field_type} "
                 )
-        cls._set_defaults_with_default_constructors(default_fields)
 
-        # builds instance
-        obj = cls()
-        return obj
+    @classmethod
+    def create_from_envs(cls, **overrides):
+        # Kept for legacy
+        # Optional to use to make the code more readable
+        # More explicit and pylance seems to get less confused
+        return cls(**overrides)

packages/settings-library/src/settings_library/email.py

@@ -1,3 +1,5 @@
+from typing import Optional
+
 from pydantic.fields import Field
 from pydantic.types import SecretStr
 
@@ -13,6 +15,6 @@ class SMTPSettings(BaseCustomSettings):
     SMTP_HOST: str
     SMTP_PORT: PortInt
 
-    SMTP_TLS_ENABLED: bool = Field(description="Enables Secure Mode")
-    SMTP_USERNAME: str
-    SMTP_PASSWORD: SecretStr
+    SMTP_TLS_ENABLED: bool = Field(False, description="Enables Secure Mode")
+    SMTP_USERNAME: Optional[str]
+    SMTP_PASSWORD: Optional[SecretStr]

packages/settings-library/src/settings_library/postgres.py

@@ -66,3 +66,17 @@ def dsn_with_query(self) -> str:
                 {"application_name": self.POSTGRES_CLIENT_NAME}
             )
         return dsn
+
+    class Config(BaseCustomSettings.Config):
+        schema_extra = {
+            "examples": [
+                # minimal
+                {
+                    "POSTGRES_HOST": "localhost",
+                    "POSTGRES_PORT": "5432",
+                    "POSTGRES_USER": "usr",
+                    "POSTGRES_PASSWORD": "secret",
+                    "POSTGRES_DB": "db",
+                }
+            ],
+        }