⬆️✅ Fixes settings tests (#6753)

Author: pcrespov

packages/models-library/tests/test__pydantic_models.py

@@ -6,7 +6,7 @@
 
 """
 
-from typing import Union, get_args, get_origin
+from typing import Any, Union, get_args, get_origin
 
 import pytest
 from models_library.projects_nodes import InputTypes, OutputTypes
@@ -173,3 +173,36 @@ class Func(BaseModel):
     print(model.model_dump_json(indent=1))
     assert model.input == {"w": 42, "z": False}
     assert model.output == [1, 2, 3, None]
+
+
+def test_nullable_fields_from_pydantic_v1():
+    # Tests issue found during migration. Pydantic v1 would default to None all nullable fields when they were not **explicitly** set with `...` as required
+    # SEE https://github.com/ITISFoundation/osparc-simcore/pull/6751
+    class MyModel(BaseModel):
+        # pydanticv1 would add a default to fields set as nullable
+        nullable_required: str | None  # <--- This was default to =None in pydantic 1 !!!
+        nullable_required_with_hyphen: str | None = Field(default=...)
+        nullable_optional: str | None = None
+
+        # but with non-nullable "required" worked both ways
+        non_nullable_required: int
+        non_nullable_required_with_hyphen: int = Field(default=...)
+        non_nullable_optional: int = 42
+
+    data: dict[str, Any] = {
+        "nullable_required_with_hyphen": "foo",
+        "non_nullable_required_with_hyphen": 1,
+        "non_nullable_required": 2,
+    }
+
+    with pytest.raises(ValidationError) as err_info:
+        MyModel.model_validate(data)
+
+    assert err_info.value.error_count() == 1
+    error = err_info.value.errors()[0]
+    assert error["type"] == "missing"
+    assert error["loc"] == ("nullable_required",)
+
+    data["nullable_required"] = None
+    model = MyModel.model_validate(data)
+    assert model.model_dump(exclude_unset=True) == data

packages/settings-library/src/settings_library/base.py

@@ -15,9 +15,9 @@
 
 _logger = logging.getLogger(__name__)
 
-_DEFAULTS_TO_NONE_MSG: Final[
+_AUTO_DEFAULT_FACTORY_RESOLVES_TO_NONE_FSTRING: Final[
     str
-] = "%s auto_default_from_env unresolved, defaulting to None"
+] = "{field_name} auto_default_from_env unresolved, defaulting to None"
 
 
 class DefaultFromEnvFactoryError(ValueError):
@@ -40,10 +40,10 @@ def _default_factory():
         except ValidationError as err:
             if is_nullable(info):
                 # e.g. Optional[PostgresSettings] would warn if defaults to None
-                _logger.warning(
-                    _DEFAULTS_TO_NONE_MSG,
-                    field_name,
+                msg = _AUTO_DEFAULT_FACTORY_RESOLVES_TO_NONE_FSTRING.format(
+                    field_name=field_name
                 )
+                _logger.warning(msg)
                 return None
             _logger.warning("Validation errors=%s", err.errors())
             raise DefaultFromEnvFactoryError(errors=err.errors()) from err
@@ -58,6 +58,9 @@ def _is_auto_default_from_env_enabled(field: FieldInfo) -> bool:
     )
 
 
+ENABLED: Final = {}
+
+
 class EnvSettingsWithAutoDefaultSource(EnvSettingsSource):
     def __init__(
         self, settings_cls: type[BaseSettings], env_settings: EnvSettingsSource
@@ -86,7 +89,7 @@ def prepare_field_value(
             _is_auto_default_from_env_enabled(field)
             and field.default_factory
             and field.default is None
-            and prepared_value == {}
+            and prepared_value == ENABLED
         ):
             prepared_value = field.default_factory()
         return prepared_value

packages/settings-library/tests/test__pydantic_settings.py

@@ -56,9 +56,13 @@ class Settings(BaseSettings):
     @classmethod
     def parse_none(cls, v, info: ValidationInfo):
         # WARNING: In nullable fields, envs equal to null or none are parsed as None !!
-        if info.field_name and is_nullable(cls.model_fields[info.field_name]):
-            if isinstance(v, str) and v.lower() in ("null", "none"):
-                return None
+        if (
+            info.field_name
+            and is_nullable(cls.model_fields[info.field_name])
+            and isinstance(v, str)
+            and v.lower() in ("null", "none")
+        ):
+            return None
         return v
 
 

packages/settings-library/tests/test_base.py

@@ -17,7 +17,7 @@
 from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_envfile
 from pytest_simcore.helpers.typing_env import EnvVarsDict
 from settings_library.base import (
-    _DEFAULTS_TO_NONE_MSG,
+    _AUTO_DEFAULT_FACTORY_RESOLVES_TO_NONE_FSTRING,
     BaseCustomSettings,
     DefaultFromEnvFactoryError,
 )
@@ -223,7 +223,12 @@ class SettingsClass(BaseCustomSettings):
 
     # Defaulting to None also logs a warning
     assert logger_warn.call_count == 1
-    assert _DEFAULTS_TO_NONE_MSG in logger_warn.call_args[0][0]
+    assert (
+        _AUTO_DEFAULT_FACTORY_RESOLVES_TO_NONE_FSTRING.format(
+            field_name="VALUE_NULLABLE_DEFAULT_ENV"
+        )
+        in logger_warn.call_args[0][0]
+    )
 
 
 def test_auto_default_to_not_none(

services/autoscaling/tests/unit/test_core_settings.py

@@ -5,12 +5,15 @@
 
 import datetime
 import json
+import logging
 import os
+from typing import Final
 
 import pytest
 from faker import Faker
 from pydantic import ValidationError
 from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict
+from settings_library.base import _AUTO_DEFAULT_FACTORY_RESOLVES_TO_NONE_FSTRING
 from simcore_service_autoscaling.core.settings import (
     ApplicationSettings,
     EC2InstancesSettings,
@@ -141,11 +144,11 @@ def test_EC2_INSTANCES_ALLOWED_TYPES_valid(  # noqa: N802
     assert settings.AUTOSCALING_EC2_INSTANCES
 
 
-@pytest.mark.xfail(
-    reason="disabling till pydantic2 migration is complete see https://github.com/ITISFoundation/osparc-simcore/pull/6705"
-)
 def test_EC2_INSTANCES_ALLOWED_TYPES_passing_invalid_image_tags(  # noqa: N802
-    app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, faker: Faker
+    app_environment: EnvVarsDict,
+    monkeypatch: pytest.MonkeyPatch,
+    faker: Faker,
+    caplog: pytest.LogCaptureFixture,
 ):
     # passing an invalid image tag name will fail
     setenvs_from_dict(
@@ -161,8 +164,18 @@ def test_EC2_INSTANCES_ALLOWED_TYPES_passing_invalid_image_tags(  # noqa: N802
             )
         },
     )
-    with pytest.raises(ValidationError):
-        ApplicationSettings.create_from_envs()
+
+    with caplog.at_level(logging.WARNING):
+
+        settings = ApplicationSettings.create_from_envs()
+        assert settings.AUTOSCALING_EC2_INSTANCES is None
+
+        assert (
+            _AUTO_DEFAULT_FACTORY_RESOLVES_TO_NONE_FSTRING.format(
+                field_name="AUTOSCALING_EC2_INSTANCES"
+            )
+            in caplog.text
+        )
 
 
 def test_EC2_INSTANCES_ALLOWED_TYPES_passing_valid_image_tags(  # noqa: N802
@@ -199,54 +212,98 @@ def test_EC2_INSTANCES_ALLOWED_TYPES_passing_valid_image_tags(  # noqa: N802
     ]
 
 
-@pytest.mark.xfail(
-    reason="disabling till pydantic2 migration is complete see https://github.com/ITISFoundation/osparc-simcore/pull/6705"
-)
+ENABLED_VALUE: Final = "{}"
+
+
 def test_EC2_INSTANCES_ALLOWED_TYPES_empty_not_allowed(  # noqa: N802
     app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch
 ):
-    assert app_environment["AUTOSCALING_EC2_INSTANCES"] == "{}"
-    monkeypatch.setenv("EC2_INSTANCES_ALLOWED_TYPES", "{}")
+    assert (
+        os.environ["AUTOSCALING_EC2_INSTANCES"] == ENABLED_VALUE
+    )  # parent field in ApplicationSettings
+    monkeypatch.setenv(
+        "EC2_INSTANCES_ALLOWED_TYPES", "{}"
+    )  # child field in EC2InstancesSettings
 
-    # test child settings
     with pytest.raises(ValidationError) as err_info:
+        # test **child** EC2InstancesSettings
         EC2InstancesSettings.create_from_envs()
 
     assert err_info.value.errors()[0]["loc"] == ("EC2_INSTANCES_ALLOWED_TYPES",)
 
 
 def test_EC2_INSTANCES_ALLOWED_TYPES_empty_not_allowed_with_main_field_env_var(  # noqa: N802
-    app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch
+    app_environment: EnvVarsDict,
+    monkeypatch: pytest.MonkeyPatch,
+    caplog: pytest.LogCaptureFixture,
 ):
-    assert os.environ["AUTOSCALING_EC2_INSTANCES"] == "{}"
-    monkeypatch.setenv("EC2_INSTANCES_ALLOWED_TYPES", "{}")
-
-    # now as part of AUTOSCALING_EC2_INSTANCES: EC2InstancesSettings | None
-    with pytest.raises(ValidationError) as exc_before:
+    assert (
+        os.environ["AUTOSCALING_EC2_INSTANCES"] == ENABLED_VALUE
+    )  # parent field in ApplicationSettings
+    monkeypatch.setenv(
+        "EC2_INSTANCES_ALLOWED_TYPES", "{}"
+    )  # child field in EC2InstancesSettings
+
+    # explicit init of parent -> fails
+    with pytest.raises(ValidationError) as exc_info:
+        # NOTE: input captured via InitSettingsSource
         ApplicationSettings.create_from_envs(AUTOSCALING_EC2_INSTANCES={})
 
-    with pytest.raises(ValidationError) as exc_after:
-        ApplicationSettings.create_from_envs()
+    assert exc_info.value.error_count() == 1
+    error = exc_info.value.errors()[0]
+
+    assert error["type"] == "value_error"
+    assert error["input"] == {}
+    assert error["loc"] == ("AUTOSCALING_EC2_INSTANCES", "EC2_INSTANCES_ALLOWED_TYPES")
+
+    # NOTE: input captured via EnvSettingsWithAutoDefaultSource
+    # default env factory -> None
+    with caplog.at_level(logging.WARNING):
 
-    assert exc_before.value.errors() == exc_after.value.errors()
+        settings = ApplicationSettings.create_from_envs()
+        assert settings.AUTOSCALING_EC2_INSTANCES is None
+
+        assert (
+            _AUTO_DEFAULT_FACTORY_RESOLVES_TO_NONE_FSTRING.format(
+                field_name="AUTOSCALING_EC2_INSTANCES"
+            )
+            in caplog.text
+        )
 
 
 def test_EC2_INSTANCES_ALLOWED_TYPES_empty_not_allowed_without_main_field_env_var(  # noqa: N802
-    app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch
+    app_environment: EnvVarsDict,
+    monkeypatch: pytest.MonkeyPatch,
+    caplog: pytest.LogCaptureFixture,
 ):
-    monkeypatch.delenv("AUTOSCALING_EC2_INSTANCES")
-    monkeypatch.setenv("EC2_INSTANCES_ALLOWED_TYPES", "{}")
+    assert os.environ["AUTOSCALING_EC2_INSTANCES"] == ENABLED_VALUE
+    monkeypatch.delenv(
+        "AUTOSCALING_EC2_INSTANCES"
+    )  # parent field in ApplicationSettings
+    monkeypatch.setenv(
+        "EC2_INSTANCES_ALLOWED_TYPES", "{}"
+    )  # child field in EC2InstancesSettings
 
     # removing any value for AUTOSCALING_EC2_INSTANCES
-    settings = ApplicationSettings.create_from_envs()
-    assert settings.AUTOSCALING_EC2_INSTANCES is None
+    caplog.clear()
+    with caplog.at_level(logging.WARNING):
 
+        settings = ApplicationSettings.create_from_envs()
+        assert settings.AUTOSCALING_EC2_INSTANCES is None
 
-@pytest.mark.xfail(
-    reason="disabling till pydantic2 migration is complete see https://github.com/ITISFoundation/osparc-simcore/pull/6705"
-)
-def test_invalid_instance_names(
-    app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, faker: Faker
+        assert (
+            _AUTO_DEFAULT_FACTORY_RESOLVES_TO_NONE_FSTRING.format(
+                field_name="AUTOSCALING_EC2_INSTANCES"
+            )
+            in caplog.text
+        )
+
+
+def test_EC2_INSTANCES_ALLOWED_TYPES_invalid_instance_names(  # noqa: N802
+    app_environment: EnvVarsDict,
+    monkeypatch: pytest.MonkeyPatch,
+    faker: Faker,
+    caplog: pytest.LogCaptureFixture,
 ):
     settings = ApplicationSettings.create_from_envs()
     assert settings.AUTOSCALING_EC2_INSTANCES
@@ -256,9 +313,24 @@ def test_invalid_instance_names(
         monkeypatch,
         {
             "EC2_INSTANCES_ALLOWED_TYPES": json.dumps(
-                {faker.pystr(): {"ami_id": faker.pystr(), "pre_pull_images": []}}
+                {
+                    faker.pystr(): {
+                        "ami_id": faker.pystr(),
+                        "pre_pull_images": [],
+                    }
+                }
             )
         },
     )
-    with pytest.raises(ValidationError):
-        ApplicationSettings.create_from_envs()
+    caplog.clear()
+    with caplog.at_level(logging.WARNING):
+
+        settings = ApplicationSettings.create_from_envs()
+        assert settings.AUTOSCALING_EC2_INSTANCES is None
+
+        assert (
+            _AUTO_DEFAULT_FACTORY_RESOLVES_TO_NONE_FSTRING.format(
+                field_name="AUTOSCALING_EC2_INSTANCES"
+            )
+            in caplog.text
+        )