Fixes deprecation of popuplate_by_name

populate_by_name usage is not recommended in v2.11+ and will be deprecated in v3. Instead, you should use the validate_by_name configuration setting.

https://docs.pydantic.dev/latest/api/config/#pydantic.config.ConfigDict.populate_by_name

Author: pcrespov

packages/models-library/src/models_library/api_schemas_webserver/_base.py

@@ -1,5 +1,5 @@
 """
-    Base model classes for schemas in OpenAPI specs (OAS) for this service
+Base model classes for schemas in OpenAPI specs (OAS) for this service
 
 """
 
@@ -29,9 +29,8 @@ class InputSchemaWithoutCamelCase(BaseModel):
     )
 
 
-class InputSchema(BaseModel):
+class InputSchema(InputSchemaWithoutCamelCase):
     model_config = ConfigDict(
-        **InputSchemaWithoutCamelCase.model_config,
         alias_generator=snake_to_camel,
     )
 

packages/models-library/tests/test__pydantic_models.py

@@ -1,18 +1,19 @@
-""" This test suite does not intend to re-test pydantic but rather
+"""This test suite does not intend to re-test pydantic but rather
 check some "corner cases" or critical setups with pydantic model such that:
 
 - we can ensure a given behaviour is preserved through updates
 - document/clarify some concept
 
 """
 
-from typing import Any, Union, get_args, get_origin
+from typing import Any, Literal, Union, get_args, get_origin
 
 import pytest
 from common_library.json_serialization import json_dumps
 from models_library.projects_nodes import InputTypes, OutputTypes
 from models_library.projects_nodes_io import SimCoreFileLink
-from pydantic import BaseModel, Field, TypeAdapter, ValidationError
+from models_library.utils.change_case import snake_to_camel
+from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
 from pydantic.types import Json
 from pydantic.version import version_short
 
@@ -120,7 +121,7 @@ class Func(BaseModel):
             {"$ref": "#/$defs/DatCoreFileLink"},
             {"$ref": "#/$defs/DownloadLink"},
             {"type": "array", "items": {}},
-            {"type": "object"},
+            {"type": "object", "additionalProperties": True},
         ],
     }
 
@@ -154,7 +155,7 @@ class Func(BaseModel):
     MINIMAL = 2  # <--- index of the example with the minimum required fields
     assert SimCoreFileLink in get_args(OutputTypes)
     example = SimCoreFileLink.model_validate(
-        SimCoreFileLink.model_config["json_schema_extra"]["examples"][MINIMAL]
+        SimCoreFileLink.model_json_schema()["examples"][MINIMAL]
     )
     model = Func.model_validate(
         {
@@ -183,7 +184,9 @@ def test_nullable_fields_from_pydantic_v1():
     # SEE https://github.com/ITISFoundation/osparc-simcore/pull/6751
     class MyModel(BaseModel):
         # pydanticv1 would add a default to fields set as nullable
-        nullable_required: str | None  # <--- This was default to =None in pydantic 1 !!!
+        nullable_required: (
+            str | None
+        )  # <--- This was default to =None in pydantic 1 !!!
         nullable_required_with_hyphen: str | None = Field(default=...)
         nullable_optional: str | None = None
 
@@ -209,3 +212,112 @@ class MyModel(BaseModel):
     data["nullable_required"] = None
     model = MyModel.model_validate(data)
     assert model.model_dump(exclude_unset=True) == data
+
+
+# BELOW some tests related to depreacated `populate_by_name` in pydantic v2.11+ !!
+#
+# https://docs.pydantic.dev/latest/api/config/#pydantic.config.ConfigDict.populate_by_name
+#
+# `populate_by_name` usage is not recommended in v2.11+ and will be deprecated in v3. Instead, you should use the validate_by_name configuration setting.
+# When validate_by_name=True and validate_by_alias=True, this is strictly equivalent to the previous behavior of populate_by_name=True.
+# In v2.11, we also introduced a validate_by_alias setting that introduces more fine grained control for validation behavior.
+# Here's how you might go about using the new settings to achieve the same behavior:
+#
+
+
+@pytest.mark.parametrize("extra", ["ignore", "allow", "forbid"])
+@pytest.mark.parametrize(
+    "validate_by_alias, validate_by_name",
+    [
+        # NOTE: (False, False) is not allowed: at least one has to be True!
+        # SEE https://docs.pydantic.dev/latest/api/config/#pydantic.config.ConfigDict.validate_by_alias
+        (False, True),
+        (True, False),
+        (True, True),
+    ],
+)
+def test_model_config_validate_by_alias_and_name(
+    validate_by_alias: bool,
+    validate_by_name: bool,
+    extra: Literal["ignore", "allow", "forbid"],
+):
+    class TestModel(BaseModel):
+        snake_case: str | None = None
+
+        model_config = ConfigDict(
+            validate_by_alias=validate_by_alias,
+            validate_by_name=validate_by_name,
+            extra=extra,
+            alias_generator=snake_to_camel,
+        )
+
+    assert TestModel.model_config.get("populate_by_name") is None
+    assert TestModel.model_config.get("validate_by_alias") is validate_by_alias
+    assert TestModel.model_config.get("validate_by_name") is validate_by_name
+    assert TestModel.model_config.get("extra") == extra
+
+    if validate_by_alias is False:
+
+        if extra == "forbid":
+            with pytest.raises(ValidationError):
+                TestModel.model_validate({"snakeCase": "foo"})
+
+        elif extra == "ignore":
+            model = TestModel.model_validate({"snakeCase": "foo"})
+            assert model.snake_case is None
+            assert model.model_dump() == {"snake_case": None}
+
+        elif extra == "allow":
+            model = TestModel.model_validate({"snakeCase": "foo"})
+            assert model.snake_case is None
+            assert model.model_dump() == {"snake_case": None, "snakeCase": "foo"}
+
+    else:
+        assert TestModel.model_validate({"snakeCase": "foo"}).snake_case == "foo"
+
+    if validate_by_name is False:
+        if extra == "forbid":
+            with pytest.raises(ValidationError):
+                TestModel.model_validate({"snake_case": "foo"})
+
+        elif extra == "ignore":
+            model = TestModel.model_validate({"snake_case": "foo"})
+            assert model.snake_case is None
+            assert model.model_dump() == {"snake_case": None}
+
+        elif extra == "allow":
+            model = TestModel.model_validate({"snake_case": "foo"})
+            assert model.snake_case is None
+            assert model.model_dump() == {"snake_case": "foo"}
+    else:
+        assert TestModel.model_validate({"snake_case": "foo"}).snake_case == "foo"
+
+
+@pytest.mark.parametrize("populate_by_name", [True, False])
+def test_model_config_populate_by_name(populate_by_name: bool):
+    # SEE https://docs.pydantic.dev/latest/api/config/#pydantic.config.ConfigDict.populate_by_name
+    class TestModel(BaseModel):
+        snake_case: str | None = None
+
+        model_config = ConfigDict(
+            populate_by_name=populate_by_name,
+            extra="forbid",  # easier to check the effect of populate_by_name!
+            alias_generator=snake_to_camel,
+        )
+
+    # checks how they are set
+    assert TestModel.model_config.get("populate_by_name") is populate_by_name
+    assert TestModel.model_config.get("extra") == "forbid"
+
+    # NOTE how defaults work with populate_by_name!!
+    assert TestModel.model_config.get("validate_by_name") == populate_by_name
+    assert TestModel.model_config.get("validate_by_alias") is True  # Default
+
+    # validate_by_alias BEHAVIUOR defaults to True
+    TestModel.model_validate({"snakeCase": "foo"})
+
+    if populate_by_name:
+        assert TestModel.model_validate({"snake_case": "foo"}).snake_case == "foo"
+    else:
+        with pytest.raises(ValidationError):
+            TestModel.model_validate({"snake_case": "foo"})

packages/settings-library/src/settings_library/base.py

@@ -1,6 +1,6 @@
 import logging
 from functools import cached_property
-from typing import Any, Final, get_origin
+from typing import Any, Final
 
 from common_library.pydantic_fields_extension import get_type, is_literal, is_nullable
 from pydantic import ValidationInfo, field_validator
@@ -15,9 +15,9 @@
 
 _logger = logging.getLogger(__name__)
 
-_AUTO_DEFAULT_FACTORY_RESOLVES_TO_NONE_FSTRING: Final[
-    str
-] = "{field_name} auto_default_from_env unresolved, defaulting to None"
+_AUTO_DEFAULT_FACTORY_RESOLVES_TO_NONE_FSTRING: Final[str] = (
+    "{field_name} auto_default_from_env unresolved, defaulting to None"
+)
 
 
 class DefaultFromEnvFactoryError(ValueError):
@@ -119,11 +119,13 @@ def _parse_none(cls, v, info: ValidationInfo):
 
     model_config = SettingsConfigDict(
         case_sensitive=True,  # All must be capitalized
-        extra="forbid",
+        env_parse_none_str="null",
+        extra="ignore",
         frozen=True,
-        validate_default=True,
         ignored_types=(cached_property,),
-        env_parse_none_str="null",
+        validate_by_alias=True,
+        validate_by_name=True,
+        validate_default=True,
     )
 
     @classmethod
@@ -133,28 +135,15 @@ def __pydantic_init_subclass__(cls, **kwargs: Any):
         for name, field in cls.model_fields.items():
             auto_default_from_env = _is_auto_default_from_env_enabled(field)
             field_type = get_type(field)
-
-            # Avoids issubclass raising TypeError. SEE test_issubclass_type_error_with_pydantic_models
-            is_not_composed = (
-                get_origin(field_type) is None
-            )  # is not composed as dict[str, Any] or Generic[Base]
             is_not_literal = not is_literal(field)
 
-            if (
-                is_not_literal
-                and is_not_composed
-                and issubclass(field_type, BaseCustomSettings)
-            ):
+            if is_not_literal and issubclass(field_type, BaseCustomSettings):
                 if auto_default_from_env:
                     # Builds a default factory `Field(default_factory=create_settings_from_env(field))`
                     field.default_factory = _create_settings_from_env(name, field)
                     field.default = None
 
-            elif (
-                is_not_literal
-                and is_not_composed
-                and issubclass(field_type, BaseSettings)
-            ):
+            elif is_not_literal and issubclass(field_type, BaseSettings):
                 msg = f"{cls}.{name} of type {field_type} must inherit from BaseCustomSettings"
                 raise ValueError(msg)
 

packages/settings-library/src/settings_library/postgres.py

@@ -10,6 +10,7 @@
     ValidationInfo,
     field_validator,
 )
+from pydantic.config import JsonDict
 from pydantic_settings import SettingsConfigDict
 
 from .base import BaseCustomSettings
@@ -41,7 +42,6 @@ class PostgresSettings(BaseCustomSettings):
         Field(
             description="Name of the application connecting the postgres database, will default to use the host hostname (hostname on linux)",
             validation_alias=AliasChoices(
-                "POSTGRES_CLIENT_NAME",
                 # This is useful when running inside a docker container, then the hostname is set each client gets a different name
                 "HOST",
                 "HOSTNAME",
@@ -103,17 +103,34 @@ def _update_query(self, uri: str) -> str:
             return urlunparse(parsed_uri._replace(query=updated_query))
         return uri
 
-    model_config = SettingsConfigDict(
-        json_schema_extra={
-            "examples": [
-                # minimal required
-                {
-                    "POSTGRES_HOST": "localhost",
-                    "POSTGRES_PORT": "5432",
-                    "POSTGRES_USER": "usr",
-                    "POSTGRES_PASSWORD": "secret",
-                    "POSTGRES_DB": "db",
-                }
-            ],
-        }
-    )
+    @staticmethod
+    def _update_json_schema_extra(schema: JsonDict) -> None:
+        schema.update(
+            {
+                "examples": [
+                    # minimal required
+                    {
+                        "POSTGRES_HOST": "localhost",
+                        "POSTGRES_PORT": "5432",
+                        "POSTGRES_USER": "usr",
+                        "POSTGRES_PASSWORD": "secret",
+                        "POSTGRES_DB": "db",
+                    },
+                    # full example
+                    {
+                        "POSTGRES_HOST": "localhost",
+                        "POSTGRES_PORT": "5432",
+                        "POSTGRES_USER": "usr",
+                        "POSTGRES_PASSWORD": "secret",
+                        "POSTGRES_DB": "db",
+                        "POSTGRES_MINSIZE": 1,
+                        "POSTGRES_MAXSIZE": 50,
+                        "POSTGRES_CLIENT_NAME": "my_app",  # first-choice
+                        "HOST": "should be ignored",
+                        "HOST_NAME": "should be ignored",
+                    },
+                ],
+            }
+        )
+
+    model_config = SettingsConfigDict(json_schema_extra=_update_json_schema_extra)

packages/settings-library/src/settings_library/utils_cli.py

@@ -26,8 +26,9 @@ def print_as_envfile(
     **pydantic_export_options,
 ):
     exclude_unset = pydantic_export_options.get("exclude_unset", False)
+    settings_cls = settings_obj.__class__
 
-    for name, field in settings_obj.model_fields.items():
+    for name, field in settings_cls.model_fields.items():
         auto_default_from_env = (
             field.json_schema_extra is not None
             and field.json_schema_extra.get("auto_default_from_env", False)

packages/settings-library/tests/test__models_examples.py

@@ -17,6 +17,13 @@ def test_all_settings_library_models_config_examples(
     model_cls: type[BaseModel], example_name: str, example_data: Any
 ):
 
+    assert (
+        model_cls.model_config.get("validate_by_alias") is True
+    ), f"validate_by_alias must be enabled in {model_cls}"
+    assert (
+        model_cls.model_config.get("validate_by_name") is True
+    ), f"validate_by_name must be enabled in {model_cls}"
+
     assert_validation_model(
         model_cls, example_name=example_name, example_data=example_data
     )

packages/settings-library/tests/test_base.py

@@ -335,28 +335,34 @@ class SettingsClassExt(SettingsClass):
     }
 
 
-def test_issubclass_type_error_with_pydantic_models():
-    # There is a problem
-    #
-    # TypeError: issubclass() arg 1 must be a class
-    #
-    # SEE https://github.com/pydantic/pydantic/issues/545
-    #
-    # >> issubclass(dict, BaseSettings)
-    # False
-    # >> issubclass(dict[str, str], BaseSettings)
-    # Traceback (most recent call last):
-    # File "<string>", line 1, in <module>
-    # File "/home/crespo/.pyenv/versions/3.10.13/lib/python3.10/abc.py", line 123, in __subclasscheck__
-    #     return _abc_subclasscheck(cls, subclass)
-    # TypeError: issubclass() arg 1 must be a class
-    #
+def test_fixed_issubclass_type_error_with_pydantic_models():
 
     assert not issubclass(dict, BaseSettings)
-
-    # NOTE: this should be fixed by pydantic at some point. When this happens, this test will fail
-    with pytest.raises(TypeError):
-        issubclass(dict[str, str], BaseSettings)
+    assert not issubclass(
+        # FIXED with
+        #
+        # pydantic                 2.11.7
+        # pydantic_core            2.33.2
+        # pydantic-extra-types     2.10.5
+        # pydantic-settings        2.7.0
+        #
+        #
+        # TypeError: issubclass() arg 1 must be a class
+        #
+        # SEE https://github.com/pydantic/pydantic/issues/545
+        #
+        # >> issubclass(dict, BaseSettings)
+        # False
+        # >> issubclass(dict[str, str], BaseSettings)
+        # Traceback (most recent call last):
+        # File "<string>", line 1, in <module>
+        # File "/home/crespo/.pyenv/versions/3.10.13/lib/python3.10/abc.py", line 123, in __subclasscheck__
+        #     return _abc_subclasscheck(cls, subclass)
+        # TypeError: issubclass() arg 1 must be a class
+        #
+        dict[str, str],
+        BaseSettings,
+    )
 
     # here reproduces the problem with our settings that ANE and PC had
     class SettingsClassThatFailed(BaseCustomSettings):