Apply source order: init > env > dotenv > secrets > defaults and pres… (#688)

Co-authored-by: Johannes Rueschel <[email protected]>
Co-authored-by: Hasan Ramezani <[email protected]>

Author: 3 people

pydantic_settings/sources/base.py

@@ -265,12 +265,27 @@ def __init__(
         init_kwarg_names = set(init_kwargs.keys())
         for field_name, field_info in settings_cls.model_fields.items():
             alias_names, *_ = _get_alias_names(field_name, field_info)
-            init_kwarg_name = init_kwarg_names & set(alias_names)
+            # When populate_by_name is True, allow using the field name as an input key,
+            # but normalize to the preferred alias to keep keys consistent across sources.
+            matchable_names = set(alias_names)
+            include_name = settings_cls.model_config.get('populate_by_name', False)
+            if include_name:
+                matchable_names.add(field_name)
+            init_kwarg_name = init_kwarg_names & matchable_names
             if init_kwarg_name:
-                preferred_alias = alias_names[0]
-                preferred_set_alias = next(alias for alias in alias_names if alias in init_kwarg_name)
+                preferred_alias = alias_names[0] if alias_names else field_name
+                # Choose provided key deterministically: prefer the first alias in alias_names order;
+                # fall back to field_name if allowed and provided.
+                provided_key = next((alias for alias in alias_names if alias in init_kwarg_names), None)
+                if provided_key is None and include_name and field_name in init_kwarg_names:
+                    provided_key = field_name
+                # provided_key should not be None here because init_kwarg_name is non-empty
+                assert provided_key is not None
                 init_kwarg_names -= init_kwarg_name
-                self.init_kwargs[preferred_alias] = init_kwargs[preferred_set_alias]
+                self.init_kwargs[preferred_alias] = init_kwargs[provided_key]
+        # Include any remaining init kwargs (e.g., extras) unchanged
+        # Note: If populate_by_name is True and the provided key is the field name, but
+        # no alias exists, we keep it as-is so it can be processed as extra if allowed.
         self.init_kwargs.update({key: val for key, val in init_kwargs.items() if key in init_kwarg_names})
 
         super().__init__(settings_cls)

tests/test_precedence_and_merging.py

@@ -0,0 +1,118 @@
+from __future__ import annotations as _annotations
+
+from pathlib import Path
+
+from pydantic import AnyHttpUrl, Field
+
+from pydantic_settings import (
+    BaseSettings,
+    PydanticBaseSettingsSource,
+    SettingsConfigDict,
+)
+
+
+def test_init_kwargs_override_env_for_alias_with_populate_by_name(env):
+    class Settings(BaseSettings):
+        abc: AnyHttpUrl = Field(validation_alias='my_abc')
+        model_config = SettingsConfigDict(populate_by_name=True, extra='allow')
+
+    env.set('MY_ABC', 'http://localhost.com')
+    # Passing by field name should be accepted (populate_by_name=True) and should
+    # override env-derived value. Also ensures init > env precedence with validation_alias.
+    assert str(Settings(abc='http://prod.localhost.com/').abc) == 'http://prod.localhost.com/'
+
+
+def test_precedence_init_over_env(tmp_path: Path, env):
+    class Settings(BaseSettings):
+        foo: str
+
+    env.set('FOO', 'from-env')
+    s = Settings(foo='from-init')
+    assert s.foo == 'from-init'
+
+
+def test_precedence_env_over_dotenv(tmp_path: Path, env):
+    env_file = tmp_path / '.env'
+    env_file.write_text('FOO=from-dotenv\n')
+
+    class Settings(BaseSettings):
+        foo: str
+
+        model_config = SettingsConfigDict(env_file=env_file)
+
+    env.set('FOO', 'from-env')
+    s = Settings()
+    assert s.foo == 'from-env'
+
+
+def test_precedence_dotenv_over_secrets(tmp_path: Path):
+    # create dotenv
+    env_file = tmp_path / '.env'
+    env_file.write_text('FOO=from-dotenv\n')
+
+    # create secrets directory with same key
+    secrets_dir = tmp_path / 'secrets'
+    secrets_dir.mkdir()
+    (secrets_dir / 'FOO').write_text('from-secrets\n')
+
+    class Settings(BaseSettings):
+        foo: str
+
+        model_config = SettingsConfigDict(env_file=env_file, secrets_dir=secrets_dir)
+
+    # No env set, dotenv should override secrets
+    s = Settings()
+    assert s.foo == 'from-dotenv'
+
+
+def test_precedence_secrets_over_defaults(tmp_path: Path):
+    secrets_dir = tmp_path / 'secrets'
+    secrets_dir.mkdir()
+    (secrets_dir / 'FOO').write_text('from-secrets\n')
+
+    class Settings(BaseSettings):
+        foo: str = 'from-default'
+
+        model_config = SettingsConfigDict(secrets_dir=secrets_dir)
+
+    s = Settings()
+    assert s.foo == 'from-secrets'
+
+
+def test_merging_preserves_earlier_values(tmp_path: Path, env):
+    # Prove that merging preserves earlier source values: init -> env -> dotenv -> secrets -> defaults
+    # We'll populate nested from dotenv and env parts, then set a default for a, and init for b
+    env_file = tmp_path / '.env'
+    env_file.write_text('NESTED={"x":1}\n')
+
+    secrets_dir = tmp_path / 'secrets'
+    secrets_dir.mkdir()
+    (secrets_dir / 'NESTED').write_text('{"y": 2}')
+
+    class Settings(BaseSettings):
+        a: int = 10
+        b: int = 0
+        nested: dict
+
+        model_config = SettingsConfigDict(env_file=env_file, secrets_dir=secrets_dir, env_nested_delimiter='__')
+
+        @classmethod
+        def settings_customise_sources(
+            cls,
+            settings_cls: type[BaseSettings],
+            init_settings: PydanticBaseSettingsSource,
+            env_settings: PydanticBaseSettingsSource,
+            dotenv_settings: PydanticBaseSettingsSource,
+            file_secret_settings: PydanticBaseSettingsSource,
+        ):
+            # normal order; we want to assert deep merging
+            return init_settings, env_settings, dotenv_settings, file_secret_settings
+
+    # env contributes nested.y and overrides dotenv nested.x=1 if set; we'll set only y to prove merge
+    env.set('NESTED__y', '3')
+    # init contributes b, defaults contribute a
+    s = Settings(b=20)
+    assert s.a == 10  # defaults preserved
+    assert s.b == 20  # init wins
+    # nested: dotenv provides x=1; env provides y=3; deep merged => {x:1, y:3}
+    assert s.nested == {'x': 1, 'y': 3}