add deep merge functionality to config file sources

Author: pmeier

pydantic_settings/sources/base.py

@@ -13,7 +13,7 @@
 from pydantic._internal._typing_extra import (  # type: ignore[attr-defined]
     get_origin,
 )
-from pydantic._internal._utils import is_model_class
+from pydantic._internal._utils import deep_update, is_model_class
 from pydantic.fields import FieldInfo
 from typing_extensions import get_args
 from typing_inspection import typing_objects
@@ -193,16 +193,17 @@ def __call__(self) -> dict[str, Any]:
 
 
 class ConfigFileSourceMixin(ABC):
-    def _read_files(self, files: PathType | None) -> dict[str, Any]:
+    def _read_files(self, files: PathType | None, deep_merge: bool = False) -> dict[str, Any]:
         if files is None:
             return {}
         if isinstance(files, (str, os.PathLike)):
             files = [files]
         vars: dict[str, Any] = {}
+        update = deep_update if deep_merge else dict.update
         for file in files:
             file_path = Path(file).expanduser()
             if file_path.is_file():
-                vars.update(self._read_file(file_path))
+                update(vars, self._read_file(file_path))
         return vars
 
     @abstractmethod

pydantic_settings/sources/providers/json.py

@@ -26,14 +26,15 @@ def __init__(
         settings_cls: type[BaseSettings],
         json_file: PathType | None = DEFAULT_PATH,
         json_file_encoding: str | None = None,
+        deep_merge: bool = False,
     ):
         self.json_file_path = json_file if json_file != DEFAULT_PATH else settings_cls.model_config.get('json_file')
         self.json_file_encoding = (
             json_file_encoding
             if json_file_encoding is not None
             else settings_cls.model_config.get('json_file_encoding')
         )
-        self.json_data = self._read_files(self.json_file_path)
+        self.json_data = self._read_files(self.json_file_path, deep_merge=deep_merge)
         super().__init__(settings_cls, self.json_data)
 
     def _read_file(self, file_path: Path) -> dict[str, Any]:

pydantic_settings/sources/providers/toml.py

@@ -50,9 +50,10 @@ def __init__(
         self,
         settings_cls: type[BaseSettings],
         toml_file: PathType | None = DEFAULT_PATH,
+        deep_merge: bool = False,
     ):
         self.toml_file_path = toml_file if toml_file != DEFAULT_PATH else settings_cls.model_config.get('toml_file')
-        self.toml_data = self._read_files(self.toml_file_path)
+        self.toml_data = self._read_files(self.toml_file_path, deep_merge=deep_merge)
         super().__init__(settings_cls, self.toml_data)
 
     def _read_file(self, file_path: Path) -> dict[str, Any]:

pydantic_settings/sources/providers/yaml.py

@@ -40,6 +40,7 @@ def __init__(
         yaml_file: PathType | None = DEFAULT_PATH,
         yaml_file_encoding: str | None = None,
         yaml_config_section: str | None = None,
+        deep_merge: bool = False,
     ):
         self.yaml_file_path = yaml_file if yaml_file != DEFAULT_PATH else settings_cls.model_config.get('yaml_file')
         self.yaml_file_encoding = (
@@ -52,7 +53,7 @@ def __init__(
             if yaml_config_section is not None
             else settings_cls.model_config.get('yaml_config_section')
         )
-        self.yaml_data = self._read_files(self.yaml_file_path)
+        self.yaml_data = self._read_files(self.yaml_file_path, deep_merge=deep_merge)
 
         if self.yaml_config_section:
             try: