feat: adding json, yaml and toml sources (#211)

Author: Smixi

.github/workflows/ci.yml

@@ -56,6 +56,15 @@ jobs:
         COVERAGE_FILE: .coverage.${{ runner.os }}-py${{ matrix.python }}
         CONTEXT: ${{ runner.os }}-py${{ matrix.python }}
 
+    - name: uninstall deps
+      run: pip uninstall -y tomlkit PyYAML
+
+    - name: test without deps
+      run: make test
+      env:
+        COVERAGE_FILE: .coverage.${{ runner.os }}-py${{ matrix.python }}-without-deps
+        CONTEXT: ${{ runner.os }}-py${{ matrix.python }}-without-deps
+
     - run: coverage combine
     - run: coverage xml
 

docs/index.md

@@ -535,6 +535,65 @@ Last, run your application inside a Docker container and supply your newly creat
 docker service create --name pydantic-with-secrets --secret my_secret_data pydantic-app:latest
 ```
 
+## Other settings source
+
+Other settings sources are available for common configuration files:
+
+- `TomlConfigSettingsSource` using `toml_file` and toml_file_encoding arguments
+- `YamlConfigSettingsSource` using `yaml_file` and yaml_file_encoding arguments
+- `JsonConfigSettingsSource` using `json_file` and `json_file_encoding` arguments
+
+You can also provide multiple files by providing a list of path:
+```py
+toml_file = ['config.default.toml', 'config.custom.toml']
+```
+To use them, you can use the same mechanism described [here](#customise-settings-sources)
+
+
+```py
+from typing import Tuple, Type
+
+from pydantic import BaseModel
+
+from pydantic_settings import (
+    BaseSettings,
+    PydanticBaseSettingsSource,
+    SettingsConfigDict,
+    TomlConfigSettingsSource,
+)
+
+
+class Nested(BaseModel):
+    nested_field: str
+
+
+class Settings(BaseSettings):
+    foobar: str
+    nested: Nested
+    model_config = SettingsConfigDict(
+        toml_file='config.toml', toml_file_encoding='utf-8'
+    )
+
+    @classmethod
+    def settings_customise_sources(
+        cls,
+        settings_cls: Type[BaseSettings],
+        init_settings: PydanticBaseSettingsSource,
+        env_settings: PydanticBaseSettingsSource,
+        dotenv_settings: PydanticBaseSettingsSource,
+        file_secret_settings: PydanticBaseSettingsSource,
+    ) -> Tuple[PydanticBaseSettingsSource, ...]:
+        return (TomlConfigSettingsSource(settings_cls),)
+```
+
+This will be able to read the following "config.toml" file, located in your working directory:
+
+```toml
+foobar = "Hello"
+[nested]
+nested_field = "world!"
+```
+
 ## Field value priority
 
 In the case where a value is specified for the same `Settings` field in multiple ways,

pydantic_settings/__init__.py

@@ -3,8 +3,11 @@
     DotEnvSettingsSource,
     EnvSettingsSource,
     InitSettingsSource,
+    JsonConfigSettingsSource,
     PydanticBaseSettingsSource,
     SecretsSettingsSource,
+    TomlConfigSettingsSource,
+    YamlConfigSettingsSource,
 )
 from .version import VERSION
 
@@ -13,9 +16,12 @@
     'DotEnvSettingsSource',
     'EnvSettingsSource',
     'InitSettingsSource',
+    'JsonConfigSettingsSource',
     'PydanticBaseSettingsSource',
     'SecretsSettingsSource',
     'SettingsConfigDict',
+    'TomlConfigSettingsSource',
+    'YamlConfigSettingsSource',
     '__version__',
 )
 

pydantic_settings/main.py

@@ -14,6 +14,7 @@
     DotenvType,
     EnvSettingsSource,
     InitSettingsSource,
+    PathType,
     PydanticBaseSettingsSource,
     SecretsSettingsSource,
 )
@@ -28,6 +29,12 @@ class SettingsConfigDict(ConfigDict, total=False):
     env_nested_delimiter: str | None
     env_parse_none_str: str | None
     secrets_dir: str | Path | None
+    json_file: PathType | None
+    json_file_encoding: str | None
+    yaml_file: PathType | None
+    yaml_file_encoding: str | None
+    toml_file: PathType | None
+    toml_file_encoding: str | None
 
 
 # Extend `config_keys` by pydantic settings config keys to
@@ -195,6 +202,12 @@ def _settings_build_values(
         env_ignore_empty=False,
         env_nested_delimiter=None,
         env_parse_none_str=None,
+        json_file=None,
+        json_file_encoding=None,
+        yaml_file=None,
+        yaml_file_encoding=None,
+        toml_file=None,
+        toml_file_encoding=None,
         secrets_dir=None,
         protected_namespaces=('model_', 'settings_'),
     )

pydantic_settings/sources.py

@@ -2,6 +2,7 @@
 
 import json
 import os
+import sys
 import warnings
 from abc import ABC, abstractmethod
 from collections import deque
@@ -19,10 +20,49 @@
 from pydantic_settings.utils import path_type_label
 
 if TYPE_CHECKING:
+    if sys.version_info >= (3, 11):
+        import tomllib
+    else:
+        tomllib = None
+    import tomlkit
+    import yaml
+
     from pydantic_settings.main import BaseSettings
+else:
+    yaml = None
+    tomllib = None
+    tomlkit = None
+
+
+def import_yaml() -> None:
+    global yaml
+    if yaml is not None:
+        return
+    try:
+        import yaml
+    except ImportError as e:
+        raise ImportError('PyYAML is not installed, run `pip install pydantic-settings[yaml]`') from e
+
+
+def import_toml() -> None:
+    global tomlkit
+    global tomllib
+    if sys.version_info < (3, 11):
+        if tomlkit is not None:
+            return
+        try:
+            import tomlkit
+        except ImportError as e:
+            raise ImportError('tomlkit is not installed, run `pip install pydantic-settings[toml]`') from e
+    else:
+        if tomllib is not None:
+            return
+        import tomllib
 
 
 DotenvType = Union[Path, str, List[Union[Path, str]], Tuple[Union[Path, str], ...]]
+PathType = Union[Path, str, List[Union[Path, str]], Tuple[Union[Path, str], ...]]
+DEFAULT_PATH: PathType = Path('')
 
 # This is used as default value for `_env_file` in the `BaseSettings` class and
 # `env_file` in `DotEnvSettingsSource` so the default can be distinguished from `None`.
@@ -674,6 +714,103 @@ def __repr__(self) -> str:
         )
 
 
+class ConfigFileSourceMixin(ABC):
+    def _read_files(self, files: PathType | None) -> dict[str, Any]:
+        if files is None:
+            return {}
+        if isinstance(files, (str, os.PathLike)):
+            files = [files]
+        vars: dict[str, Any] = {}
+        for file in files:
+            file_path = Path(file).expanduser()
+            if file_path.is_file():
+                vars.update(self._read_file(file_path))
+        return vars
+
+    @abstractmethod
+    def _read_file(self, path: Path) -> dict[str, Any]:
+        pass
+
+
+class JsonConfigSettingsSource(InitSettingsSource, ConfigFileSourceMixin):
+    """
+    A source class that loads variables from a JSON file
+    """
+
+    def __init__(
+        self,
+        settings_cls: type[BaseSettings],
+        json_file: PathType | None = DEFAULT_PATH,
+        json_file_encoding: str | None = None,
+    ):
+        self.json_file_path = json_file if json_file != DEFAULT_PATH else settings_cls.model_config.get('json_file')
+        self.json_file_encoding = (
+            json_file_encoding
+            if json_file_encoding is not None
+            else settings_cls.model_config.get('json_file_encoding')
+        )
+        self.json_data = self._read_files(self.json_file_path)
+        super().__init__(settings_cls, self.json_data)
+
+    def _read_file(self, file_path: Path) -> dict[str, Any]:
+        with open(file_path, encoding=self.json_file_encoding) as json_file:
+            return json.load(json_file)
+
+
+class TomlConfigSettingsSource(InitSettingsSource, ConfigFileSourceMixin):
+    """
+    A source class that loads variables from a JSON file
+    """
+
+    def __init__(
+        self,
+        settings_cls: type[BaseSettings],
+        toml_file: PathType | None = DEFAULT_PATH,
+        toml_file_encoding: str | None = None,
+    ):
+        self.toml_file_path = toml_file if toml_file != DEFAULT_PATH else settings_cls.model_config.get('toml_file')
+        self.toml_file_encoding = (
+            toml_file_encoding
+            if toml_file_encoding is not None
+            else settings_cls.model_config.get('toml_file_encoding')
+        )
+        self.toml_data = self._read_files(self.toml_file_path)
+        super().__init__(settings_cls, self.toml_data)
+
+    def _read_file(self, file_path: Path) -> dict[str, Any]:
+        import_toml()
+        with open(file_path, mode='rb', encoding=self.toml_file_encoding) as toml_file:
+            if sys.version_info < (3, 11):
+                return tomlkit.load(toml_file)
+            return tomllib.load(toml_file)
+
+
+class YamlConfigSettingsSource(InitSettingsSource, ConfigFileSourceMixin):
+    """
+    A source class that loads variables from a yaml file
+    """
+
+    def __init__(
+        self,
+        settings_cls: type[BaseSettings],
+        yaml_file: PathType | None = DEFAULT_PATH,
+        yaml_file_encoding: str | None = None,
+    ):
+        self.yaml_file_path = yaml_file if yaml_file != DEFAULT_PATH else settings_cls.model_config.get('yaml_file')
+        self.yaml_file_encoding = (
+            yaml_file_encoding
+            if yaml_file_encoding is not None
+            else settings_cls.model_config.get('yaml_file_encoding')
+        )
+        self.yaml_data = self._read_files(self.yaml_file_path)
+        super().__init__(settings_cls, self.yaml_data)
+
+    def _read_file(self, file_path: Path) -> dict[str, Any]:
+        import_yaml()
+        with open(file_path, encoding=self.yaml_file_encoding) as yaml_file:
+            return yaml.safe_load(yaml_file)
+
+
 def _get_env_var_key(key: str, case_sensitive: bool = False) -> str:
     return key if case_sensitive else key.lower()
 

pyproject.toml

@@ -45,6 +45,10 @@ dependencies = [
 ]
 dynamic = ['version']
 
+[project.optional-dependencies]
+yaml = ["pyyaml>=6.0.1"]
+toml = ["tomlkit>=0.12"]
+
 [project.urls]
 Homepage = 'https://github.com/pydantic/pydantic-settings'
 Funding = 'https://github.com/sponsors/samuelcolvin'

requirements/linting.in

@@ -2,4 +2,6 @@ black
 ruff
 pyupgrade
 mypy
+types-PyYAML
+pyyaml==6.0.1
 pre-commit

requirements/linting.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.10
+# This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
 #    pip-compile --output-file=requirements/linting.txt requirements/linting.in
@@ -36,16 +36,16 @@ pre-commit==2.21.0
     # via -r requirements/linting.in
 pyupgrade==3.3.2
     # via -r requirements/linting.in
-pyyaml==6.0
-    # via pre-commit
+pyyaml==6.0.1
+    # via
+    #   -r requirements/linting.in
+    #   pre-commit
 ruff==0.0.270
     # via -r requirements/linting.in
 tokenize-rt==5.0.0
     # via pyupgrade
-tomli==2.0.1
-    # via
-    #   black
-    #   mypy
+types-pyyaml==6.0.12.12
+    # via -r requirements/linting.in
 typing-extensions==4.6.2
     # via mypy
 virtualenv==20.23.0

requirements/pyproject.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
-#    pip-compile --output-file=requirements/pyproject.txt pyproject.toml
+#    pip-compile --extra=toml --extra=yaml --output-file=requirements/pyproject.txt pyproject.toml
 #
 annotated-types==0.4.0
     # via pydantic
@@ -12,6 +12,10 @@ pydantic-core==2.14.5
     # via pydantic
 python-dotenv==0.21.1
     # via pydantic-settings (pyproject.toml)
+pyyaml==6.0.1
+    # via pydantic-settings (pyproject.toml)
+tomlkit==0.12.3
+    # via pydantic-settings (pyproject.toml)
 typing-extensions==4.6.2
     # via
     #   pydantic