diff --git a/pydantic_settings/main.py b/pydantic_settings/main.py index 9a139d1e..8c716cc8 100644 --- a/pydantic_settings/main.py +++ b/pydantic_settings/main.py @@ -2,6 +2,7 @@ import asyncio import inspect +import json import threading from argparse import Namespace from types import SimpleNamespace @@ -13,6 +14,7 @@ from pydantic._internal._utils import deep_update, is_model_class from pydantic.dataclasses import is_pydantic_dataclass from pydantic.main import BaseModel +from pydantic_core import InitErrorDetails, ValidationError from .sources import ( ENV_FILE_SENTINEL, @@ -58,6 +60,7 @@ class SettingsConfigDict(ConfigDict, total=False): cli_ignore_unknown_args: bool | None cli_kebab_case: bool | None secrets_dir: PathType | None + validate_each_source: bool | None json_file: PathType | None json_file_encoding: str | None yaml_file: PathType | None @@ -257,6 +260,7 @@ def _settings_build_values( _cli_ignore_unknown_args: bool | None = None, _cli_kebab_case: bool | None = None, _secrets_dir: PathType | None = None, + _validate_each_source: bool | None = None, ) -> dict[str, Any]: # Determine settings config values case_sensitive = _case_sensitive if _case_sensitive is not None else self.model_config.get('case_sensitive') @@ -332,6 +336,12 @@ def _settings_build_values( secrets_dir = _secrets_dir if _secrets_dir is not None else self.model_config.get('secrets_dir') + validate_each_source = ( + _validate_each_source + if _validate_each_source is not None + else self.model_config.get('validate_each_source') + ) + # Configure built-in sources default_settings = DefaultSettingsSource( self.__class__, nested_model_default_partial_update=nested_model_default_partial_update @@ -400,6 +410,7 @@ def _settings_build_values( if sources: state: dict[str, Any] = {} states: dict[str, dict[str, Any]] = {} + all_line_errors: list[InitErrorDetails] = [] for source in sources: if isinstance(source, PydanticBaseSettingsSource): source._set_current_state(state) @@ -410,6 +421,33 @@ def _settings_build_values( states[source_name] = source_state state = deep_update(source_state, state) + + if source_state and validate_each_source: + try: + _ = super().__init__(**source_state) + except ValidationError as e: + line_errors = json.loads(e.json()) + for line in line_errors: + if line.get('type', '') == 'missing': + continue + line['loc'] = [source_name] + line['loc'] + ctx = line.get('ctx', {}) + ctx['source'] = source_name + line['ctx'] = ctx + all_line_errors.append(line) + + if validate_each_source: + try: + _ = super().__init__(**state) + except ValidationError as e: + line_errors = json.loads(e.json()) + all_line_errors.extend([line for line in line_errors if line.get('type', '') == 'missing']) + + if all_line_errors: + raise ValidationError.from_exception_data( + title=self.__class__.__name__, line_errors=all_line_errors + ) + return state else: # no one should mean to do this, but I think returning an empty dict is marginally preferable diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index bd4f4315..c929fe86 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -1785,11 +1785,13 @@ def _add_parser_args( if isinstance(group, dict): group = self._add_group(parser, **group) added_args += list(arg_names) - self._add_argument(group, *(f'{flag_prefix[:len(name)]}{name}' for name in arg_names), **kwargs) + self._add_argument( + group, *(f'{flag_prefix[: len(name)]}{name}' for name in arg_names), **kwargs + ) else: added_args += list(arg_names) self._add_argument( - parser, *(f'{flag_prefix[:len(name)]}{name}' for name in arg_names), **kwargs + parser, *(f'{flag_prefix[: len(name)]}{name}' for name in arg_names), **kwargs ) self._add_parser_alias_paths(parser, alias_path_args, added_args, arg_prefix, subcommand_prefix, group) @@ -2246,7 +2248,7 @@ def _load_env_vars(self) -> Mapping[str, Optional[str]]: return AzureKeyVaultMapping(secret_client) def __repr__(self) -> str: - return f'{self.__class__.__name__}(url={self._url!r}, ' f'env_nested_delimiter={self.env_nested_delimiter!r})' + return f'{self.__class__.__name__}(url={self._url!r}, env_nested_delimiter={self.env_nested_delimiter!r})' def _get_env_var_key(key: str, case_sensitive: bool = False) -> str: diff --git a/tests/test_multi_source.py b/tests/test_multi_source.py new file mode 100644 index 00000000..5e2a800c --- /dev/null +++ b/tests/test_multi_source.py @@ -0,0 +1,87 @@ +""" +Integration tests with multiple sources +""" + +from typing import Tuple, Type, Union + +import pytest +from pydantic import BaseModel, ValidationError + +from pydantic_settings import ( + BaseSettings, + JsonConfigSettingsSource, + PydanticBaseSettingsSource, + SettingsConfigDict, +) + + +def test_line_errors_from_source(monkeypatch, tmp_path): + monkeypatch.setenv('SETTINGS_NESTED__NESTED_FIELD', 'a') + p = tmp_path / 'settings.json' + p.write_text( + """ + {"foobar": 0, "null_field": null} + """ + ) + + class Nested(BaseModel): + nested_field: int + + class Settings(BaseSettings): + model_config = SettingsConfigDict( + json_file=p, env_prefix='SETTINGS_', env_nested_delimiter='__', validate_each_source=True + ) + foobar: str + nested: Nested + null_field: Union[str, None] + extra: bool + + @classmethod + def settings_customise_sources( + cls, + settings_cls: Type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> Tuple[PydanticBaseSettingsSource, ...]: + return (JsonConfigSettingsSource(settings_cls), env_settings, init_settings) + + with pytest.raises(ValidationError) as exc_info: + _ = Settings(null_field=0) + + assert exc_info.value.errors(include_url=False) == [ + { + 'ctx': {'source': 'JsonConfigSettingsSource'}, + 'input': 0, + 'loc': ( + 'JsonConfigSettingsSource', + 'foobar', + ), + 'msg': 'Input should be a valid string', + 'type': 'string_type', + }, + { + 'ctx': {'source': 'EnvSettingsSource'}, + 'input': 'a', + 'loc': ('EnvSettingsSource', 'nested', 'nested_field'), + 'msg': 'Input should be a valid integer, unable to parse string as an integer', + 'type': 'int_parsing', + }, + { + 'ctx': {'source': 'InitSettingsSource'}, + 'input': 0, + 'loc': ( + 'InitSettingsSource', + 'null_field', + ), + 'msg': 'Input should be a valid string', + 'type': 'string_type', + }, + { + 'input': {'foobar': 0, 'nested': {'nested_field': 'a'}, 'null_field': None}, + 'loc': ('extra',), + 'msg': 'Field required', + 'type': 'missing', + }, + ] diff --git a/tests/test_settings.py b/tests/test_settings.py index ba793a8e..190be2f9 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -472,6 +472,31 @@ class AnnotatedComplexSettings(BaseSettings): ] +def test_annotated_list_with_error_source(env): + class AnnotatedComplexSettings(BaseSettings, validate_each_source=True): + apples: Annotated[List[str], MinLen(2)] = [] + + env.set('apples', '["russet", "granny smith"]') + s = AnnotatedComplexSettings() + assert s.apples == ['russet', 'granny smith'] + + env.set('apples', '["russet"]') + with pytest.raises(ValidationError) as exc_info: + AnnotatedComplexSettings() + assert exc_info.value.errors(include_url=False) == [ + { + 'ctx': {'actual_length': 1, 'field_type': 'List', 'min_length': 2, 'source': 'EnvSettingsSource'}, + 'input': ['russet'], + 'loc': ( + 'EnvSettingsSource', + 'apples', + ), + 'msg': 'List should have at least 2 items after validation, not 1', + 'type': 'too_short', + } + ] + + def test_set_dict_model(env): env.set('bananas', '[1, 2, 3, 3]') env.set('CARROTS', '{"a": null, "b": 4}') @@ -1103,6 +1128,33 @@ class Settings(BaseSettings): ] +def test_env_file_with_env_prefix_invalid_with_sources(tmp_path): + p = tmp_path / '.env' + p.write_text(prefix_test_env_invalid_file) + + class Settings(BaseSettings): + a: str + b: str + c: str + + model_config = SettingsConfigDict(env_file=p, env_prefix='prefix_', validate_each_source=True) + + with pytest.raises(ValidationError) as exc_info: + Settings() + assert exc_info.value.errors(include_url=False) == [ + { + 'type': 'extra_forbidden', + 'loc': ( + 'DotEnvSettingsSource', + 'f', + ), + 'msg': 'Extra inputs are not permitted', + 'input': 'random value', + 'ctx': {'source': 'DotEnvSettingsSource'}, + } + ] + + def test_ignore_env_file_with_env_prefix_invalid(tmp_path): p = tmp_path / '.env' p.write_text(prefix_test_env_invalid_file) @@ -1879,8 +1931,7 @@ def test_builtins_settings_source_repr(): == "EnvSettingsSource(env_nested_delimiter='__', env_prefix_len=0)" ) assert repr(DotEnvSettingsSource(BaseSettings, env_file='.env', env_file_encoding='utf-8')) == ( - "DotEnvSettingsSource(env_file='.env', env_file_encoding='utf-8', " - 'env_nested_delimiter=None, env_prefix_len=0)' + "DotEnvSettingsSource(env_file='.env', env_file_encoding='utf-8', env_nested_delimiter=None, env_prefix_len=0)" ) assert ( repr(SecretsSettingsSource(BaseSettings, secrets_dir='/secrets'))