From c642e26a70decc0ea443aacf942645a8a761a841 Mon Sep 17 00:00:00 2001 From: Oleksa Date: Wed, 23 Apr 2025 13:56:06 +0200 Subject: [PATCH 1/8] [FEAT] Update Azure Key Vault source: add case insensitive option and dash-underscore translation --- pydantic_settings/sources/base.py | 15 +++++--- pydantic_settings/sources/providers/azure.py | 40 ++++++++++++++------ tests/test_settings.py | 6 +-- 3 files changed, 40 insertions(+), 21 deletions(-) diff --git a/pydantic_settings/sources/base.py b/pydantic_settings/sources/base.py index 4350f8e8..71ccaba5 100644 --- a/pydantic_settings/sources/base.py +++ b/pydantic_settings/sources/base.py @@ -412,21 +412,24 @@ class Settings(BaseSettings): if not annotation or not hasattr(annotation, 'model_fields'): values[name] = value continue + else: + model_fields: dict[str, FieldInfo] = annotation.model_fields # Find field in sub model by looking in fields case insensitively - for sub_model_field_name, f in annotation.model_fields.items(): - if not f.validation_alias and sub_model_field_name.lower() == name.lower(): - sub_model_field = f + for sub_model_field_name, sub_model_field in model_fields.items(): + aliases, _ = _get_alias_names(sub_model_field_name, sub_model_field) + _search = (alias for alias in aliases if alias.lower() == name.lower()) + if field_key := next(_search, None): break - if not sub_model_field: + if not field_key: values[name] = value continue if _lenient_issubclass(sub_model_field.annotation, BaseModel) and isinstance(value, dict): - values[sub_model_field_name] = self._replace_field_names_case_insensitively(sub_model_field, value) + values[field_key] = self._replace_field_names_case_insensitively(sub_model_field, value) else: - values[sub_model_field_name] = value + values[field_key] = value return values diff --git a/pydantic_settings/sources/providers/azure.py b/pydantic_settings/sources/providers/azure.py index c949125a..fde4277c 100644 --- a/pydantic_settings/sources/providers/azure.py +++ b/pydantic_settings/sources/providers/azure.py @@ -5,6 +5,8 @@ from collections.abc import Iterator, Mapping from typing import TYPE_CHECKING, Optional +from pydantic.fields import FieldInfo + from .env import EnvSettingsSource if TYPE_CHECKING: @@ -42,27 +44,33 @@ class AzureKeyVaultMapping(Mapping[str, Optional[str]]): def __init__( self, secret_client: SecretClient, + case_sensitive: bool, ) -> None: self._loaded_secrets = {} self._secret_client = secret_client - self._secret_names: list[str] = [ + self._case_sensitive = case_sensitive + self._secret_map: dict[str, str] = self._load_remote() + + def _load_remote(self) -> dict[str, str]: + secret_names: Iterator[str] = ( secret.name for secret in self._secret_client.list_properties_of_secrets() if secret.name and secret.enabled - ] + ) + if self._case_sensitive: + return {name: name for name in secret_names} + return {name.lower(): name for name in secret_names} def __getitem__(self, key: str) -> str | None: - if key not in self._loaded_secrets and key in self._secret_names: - try: - self._loaded_secrets[key] = self._secret_client.get_secret(key).value - except Exception: - raise KeyError(key) - + if not self._case_sensitive: + key = key.lower() + if key not in self._loaded_secrets and key in self._secret_map: + self._loaded_secrets[key] = self._secret_client.get_secret(self._secret_map[key]).value return self._loaded_secrets[key] def __len__(self) -> int: - return len(self._secret_names) + return len(self._secret_map) def __iter__(self) -> Iterator[str]: - return iter(self._secret_names) + return iter(self._secret_map.keys()) class AzureKeyVaultSettingsSource(EnvSettingsSource): @@ -74,6 +82,8 @@ def __init__( settings_cls: type[BaseSettings], url: str, credential: TokenCredential, + dash_to_underscore: bool = True, + case_sensitive: bool | None = None, env_prefix: str | None = None, env_parse_none_str: str | None = None, env_parse_enums: bool | None = None, @@ -81,9 +91,10 @@ def __init__( import_azure_key_vault() self._url = url self._credential = credential + self._dash_to_underscore = dash_to_underscore super().__init__( settings_cls, - case_sensitive=True, + case_sensitive=case_sensitive, env_prefix=env_prefix, env_nested_delimiter='--', env_ignore_empty=False, @@ -93,7 +104,12 @@ def __init__( def _load_env_vars(self) -> Mapping[str, Optional[str]]: secret_client = SecretClient(vault_url=self._url, credential=self._credential) - return AzureKeyVaultMapping(secret_client) + return AzureKeyVaultMapping(secret_client, self.case_sensitive) + + def _extract_field_info(self, field: FieldInfo, field_name: str) -> list[tuple[str, str, bool]]: + if self._dash_to_underscore: + return list((x[0], x[1].replace('_', '-'), x[2]) for x in super()._extract_field_info(field, field_name)) + return super()._extract_field_info(field, field_name) def __repr__(self) -> str: return f'{self.__class__.__name__}(url={self._url!r}, env_nested_delimiter={self.env_nested_delimiter!r})' diff --git a/tests/test_settings.py b/tests/test_settings.py index 1800d544..ccdc0189 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -2738,18 +2738,18 @@ class Settings(BaseSettings): def test_case_insensitive_nested_optional(env): class NestedSettings(BaseModel): - FOO: str + FOO: str = Field(..., alias='Foo') BaR: int class Settings(BaseSettings): model_config = SettingsConfigDict(env_nested_delimiter='__', case_sensitive=False) - nested: Optional[NestedSettings] + nEstEd: Optional[NestedSettings] = Field(..., alias='NesTed') env.set('nested__FoO', 'string') env.set('nested__bar', '123') s = Settings() - assert s.model_dump() == {'nested': {'BaR': 123, 'FOO': 'string'}} + assert s.model_dump() == {'nEstEd': {'BaR': 123, 'FOO': 'string'}} def test_case_insensitive_nested_list(env): From 96f67e478a6d8917fe506d2033b3416e4a6cd12a Mon Sep 17 00:00:00 2001 From: Oleksa Date: Wed, 23 Apr 2025 14:44:20 +0200 Subject: [PATCH 2/8] [TEST] Add dash to underscore translation test --- tests/test_source_azure_key_vault.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/test_source_azure_key_vault.py b/tests/test_source_azure_key_vault.py index 05852716..52122f33 100644 --- a/tests/test_source_azure_key_vault.py +++ b/tests/test_source_azure_key_vault.py @@ -160,3 +160,31 @@ def _raise_resource_not_found_when_getting_parent_secret_name(self, secret_name: raise ResourceNotFoundError() return key_vault_secret + + def test_dash_to_underscore_translation_enabled_by_default(self, mocker: MockerFixture) -> None: + """Test that dashes in secret names are mapped to underscores in field names.""" + + class AzureKeyVaultSettings(BaseSettings): + my_field: str + + expected_secrets = [ + type('', (), {'name': 'my-field', 'enabled': True}), + ] + expected_secret_value = 'SecretValue' + + mocker.patch( + f'{AzureKeyVaultSettingsSource.__module__}.{SecretClient.list_properties_of_secrets.__qualname__}', + return_value=expected_secrets, + ) + mocker.patch( + f'{AzureKeyVaultSettingsSource.__module__}.{SecretClient.get_secret.__qualname__}', + return_value=KeyVaultSecret(SecretProperties(), expected_secret_value), + ) + + obj = AzureKeyVaultSettingsSource( + AzureKeyVaultSettings, 'https://my-resource.vault.azure.net/', DefaultAzureCredential() + ) + + settings = obj() + + assert settings['my_field'] == expected_secret_value From feaf009919f6b1582dc770e0455acb0db8730d1d Mon Sep 17 00:00:00 2001 From: Oleksa Date: Wed, 23 Apr 2025 16:33:23 +0200 Subject: [PATCH 3/8] [REVERT] Revert changes to env source nested alias test (separated to another PR) --- tests/test_settings.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_settings.py b/tests/test_settings.py index ccdc0189..1800d544 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -2738,18 +2738,18 @@ class Settings(BaseSettings): def test_case_insensitive_nested_optional(env): class NestedSettings(BaseModel): - FOO: str = Field(..., alias='Foo') + FOO: str BaR: int class Settings(BaseSettings): model_config = SettingsConfigDict(env_nested_delimiter='__', case_sensitive=False) - nEstEd: Optional[NestedSettings] = Field(..., alias='NesTed') + nested: Optional[NestedSettings] env.set('nested__FoO', 'string') env.set('nested__bar', '123') s = Settings() - assert s.model_dump() == {'nEstEd': {'BaR': 123, 'FOO': 'string'}} + assert s.model_dump() == {'nested': {'BaR': 123, 'FOO': 'string'}} def test_case_insensitive_nested_list(env): From ffeb186546536090979957d19951ba8773a42a00 Mon Sep 17 00:00:00 2001 From: d15ky <54978533+d15ky@users.noreply.github.com> Date: Mon, 28 Apr 2025 11:11:17 +0200 Subject: [PATCH 4/8] [CHORE] Change dash_to_underscore to False by default --- pydantic_settings/sources/providers/azure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pydantic_settings/sources/providers/azure.py b/pydantic_settings/sources/providers/azure.py index fde4277c..725a875d 100644 --- a/pydantic_settings/sources/providers/azure.py +++ b/pydantic_settings/sources/providers/azure.py @@ -82,7 +82,7 @@ def __init__( settings_cls: type[BaseSettings], url: str, credential: TokenCredential, - dash_to_underscore: bool = True, + dash_to_underscore: bool = False, case_sensitive: bool | None = None, env_prefix: str | None = None, env_parse_none_str: str | None = None, From f53eeec450ea5addf5d8d5eb72a505b6ff443160 Mon Sep 17 00:00:00 2001 From: d15ky <54978533+d15ky@users.noreply.github.com> Date: Mon, 28 Apr 2025 11:16:25 +0200 Subject: [PATCH 5/8] [FIX] Update Azure KeyVault dash_to_underscore test to explicitly enable the feature --- tests/test_source_azure_key_vault.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_source_azure_key_vault.py b/tests/test_source_azure_key_vault.py index 52122f33..a700a172 100644 --- a/tests/test_source_azure_key_vault.py +++ b/tests/test_source_azure_key_vault.py @@ -161,7 +161,7 @@ def _raise_resource_not_found_when_getting_parent_secret_name(self, secret_name: return key_vault_secret - def test_dash_to_underscore_translation_enabled_by_default(self, mocker: MockerFixture) -> None: + def test_dash_to_underscore_translation(self, mocker: MockerFixture) -> None: """Test that dashes in secret names are mapped to underscores in field names.""" class AzureKeyVaultSettings(BaseSettings): @@ -182,7 +182,10 @@ class AzureKeyVaultSettings(BaseSettings): ) obj = AzureKeyVaultSettingsSource( - AzureKeyVaultSettings, 'https://my-resource.vault.azure.net/', DefaultAzureCredential() + AzureKeyVaultSettings, + 'https://my-resource.vault.azure.net/', + DefaultAzureCredential(), + dash_to_underscore=True, ) settings = obj() From c0d6382aa726259692a4e8f2d4a51354295b34d2 Mon Sep 17 00:00:00 2001 From: d15ky <54978533+d15ky@users.noreply.github.com> Date: Tue, 29 Apr 2025 11:15:20 +0200 Subject: [PATCH 6/8] [DOCS] Add documentation for dash_to_underscore option --- docs/index.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/docs/index.md b/docs/index.md index 76696dfd..92d89213 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1889,6 +1889,51 @@ class AzureKeyVaultSettings(BaseSettings): ) ``` +### Dash to underscore mapping + +The Azure Key Vault source accepts a `dash_to_underscore` option, disabled by default, to support Key Vault kebab-case secret names by mapping them to Python's snake_case field names. When enabled, dashes (`-`) in secret names are mapped to underscores (`_`) in field names during validation. + +This mapping applies only to *field names*, not to aliases. + +```py +import os + +from azure.identity import DefaultAzureCredential +from pydantic import Field + +from pydantic_settings import ( + AzureKeyVaultSettingsSource, + BaseSettings, + PydanticBaseSettingsSource, +) + + +class AzureKeyVaultSettings(BaseSettings): + field_with_underscore: str + field_with_alias: str = Field(..., alias='Alias-With-Dashes') + + @classmethod + def settings_customise_sources( + cls, + settings_cls: type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> tuple[PydanticBaseSettingsSource, ...]: + az_key_vault_settings = AzureKeyVaultSettingsSource( + settings_cls, + os.environ['AZURE_KEY_VAULT_URL'], + DefaultAzureCredential(), + ) + return (az_key_vault_settings,) +``` + +This setup will load Azure Key Vault secrets named `field-with-underscore` and `Alias-With-Dashes`, mapping them to the `field_with_underscore` and `field_with_alias` fields, respectively. + +!!! tip + Alternatively, you can configure an [alias_generator](alias.md#using-alias-generators) to map PascalCase secrets. + ## Google Cloud Secret Manager Google Cloud Secret Manager allows you to store, manage, and access sensitive information as secrets in Google Cloud Platform. This integration lets you retrieve secrets directly from GCP Secret Manager for use in your Pydantic settings. From ca02c2b1f4434191499584e2c08bdb7a3e1b2558 Mon Sep 17 00:00:00 2001 From: d15ky <54978533+d15ky@users.noreply.github.com> Date: Tue, 29 Apr 2025 11:16:06 +0200 Subject: [PATCH 7/8] [TEST] Update Azure KeyVault dash_to_underscore test to check the aliases --- tests/test_source_azure_key_vault.py | 32 ++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/tests/test_source_azure_key_vault.py b/tests/test_source_azure_key_vault.py index a700a172..89f48add 100644 --- a/tests/test_source_azure_key_vault.py +++ b/tests/test_source_azure_key_vault.py @@ -166,9 +166,29 @@ def test_dash_to_underscore_translation(self, mocker: MockerFixture) -> None: class AzureKeyVaultSettings(BaseSettings): my_field: str + alias_field: str = Field(..., alias='Secret-Alias') + + @classmethod + def settings_customise_sources( + cls, + settings_cls: type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> tuple[PydanticBaseSettingsSource, ...]: + return ( + AzureKeyVaultSettingsSource( + settings_cls, + 'https://my-resource.vault.azure.net/', + DefaultAzureCredential(), + dash_to_underscore=True, + ), + ) expected_secrets = [ type('', (), {'name': 'my-field', 'enabled': True}), + type('', (), {'name': 'Secret-Alias', 'enabled': True}), ] expected_secret_value = 'SecretValue' @@ -181,13 +201,7 @@ class AzureKeyVaultSettings(BaseSettings): return_value=KeyVaultSecret(SecretProperties(), expected_secret_value), ) - obj = AzureKeyVaultSettingsSource( - AzureKeyVaultSettings, - 'https://my-resource.vault.azure.net/', - DefaultAzureCredential(), - dash_to_underscore=True, - ) - - settings = obj() + settings = AzureKeyVaultSettings() - assert settings['my_field'] == expected_secret_value + assert settings.my_field == expected_secret_value + assert settings.alias_field == expected_secret_value From a9b7e815a2154b6b912b90e27b7fcc776c24288e Mon Sep 17 00:00:00 2001 From: d15ky <54978533+d15ky@users.noreply.github.com> Date: Wed, 30 Apr 2025 11:01:20 +0200 Subject: [PATCH 8/8] [FIX] Add dash_to_underscore parameter to the example --- docs/index.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/index.md b/docs/index.md index 92d89213..117e37f2 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1925,6 +1925,7 @@ class AzureKeyVaultSettings(BaseSettings): settings_cls, os.environ['AZURE_KEY_VAULT_URL'], DefaultAzureCredential(), + dash_to_underscore=True, ) return (az_key_vault_settings,) ```