Merge branch 'main' into cli_submodel_suppress

Author: hramezani

docs/index.md

@@ -1888,6 +1888,99 @@ class AzureKeyVaultSettings(BaseSettings):
         )
 ```
 
+## Google Cloud Secret Manager
+
+Google Cloud Secret Manager allows you to store, manage, and access sensitive information as secrets in Google Cloud Platform. This integration lets you retrieve secrets directly from GCP Secret Manager for use in your Pydantic settings.
+
+### Installation
+
+The Google Cloud Secret Manager integration requires additional dependencies:
+
+```bash
+pip install "pydantic-settings[gcp-secret-manager]"
+```
+
+### Basic Usage
+
+To use Google Cloud Secret Manager, you need to:
+
+1. Create a `GoogleSecretManagerSettingsSource`. (See [GCP Authentication](#gcp-authentication) for authentication options.)
+2. Add this source to your settings customization pipeline
+
+   ```py
+   from pydantic import BaseModel
+
+   from pydantic_settings import (
+       BaseSettings,
+       GoogleSecretManagerSettingsSource,
+       PydanticBaseSettingsSource,
+       SettingsConfigDict,
+   )
+
+
+   class Database(BaseModel):
+       password: str
+       user: str
+
+
+   class Settings(BaseSettings):
+       database: Database
+
+       model_config = SettingsConfigDict(env_nested_delimiter='__')
+
+       @classmethod
+       def settings_customise_sources(
+           cls,
+           settings_cls: type[BaseSettings],
+           init_settings: PydanticBaseSettingsSource,
+           env_settings: PydanticBaseSettingsSource,
+           dotenv_settings: PydanticBaseSettingsSource,
+           file_secret_settings: PydanticBaseSettingsSource,
+       ) -> tuple[PydanticBaseSettingsSource, ...]:
+           # Create the GCP Secret Manager settings source
+           gcp_settings = GoogleSecretManagerSettingsSource(
+               settings_cls,
+               # If not provided, will use google.auth.default()
+               # to get credentials from the environemnt
+               # credentials=your_credentials,
+               # If not provided, will use google.auth.default()
+               # to get project_id from the environemnt
+               project_id='your-gcp-project-id',
+           )
+
+           return (
+               init_settings,
+               env_settings,
+               dotenv_settings,
+               file_secret_settings,
+               gcp_settings,
+           )
+   ```
+
+### GCP Authentication
+
+The `GoogleSecretManagerSettingsSource` supports several authentication methods:
+
+1. **Default credentials** - If you don't provide credentials or project ID, it will use [`google.auth.default()`](https://google-auth.readthedocs.io/en/master/reference/google.auth.html#google.auth.default) to obtain them. This works with:
+
+   - Service account credentials from `GOOGLE_APPLICATION_CREDENTIALS` environment variable
+   - User credentials from `gcloud auth application-default login`
+   - Compute Engine, GKE, Cloud Run, or Cloud Functions default service accounts
+
+2. **Explicit credentials** - You can also provide `credentials` directly. e.g. `sa_credentials = google.oauth2.service_account.Credentials.from_service_account_file('path/to/service-account.json')` and then `GoogleSecretManagerSettingsSource(credentials=sa_credentials)`
+
+### Nested Models
+
+For nested models, Secret Manager supports the `env_nested_delimiter` setting as long as it complies with the [naming rules](https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets#create-a-secret). In the example above, you would create secrets named `database__password` and `database__user` in Secret Manager.
+
+### Important Notes
+
+1. **Case Sensitivity**: By default, secret names are case-sensitive.
+2. **Secret Naming**: Create secrets in Google Secret Manager with names that match your field names (including any prefix). According the [Secret Manager documentation](https://cloud.google.com/secret-manager/docs/creating-and-accessing-secrets#create-a-secret), a secret name can contain uppercase and lowercase letters, numerals, hyphens, and underscores. The maximum allowed length for a name is 255 characters.
+3. **Secret Versions**: The GoogleSecretManagerSettingsSource uses the "latest" version of secrets.
+
+For more details on creating and managing secrets in Google Cloud Secret Manager, see the [official Google Cloud documentation](https://cloud.google.com/secret-manager/docs).
+
 ## Other settings source
 
 Other settings sources are available for common configuration files:

pydantic_settings/__init__.py

@@ -14,6 +14,7 @@
     DotEnvSettingsSource,
     EnvSettingsSource,
     ForceDecode,
+    GoogleSecretManagerSettingsSource,
     InitSettingsSource,
     JsonConfigSettingsSource,
     NoDecode,
@@ -42,6 +43,7 @@
     'DotEnvSettingsSource',
     'EnvSettingsSource',
     'ForceDecode',
+    'GoogleSecretManagerSettingsSource',
     'InitSettingsSource',
     'JsonConfigSettingsSource',
     'NoDecode',

pydantic_settings/main.py

@@ -559,6 +559,7 @@ def run(
         if not issubclass(model_cls, BaseSettings):
 
             class CliAppBaseSettings(BaseSettings, model_cls):  # type: ignore
+                __doc__ = model_cls.__doc__
                 model_config = SettingsConfigDict(
                     nested_model_default_partial_update=True,
                     case_sensitive=True,

pydantic_settings/sources/__init__.py

@@ -21,6 +21,7 @@
 )
 from .providers.dotenv import DotEnvSettingsSource, read_env_file
 from .providers.env import EnvSettingsSource
+from .providers.gcp import GoogleSecretManagerSettingsSource
 from .providers.json import JsonConfigSettingsSource
 from .providers.pyproject import PyprojectTomlConfigSettingsSource
 from .providers.secrets import SecretsSettingsSource
@@ -45,6 +46,7 @@
     'DotenvType',
     'EnvSettingsSource',
     'ForceDecode',
+    'GoogleSecretManagerSettingsSource',
     'InitSettingsSource',
     'JsonConfigSettingsSource',
     'NoDecode',

pydantic_settings/sources/providers/__init__.py

@@ -13,6 +13,7 @@
 )
 from .dotenv import DotEnvSettingsSource
 from .env import EnvSettingsSource
+from .gcp import GoogleSecretManagerSettingsSource
 from .json import JsonConfigSettingsSource
 from .pyproject import PyprojectTomlConfigSettingsSource
 from .secrets import SecretsSettingsSource
@@ -31,6 +32,7 @@
     'CliSuppress',
     'DotEnvSettingsSource',
     'EnvSettingsSource',
+    'GoogleSecretManagerSettingsSource',
     'JsonConfigSettingsSource',
     'PyprojectTomlConfigSettingsSource',
     'SecretsSettingsSource',

pydantic_settings/sources/providers/cli.py

@@ -912,7 +912,9 @@ def _add_parser_submodels(
         is_model_suppressed = self._is_field_suppressed(field_info) or is_model_suppressed
         if not self.cli_avoid_json:
             added_args.append(arg_names[0])
-            kwargs['help'] = CLI_SUPPRESS if is_model_suppressed else f'set {arg_names[0]} from JSON string'
+            kwargs['nargs'] = '?'
+            kwargs['const'] = '{}'
+            kwargs['help'] = kwargs['help'] = CLI_SUPPRESS if is_model_suppressed else f'set {arg_names[0]} from JSON string (default: {{}})'
             model_group = self._add_group(parser, **model_group_kwargs)
             self._add_argument(model_group, *(f'{flag_prefix}{name}' for name in arg_names), **kwargs)
         for model in sub_models:

pydantic_settings/sources/providers/env.py

@@ -242,6 +242,9 @@ def explode_env_vars(self, field_name: str, field: FieldInfo, env_vars: Mapping[
             if (target_field or is_dict) and env_val:
                 if target_field:
                     is_complex, allow_json_failure = self._field_is_complex(target_field)
+                    if self.env_parse_enums:
+                        enum_val = _annotation_enum_name_to_val(target_field.annotation, env_val)
+                        env_val = env_val if enum_val is None else enum_val
                 else:
                     # nested field type is dict
                     is_complex, allow_json_failure = True, True

pydantic_settings/sources/providers/gcp.py

@@ -0,0 +1,141 @@
+from __future__ import annotations as _annotations
+
+from collections.abc import Iterator, Mapping
+from functools import cached_property
+from typing import TYPE_CHECKING, Optional
+
+from .env import EnvSettingsSource
+
+if TYPE_CHECKING:
+    from google.auth import default as google_auth_default
+    from google.auth.credentials import Credentials
+    from google.cloud.secretmanager import SecretManagerServiceClient
+
+    from pydantic_settings.main import BaseSettings
+else:
+    Credentials = None
+    SecretManagerServiceClient = None
+    google_auth_default = None
+
+
+def import_gcp_secret_manager() -> None:
+    global Credentials
+    global SecretManagerServiceClient
+    global google_auth_default
+
+    try:
+        from google.auth import default as google_auth_default
+        from google.auth.credentials import Credentials
+        from google.cloud.secretmanager import SecretManagerServiceClient
+    except ImportError as e:
+        raise ImportError(
+            'GCP Secret Namager dependencies are not installed, run `pip install pydantic-settings[gcp-secret-manager]`'
+        ) from e
+
+
+class GoogleSecretManagerMapping(Mapping[str, Optional[str]]):
+    _loaded_secrets: dict[str, str | None]
+    _secret_client: SecretManagerServiceClient
+
+    def __init__(self, secret_client: SecretManagerServiceClient, project_id: str) -> None:
+        self._loaded_secrets = {}
+        self._secret_client = secret_client
+        self._project_id = project_id
+
+    @property
+    def _gcp_project_path(self) -> str:
+        return self._secret_client.common_project_path(self._project_id)
+
+    @cached_property
+    def _secret_names(self) -> list[str]:
+        return [
+            self._secret_client.parse_secret_path(secret.name).get('secret', '')
+            for secret in self._secret_client.list_secrets(parent=self._gcp_project_path)
+        ]
+
+    def _secret_version_path(self, key: str, version: str = 'latest') -> str:
+        return self._secret_client.secret_version_path(self._project_id, key, version)
+
+    def __getitem__(self, key: str) -> str | None:
+        if key not in self._loaded_secrets:
+            # If we know the key isn't available in secret manager, raise a key error
+            if key not in self._secret_names:
+                raise KeyError(key)
+
+            try:
+                self._loaded_secrets[key] = self._secret_client.access_secret_version(
+                    name=self._secret_version_path(key)
+                ).payload.data.decode('UTF-8')
+            except Exception:
+                raise KeyError(key)
+
+        return self._loaded_secrets[key]
+
+    def __len__(self) -> int:
+        return len(self._secret_names)
+
+    def __iter__(self) -> Iterator[str]:
+        return iter(self._secret_names)
+
+
+class GoogleSecretManagerSettingsSource(EnvSettingsSource):
+    _credentials: Credentials
+    _secret_client: SecretManagerServiceClient
+    _project_id: str
+
+    def __init__(
+        self,
+        settings_cls: type[BaseSettings],
+        credentials: Credentials | None = None,
+        project_id: str | None = None,
+        env_prefix: str | None = None,
+        env_parse_none_str: str | None = None,
+        env_parse_enums: bool | None = None,
+        secret_client: SecretManagerServiceClient | None = None,
+    ) -> None:
+        # Import Google Packages if they haven't already been imported
+        if SecretManagerServiceClient is None or Credentials is None or google_auth_default is None:
+            import_gcp_secret_manager()
+
+        # If credentials or project_id are not passed, then
+        # try to get them from the default function
+        if not credentials or not project_id:
+            _creds, _project_id = google_auth_default()  # type: ignore[no-untyped-call]
+
+        # Set the credentials and/or project id if they weren't specified
+        if credentials is None:
+            credentials = _creds
+
+        if project_id is None:
+            if isinstance(_project_id, str):
+                project_id = _project_id
+            else:
+                raise AttributeError(
+                    'project_id is required to be specified either as an argument or from the google.auth.default. See https://google-auth.readthedocs.io/en/master/reference/google.auth.html#google.auth.default'
+                )
+
+        self._credentials: Credentials = credentials
+        self._project_id: str = project_id
+
+        if secret_client:
+            self._secret_client = secret_client
+        else:
+            self._secret_client = SecretManagerServiceClient(credentials=self._credentials)
+
+        super().__init__(
+            settings_cls,
+            case_sensitive=True,
+            env_prefix=env_prefix,
+            env_ignore_empty=False,
+            env_parse_none_str=env_parse_none_str,
+            env_parse_enums=env_parse_enums,
+        )
+
+    def _load_env_vars(self) -> Mapping[str, Optional[str]]:
+        return GoogleSecretManagerMapping(self._secret_client, project_id=self._project_id)
+
+    def __repr__(self) -> str:
+        return f'{self.__class__.__name__}(project_id={self._project_id!r}, env_nested_delimiter={self.env_nested_delimiter!r})'
+
+
+__all__ = ['GoogleSecretManagerSettingsSource', 'GoogleSecretManagerMapping']

pyproject.toml

@@ -51,6 +51,9 @@ yaml = ["pyyaml>=6.0.1"]
 toml = ["tomli>=2.0.1"]
 azure-key-vault = ["azure-keyvault-secrets>=4.8.0", "azure-identity>=1.16.0"]
 aws-secrets-manager = ["boto3>=1.35.0", "boto3-stubs[secretsmanager]"]
+gcp-secret-manager = [
+    "google-cloud-secret-manager>=2.23.1",
+]
 
 [project.urls]
 Homepage = 'https://github.com/pydantic/pydantic-settings'

tests/test_settings.py

@@ -2348,13 +2348,18 @@ class Settings(BaseSettings):
 
 
 def test_env_parse_enums(env):
-    class Settings(BaseSettings):
+    class NestedEnum(BaseModel):
+        fruit: FruitsEnum
+
+    class Settings(BaseSettings, env_nested_delimiter='__'):
         fruit: FruitsEnum
         union_fruit: Optional[Union[int, FruitsEnum]] = None
+        nested: NestedEnum
 
     with pytest.raises(ValidationError) as exc_info:
         env.set('FRUIT', 'kiwi')
         env.set('UNION_FRUIT', 'kiwi')
+        env.set('NESTED__FRUIT', 'kiwi')
         s = Settings()
     assert exc_info.value.errors(include_url=False) == [
         {
@@ -2385,22 +2390,43 @@ class Settings(BaseSettings):
             'msg': 'Input should be 0, 1 or 2',
             'type': 'enum',
         },
+        {
+            'ctx': {
+                'expected': '0, 1 or 2',
+            },
+            'input': 'kiwi',
+            'loc': (
+                'nested',
+                'fruit',
+            ),
+            'msg': 'Input should be 0, 1 or 2',
+            'type': 'enum',
+        },
     ]
 
     env.set('FRUIT', str(FruitsEnum.lime.value))
     env.set('UNION_FRUIT', str(FruitsEnum.lime.value))
+    env.set('NESTED__FRUIT', str(FruitsEnum.lime.value))
     s = Settings()
     assert s.fruit == FruitsEnum.lime
+    assert s.union_fruit == FruitsEnum.lime
+    assert s.nested.fruit == FruitsEnum.lime
 
     env.set('FRUIT', 'kiwi')
     env.set('UNION_FRUIT', 'kiwi')
+    env.set('NESTED__FRUIT', 'kiwi')
     s = Settings(_env_parse_enums=True)
     assert s.fruit == FruitsEnum.kiwi
+    assert s.union_fruit == FruitsEnum.kiwi
+    assert s.nested.fruit == FruitsEnum.kiwi
 
     env.set('FRUIT', str(FruitsEnum.lime.value))
     env.set('UNION_FRUIT', str(FruitsEnum.lime.value))
+    env.set('NESTED__FRUIT', str(FruitsEnum.lime.value))
     s = Settings(_env_parse_enums=True)
     assert s.fruit == FruitsEnum.lime
+    assert s.union_fruit == FruitsEnum.lime
+    assert s.nested.fruit == FruitsEnum.lime
 
 
 def test_env_parse_none_str(env):