Adds GCP Secret Manager source with doc and tests

Author: ezwiefel

docs/index.md

@@ -13,6 +13,7 @@
     DotEnvSettingsSource,
     EnvSettingsSource,
     ForceDecode,
+    GoogleSecretManagerSettingsSource,
     InitSettingsSource,
     JsonConfigSettingsSource,
     NoDecode,
@@ -40,6 +41,7 @@
     'DotEnvSettingsSource',
     'EnvSettingsSource',
     'ForceDecode',
+    'GoogleSecretManagerSettingsSource',
     'InitSettingsSource',
     'JsonConfigSettingsSource',
     'NoDecode',

pydantic_settings/__init__.py

@@ -20,6 +20,7 @@
 )
 from .providers.dotenv import DotEnvSettingsSource, read_env_file
 from .providers.env import EnvSettingsSource
+from .providers.gcp import GoogleSecretManagerSettingsSource
 from .providers.json import JsonConfigSettingsSource
 from .providers.pyproject import PyprojectTomlConfigSettingsSource
 from .providers.secrets import SecretsSettingsSource
@@ -43,6 +44,7 @@
     'DotenvType',
     'EnvSettingsSource',
     'ForceDecode',
+    'GoogleSecretManagerSettingsSource',
     'InitSettingsSource',
     'JsonConfigSettingsSource',
     'NoDecode',

pydantic_settings/sources/__init__.py

@@ -12,6 +12,7 @@
 )
 from .dotenv import DotEnvSettingsSource
 from .env import EnvSettingsSource
+from .gcp import GoogleSecretManagerSettingsSource
 from .json import JsonConfigSettingsSource
 from .pyproject import PyprojectTomlConfigSettingsSource
 from .secrets import SecretsSettingsSource
@@ -29,6 +30,7 @@
     'CliSuppress',
     'DotEnvSettingsSource',
     'EnvSettingsSource',
+    'GoogleSecretManagerSettingsSource',
     'JsonConfigSettingsSource',
     'PyprojectTomlConfigSettingsSource',
     'SecretsSettingsSource',

pydantic_settings/sources/providers/__init__.py

@@ -0,0 +1,141 @@
+from __future__ import annotations as _annotations
+
+from collections.abc import Iterator, Mapping
+from functools import cached_property
+from typing import TYPE_CHECKING, Optional
+
+from .env import EnvSettingsSource
+
+if TYPE_CHECKING:
+    from google.auth import default as google_auth_default
+    from google.auth.credentials import Credentials
+    from google.cloud.secretmanager import SecretManagerServiceClient
+
+    from pydantic_settings.main import BaseSettings
+else:
+    Credentials = None
+    SecretManagerServiceClient = None
+    google_auth_default = None
+
+
+def import_gcp_secret_manager() -> None:
+    global Credentials
+    global SecretManagerServiceClient
+    global google_auth_default
+
+    try:
+        from google.auth import default as google_auth_default
+        from google.auth.credentials import Credentials
+        from google.cloud.secretmanager import SecretManagerServiceClient
+    except ImportError as e:
+        raise ImportError(
+            'GCP Secret Namager dependencies are not installed, run `pip install pydantic-settings[gcp-secret-manager]`'
+        ) from e
+
+
+class GoogleSecretManagerMapping(Mapping[str, Optional[str]]):
+    _loaded_secrets: dict[str, str | None]
+    _secret_client: SecretManagerServiceClient
+
+    def __init__(self, secret_client: SecretManagerServiceClient, project_id: str) -> None:
+        self._loaded_secrets = {}
+        self._secret_client = secret_client
+        self._project_id = project_id
+
+    @property
+    def _gcp_project_path(self) -> str:
+        return self._secret_client.common_project_path(self._project_id)
+
+    @cached_property
+    def _secret_names(self) -> list[str]:
+        return [
+            self._secret_client.parse_secret_path(secret.name).get('secret', '')
+            for secret in self._secret_client.list_secrets(parent=self._gcp_project_path)
+        ]
+
+    def _secret_version_path(self, key: str, version: str = 'latest') -> str:
+        return self._secret_client.secret_version_path(self._project_id, key, version)
+
+    def __getitem__(self, key: str) -> str | None:
+        if key not in self._loaded_secrets:
+            # If we know the key isn't available in secret manager, raise a key error
+            if key not in self._secret_names:
+                raise KeyError(key)
+
+            try:
+                self._loaded_secrets[key] = self._secret_client.access_secret_version(
+                    name=self._secret_version_path(key)
+                ).payload.data.decode('UTF-8')
+            except Exception:
+                raise KeyError(key)
+
+        return self._loaded_secrets[key]
+
+    def __len__(self) -> int:
+        return len(self._secret_names)
+
+    def __iter__(self) -> Iterator[str]:
+        return iter(self._secret_names)
+
+
+class GoogleSecretManagerSettingsSource(EnvSettingsSource):
+    _credentials: Credentials
+    _secret_client: SecretManagerServiceClient
+    _project_id: str
+
+    def __init__(
+        self,
+        settings_cls: type[BaseSettings],
+        credentials: Credentials | None = None,
+        project_id: str | None = None,
+        env_prefix: str | None = None,
+        env_parse_none_str: str | None = None,
+        env_parse_enums: bool | None = None,
+        secret_client: SecretManagerServiceClient | None = None,
+    ) -> None:
+        # Import Google Packages if they haven't already been imported
+        if SecretManagerServiceClient is None or Credentials is None or google_auth_default is None:
+            import_gcp_secret_manager()
+
+        # If credentials or project_id are not passed, then
+        # try to get them from the default function
+        if not credentials or not project_id:
+            _creds, _project_id = google_auth_default()  # type: ignore[no-untyped-call]
+
+        # Set the credentials and/or project id if they weren't specified
+        if credentials is None:
+            credentials = _creds
+
+        if project_id is None:
+            if isinstance(_project_id, str):
+                project_id = _project_id
+            else:
+                raise AttributeError(
+                    'project_id is required to be specified either as an argument or from the google.auth.default. See https://google-auth.readthedocs.io/en/master/reference/google.auth.html#google.auth.default'
+                )
+
+        self._credentials: Credentials = credentials
+        self._project_id: str = project_id
+
+        if secret_client:
+            self._secret_client = secret_client
+        else:
+            self._secret_client = SecretManagerServiceClient(credentials=self._credentials)
+
+        super().__init__(
+            settings_cls,
+            case_sensitive=True,
+            env_prefix=env_prefix,
+            env_ignore_empty=False,
+            env_parse_none_str=env_parse_none_str,
+            env_parse_enums=env_parse_enums,
+        )
+
+    def _load_env_vars(self) -> Mapping[str, Optional[str]]:
+        return GoogleSecretManagerMapping(self._secret_client, project_id=self._project_id)
+
+    def __repr__(self) -> str:
+        return f'{self.__class__.__name__}(project_id={self._project_id!r}, env_nested_delimiter={self.env_nested_delimiter!r})'
+
+
+__all__ = ['GoogleSecretManagerSettingsSource', 'GoogleSecretManagerMapping']

pydantic_settings/sources/providers/gcp.py

@@ -50,6 +50,9 @@ dynamic = ['version']
 yaml = ["pyyaml>=6.0.1"]
 toml = ["tomli>=2.0.1"]
 azure-key-vault = ["azure-keyvault-secrets>=4.8.0", "azure-identity>=1.16.0"]
+gcp-secret-manager = [
+    "google-cloud-secret-manager>=2.23.1",
+]
 
 [project.urls]
 Homepage = 'https://github.com/pydantic/pydantic-settings'