Considering extra config in dotenv source (#39)

Author: hramezani

docs/index.md

@@ -299,6 +299,10 @@ Because python-dotenv is used to parse the file, bash-like semantics such as `ex
 (depending on your OS and environment) may allow your dotenv file to also be used with `source`,
 see [python-dotenv's documentation](https://saurabh-kumar.com/python-dotenv/#usages) for more details.
 
+Pydantic settings consider `extra` config in case of dotenv file. It means if you set the `extra=forbid`
+on `model_config` and your dotenv file contains an entry for a field that is not defined in settings model,
+it will raise `ValidationError` in settings construction.
+
 ## Secret Support
 
 Placing secret values in files is a common pattern to provide sensitive configuration to an application.

pydantic_settings/sources.py

@@ -204,7 +204,7 @@ class Settings(BaseSettings):
         return values
 
     def __call__(self) -> Dict[str, Any]:
-        d: Dict[str, Any] = {}
+        data: Dict[str, Any] = {}
 
         for field_name, field in self.settings_cls.model_fields.items():
             try:
@@ -223,11 +223,11 @@ def __call__(self) -> Dict[str, Any]:
 
             if field_value is not None:
                 if not self.config.get('case_sensitive', False) and lenient_issubclass(field.annotation, BaseModel):
-                    d[field_key] = self._replace_field_names_case_insensitively(field, field_value)
+                    data[field_key] = self._replace_field_names_case_insensitively(field, field_value)
                 else:
-                    d[field_key] = field_value
+                    data[field_key] = field_value
 
-        return d
+        return data
 
 
 class SecretsSettingsSource(PydanticBaseEnvSettingsSource):
@@ -449,12 +449,7 @@ def __init__(
         super().__init__(settings_cls, env_nested_delimiter, env_prefix_len)
 
     def _load_env_vars(self) -> Mapping[str, Optional[str]]:
-        env_vars = super()._load_env_vars()
-        dotenv_vars = self._read_env_files(self.settings_cls.model_config.get('case_sensitive', False))
-        if dotenv_vars:
-            env_vars = {**dotenv_vars, **env_vars}
-
-        return env_vars
+        return self._read_env_files(self.settings_cls.model_config.get('case_sensitive', False))
 
     def _read_env_files(self, case_sensitive: bool) -> Mapping[str, Optional[str]]:
         env_files = self.env_file
@@ -464,7 +459,7 @@ def _read_env_files(self, case_sensitive: bool) -> Mapping[str, Optional[str]]:
         if isinstance(env_files, (str, os.PathLike)):
             env_files = [env_files]
 
-        dotenv_vars = {}
+        dotenv_vars: Dict[str, Optional[str]] = {}
         for env_file in env_files:
             env_path = Path(env_file).expanduser()
             if env_path.is_file():
@@ -474,14 +469,37 @@ def _read_env_files(self, case_sensitive: bool) -> Mapping[str, Optional[str]]:
 
         return dotenv_vars
 
+    def __call__(self) -> Dict[str, Any]:
+        data: Dict[str, Any] = super().__call__()
+
+        data_lower_keys: List[str] = []
+        if not self.settings_cls.model_config.get('case_sensitive', False):
+            data_lower_keys = [x.lower() for x in data.keys()]
+
+        # As `extra` config is allowed in dotenv settings source, We have to
+        # update data with extra env variabels from dotenv file.
+        for env_name, env_value in self.env_vars.items():
+            if env_value is not None:
+                env_name_without_prefix = env_name[self.env_prefix_len :]
+                first_key, *_ = env_name_without_prefix.split(self.env_nested_delimiter)
+
+                if (data_lower_keys and first_key not in data_lower_keys) or (
+                    not data_lower_keys and first_key not in data
+                ):
+                    data[first_key] = env_value
+
+        return data
+
     def __repr__(self) -> str:
         return (
             f'DotEnvSettingsSource(env_file={self.env_file!r}, env_file_encoding={self.env_file_encoding!r}, '
             f'env_nested_delimiter={self.env_nested_delimiter!r}, env_prefix_len={self.env_prefix_len!r})'
         )
 
 
-def read_env_file(file_path: Path, *, encoding: str = None, case_sensitive: bool = False) -> Dict[str, Optional[str]]:
+def read_env_file(
+    file_path: Path, *, encoding: Optional[str] = None, case_sensitive: bool = False
+) -> Mapping[str, Optional[str]]:
     try:
         from dotenv import dotenv_values
     except ImportError as e:

tests/test_settings.py

@@ -680,12 +680,17 @@ class Settings(BaseSettings):
         b: str
         c: str
 
-        model_config = ConfigDict(env_file=p, case_sensitive=True)
+        model_config = ConfigDict(env_file=p, case_sensitive=True, extra='ignore')
 
     with pytest.raises(ValidationError) as exc_info:
         Settings()
     assert exc_info.value.errors() == [
-        {'type': 'missing', 'loc': ('a',), 'msg': 'Field required', 'input': {'b': 'better string', 'c': 'best string'}}
+        {
+            'type': 'missing',
+            'loc': ('a',),
+            'msg': 'Field required',
+            'input': {'b': 'better string', 'c': 'best string', 'A': 'good string'},
+        }
     ]
 
 
@@ -1512,3 +1517,68 @@ class Settings(BaseSettings):
     assert s.nested.SUB_sub.Val2 == 'v2'
     assert s.nested.SUB_sub.SUB_sub_SuB.VaL3 == 'v3'
     assert s.nested.SUB_sub.SUB_sub_SuB.val4 == 'v4'
+
+
+@pytest.mark.skipif(not dotenv, reason='python-dotenv not installed')
+def test_dotenv_extra_allow(tmp_path):
+    p = tmp_path / '.env'
+    p.write_text('a=b\nx=y')
+
+    class Settings(BaseSettings):
+        a: str
+
+        model_config = ConfigDict(env_file=p, extra='allow')
+
+    s = Settings()
+    assert s.a == 'b'
+    assert s.x == 'y'
+
+
+@pytest.mark.skipif(not dotenv, reason='python-dotenv not installed')
+def test_dotenv_extra_forbid(tmp_path):
+    p = tmp_path / '.env'
+    p.write_text('a=b\nx=y')
+
+    class Settings(BaseSettings):
+        a: str
+
+        model_config = ConfigDict(env_file=p, extra='forbid')
+
+    with pytest.raises(ValidationError) as exc_info:
+        Settings()
+    assert exc_info.value.errors() == [
+        {'type': 'extra_forbidden', 'loc': ('x',), 'msg': 'Extra inputs are not permitted', 'input': 'y'}
+    ]
+
+
+@pytest.mark.skipif(not dotenv, reason='python-dotenv not installed')
+def test_dotenv_extra_case_insensitive(tmp_path):
+    p = tmp_path / '.env'
+    p.write_text('a=b')
+
+    class Settings(BaseSettings):
+        A: str
+
+        model_config = ConfigDict(env_file=p, extra='forbid')
+
+    s = Settings()
+    assert s.A == 'b'
+
+
+@pytest.mark.skipif(not dotenv, reason='python-dotenv not installed')
+def test_dotenv_extra_sub_model_case_insensitive(tmp_path):
+    p = tmp_path / '.env'
+    p.write_text('a=b\nSUB_model={"v": "v1"}')
+
+    class SubModel(BaseModel):
+        v: str
+
+    class Settings(BaseSettings):
+        A: str
+        sub_MODEL: SubModel
+
+        model_config = ConfigDict(env_file=p, extra='forbid')
+
+    s = Settings()
+    assert s.A == 'b'
+    assert s.sub_MODEL.v == 'v1'