Fix a regression in dotenv optional nested field

Author: hramezani

pydantic_settings/sources.py

@@ -986,10 +986,19 @@ def __call__(self) -> dict[str, Any]:
             for field_name, field in self.settings_cls.model_fields.items():
                 for _, field_env_name, _ in self._extract_field_info(field, field_name):
                     if env_name == field_env_name or (
-                        lenient_issubclass(field.annotation, BaseModel) and env_name.startswith(field_env_name)
+                        (
+                            _annotation_is_complex(field.annotation, field.metadata)
+                            or (
+                                origin_is_union(get_origin(field.annotation))
+                                and _union_is_complex(field.annotation, field.metadata)
+                            )
+                        )
+                        and env_name.startswith(field_env_name)
                     ):
                         env_used = True
                         break
+                if env_used:
+                    break
             if not env_used:
                 if is_extra_allowed and env_name.startswith(self.env_prefix):
                     # env_prefix should be respected and removed from the env_name

tests/test_settings.py

@@ -5218,3 +5218,25 @@ class Settings(BaseSettings):
 
     s = Settings()
     assert s.model_dump() == {'nested': {'foo': ['one', 'two']}}
+
+
+def test_dotenv_optional_nested(tmp_path):
+    p = tmp_path / '.env'
+    p.write_text('not_nested=works\nNESTED__A=fails\nNESTED__b=2')
+
+    class NestedSettings(BaseModel):
+        A: str
+        b: int
+
+    class Settings(BaseSettings):
+        model_config = SettingsConfigDict(
+            env_file='.env',
+            env_nested_delimiter='__',
+            extra='forbid',
+        )
+
+        not_nested: str
+        NESTED: NestedSettings | None
+
+    s = Settings()
+    assert s.model_dump() == {'not_nested': 'works', 'NESTED': {'A': 'fails', 'b': 2}}