diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1fc5056b..02299167 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,6 +14,13 @@ paths are considered internals and can change in minor and patch releases. v4.40.1 (2025-05-??) -------------------- +Added +^^^^^ +- Support for Pydantic models with ``extra`` field configuration (``allow``, + ``forbid``, ``ignore``). Models with ``extra="allow"`` now accept additional + fields, while ``extra="forbid"`` properly rejects them and ``extra="ignore"`` + accepts but ignores extra fields during instantiation. + Fixed ^^^^^ - ``print_shtab`` incorrectly parsed from environment variable (`#725 diff --git a/jsonargparse/_core.py b/jsonargparse/_core.py index 0dcc3b29..d2cbc881 100644 --- a/jsonargparse/_core.py +++ b/jsonargparse/_core.py @@ -1147,8 +1147,24 @@ def check_values(cfg): group_key = next((g for g in self.groups if key.startswith(g + ".")), None) if group_key: subkey = key[len(group_key) + 1 :] - raise NSKeyError(f"Group '{group_key}' does not accept nested key '{subkey}'") - raise NSKeyError(f"Key '{key}' is not expected") + # Check if this is a Pydantic model with extra configuration + group = self.groups[group_key] + should_raise_error = True + if hasattr(group, "group_class") and group.group_class: + from ._optionals import get_pydantic_extra_config + + extra_config = get_pydantic_extra_config(group.group_class) + if extra_config == "allow": + # Allow extra fields - don't raise an error + should_raise_error = False + elif extra_config == "ignore": + # Ignore extra fields - don't raise an error, Pydantic will ignore during instantiation + should_raise_error = False + # For 'forbid' or None (default), raise error + if should_raise_error: + raise NSKeyError(f"Group '{group_key}' does not accept nested key '{subkey}'") + else: + raise NSKeyError(f"Key '{key}' is not expected") try: with parser_context(load_value_mode=self.parser_mode): diff --git a/jsonargparse/_optionals.py b/jsonargparse/_optionals.py index de7becc2..62a4be71 100644 --- a/jsonargparse/_optionals.py +++ b/jsonargparse/_optionals.py @@ -361,3 +361,58 @@ def validate_annotated(value, typehint: type): from pydantic import TypeAdapter return TypeAdapter(typehint).validate_python(value) + + +def get_pydantic_extra_config(class_type) -> Optional[str]: + """Get the 'extra' configuration from a Pydantic model. + + Args: + class_type: The class to check for Pydantic extra configuration. + + Returns: + The extra configuration ('allow', 'forbid', 'ignore') or None if not a Pydantic model. + """ + pydantic_model_version = is_pydantic_model(class_type) + if not pydantic_model_version: + return None + + try: + + # Handle Pydantic v2 models + if pydantic_model_version > 1: + # Check for model_config attribute (Pydantic v2 style) + if hasattr(class_type, "model_config"): + config = class_type.model_config + if hasattr(config, "get"): + # ConfigDict case + return config.get("extra") + elif hasattr(config, "extra"): + # Direct attribute access + return config.extra + + # Check for __config__ attribute (legacy support in v2) + if hasattr(class_type, "__config__"): + config = class_type.__config__ + if hasattr(config, "extra"): + return config.extra + + # Handle Pydantic v1 models (including v1 compatibility mode in v2) + else: + if hasattr(class_type, "__config__"): + config = class_type.__config__ + if hasattr(config, "extra"): + extra_value = config.extra + # Handle Pydantic v1 Extra enum + if hasattr(extra_value, "value"): + return extra_value.value + elif isinstance(extra_value, str): + return extra_value + else: + # Convert enum to string by taking the last part after the dot + return str(extra_value).split(".")[-1] + + except Exception: + # If anything goes wrong, return None to fall back to default behavior + pass + + return None diff --git a/jsonargparse_tests/test_dataclass_like.py b/jsonargparse_tests/test_dataclass_like.py index e8fdf548..f3ee4158 100644 --- a/jsonargparse_tests/test_dataclass_like.py +++ b/jsonargparse_tests/test_dataclass_like.py @@ -825,6 +825,62 @@ class NestedModel(pydantic.BaseModel): class PydanticNestedDict(pydantic.BaseModel): nested: Optional[Dict[str, NestedModel]] = None + # Helper function to create test models dynamically based on current pydantic version + def _create_extra_models(): + """Create pydantic models with extra field handling based on the current pydantic version.""" + # Check if the actual pydantic module supports v2 syntax + # In pydantic v1, ConfigDict is just dict, in v2 it's a special class + # In pydantic.v1 compatibility mode, ConfigDict exists but is in pydantic.v1.config module + _pydantic_v2_syntax = ( + hasattr(pydantic, "ConfigDict") + and hasattr(pydantic.ConfigDict, "__module__") + and "pydantic.config" in pydantic.ConfigDict.__module__ + and "v1" not in pydantic.ConfigDict.__module__ + ) + + if _pydantic_v2_syntax: + from pydantic import ConfigDict + + class PydanticExtraAllow(pydantic.BaseModel): + model_config = ConfigDict(extra="allow") + name: str + age: int = 25 + + class PydanticExtraForbid(pydantic.BaseModel): + model_config = ConfigDict(extra="forbid") + name: str + age: int = 25 + + class PydanticExtraIgnore(pydantic.BaseModel): + model_config = ConfigDict(extra="ignore") + name: str + age: int = 25 + + else: + # Pydantic v1 style (including v1 compatibility mode) + class PydanticExtraAllow(pydantic.BaseModel): + name: str + age: int = 25 + + class Config: + extra = "allow" + + class PydanticExtraForbid(pydantic.BaseModel): + name: str + age: int = 25 + + class Config: + extra = "forbid" + + class PydanticExtraIgnore(pydantic.BaseModel): + name: str + age: int = 25 + + class Config: + extra = "ignore" + + return PydanticExtraAllow, PydanticExtraForbid, PydanticExtraIgnore + def none(x): return x @@ -989,6 +1045,295 @@ def test_nested_dict(self, parser): assert isinstance(init.model, PydanticNestedDict) assert isinstance(init.model.nested["key"], NestedModel) + def test_extra_allow(self, parser): + """Test that extra='allow' accepts and includes extra fields.""" + PydanticExtraAllow, _, _ = _create_extra_models() + parser.add_argument("--model", type=PydanticExtraAllow, default=PydanticExtraAllow(name="default")) + + # Test with parse_object (where validation logic applies) + cfg = parser.parse_object({"model": {"name": "John", "age": 30, "extra_field": "extra_value"}}) + + # Check that extra field is in the namespace + assert cfg.model.name == "John" + assert cfg.model.age == 30 + assert cfg.model.extra_field == "extra_value" + + # Check that instantiation includes the extra field + init = parser.instantiate_classes(cfg) + assert isinstance(init.model, PydanticExtraAllow) + assert init.model.name == "John" + assert init.model.age == 30 + assert hasattr(init.model, "extra_field") + assert init.model.extra_field == "extra_value" + + def test_extra_forbid(self, parser): + """Test that extra='forbid' rejects extra fields with appropriate error.""" + _, PydanticExtraForbid, _ = _create_extra_models() + parser.add_argument("--model", type=PydanticExtraForbid, default=PydanticExtraForbid(name="default")) + + # Test with parse_object (where validation logic applies) + with pytest.raises(ArgumentError) as ctx: + parser.parse_object({"model": {"name": "John", "age": 30, "extra_field": "extra_value"}}) + assert "does not accept nested key 'extra_field'" in str(ctx.value) + + def test_extra_ignore(self, parser): + """Test that extra='ignore' accepts but ignores extra fields.""" + _, _, PydanticExtraIgnore = _create_extra_models() + parser.add_argument("--model", type=PydanticExtraIgnore, default=PydanticExtraIgnore(name="default")) + + # Test with parse_object (where validation logic applies) + cfg = parser.parse_object({"model": {"name": "John", "age": 30, "extra_field": "extra_value"}}) + + # Check that extra field is in the namespace (parsing succeeded) + assert cfg.model.name == "John" + assert cfg.model.age == 30 + assert cfg.model.extra_field == "extra_value" + + # Check that instantiation ignores the extra field + init = parser.instantiate_classes(cfg) + assert isinstance(init.model, PydanticExtraIgnore) + assert init.model.name == "John" + assert init.model.age == 30 + assert not hasattr(init.model, "extra_field") + + def test_extra_default_behavior(self, parser): + """Test that models without explicit extra config behave according to their Pydantic version defaults.""" + parser.add_argument("--model", type=PydanticModel, default=PydanticModel(p1="default")) + + from jsonargparse._optionals import is_pydantic_model + + model_version = is_pydantic_model(PydanticModel) + + if model_version == 1: + # Pydantic v1 models (including v1 compatibility mode) default to 'ignore' + cfg = parser.parse_object({"model": {"p1": "test", "p2": 5, "extra_field": "extra_value"}}) + assert cfg.model.p1 == "test" + assert cfg.model.p2 == 5 + assert cfg.model.extra_field == "extra_value" + + # Check that instantiation ignores the extra field (Pydantic v1 default behavior) + init = parser.instantiate_classes(cfg) + assert isinstance(init.model, PydanticModel) + assert init.model.p1 == "test" + assert init.model.p2 == 5 + assert not hasattr(init.model, "extra_field") + else: + # Pydantic v2 models default to 'forbid' + with pytest.raises(ArgumentError) as ctx: + parser.parse_object({"model": {"p1": "test", "p2": 5, "extra_field": "extra_value"}}) + assert "does not accept nested key 'extra_field'" in str(ctx.value) + + def test_extra_with_class_arguments(self, parser): + """Test extra field handling when using add_class_arguments.""" + PydanticExtraAllow, _, _ = _create_extra_models() + parser.add_class_arguments(PydanticExtraAllow, "model") + + # Test with parse_object to include extra field + cfg = parser.parse_object({"model": {"name": "John", "age": 30, "extra_field": "extra_value"}}) + + assert cfg.model.name == "John" + assert cfg.model.age == 30 + assert cfg.model.extra_field == "extra_value" + + # Test instantiation + init = parser.instantiate_classes(cfg) + assert isinstance(init.model, PydanticExtraAllow) + assert init.model.extra_field == "extra_value" + + def test_extra_config_function_coverage(self, parser): + """Test edge cases in get_pydantic_extra_config function for coverage.""" + from jsonargparse._optionals import get_pydantic_extra_config + + # Test with non-pydantic class + class NonPydanticClass: + pass + + assert get_pydantic_extra_config(NonPydanticClass) is None + + # Test with pydantic model that has no extra config + class PydanticNoExtra(pydantic.BaseModel): + name: str + + result = get_pydantic_extra_config(PydanticNoExtra) + # In pydantic v1, models without explicit extra config default to 'ignore' + # In pydantic v2, they default to 'forbid' (but our function returns None for default) + from jsonargparse._optionals import is_pydantic_model + + model_version = is_pydantic_model(PydanticNoExtra) + if model_version == 1: + # Pydantic v1 has a default extra='ignore' behavior + assert result in [None, "ignore"] # Allow both since it depends on implementation details + else: + # Pydantic v2 models without explicit extra config + assert result is None + + # Test with a model that has __config__ but no extra + class PydanticConfigNoExtra(pydantic.BaseModel): + name: str + + class Config: + validate_assignment = True + + result = get_pydantic_extra_config(PydanticConfigNoExtra) + # This should return None since no extra is specified + if model_version == 1: + assert result in [None, "ignore"] # v1 might have default behavior + else: + assert result is None + + # Test with pydantic v1 enum if available + try: + # Import pydantic v1 directly to avoid regex replacement issues + from pydantic import v1 as pydantic_v1 + + class PydanticV1ExtraEnum(pydantic_v1.BaseModel): + name: str + + class Config: + extra = pydantic_v1.Extra.allow + + result = get_pydantic_extra_config(PydanticV1ExtraEnum) + assert result == "allow" + + except (ImportError, AttributeError): + # pydantic v1 not available, skip this test + pass + + # Test with a class that might cause an exception (edge case) + class ProblematicClass: + """A class that might cause issues in the function.""" + + def __init__(self): + pass + + # This should not raise an exception and should return None + result = get_pydantic_extra_config(ProblematicClass) + assert result is None + + def test_pydantic_extra_config_v2_direct_attribute(self, monkeypatch): + """Test get_pydantic_extra_config with direct attribute access for model_config.extra.""" + + from jsonargparse._optionals import get_pydantic_extra_config, is_pydantic_model + + # Create a mock class that looks like a Pydantic model + class MockPydanticModel: + pass + + class MockConfig: + extra = "allow" + + mock_model_v2 = MockPydanticModel() + mock_model_v2.model_config = MockConfig() + + # Patch is_pydantic_model to return 2 (v2) for our mock + monkeypatch.setattr( + "jsonargparse._optionals.is_pydantic_model", + lambda cls: 2 if cls is mock_model_v2 else is_pydantic_model(cls), + ) + + # This should hit the direct attribute access branch + result = get_pydantic_extra_config(mock_model_v2) + assert result == "allow" + + def test_pydantic_extra_config_v2_legacy_config(self, monkeypatch): + """Test get_pydantic_extra_config with legacy __config__ in v2.""" + from jsonargparse._optionals import get_pydantic_extra_config, is_pydantic_model + + # Create a mock class that looks like a Pydantic model + class MockPydanticModel: + pass + + class MockConfig: + extra = "allow" + + mock_model_v2_legacy = MockPydanticModel() + mock_model_v2_legacy.__config__ = MockConfig() + + # Patch is_pydantic_model to return 2 (v2) for our mock + monkeypatch.setattr( + "jsonargparse._optionals.is_pydantic_model", + lambda cls: 2 if cls is mock_model_v2_legacy else is_pydantic_model(cls), + ) + + # This should hit the legacy __config__ branch for v2 + result = get_pydantic_extra_config(mock_model_v2_legacy) + assert result == "allow" + + def test_pydantic_extra_config_v1_string_extra(self, monkeypatch): + """Test get_pydantic_extra_config with string extra_value.""" + import types + + from jsonargparse._optionals import get_pydantic_extra_config, is_pydantic_model + + # Create a mock class that looks like a Pydantic model + class MockPydanticModel: + pass + + mock_model_v1 = MockPydanticModel() + mock_model_v1.__config__ = types.SimpleNamespace(extra="ignore") + + # Patch is_pydantic_model to return 1 (v1) for our mock + monkeypatch.setattr( + "jsonargparse._optionals.is_pydantic_model", + lambda cls: 1 if cls is mock_model_v1 else is_pydantic_model(cls), + ) + + # This should hit the string extra_value branch + result = get_pydantic_extra_config(mock_model_v1) + assert result == "ignore" + + def test_pydantic_extra_config_v1_enum_conversion(self, monkeypatch): + """Test get_pydantic_extra_config with enum-like extra_value that needs string conversion.""" + import types + + from jsonargparse._optionals import get_pydantic_extra_config, is_pydantic_model + + # Create a mock class that looks like a Pydantic model + class MockPydanticModel: + pass + + class MockEnum: + def __str__(self): + return "Extra.forbid" + + mock_model_v1_enum = MockPydanticModel() + mock_model_v1_enum.__config__ = types.SimpleNamespace(extra=MockEnum()) + + # Patch is_pydantic_model to return 1 (v1) for our mock + monkeypatch.setattr( + "jsonargparse._optionals.is_pydantic_model", + lambda cls: 1 if cls is mock_model_v1_enum else is_pydantic_model(cls), + ) + + # This should hit the string conversion branch + result = get_pydantic_extra_config(mock_model_v1_enum) + assert result == "forbid" + + def test_pydantic_extra_config_exception_handling(self, monkeypatch): + """Test get_pydantic_extra_config exception handling.""" + from jsonargparse._optionals import get_pydantic_extra_config, is_pydantic_model + + class ExceptionModel: + @property + def model_config(self): + raise ValueError("Simulated error") + + @property + def __config__(self): + raise AttributeError("Another simulated error") + + exception_model = ExceptionModel() + + # Patch is_pydantic_model to return 2 (v2) for our mock + monkeypatch.setattr( + "jsonargparse._optionals.is_pydantic_model", + lambda cls: 2 if cls is exception_model else is_pydantic_model(cls), + ) + + # This should hit the exception handling branch + result = get_pydantic_extra_config(exception_model) + assert result is None + # attrs tests