From eebec052e28141b2a45e9601695f5182223831b5 Mon Sep 17 00:00:00 2001 From: Ritvik Nag Date: Sun, 24 Nov 2024 10:17:08 -0500 Subject: [PATCH 1/6] Fix: using `auto_assign_tags` and `raise_on_unknown_json_key` together * Fix for using `auto_assign_tags` and `raise_on_unknown_json_key` together * Performance fix: In `UnionParser`, ensure that `get_parser()` is only called once per type * Add test case to confirm intended behavior --- dataclass_wizard/loaders.py | 5 +++ dataclass_wizard/parsers.py | 57 +++++++++++++++++---------------- tests/unit/test_load.py | 64 +++++++++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 27 deletions(-) diff --git a/dataclass_wizard/loaders.py b/dataclass_wizard/loaders.py index 6882ccb9..9416cd6f 100644 --- a/dataclass_wizard/loaders.py +++ b/dataclass_wizard/loaders.py @@ -640,6 +640,11 @@ def load_func_for_dataclass( catch_all_field = json_to_field.get(CATCH_ALL) has_catch_all = catch_all_field is not None + # Fix for using `auto_assign_tags` and `raise_on_unknown_json_key` together + # See https://github.com/rnag/dataclass-wizard/issues/137 + if meta.tag is not None: + json_to_field[meta.tag_key] = ExplicitNull + _locals = { 'cls': cls, 'py_case': cls_loader.transform_json_field, diff --git a/dataclass_wizard/parsers.py b/dataclass_wizard/parsers.py index 4ccdab6e..55a32f47 100644 --- a/dataclass_wizard/parsers.py +++ b/dataclass_wizard/parsers.py @@ -238,36 +238,39 @@ def __post_init__(self, cls: Type, self.tag_key = TAG auto_assign_tags = False - # noinspection PyUnboundLocalVariable - self.parsers = tuple( - parser - for t in self.base_type - if t is not NoneType - and isinstance(parser := get_parser(t, cls, extras), AbstractParser)) - + parsers_list = [] self.tag_to_parser = {} + for t in self.base_type: t = eval_forward_ref_if_needed(t, cls) - if is_dataclass(t): - meta = get_meta(t) - tag = meta.tag - if not tag and (auto_assign_tags or meta.auto_assign_tags): - cls_name = t.__name__ - tag = cls_name - # We don't want to mutate the base Meta class here - if meta is AbstractMeta: - from .bases_meta import BaseJSONWizardMeta - cls_dict = {'__slots__': (), 'tag': tag} - # noinspection PyTypeChecker - meta: type[M] = type(cls_name + 'Meta', (BaseJSONWizardMeta, ), cls_dict) - _META[t] = meta - else: - meta.tag = cls_name - if tag: - # TODO see if we can use a mapping of dataclass type to - # load func (maybe one passed in to __post_init__), - # rather than generating one on the fly like this. - self.tag_to_parser[tag] = get_parser(t, cls, extras) + if t is not NoneType: + parser = get_parser(t, cls, extras) + + if isinstance(parser, AbstractParser): + parsers_list.append(parser) + + elif is_dataclass(t): + meta = get_meta(t) + tag = meta.tag + if not tag and (auto_assign_tags or meta.auto_assign_tags): + cls_name = t.__name__ + tag = cls_name + # We don't want to mutate the base Meta class here + if meta is AbstractMeta: + from .bases_meta import BaseJSONWizardMeta + cls_dict = {'__slots__': (), 'tag': tag} + # noinspection PyTypeChecker + meta: type[M] = type(cls_name + 'Meta', (BaseJSONWizardMeta, ), cls_dict) + _META[t] = meta + else: + meta.tag = cls_name + if tag: + # TODO see if we can use a mapping of dataclass type to + # load func (maybe one passed in to __post_init__), + # rather than generating one on the fly like this. + self.tag_to_parser[tag] = parser + + self.parsers = tuple(parsers_list) def __contains__(self, item): """Check if parser is expected to handle the specified item type.""" diff --git a/tests/unit/test_load.py b/tests/unit/test_load.py index 2f9e1fc3..a370b344 100644 --- a/tests/unit/test_load.py +++ b/tests/unit/test_load.py @@ -2156,3 +2156,67 @@ class _(JSONWizard.Meta): } }, } + + +def test_auto_assign_tags_and_raise_on_unknown_json_key(): + + @dataclass + class A: + mynumber: int + + @dataclass + class B: + mystring: str + + @dataclass + class Container(JSONWizard): + obj2: Union[A, B] + + class _(JSONWizard.Meta): + auto_assign_tags = True + raise_on_unknown_json_key = True + + c = Container(obj2=B("bar")) + + output_dict = c.to_dict() + + assert output_dict == { + "obj2": { + "mystring": "bar", + "__tag__": "B" + } + } + + assert c == Container.from_dict(output_dict) + + +# def test_auto_assign_tags_and_catch_all(): +# +# @dataclass +# class A: +# mynumber: int +# +# @dataclass +# class B: +# mystring: str +# +# @dataclass +# class Container(JSONWizard, debug=True): +# obj2: Union[A, B] +# extra: CatchAll = None +# +# class _(JSONWizard.Meta): +# auto_assign_tags = True +# +# c = Container(obj2=B("bar")) +# +# output_dict = c.to_dict() +# +# assert output_dict == { +# "obj2": { +# "mystring": "bar", +# "__tag__": "B" +# } +# } +# +# assert c == Container.from_dict(output_dict) From 68418f5167d7bcea71bcc0f22da53edce67b060a Mon Sep 17 00:00:00 2001 From: Ritvik Nag Date: Sun, 24 Nov 2024 10:43:12 -0500 Subject: [PATCH 2/6] Fix bug for `CatchAll` field with default value, but `skip_defaults=False` --- dataclass_wizard/dumpers.py | 5 ++- tests/unit/test_load.py | 62 +++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/dataclass_wizard/dumpers.py b/dataclass_wizard/dumpers.py index d66d380e..48def6cc 100644 --- a/dataclass_wizard/dumpers.py +++ b/dataclass_wizard/dumpers.py @@ -414,7 +414,10 @@ def dump_func_for_dataclass(cls: Type[T], f' paths{key_part} = asdict(o.{field},dict_factory,hooks,config,cls_to_asdict)') elif has_catch_all and catch_all_field == field: - field_assignments.append(f"if not {skip_field}:") + if field in field_to_default: + field_assignments.append(f"if o.{field} != {default_value} and not {skip_field}:") + else: + field_assignments.append(f"if not {skip_field}:") field_assignments.append(f" for k, v in o.{field}.items():") field_assignments.append(" result.append((k," "asdict(v,dict_factory,hooks,config,cls_to_asdict)))") diff --git a/tests/unit/test_load.py b/tests/unit/test_load.py index a370b344..d4e1f631 100644 --- a/tests/unit/test_load.py +++ b/tests/unit/test_load.py @@ -1901,6 +1901,68 @@ class MyData(TOMLWizard): def test_catch_all_with_default(): """'Catch All' support with a default field value.""" + @dataclass + class MyData(JSONWizard): + my_str: str + my_float: float + extra_data: CatchAll = False + + # Case 1: Extra Data is provided + + input_dict = { + 'my_str': "test", + 'my_float': 3.14, + 'my_other_str': "test!", + 'my_bool': True + } + + # Load from TOML string + data = MyData.from_dict(input_dict) + + assert data.extra_data == {'my_other_str': 'test!', 'my_bool': True} + + # Save to TOML file + output_dict = data.to_dict() + + assert output_dict == { + "myStr": "test", + "myFloat": 3.14, + "my_other_str": "test!", + "my_bool": True + } + + new_data = MyData.from_dict(output_dict) + + assert new_data.extra_data == {'my_other_str': 'test!', 'my_bool': True} + + # Case 2: Extra Data is not provided + + input_dict = { + 'my_str': "test", + 'my_float': 3.14, + } + + # Load from TOML string + data = MyData.from_dict(input_dict) + + assert data.extra_data is False + + # Save to TOML file + output_dict = data.to_dict() + + assert output_dict == { + "myStr": "test", + "myFloat": 3.14, + } + + new_data = MyData.from_dict(output_dict) + + assert new_data.extra_data is False + + +def test_catch_all_with_skip_defaults(): + """'Catch All' support with a default field value and `skip_defaults`.""" + @dataclass class MyData(JSONWizard): class _(JSONWizard.Meta): From fc3edfb3ab27f785c351b6ff30551ff836cfda51 Mon Sep 17 00:00:00 2001 From: Ritvik Nag Date: Sun, 24 Nov 2024 10:49:21 -0500 Subject: [PATCH 3/6] Update HISTORY.rst --- HISTORY.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 9bcf4bbf..0434a8c3 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,6 +2,16 @@ History ======= +0.29.2 (2024-11-24) +------------------- + +**Bugfixes** + +* Fixed issue with using `Meta.auto_assign_tags` and `Meta.raise_on_unknown_json_key` together. +* Resolved problem when `CatchAll` field is specified with a default value, but serializing with `skip_defaults=False`. +* Improved performance in `UnionParser`: ensured that `get_parser()` is only called once per type. +* Added test case(s) to confirm intended behavior. + 0.29.1 (2024-11-23) ------------------- From 8e6a8d7b0a298bad8ed454bca5dcbcc11a362023 Mon Sep 17 00:00:00 2001 From: Ritvik Nag Date: Sun, 24 Nov 2024 10:51:06 -0500 Subject: [PATCH 4/6] Update HISTORY.rst --- HISTORY.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 0434a8c3..c6002bd8 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,9 +7,9 @@ History **Bugfixes** -* Fixed issue with using `Meta.auto_assign_tags` and `Meta.raise_on_unknown_json_key` together. -* Resolved problem when `CatchAll` field is specified with a default value, but serializing with `skip_defaults=False`. -* Improved performance in `UnionParser`: ensured that `get_parser()` is only called once per type. +* Fixed issue with using :attr:`Meta.auto_assign_tags` and :attr:`Meta.raise_on_unknown_json_key` together. +* Resolved problem when :type:`CatchAll` field is specified with a default value, but serializing with :attr:`skip_defaults=False`. +* Improved performance in :class:`UnionParser`: ensured that :func:`get_parser` is only called once per annotated type. * Added test case(s) to confirm intended behavior. 0.29.1 (2024-11-23) From a83e88db03daaf4167ed8111cc61fbb05fbb3b97 Mon Sep 17 00:00:00 2001 From: Ritvik Nag Date: Sun, 24 Nov 2024 11:14:42 -0500 Subject: [PATCH 5/6] Fix `JSONWizard.debug` so it does not overwrite existing class meta --- HISTORY.rst | 1 + dataclass_wizard/serial_json.py | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index c6002bd8..40f83c3c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,6 +8,7 @@ History **Bugfixes** * Fixed issue with using :attr:`Meta.auto_assign_tags` and :attr:`Meta.raise_on_unknown_json_key` together. +* Fix ``JSONWizard.debug`` so it does not overwrite existing class meta. * Resolved problem when :type:`CatchAll` field is specified with a default value, but serializing with :attr:`skip_defaults=False`. * Improved performance in :class:`UnionParser`: ensured that :func:`get_parser` is only called once per annotated type. * Added test case(s) to confirm intended behavior. diff --git a/dataclass_wizard/serial_json.py b/dataclass_wizard/serial_json.py index 3fc1326e..a2e8e4e3 100644 --- a/dataclass_wizard/serial_json.py +++ b/dataclass_wizard/serial_json.py @@ -2,8 +2,9 @@ import logging from .abstractions import AbstractJSONWizard +from .bases import AbstractMeta from .bases_meta import BaseJSONWizardMeta, LoadMeta -from .class_helper import call_meta_initializer_if_needed +from .class_helper import call_meta_initializer_if_needed, get_meta from .dumpers import asdict from .loaders import fromdict, fromlist # noinspection PyProtectedMember @@ -69,7 +70,11 @@ def __init_subclass__(cls, str=True, debug=False): # minimum logging level for logs by this library min_level = default_lvl if isinstance(debug, bool) else debug # set `debug_enabled` flag for the class's Meta - LoadMeta(debug_enabled=min_level).bind_to(cls) + cls_meta = get_meta(cls) + if cls_meta is not AbstractMeta: + cls_meta.debug_enabled = min_level + else: + LoadMeta(debug_enabled=min_level).bind_to(cls) def _str_fn(): From 47a2c0d236c859aeee3566624264b223deca4c6e Mon Sep 17 00:00:00 2001 From: Ritvik Nag Date: Sun, 24 Nov 2024 11:56:11 -0500 Subject: [PATCH 6/6] Fix: Using both `auto_assign_tags` and `CatchAll` does not save tag key in `CatchAll`. --- HISTORY.rst | 9 +-- dataclass_wizard/loaders.py | 12 +++- dataclass_wizard/utils/function_builder.py | 16 +++++ tests/unit/test_load.py | 70 ++++++++++++---------- 4 files changed, 70 insertions(+), 37 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 40f83c3c..1a12f828 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,10 +7,11 @@ History **Bugfixes** -* Fixed issue with using :attr:`Meta.auto_assign_tags` and :attr:`Meta.raise_on_unknown_json_key` together. -* Fix ``JSONWizard.debug`` so it does not overwrite existing class meta. -* Resolved problem when :type:`CatchAll` field is specified with a default value, but serializing with :attr:`skip_defaults=False`. -* Improved performance in :class:`UnionParser`: ensured that :func:`get_parser` is only called once per annotated type. +* Fixed issue with using :attr:`Meta.auto_assign_tags` and :attr:`Meta.raise_on_unknown_json_key` together (:issue:`137`). +* Fixed :attr:`JSONWizard.debug` to prevent overwriting existing class meta. +* Resolved issue where both :attr:`auto_assign_tags` and :type:`CatchAll` resulted in the tag key being incorrectly saved in :type:`CatchAll`. +* Fixed issue when :type:`CatchAll` field was specified with a default value but serialized with :attr:`skip_defaults=False`. +* Improved performance in :class:`UnionParser`: ensured that :func:`get_parser` is called only once per annotated type. * Added test case(s) to confirm intended behavior. 0.29.1 (2024-11-23) diff --git a/dataclass_wizard/loaders.py b/dataclass_wizard/loaders.py index 9416cd6f..765a7088 100644 --- a/dataclass_wizard/loaders.py +++ b/dataclass_wizard/loaders.py @@ -642,7 +642,8 @@ def load_func_for_dataclass( # Fix for using `auto_assign_tags` and `raise_on_unknown_json_key` together # See https://github.com/rnag/dataclass-wizard/issues/137 - if meta.tag is not None: + has_tag_assigned = meta.tag is not None + if has_tag_assigned: json_to_field[meta.tag_key] = ExplicitNull _locals = { @@ -762,8 +763,13 @@ def load_func_for_dataclass( fn_gen.add_line("raise") if has_catch_all: - with fn_gen.else_(): - fn_gen.add_line('catch_all[json_key] = o[json_key]') + line = 'catch_all[json_key] = o[json_key]' + if has_tag_assigned: + with fn_gen.elif_(f'json_key != {meta.tag_key!r}'): + fn_gen.add_line(line) + else: + with fn_gen.else_(): + fn_gen.add_line(line) with fn_gen.except_(TypeError): # If the object `o` is None, then raise an error with the relevant info included. diff --git a/dataclass_wizard/utils/function_builder.py b/dataclass_wizard/utils/function_builder.py index 3f5e4a64..4fb5cd06 100644 --- a/dataclass_wizard/utils/function_builder.py +++ b/dataclass_wizard/utils/function_builder.py @@ -79,6 +79,22 @@ def if_(self, condition: str) -> 'FunctionBuilder': """ return self._with_new_block('if', condition) + def elif_(self, condition: str) -> 'FunctionBuilder': + """Equivalent to the `elif` statement in Python. + + Sample Usage: + + >>> with FunctionBuilder().elif_('something is True'): + >>> ... + + Will generate the following code: + + >>> elif something is True: + >>> ... + + """ + return self._with_new_block('elif', condition) + def else_(self) -> 'FunctionBuilder': """Equivalent to the `else` statement in Python. diff --git a/tests/unit/test_load.py b/tests/unit/test_load.py index d4e1f631..1933d7f6 100644 --- a/tests/unit/test_load.py +++ b/tests/unit/test_load.py @@ -2252,33 +2252,43 @@ class _(JSONWizard.Meta): assert c == Container.from_dict(output_dict) -# def test_auto_assign_tags_and_catch_all(): -# -# @dataclass -# class A: -# mynumber: int -# -# @dataclass -# class B: -# mystring: str -# -# @dataclass -# class Container(JSONWizard, debug=True): -# obj2: Union[A, B] -# extra: CatchAll = None -# -# class _(JSONWizard.Meta): -# auto_assign_tags = True -# -# c = Container(obj2=B("bar")) -# -# output_dict = c.to_dict() -# -# assert output_dict == { -# "obj2": { -# "mystring": "bar", -# "__tag__": "B" -# } -# } -# -# assert c == Container.from_dict(output_dict) +def test_auto_assign_tags_and_catch_all(): + """Using both `auto_assign_tags` and `CatchAll` does not save tag key in `CatchAll`.""" + @dataclass + class A: + mynumber: int + extra: CatchAll = None + + @dataclass + class B: + mystring: str + extra: CatchAll = None + + @dataclass + class Container(JSONWizard, debug=False): + obj2: Union[A, B] + extra: CatchAll = None + + class _(JSONWizard.Meta): + auto_assign_tags = True + tag_key = 'type' + + c = Container(obj2=B("bar")) + + output_dict = c.to_dict() + + assert output_dict == { + "obj2": { + "mystring": "bar", + "type": "B" + } + } + + c2 = Container.from_dict(output_dict) + assert c2 == c == Container(obj2=B(mystring='bar', extra=None), extra=None) + + assert c2.to_dict() == { + "obj2": { + "mystring": "bar", "type": "B" + } + }