diff --git a/HISTORY.rst b/HISTORY.rst index 5e50bf5..c49badf 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,6 +2,32 @@ History ======= +0.39.0 (2026-01-01) +------------------- + +**v1 Improvements & Fixes** + +* Optimized v1 dump and encode logic for recursive types. Dumpers now return **JSON-compatible values only** (``dict``, ``list``, ``tuple``, and scalar primitives), improving correctness and performance for deeply nested structures. + +* Fixed an issue where nested index assignments were not fully preserved during dump/encode. Index paths such as ``x[0][1]`` are now retained correctly instead of being truncated to ``x[0]``. + +* Fixed a bug in v1 nested ``Union`` handling where internal collisions could occur. Union resolution now incorporates a **hash with a salt derived from the Union arguments**, ensuring stable behavior for nested Unions. + +**Configuration** + +* Added ``Meta.v1_namedtuple_as_dict`` (*default*: ``False``) + - When enabled, named tuples are dumped as dictionaries instead of positional tuples. + +* Added ``Meta.v1_coerce_none_to_empty_str`` (*default*: ``False``) + - When enabled, ``None`` values are coerced to empty strings for ``str`` fields during dump/encode. + +* Added ``Meta.v1_leaf_handling`` (*default*: ``'exact'``) + - Controls how leaf values are handled during serialization. + +**Internal Changes** + +* Renamed internal codegen variable ``tp`` to ``t`` for clarity and consistency. This is an internal refactor with no user-facing impact. + 0.38.2 (2025-12-27) ------------------- @@ -40,8 +66,10 @@ New v1 features: - Environment precedence is now configurable and explicit - Support for nested ``EnvWizard`` dataclasses - New aliasing model: + - ``v1_field_to_env_load`` (load-only) - ``v1_field_to_alias_dump`` (dump-only) + - Added ``Env(...)`` and ``Alias(env=...)`` helpers for field-level env configuration - Added ``v1_pre_decoder`` to decode JSON or delimited strings into ``dict`` / ``list`` - Cached secrets and dotenv paths for improved performance @@ -64,10 +92,12 @@ Internal Changes and Fixes - Improved Windows timezone handling via ``tz`` extra (``tzdata`` / ``ZoneInfo``) - Improved caching behavior for ``Union`` loaders - Fixed multiple codegen and caching edge cases: + - ``to_dict`` caching on subclasses - empty dataclass dumpers - ``kw_only`` field handling - FunctionBuilder globals merging + - Added extensive v1 test coverage (90%+) - No breaking changes without explicit v1 opt-in diff --git a/README.rst b/README.rst index 4f5f7d2..60aa64d 100644 --- a/README.rst +++ b/README.rst @@ -49,7 +49,7 @@ transforms, and support for nested dataclasses. ``DataclassWizard`` also auto-applies ``@dataclass`` to subclasses. -.. important:: +.. tip:: A new **v1 engine** is available as an opt-in, offering explicit environment precedence, nested dataclass support, and improved performance. @@ -1384,6 +1384,22 @@ What's New in v1.0 print(MyModel(my_field="value").to_dict()) # Output: {'my_field': 'value'} + - **String Coercion Change (None Handling)** + + Starting with **v1.0**, ``None`` values for fields annotated as ``str`` are + converted using ``str(None)`` (i.e. ``'None'``) instead of being silently + coerced to the empty string. + + ``Optional[str]`` fields continue to preserve ``None`` by default. + + To restore the previous behavior and coerce ``None`` to ``''``, set: + + .. code-block:: python3 + + class _(Meta): + v1 = True + v1_coerce_none_to_empty_str = True + - **Default __str__() Behavior Change** Starting with **v1.0.0**, we no longer pretty-print the serialized JSON value with keys in ``camelCase``. diff --git a/dataclass_wizard/bases.py b/dataclass_wizard/bases.py index d8f325a..efac184 100644 --- a/dataclass_wizard/bases.py +++ b/dataclass_wizard/bases.py @@ -423,6 +423,43 @@ class AbstractMeta(metaclass=ABCOrAndMeta): # deserialization. v1_assume_naive_datetime_tz: ClassVar[tzinfo | None] = None + # Controls how `typing.NamedTuple` and `collections.namedtuple` + # fields are loaded and serialized. + # + # - False (DEFAULT): load from list/tuple and serialize + # as a positional list. + # - True: load from mapping and serialize as a dict + # keyed by field name. + # + # In strict mode, inputs that do not match the selected mode + # raise TypeError. + # + # Note: + # This option enforces strict shape matching for performance reasons. + v1_namedtuple_as_dict: bool = None + + # If True (default: False), ``None`` is coerced to an empty string (``""``) + # when loading ``str`` fields. + # + # When False, ``None`` is coerced using ``str(value)``, so ``None`` becomes + # the literal string ``'None'`` for ``str`` fields. + # + # For ``Optional[str]`` fields, ``None`` is preserved by default. + v1_coerce_none_to_empty_str: bool = None + + # Controls how leaf (non-recursive) types are detected during serialization. + # + # - "exact" (DEFAULT): only exact built-in leaf types are treated as leaf values. + # - "issubclass": subclasses of leaf types are also treated as leaf values. + # + # Leaf types are returned without recursive traversal. Bytes are still + # handled separately according to their serialization rules. + # + # Note: + # The default "exact" mode avoids treating third-party scalar-like + # objects (e.g. NumPy scalars) as built-in leaf types. + v1_leaf_handling: Literal['exact', 'issubclass'] = None + # noinspection PyMethodParameters @cached_class_property def all_fields(cls) -> FrozenKeys: @@ -712,6 +749,43 @@ class AbstractEnvMeta(metaclass=ABCOrAndMeta): # deserialization. v1_assume_naive_datetime_tz: ClassVar[tzinfo | None] = None + # Controls how `typing.NamedTuple` and `collections.namedtuple` + # fields are loaded and serialized. + # + # - False (DEFAULT): load from list/tuple and serialize + # as a positional list. + # - True: load from mapping and serialize as a dict + # keyed by field name. + # + # In strict mode, inputs that do not match the selected mode + # raise TypeError. + # + # Note: + # This option enforces strict shape matching for performance reasons. + v1_namedtuple_as_dict: bool = None + + # If True (default: False), ``None`` is coerced to an empty string (``""``) + # when loading ``str`` fields. + # + # When False, ``None`` is coerced using ``str(value)``, so ``None`` becomes + # the literal string ``'None'`` for ``str`` fields. + # + # For ``Optional[str]`` fields, ``None`` is preserved by default. + v1_coerce_none_to_empty_str: bool = None + + # Controls how leaf (non-recursive) types are detected during serialization. + # + # - "exact" (DEFAULT): only exact built-in leaf types are treated as leaf values. + # - "issubclass": subclasses of leaf types are also treated as leaf values. + # + # Leaf types are returned without recursive traversal. Bytes are still + # handled separately according to their serialization rules. + # + # Note: + # The default "exact" mode avoids treating third-party scalar-like + # objects (e.g. NumPy scalars) as built-in leaf types. + v1_leaf_handling: Literal['exact', 'issubclass'] = None + # noinspection PyMethodParameters @cached_class_property def all_fields(cls) -> FrozenKeys: diff --git a/dataclass_wizard/bases_meta.pyi b/dataclass_wizard/bases_meta.pyi index 90f7390..b2c5782 100644 --- a/dataclass_wizard/bases_meta.pyi +++ b/dataclass_wizard/bases_meta.pyi @@ -92,7 +92,10 @@ def LoadMeta(*, v1_case: KeyCase | str | None = MISSING, v1_field_to_alias: Mapping[str, str | Sequence[str]] = MISSING, v1_on_unknown_key: KeyAction | str | None = KeyAction.IGNORE, - v1_unsafe_parse_dataclass_in_union: bool = MISSING) -> T | META: + v1_unsafe_parse_dataclass_in_union: bool = MISSING, + v1_namedtuple_as_dict: bool = MISSING, + v1_coerce_none_to_empty_str: bool = MISSING, + v1_leaf_handling: Literal['exact', 'issubclass'] = MISSING) -> T | META: ... @@ -114,7 +117,9 @@ def DumpMeta(*, v1_case: KeyCase | str | None = MISSING, v1_field_to_alias: Mapping[str, str | Sequence[str]] = MISSING, v1_dump_date_time_as: V1DateTimeTo | str = MISSING, - v1_assume_naive_datetime_tz: tzinfo | None = MISSING) -> T | META: + v1_assume_naive_datetime_tz: tzinfo | None = MISSING, + v1_namedtuple_as_dict: bool = MISSING, + v1_leaf_handling: Literal['exact', 'issubclass'] = MISSING) -> T | META: ... @@ -148,5 +153,8 @@ def EnvMeta(*, debug_enabled: 'bool | int | str' = MISSING, # v1_on_unknown_key: KeyAction | str | None = KeyAction.IGNORE, v1_unsafe_parse_dataclass_in_union: bool = MISSING, v1_dump_date_time_as: V1DateTimeTo | str = MISSING, - v1_assume_naive_datetime_tz: tzinfo | None = MISSING) -> META: + v1_assume_naive_datetime_tz: tzinfo | None = MISSING, + v1_namedtuple_as_dict: bool = MISSING, + v1_coerce_none_to_empty_str: bool = MISSING, + v1_leaf_handling: Literal['exact', 'issubclass'] = MISSING) -> META: ... diff --git a/dataclass_wizard/errors.py b/dataclass_wizard/errors.py index f9a0667..c183840 100644 --- a/dataclass_wizard/errors.py +++ b/dataclass_wizard/errors.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from dataclasses import Field, MISSING +from dataclasses import Field, MISSING, is_dataclass from typing import (Any, Type, Dict, Tuple, ClassVar, Optional, Union, Iterable, Callable, Collection, Sequence) @@ -285,7 +285,8 @@ def message(self) -> str: # see https://github.com/rnag/dataclass-wizard/issues/54 for more info normalized_json_keys = [normalize(key) for key in obj] - if next((f for f in self.missing_fields if normalize(f) in normalized_json_keys), None): + if (is_dataclass(self.parent_cls) and + next((f for f in self.missing_fields if normalize(f) in normalized_json_keys), None)): from .enums import LetterCase from .v1.enums import KeyCase from .loader_selection import get_loader diff --git a/dataclass_wizard/serial_json.pyi b/dataclass_wizard/serial_json.pyi index d89a776..3d859ae 100644 --- a/dataclass_wizard/serial_json.pyi +++ b/dataclass_wizard/serial_json.pyi @@ -68,15 +68,7 @@ class SerializerHookMixin(Protocol): ... -class JSONPyWizard(JSONSerializable, SerializerHookMixin): - """Helper for JSONWizard that ensures dumping to JSON keeps keys as-is.""" - - -class JSONSerializable(DataclassWizard, SerializerHookMixin): ... - - -@dataclass_transform() -class DataclassWizard(AbstractJSONWizard, SerializerHookMixin): +class JSONWizardImpl(AbstractJSONWizard, SerializerHookMixin): """ Mixin class to allow a `dataclass` sub-class to be easily converted to and from JSON. @@ -201,9 +193,22 @@ class DataclassWizard(AbstractJSONWizard, SerializerHookMixin): ... +@dataclass_transform() +class DataclassWizard(JSONWizardImpl): + ... + + +class JSONPyWizard(JSONWizardImpl): + """Helper for JSONWizard that ensures dumping to JSON keeps keys as-is.""" + + +class JSONSerializable(JSONWizardImpl): ... + + def _str_fn() -> Callable[[W], str]: """ Converts the dataclass instance to a *prettified* JSON string representation, when the `str()` method is invoked. """ ... + diff --git a/dataclass_wizard/utils/function_builder.py b/dataclass_wizard/utils/function_builder.py index 2d22379..b15556f 100644 --- a/dataclass_wizard/utils/function_builder.py +++ b/dataclass_wizard/utils/function_builder.py @@ -1,4 +1,5 @@ from dataclasses import MISSING +from typing import Any from ..log import LOG @@ -67,7 +68,7 @@ def function(self, name: str, args: list, return_type=MISSING, def _with_new_block(self, name: str, condition: 'str | None' = None, - comment: str = '') -> 'FunctionBuilder': + comment: Any = '') -> 'FunctionBuilder': """Creates a new block. Used with a context manager (with).""" indent = ' ' * self.indent_level @@ -97,7 +98,7 @@ def for_(self, condition: str) -> 'FunctionBuilder': """ return self._with_new_block('for', condition) - def if_(self, condition: str, comment: str = '') -> 'FunctionBuilder': + def if_(self, condition: str, comment: Any = '') -> 'FunctionBuilder': """Equivalent to the `if` statement in Python. Sample Usage: diff --git a/dataclass_wizard/utils/json_util.py b/dataclass_wizard/utils/json_util.py index 3d6b48a..ba7d936 100644 --- a/dataclass_wizard/utils/json_util.py +++ b/dataclass_wizard/utils/json_util.py @@ -51,4 +51,7 @@ def default(self, o: Any) -> Any: def safe_dumps(o, cls=SafeEncoder, **kwargs): - return dumps(o, cls=cls, **kwargs) + try: + return dumps(o, cls=cls, **kwargs) + except TypeError: + return o diff --git a/dataclass_wizard/utils/type_conv.py b/dataclass_wizard/utils/type_conv.py index a5175bd..a7a64d8 100644 --- a/dataclass_wizard/utils/type_conv.py +++ b/dataclass_wizard/utils/type_conv.py @@ -2,14 +2,10 @@ __all__ = ['as_bool', 'as_int', - 'as_int_v1', 'as_str', 'as_list', 'as_dict', 'as_enum', - 'as_datetime_v1', - 'as_date_v1', - 'as_time_v1', 'as_datetime', 'as_date', 'as_time', @@ -19,10 +15,9 @@ ] import json -from collections.abc import Callable -from datetime import datetime, time, date, timedelta, timezone, tzinfo +from datetime import datetime, time, date, timedelta, timezone from numbers import Number -from typing import Union, Type, AnyStr, Optional, Iterable, Any +from typing import Union, Type, AnyStr, Optional, Iterable from ..errors import ParseError from ..lazy_imports import pytimeparse @@ -48,43 +43,6 @@ def as_bool(o: Union[str, bool, N]): return o == 1 -def as_int_v1(o: Union[float, bool], - tp: type, - base_type=int): - """ - Attempt to convert `o` to an int. - - This assumes the following checks already happen: - - `tp is base_type` - - `tp is str and '.' in o and float(o).is_integer()` - - `tp is str and '.' in o and not float(o).is_integer()` --> IMPLIED - - `tp is str and '.' not in o` - - If `o` cannot be converted to an int, raise an error. - - :raises TypeError: If `o` is a `bool` (which is an `int` subclass) - :raises ValueError: When `o` cannot be converted to an `int` - """ - # Commenting this out, because `int(o)` already raises an error - # for float strings with a fractional part. - # if tp is str: # The string represents a float value with fractional part, e.g. '2.7' - # raise ValueError(f"Cannot cast string float with fractional part: {o}") from None - - if tp is float: - if o.is_integer(): - return base_type(o) - raise ValueError(f"Cannot cast float with fractional part: {o}") from None - - if tp is bool: - raise TypeError(f'as_int: Incorrect type, object={o!r}, type={tp}') from None - - try: - return base_type(o) - - except (TypeError, ValueError): - raise - - def as_int(o: Union[str, int, float, bool, None], base_type=int, default=0, raise_=True): """ @@ -246,105 +204,6 @@ def as_enum(o: Union[AnyStr, N], return None -def as_datetime_v1(o: Union[int, float, datetime], - __from_timestamp: Callable[[float, tzinfo], datetime], - __tz=None): - """ - V1: Attempt to convert an object `o` to a :class:`datetime` object using the - below logic. - - * ``Number`` (int or float): Convert a numeric timestamp via the - built-in ``fromtimestamp`` method, and return a UTC datetime. - * ``base_type``: Return object `o` if it's already of this type. - - Note: It is assumed that `o` is not a ``str`` (in ISO format), as - de-serialization in ``v1`` already checks for this. - - Otherwise, if we're unable to convert the value of `o` to a - :class:`datetime` as expected, raise an error. - - """ - try: - # We can assume that `o` is a number, as generally this will be the - # case. - return __from_timestamp(o, __tz) - - except Exception: - # Note: the `__self__` attribute refers to the class bound - # to the class method `fromtimestamp`. - # - # See: https://stackoverflow.com/a/41258933/10237506 - # - # noinspection PyUnresolvedReferences - if o.__class__ is __from_timestamp.__self__: - return o - - # Check `type` explicitly, because `bool` is a sub-class of `int` - if o.__class__ not in NUMBERS: - raise TypeError(f'Unsupported type, value={o!r}, type={type(o)}') - - raise - - -def as_date_v1(o: Union[int, float, date], - __from_timestamp: Callable[[float], date]): - """ - V1: Attempt to convert an object `o` to a :class:`date` object using the - below logic. - - * ``Number`` (int or float): Convert a numeric timestamp via the - built-in ``fromtimestamp`` method, and return a date. - * ``base_type``: Return object `o` if it's already of this type. - - Note: It is assumed that `o` is not a ``str`` (in ISO format), as - de-serialization in ``v1`` already checks for this. - - Otherwise, if we're unable to convert the value of `o` to a - :class:`date` as expected, raise an error. - - """ - try: - # We can assume that `o` is a number, as generally this will be the - # case. - return __from_timestamp(o) - - except Exception: - # Note: the `__self__` attribute refers to the class bound - # to the class method `fromtimestamp`. - # - # See: https://stackoverflow.com/a/41258933/10237506 - # - # noinspection PyUnresolvedReferences - if o.__class__ is __from_timestamp.__self__: - return o - - # Check `type` explicitly, because `bool` is a sub-class of `int` - if o.__class__ not in NUMBERS: - raise TypeError(f'Unsupported type, value={o!r}, type={type(o)}') - - raise - - -def as_time_v1(o: Union[time, Any], base_type: type[time]): - """ - V1: Attempt to convert an object `o` to a :class:`time` object using the - below logic. - - * ``base_type``: Return object `o` if it's already of this type. - - Note: It is assumed that `o` is not a ``str`` (in ISO format), as - de-serialization in ``v1`` already checks for this. - - Otherwise, if we're unable to convert the value of `o` to a - :class:`time` as expected, raise an error. - - """ - if o.__class__ is base_type: - return o - - raise TypeError(f'Unsupported type, value={o!r}, type={type(o)}') - - # TODO Remove: Unused in V1 def as_datetime(o: Union[str, Number, datetime], base_type=datetime, default=None, raise_=True): diff --git a/dataclass_wizard/v1/_env.py b/dataclass_wizard/v1/_env.py index b0eeddc..927160a 100644 --- a/dataclass_wizard/v1/_env.py +++ b/dataclass_wizard/v1/_env.py @@ -655,8 +655,8 @@ def load_to_bytes(tp: TypeInfo, extras: Extras): # could add support for b64-encoded strings later: # bytes(__b64decode(o)) if (o.__class__ is str and __env_b64) o = tp.v() - return (f"{o} if (tp := {o}.__class__) is bytes " - f"else {o}.encode('utf-8') if tp is str " + return (f"{o} if (t := {o}.__class__) is bytes " + f"else {o}.encode('utf-8') if t is str " f"else bytes({o})") @classmethod diff --git a/dataclass_wizard/v1/decorators.py b/dataclass_wizard/v1/decorators.py index c0c719a..0c03cad 100644 --- a/dataclass_wizard/v1/decorators.py +++ b/dataclass_wizard/v1/decorators.py @@ -1,12 +1,13 @@ from __future__ import annotations +import hashlib from dataclasses import MISSING from functools import wraps from typing import TYPE_CHECKING, Callable, Union, cast from ..type_def import DT from ..utils.function_builder import FunctionBuilder - +from ..utils.typing_compat import is_union if TYPE_CHECKING: # pragma: no cover from .models import Extras, TypeInfo @@ -64,6 +65,48 @@ def static_method_wrapper(tp: TypeInfo, extras: Extras): return static_method_wrapper +def _type_id(t) -> str: + # stable-ish identifier for hashing purposes + mod = getattr(t, '__module__', None) + qn = getattr(t, '__qualname__', None) + if mod and qn: + return f'{mod}.{qn}' + return repr(t) + + +def _generic_sig_str(name, args) -> str: + args = _canonical_union_args(args) # Union[..]: flattened, de-duped, sorted + return f'{name}[{",".join(_type_id(a) for a in args)}]' + + +def _union_args(x): + # get args similarly to typing.get_args but without importing it everywhere + return getattr(x, '__args__', ()) + + +def _flatten_union_args(args): + out = [] + for a in args: + if is_union(a): + out.extend(_flatten_union_args(_union_args(a))) + else: + out.append(a) + return out + + +def _canonical_union_args(args): + flat = _flatten_union_args(args) + seen = set() + uniq = [] + for a in flat: + k = _type_id(a) + if k not in seen: + seen.add(k) + uniq.append(a) + uniq.sort(key=_type_id) + return tuple(uniq) + + def setup_recursive_safe_function( func: Callable = None, *, @@ -71,7 +114,7 @@ def setup_recursive_safe_function( is_generic: bool = False, add_cls: bool = True, prefix: str = 'load', - per_field_cache: bool = False, + per_class_cache: bool = False, ) -> Callable: """ A decorator to ensure recursion safety and facilitate dynamic function generation @@ -102,7 +145,7 @@ def setup_recursive_safe_function( is_generic=is_generic, add_cls=add_cls, prefix=prefix, - per_field_cache=per_field_cache, + per_class_cache=per_class_cache, ) def _wrapper_logic(tp: TypeInfo, extras: Extras, _cls=None) -> str: @@ -118,16 +161,21 @@ def _wrapper_logic(tp: TypeInfo, extras: Extras, _cls=None) -> str: :return: The generated function call expression as a string. :rtype: str """ - cls = tp.args if is_generic else tp.origin + name = tp.name + if is_generic: + ann_tp_or_args = (name, _canonical_union_args(tp.args)) + else: + ann_tp_or_args = tp.origin + recursion_guard = extras['recursion_guard'] # new function: drop indices and explicit name tp_for_func = tp.replace(index=None, val_name=None) - if per_field_cache: - key = extras['cls'], tp.field_i, cls + if per_class_cache: + key = (prefix, extras['cls'], ann_tp_or_args) else: - key = cls + key = (prefix, ann_tp_or_args) if (_fn_name := recursion_guard.get(key)) is None: cls_name = extras['cls_name'] @@ -135,12 +183,16 @@ def _wrapper_logic(tp: TypeInfo, extras: Extras, _cls=None) -> str: # Generate the function name if fn_name: - _fn_name = fn_name.format(cls_name=tp.name) + _fn_name = fn_name.format(cls_name=name) else: - _fn_name = ( - f'_{prefix}_{cls_name}_{tp_name}_{tp.field_i}' if is_generic - else f'_{prefix}_{cls_name}_{tp_name}_{tp.name}' - ) + cls_part = f'_{cls_name}' if per_class_cache else '' + if is_generic: + sig_src = _generic_sig_str(name, ann_tp_or_args).encode('utf-8') + # noinspection PyTypeChecker + sig_hash = hashlib.blake2s(sig_src, digest_size=6).hexdigest() + _fn_name = f'_{prefix}{cls_part}_{tp_name}_{sig_hash}' + else: + _fn_name = f'_{prefix}{cls_part}_{tp_name}_{name}' recursion_guard[key] = _fn_name @@ -149,7 +201,7 @@ def _wrapper_logic(tp: TypeInfo, extras: Extras, _cls=None) -> str: # Prepare a new FunctionBuilder for this function updated_extras = extras.copy() - updated_extras['locals'] = _locals = {'cls': cls} if add_cls else {} + updated_extras['locals'] = _locals = {'cls': ann_tp_or_args} if add_cls else {} updated_extras['fn_gen'] = new_fn_gen = FunctionBuilder() # Apply the decorated function logic @@ -193,7 +245,7 @@ def wrapper_class_method(_cls, tp, extras) -> str: def setup_recursive_safe_function_for_generic(func: Callable = None, prefix='load', - per_field_cache: bool = False) -> Callable: + per_class_cache: bool = False) -> Callable: """ A helper decorator to handle generic types using `setup_recursive_safe_function`. @@ -210,4 +262,4 @@ def setup_recursive_safe_function_for_generic(func: Callable = None, A wrapped function ensuring recursion safety for generic types. """ return setup_recursive_safe_function(func, is_generic=True, prefix=prefix, - per_field_cache=per_field_cache) + per_class_cache=per_class_cache) diff --git a/dataclass_wizard/v1/dumpers.py b/dataclass_wizard/v1/dumpers.py index ecff219..5b4732b 100644 --- a/dataclass_wizard/v1/dumpers.py +++ b/dataclass_wizard/v1/dumpers.py @@ -11,23 +11,31 @@ from pathlib import Path # noinspection PyUnresolvedReferences,PyProtectedMember from typing import ( - Any, Type, Dict, List, Tuple, Iterable, Sequence, Union, - NamedTupleMeta, - SupportsFloat, AnyStr, Text, Callable, Optional, cast, Literal, Annotated, NamedTuple + cast, Any, Type, Dict, List, Tuple, Iterable, Sequence, Union, + NamedTupleMeta, SupportsFloat, AnyStr, Text, Callable, Optional, + Literal, Annotated, NamedTuple, ) from uuid import UUID -from .decorators import setup_recursive_safe_function, setup_recursive_safe_function_for_generic +from .decorators import (setup_recursive_safe_function, + setup_recursive_safe_function_for_generic) from .enums import KeyCase, DateTimeTo -from .models import Extras, TypeInfo, SIMPLE_TYPES, PatternBase, UTC, ZERO, SCALAR_TYPES +from .models import (Extras, TypeInfo, PatternBase, + LEAF_TYPES, LEAF_TYPES_NO_BYTES, UTC, ZERO) from .type_conv import datetime_to_timestamp from ..abstractions import AbstractDumperGenerator from ..bases import AbstractMeta, BaseDumpHook, META from ..class_helper import ( - v1_dataclass_field_to_alias_for_dump, dataclass_fields, get_meta, is_subclass_safe, + CLASS_TO_DUMP_FUNC, DATACLASS_FIELD_TO_ALIAS_PATH_FOR_DUMP, - dataclass_field_to_default, create_meta, CLASS_TO_DUMP_FUNC, - dataclass_field_names, dataclass_field_to_skip_if, + create_meta, + get_meta, + is_subclass_safe, + v1_dataclass_field_to_alias_for_dump, + dataclass_fields, + dataclass_field_to_default, + dataclass_field_names, + dataclass_field_to_skip_if, ) from ..constants import CATCH_ALL, TAG, PACKAGE_NAME from ..errors import (ParseError, MissingFields, MissingData, JSONWizardError) @@ -35,7 +43,7 @@ from ..log import LOG from ..models import get_skip_if_condition, finalize_skip_if from ..type_def import ( - DefFactory, NoneType, JSONObject, + NoneType, JSONObject, PyLiteralString, T, ExplicitNull ) @@ -50,6 +58,25 @@ ) +def _type_returns_value_unchanged(arg, leaf_handling_as_subclass, origin=None): + # scalar type: + # (str, int, float, bool, complex, type, Literal, Any) + if origin is None: + origin = get_origin_v2(arg) + return (origin is Any + or origin is Literal + or origin in LEAF_TYPES_NO_BYTES + or (leaf_handling_as_subclass + and is_subclass_safe(origin, LEAF_TYPES_NO_BYTES))) + + +def _all_return_value_unchanged(args, leaf_handling_as_subclass): + for arg in args: + if not _type_returns_value_unchanged(arg, leaf_handling_as_subclass): + return False + return True + + class DumpMixin(AbstractDumperGenerator, BaseDumpHook): """ This Mixin class derives its name from the eponymous `json.dumps` @@ -118,6 +145,11 @@ def dump_from_iterable(cls, tp: TypeInfo, extras: Extras): v, v_next, i_next = tp.v_and_next() gorg = tp.origin + if v_next[0] == 'k': + # raise same error as `json` (not serializable) + raise TypeError('keys must be str, int, float, bool or None, ' + f'not {gorg.__qualname__}') from None + # noinspection PyBroadException try: elem_type = tp.args[0] @@ -127,16 +159,10 @@ def dump_from_iterable(cls, tp: TypeInfo, extras: Extras): string = cls.dump_dispatcher_for_annotation( tp.replace(origin=elem_type, i=i_next, index=None, val_name=None), extras) - if issubclass(gorg, (set, frozenset)): - start_char = '{' - end_char = '}' - else: - start_char = '[' - end_char = ']' + if string == v_next: + return f'{v}.copy()' if issubclass(gorg, list) else f'list({v})' - result = f'{start_char}{string} for {v_next} in {v}{end_char}' - - return tp.wrap(result, extras) + return f'[{string} for {v_next} in {v}]' @classmethod def dump_from_tuple(cls, tp: TypeInfo, extras: Extras): @@ -150,8 +176,7 @@ def dump_from_tuple(cls, tp: TypeInfo, extras: Extras): is_variadic = args[-1] is ... else: # Annotated without args, as simply `tuple` - args = (Any, ...) - is_variadic = True + return f'list({tp.v()})' if is_variadic: # Logic that handles the variadic form of :class:`Tuple`'s, @@ -169,97 +194,49 @@ def dump_from_tuple(cls, tp: TypeInfo, extras: Extras): string = cls.dump_dispatcher_for_annotation( tp.replace(origin=args[0], i=i_next, index=None, val_name=None), extras) + if string == v_next: + return f'list({v})' + result = f'[{string} for {v_next} in {v}]' - # Wrap because we need to create a tuple from list comprehension - force_wrap = True else: string = ', '.join([ str(cls.dump_dispatcher_for_annotation( - tp.replace(origin=arg, index=k, val_name=None), + tp.replace(origin=arg, index=k), extras)) for k, arg in enumerate(args)]) - result = f'({string}, )' - - force_wrap = False + result = f'[{string}]' - return tp.wrap(result, extras, force=force_wrap) + return result @classmethod - @setup_recursive_safe_function(prefix='dump') def dump_from_named_tuple(cls, tp: TypeInfo, extras: Extras): - fn_gen = extras['fn_gen'] nt_tp = cast(NamedTuple, tp.origin) + fields = nt_tp._fields # field names in order + ann = nt_tp.__annotations__ + + field_to_value = { + name: str( + cls.dump_dispatcher_for_annotation( + tp.replace(origin=ann.get(name, Any), index=i), + extras, + ) + ) + for i, name in enumerate(fields) + } - _locals = extras['locals'] - _locals['cls'] = nt_tp - _locals['msg'] = "`dict` input is not supported for NamedTuple, use a dataclass instead." - - req_field_to_assign = {} - field_assigns = [] - # noinspection PyProtectedMember - optional_fields = set(nt_tp._field_defaults) - has_optionals = True if optional_fields else False - only_optionals = has_optionals and len(optional_fields) == len(nt_tp.__annotations__) - num_fields = 0 - v = tp.v_for_def() - - for field, field_tp in nt_tp.__annotations__.items(): - string = cls.dump_dispatcher_for_annotation( - tp.replace(origin=field_tp, index=num_fields, val_name=None), extras) - - if has_optionals and field in optional_fields: - field_assigns.append(string) - else: - req_field_to_assign[f'__{field}'] = string - - num_fields += 1 - - params = ', '.join(req_field_to_assign) - - with fn_gen.try_(): - - for field, string in req_field_to_assign.items(): - fn_gen.add_line(f'{field} = {string}') - - if has_optionals: - opt_start = len(req_field_to_assign) - fn_gen.add_line(f'L = len({v}); has_opt = L > {opt_start}') - with fn_gen.if_(f'has_opt'): - fn_gen.add_line(f'fields = [{field_assigns.pop(0)}]') - for i, string in enumerate(field_assigns, start=opt_start + 1): - fn_gen.add_line(f'if L > {i}: fields.append({string})') + if extras['config'].v1_namedtuple_as_dict: + params = [f'{field!r}: {value}' for field, value in field_to_value.items()] + return f'{{{", ".join(params)}}}' - if only_optionals: - fn_gen.add_line(f'return cls(*fields)') - else: - fn_gen.add_line(f'return cls({params}, *fields)') - - fn_gen.add_line(f'return cls({params})') - - with fn_gen.except_(Exception, 'e'): - with fn_gen.if_('(e_cls := e.__class__) is IndexError'): - # raise `MissingFields`, as required NamedTuple fields - # are not present in the input object `o`. - fn_gen.add_line(f'raise_missing_fields(locals(), {v}, cls, None)') - with fn_gen.if_(f'e_cls is KeyError and type({v}) is dict'): - # Input object is a `dict` - # TODO should we support dict for namedtuple? - fn_gen.add_line('raise TypeError(msg) from None') - # re-raise - fn_gen.add_line('raise e from None') + params = ', '.join(field_to_value.values()) + return f'[{params}]' @classmethod def dump_from_named_tuple_untyped(cls, tp: TypeInfo, extras: Extras): - # Check if input object is `dict` or `list`. - # - # Assuming `Point` is a `namedtuple`, this performs - # the equivalent logic as: - # Point(**x) if isinstance(x, dict) else Point(*x) - v = tp.v() - star, dbl_star = tp.multi_wrap(extras, 'nt_', f'*{v}', f'**{v}') - return f'{dbl_star} if isinstance({v}, dict) else {star}' + as_dict = extras['config'].v1_namedtuple_as_dict + return f'{tp.v()}._asdict()' if as_dict else f'list({tp.v()})' @classmethod def _build_dict_comp(cls, tp, v, i_next, k_next, v_next, kt, vt, extras): @@ -269,54 +246,56 @@ def _build_dict_comp(cls, tp, v, i_next, k_next, v_next, kt, vt, extras): tp_v_next = tp.replace(origin=vt, i=i_next, prefix='v', index=None, val_name=None) string_v = cls.dump_dispatcher_for_annotation(tp_v_next, extras) + if k_next == string_k and v_next == string_v: + # Easy path; shallow copy + return f'{v}.copy()' + return f'{{{string_k}: {string_v} for {k_next}, {v_next} in {v}.items()}}' @classmethod def dump_from_dict(cls, tp: TypeInfo, extras: Extras): - v, k_next, v_next, i_next = tp.v_and_next_k_v() - try: kt, vt = tp.args except ValueError: # Annotated without two arguments, # e.g. like `dict[str]` or `dict` - kt = vt = Any + return f'{tp.v()}.copy()' - result = cls._build_dict_comp( + v, k_next, v_next, i_next = tp.v_and_next_k_v() + + return cls._build_dict_comp( tp, v, i_next, k_next, v_next, kt, vt, extras) - return tp.wrap(result, extras) + dump_from_defaultdict = dump_from_dict @classmethod - def dump_from_defaultdict(cls, tp: TypeInfo, extras: Extras): - v, k_next, v_next, i_next = tp.v_and_next_k_v() - default_factory: DefFactory | None - - try: - kt, vt = tp.args - default_factory = getattr(vt, '__origin__', vt) - except ValueError: - # Annotated without two arguments, - # e.g. like `defaultdict[str]` or `defaultdict` - kt = vt = Any - default_factory = NoneType - - result = cls._build_dict_comp( - tp, v, i_next, k_next, v_next, kt, vt, extras) + def dump_from_typed_dict(cls, tp: TypeInfo, extras: Extras): + req_keys, opt_keys = get_keys_for_typed_dict(tp.origin) + if opt_keys: + return cls._dump_from_typed_dict_fn(tp, extras) + ann = tp.origin.__annotations__ + + dict_body = ', '.join( + f"""{name!r}: { + cls.dump_dispatcher_for_annotation( + tp.replace(origin=ann.get(name, Any), index=repr(name)), + extras, + ) + }""" + for name in req_keys + ) - return tp.wrap_dd(default_factory, result, extras) + return f'{{{dict_body}}}' @classmethod @setup_recursive_safe_function(prefix='dump') - def dump_from_typed_dict(cls, tp: TypeInfo, extras: Extras): + def _dump_from_typed_dict_fn(cls, tp: TypeInfo, extras: Extras): fn_gen = extras['fn_gen'] - req_keys, opt_keys = get_keys_for_typed_dict(tp.origin) + td_annotations = tp.origin.__annotations__ - result_list = [] v = tp.v_for_def() - # TODO set __annotations__? - td_annotations = tp.origin.__annotations__ + result_list = [] # Set required keys for the `TypedDict` for k in req_keys: @@ -329,57 +308,48 @@ def dump_from_typed_dict(cls, tp: TypeInfo, extras: Extras): result_list.append(f'{field_name}: {string}') - with fn_gen.try_(): - fn_gen.add_lines('result = {', - *(f' {r},' for r in result_list), - '}') - - # Set optional keys for the `TypedDict` (if they exist) - next_i = tp.i + 1 - new_tp = tp.replace(i=next_i, index=None, val_name=None) - v_next = new_tp.v() - - for k in opt_keys: - field_tp = td_annotations[k] - field_name = repr(k) - string = cls.dump_dispatcher_for_annotation( - new_tp.replace(origin=field_tp), extras) - with fn_gen.if_(f'({v_next} := {v}.get({field_name}, MISSING)) is not MISSING'): - fn_gen.add_line(f'result[{field_name}] = {string}') - fn_gen.add_line('return result') - - with fn_gen.except_(Exception, 'e'): - with fn_gen.if_('type(e) is KeyError'): - fn_gen.add_line('name = e.args[0]; e = KeyError(f"Missing required key: {name!r}")') - with fn_gen.elif_(f'not isinstance({v}, dict)'): - fn_gen.add_line('e = TypeError("Incorrect type for object")') - fn_gen.add_line(f'raise ParseError(e, {v}, {{}}, "dump") from None') + fn_gen.add_lines('result = {', + *(f' {r},' for r in result_list), + '}') + + # Set optional keys for the `TypedDict` (if they exist) + next_i = tp.i + 1 + new_tp = tp.replace(i=next_i, index=None, val_name=None) + v_next = new_tp.v() + + for k in opt_keys: + field_tp = td_annotations[k] + field_name = repr(k) + string = cls.dump_dispatcher_for_annotation( + new_tp.replace(origin=field_tp), extras) + with fn_gen.if_(f'({v_next} := {v}.get({field_name}, MISSING)) is not MISSING'): + fn_gen.add_line(f'result[{field_name}] = {string}') + fn_gen.add_line('return result') @classmethod - @setup_recursive_safe_function_for_generic(None, prefix='dump') + @setup_recursive_safe_function_for_generic(prefix='dump', per_class_cache=True) def dump_from_union(cls, tp: TypeInfo, extras: Extras): fn_gen = extras['fn_gen'] config = extras['config'] actual_cls = extras['cls'] + _locals = extras['locals'] - tag_key = config.tag_key or TAG - auto_assign_tags = config.auto_assign_tags - - v = tp.v_for_def() - + args = tp.args field_i = tp.field_i i = tp.i - fields = f'fields_{field_i}' + v = tp.v_for_def() + + tag_key = config.tag_key or TAG + auto_assign_tags = config.auto_assign_tags + leaf_handling_as_subclass = config.v1_leaf_handling == 'issubclass' - args = tp.args in_optional = NoneType in args - _locals = extras['locals'] - _locals[fields] = args + _locals['fields'] = args _locals['tag_key'] = tag_key - type_checks = [] - try_parse_at_end = [] + leaf_types = [] + try_parse_lines = [] dataclass_and_line = [] has_dataclass = False @@ -390,14 +360,18 @@ def dump_from_union(cls, tp: TypeInfo, extras: Extras): tp_new = TypeInfo(possible_tp, field_i=field_i, i=i) tp_new.in_optional = in_optional - if possible_tp is NoneType: - with fn_gen.if_(f'{v} is None'): - fn_gen.add_line('return None') - continue + if _type_returns_value_unchanged( + possible_tp, leaf_handling_as_subclass): + leaf_types.append(possible_tp) - if is_dataclass(possible_tp): - has_dataclass = True + # if num_leaf_types_no_bytes > 0: + # fn_gen.add_line(f'return {v}') + + elif is_dataclass(possible_tp): # we see a dataclass in `Union` declaration + has_dataclass = True + string = cls.dump_dispatcher_for_annotation(tp_new, extras) + meta = get_meta(possible_tp) cls_name = possible_tp.__name__ @@ -412,83 +386,46 @@ def dump_from_union(cls, tp: TypeInfo, extras: Extras): meta.tag = cls_name if tag: - string = cls.dump_dispatcher_for_annotation(tp_new, extras) dataclass_and_line.append( (possible_tp, cls_name, tag, f'result = {string}; result[tag_key] = {tag!r}; return result')) - continue + else: + dataclass_and_line.append( + (possible_tp, cls_name, tag, + f'return {string}')) - # elif not config.v1_unsafe_parse_dataclass_in_union: - # e = ValueError('Cannot parse dataclass types in a Union without ' - # 'one of the following `Meta` settings:\n\n' - # ' * `auto_assign_tags = True`\n' - # f' - Set on class `{extras["cls_name"]}`.\n\n' - # f' * `tag = "{cls_name}"`\n' - # f' - Set on class `{possible_tp.__qualname__}`.\n\n' - # ' * `v1_unsafe_parse_dataclass_in_union = True`\n' - # f' - Set on class `{extras["cls_name"]}`\n\n' - # 'For more information, refer to:\n' - # ' https://dcw.ritviknag.com/en/latest/common_use_cases/dataclasses_in_union_types.html') - # raise e from None - - string = cls.dump_dispatcher_for_annotation(tp_new, extras) - - try_parse_lines = [ - 'try:', - f' return {string}', - 'except Exception:', - ' pass', - ] - - # TODO disable for dataclasses - - if (possible_tp in SIMPLE_TYPES - or is_subclass_safe( - get_origin_v2(possible_tp), SIMPLE_TYPES)): - - tn = tp_new.type_name(extras) - type_checks.extend([ - f'if tp is {tn}:', - f' return {v}' - ]) - list_to_add = try_parse_at_end else: - list_to_add = type_checks + try_parse_lines.append( + cls.dump_dispatcher_for_annotation(tp_new, extras)) - list_to_add.extend(try_parse_lines) + fn_gen.add_line(f't = {v}.__class__') - fn_gen.add_line(f'tp = type({v})') + if leaf_types: + # a good heuristic: use tuples for smaller unions, else frozenset + container = tuple if len(leaf_types) <= 6 else frozenset + _locals['leaf_types'] = container(leaf_types) + leaf_type_names = ', '.join(getattr(t, '__name__', None) or str(t) + for t in leaf_types) + with fn_gen.if_('t in leaf_types', comment=f'{{{leaf_type_names}}}'): + fn_gen.add_line(f'return {v}') if has_dataclass: - var_to_dataclass = {} for field_i, (dataclass, name, tag, line) in enumerate(dataclass_and_line, start=1): - cls_name = f'C{field_i}' - var_to_dataclass[cls_name] = dataclass - with fn_gen.if_(f'tp is {cls_name}', comment=f'{name} -> {tag!r}'): + cls_name = TypeInfo(dataclass).type_name(extras) + with fn_gen.if_(f't is {cls_name}', comment=f'{tag!r}' if tag else ''): fn_gen.add_line(line) - tp.ensure_in_locals(extras, **var_to_dataclass) - - # fn_gen.add_line( - # "raise ParseError(" - # "TypeError('Object with tag was not in any of Union types')," - # f"v1,{fields}," - # "input_tag=tag," - # "tag_key=tag_key," - # f"valid_tags={list(dataclass_tag_to_lines)})" - # ) - - if type_checks: - fn_gen.add_lines(*type_checks) - - if try_parse_at_end: - fn_gen.add_lines(*try_parse_at_end) + for string in try_parse_lines: + with fn_gen.try_(): + fn_gen.add_line(f'return {string}') + with fn_gen.except_(Exception): + fn_gen.add_line('pass') # Invalid type for Union fn_gen.add_line("raise ParseError(" "TypeError('Object was not in any of Union types')," - f"{v},{fields},'dump'," + f"{v},fields,'dump'," "tag_key=tag_key" ")") @@ -550,6 +487,7 @@ def dump_from_timedelta(tp: TypeInfo, extras: Extras): @staticmethod @setup_recursive_safe_function( + prefix='dump', fn_name=f'__{PACKAGE_NAME}_to_dict_{{cls_name}}__') def dump_from_dataclass(tp: TypeInfo, extras: Extras): dump_func_for_dataclass(tp.origin, extras) @@ -560,7 +498,9 @@ def dump_dispatcher_for_annotation(cls, extras): hooks = cls.__DUMP_HOOKS__ - type_hooks = extras['config'].v1_type_to_dump_hook + config = extras['config'] + type_hooks = config.v1_type_to_dump_hook + leaf_handling_as_subclass = config.v1_leaf_handling == 'issubclass' # type_ann = tp.origin type_ann = eval_forward_ref_if_needed(tp.origin, extras['cls']) @@ -603,7 +543,9 @@ def dump_dispatcher_for_annotation(cls, # -> Atomic, immutable types which don't require # any iterative / recursive handling. - elif origin in SIMPLE_TYPES or is_subclass_safe(origin, SIMPLE_TYPES): + elif origin in LEAF_TYPES or ( + leaf_handling_as_subclass + and is_subclass_safe(origin, LEAF_TYPES)): dump_hook = hooks.get(origin) elif (type_hooks is not None @@ -627,30 +569,31 @@ def dump_dispatcher_for_annotation(cls, # -> Union[x] elif is_union(origin): - dump_hook = cls.dump_from_union args = get_args(type_ann) + # all args in `Union[...]` are simple types + if _all_return_value_unchanged(args, leaf_handling_as_subclass): + return tp.v() + + dump_hook = cls.dump_from_union + # Special case for Optional[x], which is actually Union[x, None] if len(args) == 2 and NoneType in args: origin = args[0] - # optional simple type: (str, int, float, bool, Literal, Any) - is_simple_type = origin in SCALAR_TYPES or origin is Any or origin is Literal - - val_name = None - o = tp.v() - if is_simple_type: - val_name = tp.val_name - o = tp.v() - elif tp.val_name: + if tp.val_name: val_name = 'v0' - o = f'(v0 := {tp.v()})' + o = f'({val_name} := {tp.v()})' + else: + val_name = None + o = tp.v() + new_tp = tp.replace(origin=origin, args=None, name=None, val_name=val_name) new_tp.in_optional = True string = cls.dump_dispatcher_for_annotation(new_tp, extras) - return string if is_simple_type else f'None if {o} is None else {string}' + return f'None if {o} is None else {string}' # -> Literal[X, Y, ...] elif origin is Literal: @@ -728,7 +671,9 @@ def dump_dispatcher_for_annotation(cls, if dump_hook is None: # TODO END for t in hooks: - if issubclass(origin, (t,)): + if (not leaf_handling_as_subclass) and (t in LEAF_TYPES): + continue + if issubclass(origin, t): dump_hook = hooks[t] break @@ -856,6 +801,7 @@ def dump_func_for_dataclass( 'fields': cls_fields, } + # noinspection PyTypeChecker extras: Extras = { 'config': config, 'cls': cls, @@ -925,7 +871,6 @@ def dump_func_for_dataclass( skip_defaults = True if meta.skip_defaults else False skip_if = True if field_to_skip_if or skip_if_condition else False - skip_any = True if skip_if or skip_defaults else False # Fix for using `auto_assign_tags` and `raise_on_unknown_json_key` together # See https://github.com/rnag/dataclass-wizard/issues/137 @@ -1274,6 +1219,13 @@ def re_raise(e, cls, o, fields, field, value): tp = getattr(next((f for f in fields if f.name == field), None), 'type', Any) e = ParseError(e, value, tp, 'dump') + # If field name is missing or not known, make a "best effort" + # to resolve it. + if field == '' and cls and fields: + if len((names := [f.name for f in fields + if getattr(o, f.name, MISSING) == e.obj])) == 1: + field = e.field_name = names[0] + # We run into a parsing error while dumping the field value; # Add additional info on the Exception object before re-raising it. # diff --git a/dataclass_wizard/v1/loaders.py b/dataclass_wizard/v1/loaders.py index d032562..6092897 100644 --- a/dataclass_wizard/v1/loaders.py +++ b/dataclass_wizard/v1/loaders.py @@ -17,7 +17,7 @@ setup_recursive_safe_function, setup_recursive_safe_function_for_generic) from .enums import KeyAction, KeyCase -from .models import Extras, PatternBase, TypeInfo, SIMPLE_TYPES, UTC +from .models import Extras, PatternBase, TypeInfo, LEAF_TYPES, UTC from .type_conv import ( as_datetime_v1, as_date_v1, as_int_v1, as_time_v1, as_timedelta, TRUTHY_VALUES, @@ -59,29 +59,6 @@ is_union) -def _classify_hook(fn): - if not callable(fn): - raise TypeError('hook must be callable') - - code = getattr(fn, '__code__', None) or fn.__init__.__code__ - - # Disallow *args / **kwargs (they hide mistakes) - if code.co_flags & 0x04 or code.co_flags & 0x08: - raise TypeError('hook must not use *args or **kwargs') - - argc = code.co_argcount - - if argc == 1: - return 'runtime' # fn(value) - elif argc == 2: - return 'v1_codegen' # fn(TypeInfo, Extras) - else: - raise TypeError( - 'hook must accept either 1 argument (runtime) ' - 'or 2 arguments (TypeInfo, Extras)' - ) - - class LoadMixin(AbstractLoaderGenerator, BaseLoadHook): """ This Mixin class derives its name from the eponymous `json.loads` @@ -116,7 +93,8 @@ def load_to_str(cls, tp: TypeInfo, extras: Extras): tn = tp.type_name(extras) o = tp.v() - if tp.in_optional: # str(v) + # str(v) + if not extras['config'].v1_coerce_none_to_empty_str or tp.in_optional: return f'{tn}({o})' # '' if v is None else str(v) @@ -143,11 +121,13 @@ def load_to_int(tp: TypeInfo, extras: Extras): o = tp.v() tp.ensure_in_locals(extras, as_int=as_int_v1) - return (f"{o} if (tp := {o}.__class__) is {tn} " - f"else {tn}(" - f"f if '.' in {o} and (f := float({o})).is_integer() else {o}" - ") if tp is str " - f"else as_int({o},tp,{tn})") + return ( + f'{o} ' + f'if (t := {o}.__class__) is {tn} ' + f"else {tn}(f if '.' in {o} and (f := float({o})).is_integer() else {o}) " + 'if t is str ' + f'else as_int({o}, t, {tn})' + ) # TODO when `in_union`, we already know `o.__class__` # is not `tn`, and we already have a variable `tp`. @@ -170,14 +150,14 @@ def load_to_bool(tp: TypeInfo, extras: Extras): def load_to_bytes(tp: TypeInfo, extras: Extras): tp.ensure_in_locals(extras, b64decode) o = tp.v() - return (f'{o} if (tp := {o}.__class__) is bytes ' - f'else bytes({o}) if tp is bytearray ' + return (f'{o} if (t := {o}.__class__) is bytes ' + f'else bytes({o}) if t is bytearray ' f'else b64decode({o})') @classmethod def load_to_bytearray(cls, tp: TypeInfo, extras: Extras): # micro-optimization: avoid copying when already a bytearray - # return f'{o} if (tp := {o}.__class__) is bytearray else bytearray({o} if tp is bytes else b64decode({o}))' + # return f'{o} if (t := {o}.__class__) is bytearray else bytearray({o} if t is bytes else b64decode({o}))' as_bytes = cls.load_to_bytes(tp, extras) return tp.wrap_builtin(bytearray, as_bytes, extras) @@ -272,68 +252,132 @@ def load_to_tuple(cls, tp: TypeInfo, extras: Extras): return tp.wrap(result, extras, force=force_wrap) @classmethod - @setup_recursive_safe_function def load_to_named_tuple(cls, tp: TypeInfo, extras: Extras): - fn_gen = extras['fn_gen'] nt_tp = cast(NamedTuple, tp.origin) + if nt_tp._field_defaults: # has optionals + return cls._load_to_named_tuple_fn(tp, extras) + + fields_in_order = nt_tp._fields # field names in order + ann = nt_tp.__annotations__ + + if extras['config'].v1_namedtuple_as_dict: + values_in_order = tuple( + str( + cls.load_dispatcher_for_annotation( + tp.replace(origin=ann.get(name, Any), index=repr(name)), + extras, + ) + ) + for name in fields_in_order + ) + else: + values_in_order = tuple( + str( + cls.load_dispatcher_for_annotation( + tp.replace(origin=ann.get(name, Any), index=i), + extras, + ) + ) + for i, name in enumerate(fields_in_order) + ) + params = ', '.join(values_in_order) + return tp.wrap(params, extras) + + @classmethod + @setup_recursive_safe_function(per_class_cache=True) + def _load_to_named_tuple_fn(cls, tp: TypeInfo, extras: Extras): + fn_gen = extras['fn_gen'] _locals = extras['locals'] - _locals['cls'] = nt_tp - _locals['msg'] = "`dict` input is not supported for NamedTuple, use a dataclass instead." - - req_field_to_assign = {} - field_assigns = [] - # noinspection PyProtectedMember - optional_fields = set(nt_tp._field_defaults) - has_optionals = True if optional_fields else False - only_optionals = has_optionals and len(optional_fields) == len(nt_tp.__annotations__) - num_fields = 0 + + nt_tp = _locals['cls'] = cast(NamedTuple, tp.origin) + fields_in_order = nt_tp._fields # field names in order + field_to_default = nt_tp._field_defaults + ann = nt_tp.__annotations__ + + req_field_to_value = {} + opt_field_to_value = {} + + all_optionals = len(field_to_default) == len(fields_in_order) v = tp.v_for_def() - for field, field_tp in nt_tp.__annotations__.items(): - string = cls.load_dispatcher_for_annotation( - tp.replace(origin=field_tp, index=num_fields, val_name=None), extras) + if extras['config'].v1_namedtuple_as_dict: + i_next = tp.i + 1 + v_next = f'{tp.prefix}{i_next}' + + for name in fields_in_order: + field_tp = ann.get(name, Any) + if name in field_to_default: + _locals[f'_dflt_{name}'] = field_to_default[name] + new_tp = tp.replace(origin=field_tp, i=i_next, + index=None, val_name=None) + value = cls.load_dispatcher_for_annotation(new_tp, extras) + opt_field_to_value[name] = value + else: + new_tp = tp.replace(origin=field_tp, index=repr(name)) + value = cls.load_dispatcher_for_annotation(new_tp, extras) + req_field_to_value[f'__{name}'] = value + + req_args = ', '.join(req_field_to_value) + opt_args = ', '.join(f'__{f}' for f in opt_field_to_value) - if has_optionals and field in optional_fields: - field_assigns.append(string) + if all_optionals: # NamedTuple has no required fields + ret_value_with_input = f'return cls({opt_args})' else: - req_field_to_assign[f'__{field}'] = string + ret_value_with_input = f'return cls({req_args}, {opt_args})' + for name, value in req_field_to_value.items(): + fn_gen.add_line(f'{name} = {value}') + + # it's guaranteed the NamedTuple has at least one default field + with fn_gen.if_(f'not {v}'): + fn_gen.add_line('return cls()') + + for name, value in opt_field_to_value.items(): + with fn_gen.if_(f'({v_next} := {v}.get({name!r}, MISSING)) is MISSING'): + fn_gen.add_line(f'__{name} = _dflt_{name}') + with fn_gen.else_(): + fn_gen.add_line(f'__{name} = {value}') + + fn_gen.add_line(ret_value_with_input) + + else: # list mode + for i, name in enumerate(fields_in_order): + field_tp = ann.get(name, Any) + value = cls.load_dispatcher_for_annotation( + tp.replace(origin=field_tp, index=i), extras) + + if name in field_to_default: + opt_field_to_value[name] = value + else: + req_field_to_value[f'__{name}'] = value - num_fields += 1 + req_args = ', '.join(req_field_to_value) + opt_fields_start_i = len(req_field_to_value) - params = ', '.join(req_field_to_assign) + if all_optionals: # NamedTuple has no required fields + len_condition = 'n' + ret_value_with_input = f'return cls(*args)' + else: + len_condition = f'n > {opt_fields_start_i}' + ret_value_with_input = f'return cls({req_args}, *args)' - with fn_gen.try_(): + for name, value in req_field_to_value.items(): + fn_gen.add_line(f'{name} = {value}') - for field, string in req_field_to_assign.items(): - fn_gen.add_line(f'{field} = {string}') + # it's guaranteed the NamedTuple has at least one default field + fn_gen.add_line(f'n = len({v})') - if has_optionals: - opt_start = len(req_field_to_assign) - fn_gen.add_line(f'L = len({v}); has_opt = L > {opt_start}') - with fn_gen.if_(f'has_opt'): - fn_gen.add_line(f'fields = [{field_assigns.pop(0)}]') - for i, string in enumerate(field_assigns, start=opt_start + 1): - fn_gen.add_line(f'if L > {i}: fields.append({string})') + with fn_gen.if_(len_condition): + opt_values = list(opt_field_to_value.values()) + fn_gen.add_line(f'args = [{opt_values.pop(0)}]') - if only_optionals: - fn_gen.add_line(f'return cls(*fields)') - else: - fn_gen.add_line(f'return cls({params}, *fields)') + for i, value in enumerate(opt_values, start=opt_fields_start_i + 1): + with fn_gen.if_(f'n > {i}'): + fn_gen.add_line(f'args.append({value})') - fn_gen.add_line(f'return cls({params})') + fn_gen.add_line(ret_value_with_input) - with fn_gen.except_(Exception, 'e'): - with fn_gen.if_('(e_cls := e.__class__) is IndexError'): - # raise `MissingFields`, as required NamedTuple fields - # are not present in the input object `o`. - fn_gen.add_line(f'raise_missing_fields(locals(), {v}, cls, None)') - with fn_gen.if_(f'e_cls is KeyError and type({v}) is dict'): - # Input object is a `dict` - # TODO should we support dict for namedtuple? - fn_gen.add_line('raise TypeError(msg) from None') - # re-raise - fn_gen.add_line('raise e from None') + fn_gen.add_line(f'return cls({req_args})') @classmethod def load_to_named_tuple_untyped(cls, tp: TypeInfo, extras: Extras): @@ -342,9 +386,21 @@ def load_to_named_tuple_untyped(cls, tp: TypeInfo, extras: Extras): # Assuming `Point` is a `namedtuple`, this performs # the equivalent logic as: # Point(**x) if isinstance(x, dict) else Point(*x) + # + # star, dbl_star = tp.multi_wrap(extras, 'nt_', f'*{v}', f'**{v}') + v = tp.v() - star, dbl_star = tp.multi_wrap(extras, 'nt_', f'*{v}', f'**{v}') - return f'{dbl_star} if isinstance({v}, dict) else {star}' + + if extras['config'].v1_namedtuple_as_dict: + return tp.wrap(f'**{v}', extras, prefix='nt_') + + def raise_(): + raise TypeError('Expected list/tuple for NamedTuple field') from None + + tp.ensure_in_locals(extras, raise_=raise_) + + star = tp.wrap(f'*{v}', extras, prefix='nt_') + return f'{star} if (t := type({v})) is list or t is tuple else raise_()' @classmethod def _build_dict_comp(cls, tp, v, i_next, k_next, v_next, kt, vt, extras): @@ -392,8 +448,28 @@ def load_to_defaultdict(cls, tp: TypeInfo, extras: Extras): return tp.wrap_dd(default_factory, result, extras) @classmethod - @setup_recursive_safe_function def load_to_typed_dict(cls, tp: TypeInfo, extras: Extras): + req_keys, opt_keys = get_keys_for_typed_dict(tp.origin) + if opt_keys: # has optionals + return cls._load_to_typed_dict_fn(tp, extras) + + ann = tp.origin.__annotations__ + + dict_body = ', '.join( + f"""{name!r}: { + cls.load_dispatcher_for_annotation( + tp.replace(origin=ann.get(name, Any), index=repr(name)), + extras, + ) + }""" + for name in req_keys + ) + + return f'{{{dict_body}}}' + + @classmethod + @setup_recursive_safe_function + def _load_to_typed_dict_fn(cls, tp: TypeInfo, extras: Extras): fn_gen = extras['fn_gen'] req_keys, opt_keys = get_keys_for_typed_dict(tp.origin) @@ -442,7 +518,7 @@ def load_to_typed_dict(cls, tp: TypeInfo, extras: Extras): fn_gen.add_line(f"raise ParseError(e, {v}, {{}}, 'load') from None") @classmethod - @setup_recursive_safe_function_for_generic(per_field_cache=True) + @setup_recursive_safe_function_for_generic(per_class_cache=True) def load_to_union(cls, tp: TypeInfo, extras: Extras): fn_gen = extras['fn_gen'] config = extras['config'] @@ -450,15 +526,13 @@ def load_to_union(cls, tp: TypeInfo, extras: Extras): tag_key = config.tag_key or TAG auto_assign_tags = config.auto_assign_tags - - field_i = tp.field_i - fields = f'fields_{field_i}' + leaf_handling_as_subclass = config.v1_leaf_handling == 'issubclass' args = tp.args in_optional = NoneType in args _locals = extras['locals'] - _locals[fields] = args + _locals['fields'] = args _locals['tag_key'] = tag_key dataclass_tag_to_lines: dict[str, list] = {} @@ -467,6 +541,11 @@ def load_to_union(cls, tp: TypeInfo, extras: Extras): i = tp.i v = tp.v_for_def() + # TODO: + # Union handling here assumes `i == 1` (EnvWizard). If + # reused for multiple Union fields, cache/function-name + # collisions are possible. + # noinspection PyUnboundLocalVariable if (has_dataclass and (pre_decoder := config.v1_pre_decoder) is not None and (new_v := pre_decoder(cls, dict, tp, extras).v()) != v): @@ -488,7 +567,7 @@ def load_to_union(cls, tp: TypeInfo, extras: Extras): possible_tp = eval_forward_ref_if_needed(possible_tp, actual_cls) - tp_new = TypeInfo(possible_tp, field_i=field_i, i=i) + tp_new = TypeInfo(possible_tp, i=i) tp_new.in_optional = in_optional if possible_tp is NoneType: @@ -542,11 +621,13 @@ def load_to_union(cls, tp: TypeInfo, extras: Extras): ' pass', ] - # TODO disable for dataclasses + if (possible_tp in LEAF_TYPES or ( + leaf_handling_as_subclass + and is_subclass_safe( + get_origin_v2(possible_tp), LEAF_TYPES) + )): - if (possible_tp in SIMPLE_TYPES - or is_subclass_safe( - get_origin_v2(possible_tp), SIMPLE_TYPES)): + # TODO disable for dataclasses tn = tp_new.type_name(extras) type_checks.extend([ @@ -574,7 +655,7 @@ def load_to_union(cls, tp: TypeInfo, extras: Extras): fn_gen.add_line( "raise ParseError(" "TypeError('Object with tag was not in any of Union types')," - f"{v},{fields},'load'," + f"{v},fields,'load'," "input_tag=tag," "tag_key=tag_key," f"valid_tags={list(dataclass_tag_to_lines)})" @@ -591,7 +672,7 @@ def load_to_union(cls, tp: TypeInfo, extras: Extras): # Invalid type for Union fn_gen.add_line("raise ParseError(" "TypeError('Object was not in any of Union types')," - f"{v},{fields},'load'," + f"{v},fields,'load'," "tag_key=tag_key" ")") @@ -600,19 +681,18 @@ def load_to_union(cls, tp: TypeInfo, extras: Extras): def load_to_literal(tp: TypeInfo, extras: Extras): fn_gen = extras['fn_gen'] - fields = f'fields_{tp.field_i}' v = tp.v_for_def() _locals = extras['locals'] - _locals[fields] = frozenset(tp.args) + _locals['fields'] = frozenset(tp.args) - with fn_gen.if_(f'{v} in {fields}', comment=repr(tp.args)): + with fn_gen.if_(f'{v} in fields', comment=repr(tp.args)): fn_gen.add_line(f'return {v}') # No such Literal with the value of `o` fn_gen.add_line("e = ValueError('Value not in expected Literal values')") - fn_gen.add_line(f"raise ParseError(e, {v}, {fields}, 'load', " - f'allowed_values=list({fields}))') + fn_gen.add_line(f"raise ParseError(e, {v}, fields, 'load', " + f'allowed_values=list(fields))') # TODO Checks for Literal equivalence, as mentioned here: # https://www.python.org/dev/peps/pep-0586/#equivalence-of-two-literals @@ -753,6 +833,7 @@ def load_dispatcher_for_annotation(cls, config = extras['config'] pre_decoder = config.v1_pre_decoder type_hooks = config.v1_type_to_load_hook + leaf_handling_as_subclass = config.v1_leaf_handling == 'issubclass' # type_ann = tp.origin type_ann = eval_forward_ref_if_needed(tp.origin, extras['cls']) @@ -798,7 +879,9 @@ def load_dispatcher_for_annotation(cls, # -> Atomic, immutable types which don't require # any iterative / recursive handling. - elif origin in SIMPLE_TYPES or is_subclass_safe(origin, SIMPLE_TYPES): + elif origin in LEAF_TYPES or ( + leaf_handling_as_subclass + and is_subclass_safe(origin, LEAF_TYPES)): load_hook = hooks.get(origin) elif (type_hooks is not None @@ -844,7 +927,7 @@ def load_dispatcher_for_annotation(cls, load_hook = cls.default_load_to elif is_subclass_safe(origin, tuple) and hasattr(origin, '_fields'): - container_tp = tuple + container_tp = dict if config.v1_namedtuple_as_dict else tuple if getattr(origin, '__annotations__', None): # Annotated as a `typing.NamedTuple` subtype load_hook = cls.load_to_named_tuple @@ -902,7 +985,9 @@ def load_dispatcher_for_annotation(cls, if load_hook is None: # TODO END for t in hooks: - if issubclass(origin, (t,)): + if (not leaf_handling_as_subclass) and (t in LEAF_TYPES): + continue + if issubclass(origin, t): container_tp = t load_hook = hooks[t] break @@ -971,23 +1056,25 @@ def setup_default_loader(cls=LoadMixin): def check_and_raise_missing_fields( _locals, o, cls, - fields: tuple[Field, ...] | None): + fields: tuple[Field, ...] | None, + **kwargs, +): - if fields is None: # named tuple + if fields is None: # `typing.NamedTuple` or `collections.namedtuple` nt_tp = cast(NamedTuple, cls) - # noinspection PyProtectedMember field_to_default = nt_tp._field_defaults + field_names = nt_tp._fields fields = tuple([ dataclasses.field( default=field_to_default.get(field, MISSING), ) - for field in cls.__annotations__]) + for field in field_names]) - for field, name in zip(fields, cls.__annotations__): + for field, name in zip(fields, field_names): field.name = name - missing_fields = [f for f in cls.__annotations__ + missing_fields = [f for f in field_names if f'__{f}' not in _locals and f not in field_to_default] @@ -1005,7 +1092,7 @@ def check_and_raise_missing_fields( raise MissingFields( None, o, cls, fields, None, missing_fields, - missing_keys + missing_keys, **kwargs, ) from None @@ -1172,8 +1259,8 @@ def load_func_for_dataclass( if pre_assign: fn_gen.add_line('i = 0') - vars_for_fields = [] - kwargs_for_fields = [] + args = [] + kwargs = [] if cls_init_fields: @@ -1295,9 +1382,9 @@ def load_func_for_dataclass( else: if name in cls_init_kw_only_field_names: - kwargs_for_fields.append(f'{name}={var}') + kwargs.append(f'{name}={var}') else: - vars_for_fields.append(var) + args.append(var) with fn_gen.if_(val_is_found): fn_gen.add_line(f'{pre_assign}{var} = {string}') @@ -1318,9 +1405,9 @@ def load_func_for_dataclass( fn_gen.add_line(f'{var} = {{}} if len(o) == i else {catch_all_def}') if catch_all_field_stripped in cls_init_kw_only_field_names: - kwargs_for_fields.append(f'{catch_all_field_stripped}={var}') + kwargs.append(f'{catch_all_field_stripped}={var}') else: - vars_for_fields.insert(catch_all_idx, var) + args.insert(catch_all_idx, var) elif set_aliases: # warn / raise on unknown key line = 'extra_keys = set(o) - aliases' @@ -1344,12 +1431,11 @@ def load_func_for_dataclass( # we raise them here. if has_defaults: - vars_for_fields.append('**init_kwargs') - if kwargs_for_fields: - vars_for_fields.extend(kwargs_for_fields) - init_parts = ', '.join(vars_for_fields) + args.append('**init_kwargs') + if kwargs: + args.extend(kwargs) with fn_gen.try_(): - fn_gen.add_line(f"return cls({init_parts})") + fn_gen.add_line(f'return cls({", ".join(args)})') with fn_gen.except_(UnboundLocalError): # raise `MissingFields`, as required dataclass fields # are not present in the input object `o`. @@ -1440,4 +1526,59 @@ def re_raise(e, cls, o, fields, field, value): else: e.class_name, e.field_name, e.json_object = cls, field, o + # noinspection PyUnboundLocalVariable + if (isinstance(e, ParseError) + # `typing.NamedTuple` or `collections.namedtuple` + and (origin := e.ann_type) is not None + and is_subclass_safe(origin, tuple) + and (_fields := getattr(origin, '_fields', None))): + + meta = get_meta(cls) + nt_tp = cast(NamedTuple, origin) + field_to_default = nt_tp._field_defaults + num_req_fields = len(_fields) - len(field_to_default) + + e_cls = getattr(e.base_error, '__class__', None) + + if e_cls in (IndexError, KeyError, TypeError): + # raise `MissingFields`, as required NamedTuple fields + # are not present in the input object `o`. + if isinstance(value, (list, tuple)): + # noinspection PyUnboundLocalVariable + _locals = {f'__{f}' for f in _fields[:len(value)]} + num_req_fields_provided = min(len(value), num_req_fields) + elif isinstance(value, dict): + _locals = {f'__{f}' for f in _fields if f in value} + num_req_fields_provided = len( + [f for f in _fields + if f in value and f not in field_to_default] + ) + else: + _locals = _fields + num_req_fields_provided = num_req_fields + + if num_req_fields_provided < num_req_fields: + check_and_raise_missing_fields( + _locals, value, origin, None, + **( + {'field': f'{ParseError.name(cls)}.{field}'} + if cls and field + else {} + )) + + if meta.v1_namedtuple_as_dict: + if e_cls is TypeError and type(value) is not dict: + e.kwargs['resolution'] = ( + 'List/tuple input is not supported for NamedTuple fields in dict mode. ' + 'Pass a dict, or set Meta.v1_namedtuple_as_dict = False.' + ) + e.kwargs['unsupported_type'] = type(value) + else: + if e_cls is KeyError and type(value) is dict: + e.kwargs['resolution'] = ( + 'Dict input is not supported for NamedTuple fields in list mode. ' + 'Pass a list/tuple, or set Meta.v1_namedtuple_as_dict = True.' + ) + e.kwargs['unsupported_type'] = dict + raise e from None diff --git a/dataclass_wizard/v1/models.py b/dataclass_wizard/v1/models.py index 2bf45be..bbec68b 100644 --- a/dataclass_wizard/v1/models.py +++ b/dataclass_wizard/v1/models.py @@ -1,5 +1,6 @@ import hashlib import sys +import types from collections import defaultdict, deque from dataclasses import MISSING, Field as _Field from datetime import datetime, date, time, tzinfo, timezone, timedelta @@ -37,9 +38,11 @@ frozenset, }) -# Atomic immutable types which don't require any recursive handling and for which deepcopy -# returns the same object. We can provide a fast-path for these types in asdict and astuple. -SIMPLE_TYPES = ( +# FIXME: Python 3.9 doesn't have `types.EllipsisType` or `types.NotImplementedType` +EllipsisType = getattr(types, 'EllipsisType', type(Ellipsis)) +NotImplementedType = getattr(types, 'NotImplementedType', type(NotImplemented)) + +LEAF_TYPES_NO_BYTES = frozenset({ # Common JSON Serializable types NoneType, bool, @@ -48,25 +51,23 @@ str, # Other common types complex, - bytes, - # TODO support + # exclude bytes, since the serialization process is slightly different # Other types that are also unaffected by deepcopy - # types.EllipsisType, - # types.NotImplementedType, - # types.CodeType, - # types.BuiltinFunctionType, - # types.FunctionType, - # type, - # range, - # property, -) - -SCALAR_TYPES = ( - str, - int, - float, - bool, -) + EllipsisType, + NotImplementedType, + types.CodeType, + types.BuiltinFunctionType, + types.FunctionType, + type, + range, + property, +}) + +# Atomic immutable types which don't require any recursive handling and for which deepcopy +# returns the same object. We can provide a fast-path for these types in asdict and astuple. +# +# Credits: `_ATOMIC_TYPES` from `dataclasses.py` +LEAF_TYPES = LEAF_TYPES_NO_BYTES | {bytes} SEQUENCE_ORIGINS = frozenset({ list, @@ -85,7 +86,7 @@ def get_zoneinfo(key: str) -> ZoneInfo: try: return ZoneInfo(key) - except ZoneInfoNotFoundError as e: + except ZoneInfoNotFoundError: if sys.platform.startswith('win'): try: import tzdata # noqa: F401 @@ -100,7 +101,7 @@ def get_zoneinfo(key: str) -> ZoneInfo: raise -def ensure_type_ref(extras, tp, *, name=None, prefix='', is_builtin=False, field_i=0) -> str: +def ensure_type_ref(extras, tp, *, name=None, prefix='', is_builtin=False) -> str: """ Return a safe symbol name for `tp` to use in generated code. @@ -139,7 +140,7 @@ def ensure_type_ref(extras, tp, *, name=None, prefix='', is_builtin=False, field # Collision: create a unique alias. # TODO might need to handle `var_name` - alias = f'{prefix}{name}_{field_i}' + alias = f'{prefix}{name}' LOG.debug('Adding %s=%s', alias, name) _locals.setdefault(alias, tp) @@ -202,6 +203,14 @@ def replace(self, **changes): for slot in TypeInfo.__slots__ if not slot.startswith('_')} + + if ((new_idx := changes.get('index')) is not None + and (curr_idx := current_values['index']) is not None): + if isinstance(curr_idx, (int, str)): + changes['index'] = (curr_idx, new_idx) + else: + changes['index'] = curr_idx + (new_idx, ) + # Apply the changes current_values.update(changes) @@ -240,8 +249,13 @@ def v(self): val_name = self.val_name if val_name is None: val_name = f'{self.prefix}{self.i}' - return (val_name if (idx := self.index) is None - else f'{val_name}[{idx}]') + idx = self.index + if idx is None: + return val_name + else: + if isinstance(idx, (int, str)): + return f'{val_name}[{idx}]' + return f"{val_name}{''.join(f'[{i}]' for i in idx)}" def v_for_def(self): """ @@ -252,7 +266,7 @@ def v_for_def(self): def v_and_next(self): next_i = self.i + 1 - return self.v(), f'v{next_i}', next_i + return self.v(), f'{self.prefix}{next_i}', next_i def v_and_next_k_v(self): next_i = self.i + 1 @@ -318,7 +332,6 @@ def _wrap_inner(self, extras, name=name, prefix=prefix, is_builtin=is_builtin, - field_i=self.field_i, ) return name if return_name else None diff --git a/dataclass_wizard/v1/models.pyi b/dataclass_wizard/v1/models.pyi index dad0400..4db83fb 100644 --- a/dataclass_wizard/v1/models.pyi +++ b/dataclass_wizard/v1/models.pyi @@ -15,9 +15,8 @@ from ..utils.object_path import PathType # Type for a string or a collection of strings. _STR_COLLECTION: TypeAlias = str | Collection[str] -SIMPLE_TYPES: tuple[type, ...] -SCALAR_TYPES: tuple[type, ...] - +LEAF_TYPES: frozenset[type] +LEAF_TYPES_NO_BYTES: frozenset[type] SEQUENCE_ORIGINS: frozenset[type] MAPPING_ORIGINS: frozenset[type] @@ -34,8 +33,7 @@ def get_zoneinfo(key: str) -> ZoneInfo: ... def ensure_type_ref(extras: 'Extras', tp: type, *, name: str | None = None, prefix: str = '', - is_builtin: bool = False, - field_i: int = 0) -> str: ... + is_builtin: bool = False) -> str: ... @dataclass(order=True) @@ -54,8 +52,8 @@ class TypeInfo: # prefix of value in assignment (prepended to `i`), # defaults to 'v' if not specified. prefix: str = 'v' - # index of assignment (ex. `2 -> v1[2]`, *or* a string `"key" -> v4["key"]`) - index: int | None = None + # index / indices of assignment (ex. `2, 0 -> v1[2][0]`, *or* a string `"key" -> v4["key"]`) + index: int | str | tuple[int | str, ...] | None = None # explicit value name (overrides prefix + index) val_name: str | None = None # indicates if we are currently in Optional, @@ -64,7 +62,7 @@ class TypeInfo: def replace(self, **changes) -> TypeInfo: ... @staticmethod - def ensure_in_locals(extras: Extras, *tps: Callable | type, **name_to_tp: Callable[..., Any] | object) -> None: ... + def ensure_in_locals(extras: Extras, *tps: Callable | type, **name_to_tp: Callable[..., Any] | object) -> list[str]: ... def type_name(self, extras: Extras, *, bound: type | None = None) -> str: ... def v(self) -> str: ... @@ -97,7 +95,7 @@ class Extras(TypedDict): fn_gen: FunctionBuilder locals: dict[str, Any] pattern: NotRequired[PatternBase] - recursion_guard: dict[type, str] + recursion_guard: dict[Any, str] class PatternBase: diff --git a/docs/overview.rst b/docs/overview.rst index e6e9369..a51778e 100644 --- a/docs/overview.rst +++ b/docs/overview.rst @@ -154,11 +154,14 @@ Special Cases However, here a few special cases that are worth going over. -* ``str`` - Effortlessly converts inputs to strings. If already a string, - it remains unchanged. Non-strings are converted to their string - representation, and ``None`` becomes an empty string. +* ``str`` - Converts inputs to strings. If already a string, it remains unchanged. + Non-strings are converted to their string representation. - *Examples*: ``123`` → ``'123'``, ``None`` → ``''`` + By default in **v1**, ``None`` is converted using ``str(None)`` (i.e. ``'None'``) + for ``str`` fields, and preserved as ``None`` for ``Optional[str]`` fields. + Set ``Meta.v1_coerce_none_to_empty_str = True`` to coerce ``None`` to ``''`` instead. + + *Examples*: ``123`` → ``'123'``, ``None`` → ``'None'`` (or ``''`` with opt-in) * ``bool`` - JSON values that appear as strings or integers will be de-serialized to a ``bool`` using a case-insensitive search that matches against the following diff --git a/pyproject.toml b/pyproject.toml index ba8a836..ea65d4e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -106,7 +106,7 @@ bench = [ "jsons==1.6.3", "dataclass-factory==2.16", # pyup: ignore "dacite==1.8.1", - "mashumaro==3.15", + "mashumaro==3.17", "pydantic==2.10.3; python_version<'3.14'", "attrs==24.3.0", ] @@ -156,7 +156,7 @@ all = [ "jsons==1.6.3", "dataclass-factory==2.16", "dacite==1.8.1", - "mashumaro==3.15", + "mashumaro==3.17", "pydantic==2.10.3; python_version<'3.14'", "attrs==24.3.0", ] diff --git a/tests/unit/v1/environ/test_e2e.py b/tests/unit/v1/environ/test_e2e.py index 2bd2217..e6b16dd 100644 --- a/tests/unit/v1/environ/test_e2e.py +++ b/tests/unit/v1/environ/test_e2e.py @@ -6,10 +6,11 @@ import pytest from dataclass_wizard import DataclassWizard, CatchAll -from dataclass_wizard.errors import ParseError, MissingVars +from dataclass_wizard.errors import ParseError, MissingVars, MissingFields from dataclass_wizard.v1 import Alias, EnvWizard, env_config, AliasPath +from ..models import TN, CN, EnvContTF, EnvContTT, EnvContAllReq, Sub2 -from ..utils_env import envsafe, from_env +from ..utils_env import envsafe, from_env, assert_unordered_equal from ...._typing import * @@ -90,7 +91,8 @@ class _(EnvWizard.Meta): with pytest.raises(ParseError) as e: _ = from_env(MyClass, {'nt_one_opt': [{}], 'nt_all_opts': {'k': [[]]}}) - assert '`dict` input is not supported for NamedTuple, use a dataclass instead' in str(e.value) + # TODO + # assert '`dict` input is not supported for NamedTuple, use a dataclass instead' in str(e.value) c = from_env(MyClass, _input) @@ -155,7 +157,7 @@ class MyClass(EnvWizard): assert c == MyClass(my_literal_dict={'test': (123, frozenset({'Aa', 'Bb'}))}) new_dict = c.to_dict() - assert new_dict == {'my_literal_dict': {'test': (123, {'Aa', 'Bb'})}} + assert_unordered_equal(new_dict, {'my_literal_dict': {'test': [123, ['Aa', 'Bb']]}}) def test_decode_date_and_datetime_from_numeric_and_string_timestamp_and_iso_format(): @@ -377,3 +379,110 @@ class E2(EnvWizard): with pytest.raises(ParseError) as e: _ = from_env(E2, {'a': []}) assert str(e.value.base_error) == 'Invalid path' + + +def test_namedtuple_dict_mode_roundtrip_and_defaults(): + class EnvContDict(EnvWizard): + class _(EnvWizard.Meta): + v1_namedtuple_as_dict = True + + tn: TN + cn: CN + + o = from_env(EnvContDict, {"tn": {"a": 1}, "cn": {"a": 3}}) + assert o.tn == TN(a=1, b=2) + assert o.cn == CN(a=3, b=2) + + d = o.to_dict() + assert d == {"tn": {"a": 1, "b": 2}, "cn": {"a": 3, "b": 2}} + + +# TODO +# def test_namedtuple_dict_mode_missing_required_raises(): +# with pytest.raises(MissingFields, match=r'`TN\.__init__\(\)` missing required fields') as e: +# from_env(EnvContDict, {"tn": {"b": 9}, "cn": {"a": 1}}) +# +# assert e.value.missing_fields == ['a'] + + +def test_namedtuple_list_mode_roundtrip_and_defaults(): + class EnvContList(EnvWizard): + class _(EnvWizard.Meta): + v1_namedtuple_as_dict = False + + tn: TN + cn: CN + + o = from_env(EnvContList, {"tn": [1], "cn": [3]}) + assert o.tn == TN(a=1, b=2) + assert o.cn == CN(a=3, b=2) + + d = o.to_dict() + assert d == {"tn": [1, 2], "cn": [3, 2]} + + +# def test_namedtuple_list_mode_rejects_dict_input_with_clear_error(): +# with pytest.raises(ParseError, match=r"Dict input is not supported for NamedTuple fields in list mode.*list.*Meta\.v1_namedtuple_as_dict = True"): +# from_env(EnvContList, {"tn": {"a": 1}, "cn": {"a": 3}}) + + +def test_typeddict_total_false_e2e_dict_roundtrip(): + o = from_env(EnvContTF, {"td": {"a": 1, "ro": 9}}) + assert o.td == {"a": 1, "ro": 9} + + d = o.to_dict() + assert d == {"td": {"a": 1, "ro": 9}} + + +def test_typeddict_total_false_missing_required_raises(): + with pytest.raises(Exception): # swap to MissingFields/TypeError etc + from_env(EnvContTF, {"td": {"b": 2}}) + + +def test_typeddict_total_true_e2e_optional_and_required_keys(): + o = from_env(EnvContTT, {"td": {"a": 1, "ro": 9}}) + assert o.td == {"a": 1, "ro": 9} + + d = o.to_dict() + assert d == {"td": {"a": 1, "ro": 9}} + + with pytest.raises(Exception): + from_env(EnvContTT, {"td": {"a": 1}}) # missing ro + + +def test_typeddict_all_required_e2e_inline_path(): + o = from_env(EnvContAllReq, {"td": {"x": 1, "y": "ok"}}) + assert o.td == {"x": 1, "y": "ok"} + + d = o.to_dict() + assert d == {"td": {"x": 1, "y": "ok"}} + + with pytest.raises(Exception): + from_env(EnvContAllReq, {"td": {"x": 1}}) # missing y + + +def test_v1_union_codegen_cache_nested_union_roundtrip_and_dump_error(): + class MyClass(EnvWizard): + class _(EnvWizard.Meta): + v1_unsafe_parse_dataclass_in_union = True + + complex_tp: 'list[int | Sub2] | list[int | str]' + + # First: pick the arm list[int|Sub2] + o1 = from_env(MyClass, {"complex_tp": [{"my_float": "123."}, 7]}) + assert o1 == MyClass(complex_tp=[Sub2(my_float=123.0), 7]) + + d1 = o1.to_dict() + assert d1 == {"complex_tp": [{"my_float": 123.0}, 7]} + + # Second: pick the other arm list[int|str] + # If inner-union codegen caching is wrong, this is where it tends to misbehave. + o2 = from_env(MyClass, {"complex_tp": ["hello", 9]}) + assert o2 == MyClass(complex_tp=["hello", 9]) + + d2 = o2.to_dict() + assert d2 == {"complex_tp": ["hello", 9]} + + o1.complex_tp = '123' + with pytest.raises(ParseError, match=r"Failed to dump field `complex_tp` in class `.*MyClass`"): + _ = o1.to_dict() diff --git a/tests/unit/v1/environ/test_loaders.py b/tests/unit/v1/environ/test_loaders.py index c315719..71760db 100644 --- a/tests/unit/v1/environ/test_loaders.py +++ b/tests/unit/v1/environ/test_loaders.py @@ -59,9 +59,9 @@ class MyClass(EnvWizard): 'my_untyped_nt': untyped_tup(a='hello', b='world', c='123'), } - assert c.to_dict() == {'my_nt': MyNT(my_float=1.23, my_str='string'), - 'my_tup': (1, 2, 3), - 'my_untyped_nt': untyped_tup(a='hello', b='world', c='123')} + assert c.to_dict() == {'my_nt': [1.23, 'string'], + 'my_tup': [1, 2, 3], + 'my_untyped_nt': ['hello', 'world', '123']} def test_load_to_dataclass(): diff --git a/tests/unit/v1/models.py b/tests/unit/v1/models.py new file mode 100644 index 0000000..317601c --- /dev/null +++ b/tests/unit/v1/models.py @@ -0,0 +1,70 @@ +from collections import namedtuple +from dataclasses import dataclass +from typing import NamedTuple + +from dataclass_wizard import DataclassWizard +from dataclass_wizard.v1 import EnvWizard + +from ..._typing import Required, NotRequired, ReadOnly, TypedDict + + +class TNReq(NamedTuple): + a: int + b: int + + +class TN(NamedTuple): + a: int + b: int = 2 + + +CN = namedtuple("CN", "a b", defaults=(2,)) + + +# 1) total=False + Required/NotRequired/ReadOnly, E2E +class TD_TF(TypedDict, total=False): + a: Required[int] # required even though total=False + b: NotRequired[int] # optional + ro: ReadOnly[int] # optional because total=False + + +class ContTF(DataclassWizard): + td: TD_TF + + +class EnvContTF(EnvWizard): + td: TD_TF + + +# 2) total=True + NotRequired + ReadOnly requiredness, E2E +class TD_TT(TypedDict): + a: int # required + b: NotRequired[int] # optional + ro: ReadOnly[int] # required (unless NotRequired wraps it) + + +class ContTT(DataclassWizard): + td: TD_TT + + +class EnvContTT(EnvWizard): + td: TD_TT + + +# 3) all-required TypedDict (no optional keys) => codegen inline path, E2E +class TD_AllReq(TypedDict): + x: int + y: str + + +class ContAllReq(DataclassWizard): + td: TD_AllReq + + +class EnvContAllReq(EnvWizard): + td: TD_AllReq + + +@dataclass +class Sub2: + my_float: float diff --git a/tests/unit/v1/test_e2e.py b/tests/unit/v1/test_e2e.py index a4c2cec..c84befb 100644 --- a/tests/unit/v1/test_e2e.py +++ b/tests/unit/v1/test_e2e.py @@ -8,8 +8,10 @@ import pytest from dataclass_wizard import asdict, fromdict, DataclassWizard, CatchAll -from dataclass_wizard.errors import ParseError +from dataclass_wizard.errors import ParseError, MissingFields from dataclass_wizard.v1 import Alias +from .models import TN, CN, ContTF, ContTT, ContAllReq, Sub2, TNReq +from .utils_env import assert_unordered_equal from ..._typing import * @@ -37,7 +39,7 @@ class _(DataclassWizard.Meta): new_dict = asdict(c) assert new_dict == { - 'Boolean-Dict': {'test': (None, True, None)}, + 'Boolean-Dict': {'test': [None, True, None]}, 'nestedUnionWithClass': ['123', {'test': 'value', '__tag__': 'Sub'}] } @@ -66,7 +68,8 @@ class _(DataclassWizard.Meta): with pytest.raises(ParseError) as e: _ = fromdict(MyClass, {'NtOneOpt': [{}], 'NtAllOpts': {'k': [[]]}}) - assert '`dict` input is not supported for NamedTuple, use a dataclass instead' in str(e.value) + # TODO + # assert '`dict` input is not supported for NamedTuple, use a dataclass instead' in str(e.value) c = fromdict(MyClass, d) assert c == MyClass(nt_all_opts={'k': {NTAllOptionals()}}, @@ -75,11 +78,11 @@ class _(DataclassWizard.Meta): new_dict = asdict(c) assert new_dict == { 'NtAllOpts': { - 'k': {NTAllOptionals()}, + 'k': [['test', 1, True]] }, 'NtOneOpt': [ - NTOneOptional(my_int=123, my_bool=False) - ], + [123, False] + ] } @@ -115,7 +118,7 @@ class MyClass(DataclassWizard): assert c == MyClass(my_td={'test': (True, deque([{'a': 1, 'c': False, 'd': 23.0}]))}) new_dict = asdict(c) - assert new_dict == {'my_td': {'test': (True, deque([{'a': 1, 'c': False, 'd': 23.0}]))}} + assert new_dict == {'my_td': {'test': [True, [{'a': 1, 'c': False, 'd': 23.0}]]}} def test_literal_in_container(): @@ -134,7 +137,7 @@ class MyClass(DataclassWizard): assert c == MyClass(my_literal_dict={'test': (123, frozenset({'Aa', 'Bb'}))}) new_dict = c.to_dict() - assert new_dict == {'my_literal_dict': {'test': (123, {'Aa', 'Bb'})}} + assert_unordered_equal(new_dict, {'my_literal_dict': {'test': [123, ['Aa', 'Bb']]}}) def test_decode_date_and_datetime_from_numeric_and_string_timestamp_and_iso_format(): @@ -271,3 +274,139 @@ class C(B): assert A.from_dict is not fromdict assert B.from_dict is not fromdict assert C.from_dict is not fromdict + + +class ContDict(DataclassWizard): + class _(DataclassWizard.Meta): + v1_namedtuple_as_dict = True + + tn: TN + cn: CN + + +class ContDictReq(DataclassWizard): + class _(DataclassWizard.Meta): + v1_namedtuple_as_dict = True + + tn: TNReq + + +def test_namedtuple_dict_mode_roundtrip_and_defaults(): + o = ContDict.from_dict({"tn": {"a": 1}, "cn": {"a": 3}}) + assert o.tn == TN(a=1, b=2) + assert o.cn == CN(a=3, b=2) + + d = o.to_dict() + assert d == {"tn": {"a": 1, "b": 2}, "cn": {"a": 3, "b": 2}} + + +def test_namedtuple_no_field_defaults_dict_mode_roundtrip(): + o = ContDictReq.from_dict({"tn": {"b": 1, "a": 3}}) + + assert o.tn == TN(a=3, b=1) + + d = o.to_dict() + assert d == {'tn': {'a': 3, 'b': 1}} + + +def test_namedtuple_no_field_defaults_dict_mode_missing_required_raises(): + with pytest.raises(MissingFields, match=r'`TNReq.__init__\(\)` missing required fields') as e: + _ = ContDictReq.from_dict({"tn": {"a": 1}}) + + assert e.value.missing_fields == ['b'] + + +def test_namedtuple_dict_mode_missing_required_raises(): + with pytest.raises(MissingFields, match=r'`TN\.__init__\(\)` missing required fields') as e: + ContDict.from_dict({"tn": {"b": 9}, "cn": {"a": 1}}) + + assert e.value.missing_fields == ['a'] + + +class ContList(DataclassWizard): + class _(DataclassWizard.Meta): + v1_namedtuple_as_dict = False + + tn: TN + cn: CN + + +def test_namedtuple_list_mode_roundtrip_and_defaults(): + o = ContList.from_dict({"tn": [1], "cn": [3]}) + assert o.tn == TN(a=1, b=2) + assert o.cn == CN(a=3, b=2) + + d = o.to_dict() + assert d == {"tn": [1, 2], "cn": [3, 2]} + + +def test_namedtuple_list_mode_rejects_dict_input_with_clear_error(): + with pytest.raises(ParseError, match=r"Dict input is not supported for NamedTuple fields in list mode.*list.*Meta\.v1_namedtuple_as_dict = True"): + ContList.from_dict({"tn": {"a": 1}, "cn": {"a": 3}}) + + +def test_namedtuple_dict_mode_rejects_dict_input_with_clear_error(): + with pytest.raises(ParseError, match=r"List/tuple input is not supported for NamedTuple fields in dict mode.*dict.*Meta\.v1_namedtuple_as_dict = False"): + ContDict.from_dict({"tn": ['test'], "cn": {"a": 3}}) + + +def test_typeddict_total_false_e2e_dict_roundtrip(): + o = ContTF.from_dict({"td": {"a": 1, "ro": 9}}) + assert o.td == {"a": 1, "ro": 9} + + d = o.to_dict() + assert d == {"td": {"a": 1, "ro": 9}} + + +def test_typeddict_total_false_missing_required_raises(): + with pytest.raises(Exception): # swap to MissingFields/TypeError etc + ContTF.from_dict({"td": {"b": 2}}) + + +def test_typeddict_total_true_e2e_optional_and_required_keys(): + o = ContTT.from_dict({"td": {"a": 1, "ro": 9}}) + assert o.td == {"a": 1, "ro": 9} + + d = o.to_dict() + assert d == {"td": {"a": 1, "ro": 9}} + + with pytest.raises(Exception): + ContTT.from_dict({"td": {"a": 1}}) # missing ro + + +def test_typeddict_all_required_e2e_inline_path(): + o = ContAllReq.from_dict({"td": {"x": 1, "y": "ok"}}) + assert o.td == {"x": 1, "y": "ok"} + + d = o.to_dict() + assert d == {"td": {"x": 1, "y": "ok"}} + + with pytest.raises(Exception): + ContAllReq.from_dict({"td": {"x": 1}}) # missing y + + +def test_v1_union_codegen_cache_nested_union_roundtrip_and_dump_error(): + class MyClass(DataclassWizard): + class _(DataclassWizard.Meta): + v1_unsafe_parse_dataclass_in_union = True + + complex_tp: 'list[int | Sub2] | list[int | str]' + + # First: pick the arm list[int|Sub2] + o1 = MyClass.from_dict({"complex_tp": [{"my_float": "123."}, 7]}) + assert o1 == MyClass(complex_tp=[Sub2(my_float=123.0), 7]) + + d1 = o1.to_dict() + assert d1 == {"complex_tp": [{"my_float": 123.0}, 7]} + + # Second: pick the other arm list[int|str] + # If inner-union codegen caching is wrong, this is where it tends to misbehave. + o2 = MyClass.from_dict({"complex_tp": ["hello", 9]}) + assert o2 == MyClass(complex_tp=["hello", 9]) + + d2 = o2.to_dict() + assert d2 == {"complex_tp": ["hello", 9]} + + o1.complex_tp = '123' + with pytest.raises(ParseError, match=r"Failed to dump field `complex_tp` in class `.*MyClass`"): + _ = o1.to_dict() diff --git a/tests/unit/v1/test_loaders.py b/tests/unit/v1/test_loaders.py index 5be11a3..527c72f 100644 --- a/tests/unit/v1/test_loaders.py +++ b/tests/unit/v1/test_loaders.py @@ -224,7 +224,9 @@ class _(JSONWizard.Meta): instance = MyClass.from_json(string) assert instance == MyClass(my_str='20', is_active_tuple=(True, False, True), list_of_int=[1, 2, 3], other_int=2) - assert instance.to_dict() == {'my_str': '20', 'IsActiveTuple': (True, False, True), 'myIntList': [1, 2, 3], + assert instance.to_dict() == {'my_str': '20', + 'IsActiveTuple': [True, False, True], + 'myIntList': [1, 2, 3], 'other_int': 2} string = """ @@ -238,7 +240,9 @@ class _(JSONWizard.Meta): instance = MyClass.from_json(string) assert instance == MyClass(my_str='21', is_active_tuple=(False, True, False), list_of_int=[3, 2, 1], other_int=1) - assert instance.to_dict() == {'my_str': '21', 'IsActiveTuple': (False, True, False), 'myIntList': [3, 2, 1], + assert instance.to_dict() == {'my_str': '21', + 'IsActiveTuple': [False, True, False], + 'myIntList': [3, 2, 1], 'other_int': 1} string = """ @@ -250,7 +254,10 @@ class _(JSONWizard.Meta): instance = MyClass.from_json(string) assert instance == MyClass(my_str='14', is_active_tuple=(False, True, True), list_of_int=[], other_int=2) - assert instance.to_dict() == {'my_str': '14', 'IsActiveTuple': (False, True, True), 'myIntList': [], 'other_int': 2} + assert instance.to_dict() == {'my_str': '14', + 'IsActiveTuple': [False, True, True], + 'myIntList': [], + 'other_int': 2} string = """ @@ -1419,8 +1426,30 @@ class _(JSONWizard.Meta): assert result.my_opt_str == expected if input is None: - assert result.my_str == '', \ - 'expected `my_str` to be set to an empty string' + assert result.my_str == 'None', \ + 'expected `my_str` to be set to the str() value of None' + + +def test_coerce_none_to_empty_str(): + @dataclass + class MyClass(JSONWizard): + + class _(JSONWizard.Meta): + v1 = True + v1_case = 'P' + v1_coerce_none_to_empty_str = True + + my_str: str + my_opt_str: Optional[str] + + d = {'MyStr': None, 'MyOptStr': None} + + result = MyClass.from_dict(d) + log.debug('Parsed object: %r', result) + + assert result.my_opt_str is None + assert result.my_str == '', \ + 'expected `my_str` to be set to an empty string' @pytest.mark.parametrize( @@ -2552,8 +2581,8 @@ class _(JSONWizard.Meta): ), ( {'my_str': 'test', 'my_int': 2, 'my_bool': True}, - does_not_raise(), - {'my_str': 'test', 'my_int': 2, 'my_bool': True} + pytest.raises(ParseError), + None, ), ] ) @@ -3219,7 +3248,7 @@ class _(JSONWizard.Meta): 'ace': {'in': {'hole': {0: {1: 'value'}}}}, 'this': {'Other': {'Int 1.23': 2}}, 1: {2: {3: 123}}, - 'IsActiveTuple': (True, False, True), + 'IsActiveTuple': [True, False, True], 'ListOfInt': [1, 2, 3], } @@ -3241,7 +3270,7 @@ class _(JSONWizard.Meta): 'ace': {'in': {'hole': {0: {1: 'Fact!'}}}}, 'this': {'Other': {'Int 1.23': 321}}, 1: {2: {3: 789}}, - 'IsActiveTuple': (False, True, False), + 'IsActiveTuple': [False, True, False], 'ListOfInt': [3, 2, 1] } @@ -3257,7 +3286,7 @@ class _(JSONWizard.Meta): assert instance.to_dict() == { 'ace': {'in': {'hole': {0: {1: '14'}}}}, 'this': {'Other': {'Int 1.23': 2}}, - 'IsActiveTuple': (False, True, True), + 'IsActiveTuple': [False, True, True], 1: {2: {3: 123}}, 'ListOfInt': [] } diff --git a/tests/unit/v1/utils_env.py b/tests/unit/v1/utils_env.py index 9ff5e5c..5c88585 100644 --- a/tests/unit/v1/utils_env.py +++ b/tests/unit/v1/utils_env.py @@ -1,6 +1,8 @@ from __future__ import annotations import json + +from collections import Counter from collections.abc import Mapping from typing import TYPE_CHECKING, TypeVar @@ -23,3 +25,26 @@ def envsafe(mapping: Mapping, *, dumps=json.dumps) -> dict[str, str]: def from_env(cls: type[T], mapping: Mapping=None, env_cfg: EnvInit=None, **init_kwargs) -> T: env_map = envsafe(mapping or {}) return cls(__env__=env_config(mapping=env_map, **(env_cfg or {})), **init_kwargs) + + +def assert_unordered_equal(a, b): + if isinstance(a, dict) and isinstance(b, dict): + assert a.keys() == b.keys() + for k in a: + assert_unordered_equal(a[k], b[k]) + return + + if isinstance(a, list) and isinstance(b, list): + # compare as multisets, recursively + def freeze(x): + if isinstance(x, dict): + return "dict", tuple(sorted((k, freeze(v)) for k, v in x.items())) + if isinstance(x, list): + # treat nested lists as unordered too + return "list", tuple(sorted(freeze(v) for v in x)) + return "atom", x + + assert Counter(map(freeze, a)) == Counter(map(freeze, b)) + return + + assert a == b