diff --git a/docs/source/structs.rst b/docs/source/structs.rst index 03667d0c..75005a8c 100644 --- a/docs/source/structs.rst +++ b/docs/source/structs.rst @@ -931,6 +931,58 @@ has a signature similar to `dataclasses.make_dataclass`. See Point(x=1.0, y=2.0) +Metaclasses +----------- + +You can define project-wide :class:`msgspec.Struct` policies at class-creation +time by extending the :class:`msgspec.StructMeta` metaclass. + +In the following example, we flip the default value of ``kw_only`` to ``True`` +in all subclasses of ``KwOnlyStruct``. + +.. code-block:: python + + >>> from msgspec import Struct, StructMeta + + >>> class KwOnlyStructMeta(StructMeta): + ... def __new__(mcls, name, bases, namespace, **kwargs): + ... kwargs.setdefault("kw_only", True) + ... return super().__new__(mcls, name, bases, namespace, **kwargs) + + >>> class KwOnlyStruct(Struct, metaclass=KwOnlyStructMeta): ... + + >>> class Example(KwOnlyStruct): + ... a: str = "" + ... b: int + + >>> Example() + Traceback (most recent call last): + File "", line 1, in + Example() + ~~~~~~~^^ + TypeError: Missing required argument 'b' + + >>> Example(b=123) + Example(a='', b=123) + +You can also mix :class:`msgspec.StructMeta` with other metaclasses. For +example, here we create an abstract :class:`msgspec.Struct` class: + +.. code-block:: python + + >>> from msgspec import Struct, StructMeta + + >>> from abc import ABCMeta + + >>> class ABCStructMeta(StructMeta, ABCMeta): ... + + >>> class StructABC(Struct, metaclass=ABCStructMeta): ... + +.. note:: + + When inheriting from multiple metaclasses, put :class:`msgspec.StructMeta` first. + + .. _struct-gc: Disabling Garbage Collection (Advanced) @@ -1013,3 +1065,4 @@ collected (leading to a memory leak). .. _rich: https://rich.readthedocs.io/en/stable/pretty.html .. _keyword-only parameters: https://docs.python.org/3/glossary.html#term-parameter .. _lambda: https://docs.python.org/3/tutorial/controlflow.html#lambda-expressions +.. _metaclass: https://docs.python.org/3/reference/datamodel.html#metaclasses diff --git a/msgspec/__init__.py b/msgspec/__init__.py index 421e4311..95079e7d 100644 --- a/msgspec/__init__.py +++ b/msgspec/__init__.py @@ -6,6 +6,7 @@ MsgspecError, Raw, Struct, + StructMeta, UnsetType, UNSET, NODEFAULT, diff --git a/msgspec/__init__.pyi b/msgspec/__init__.pyi index 4d2d8744..4b2990c9 100644 --- a/msgspec/__init__.pyi +++ b/msgspec/__init__.pyi @@ -1,4 +1,5 @@ import enum +from inspect import Signature from typing import ( Any, Callable, @@ -20,6 +21,50 @@ from typing_extensions import dataclass_transform, Buffer from . import inspect, json, msgpack, structs, toml, yaml +# PEP 673 explicitly rejects using Self in metaclass definitions: +# https://peps.python.org/pep-0673/#valid-locations-for-self +# +# Typeshed works around this by using a type variable as well: +# https://github.com/python/typeshed/blob/17bde1bd5e556de001adde3c2f340ba1c3581bd2/stdlib/abc.pyi#L14-L19 +_SM = TypeVar("_SM", bound="StructMeta") + +class StructMeta(type): + __struct_fields__: ClassVar[Tuple[str, ...]] + __struct_defaults__: ClassVar[Tuple[Any, ...]] + __struct_encode_fields__: ClassVar[Tuple[str, ...]] + __match_args__: ClassVar[Tuple[str, ...]] + @property + def __signature__(self) -> Signature: ... + @property + def __struct_config__(self) -> structs.StructConfig: ... + def __new__( + mcls: Type[_SM], + name: str, + bases: Tuple[type, ...], + namespace: Dict[str, Any], + *, + tag_field: Optional[str] = None, + tag: Union[None, bool, str, int, Callable[[str], Union[str, int]]] = None, + rename: Union[ + None, + Literal["lower", "upper", "camel", "pascal", "kebab"], + Callable[[str], Optional[str]], + Mapping[str, str], + ] = None, + omit_defaults: bool = False, + forbid_unknown_fields: bool = False, + frozen: bool = False, + eq: bool = True, + order: bool = False, + kw_only: bool = False, + repr_omit_defaults: bool = False, + array_like: bool = False, + gc: bool = True, + weakref: bool = False, + dict: bool = False, + cache_hash: bool = False, + ) -> _SM: ... + T = TypeVar("T") class UnsetType(enum.Enum): @@ -39,7 +84,7 @@ def field(*, default_factory: Callable[[], T], name: Optional[str] = None) -> T: @overload def field(*, name: Optional[str] = None) -> Any: ... @dataclass_transform(field_specifiers=(field,)) -class Struct: +class Struct(metaclass=StructMeta): __struct_fields__: ClassVar[Tuple[str, ...]] __struct_config__: ClassVar[structs.StructConfig] __match_args__: ClassVar[Tuple[str, ...]] diff --git a/msgspec/_core.c b/msgspec/_core.c index 6de39d6d..7399752c 100644 --- a/msgspec/_core.c +++ b/msgspec/_core.c @@ -4943,8 +4943,8 @@ typenode_collect_type(TypeNodeCollectState *state, PyObject *obj) { out = typenode_collect_typevar(state, t); } else if ( - Py_TYPE(t) == &StructMetaType || - (origin != NULL && Py_TYPE(origin) == &StructMetaType) + PyType_IsSubtype(Py_TYPE(t), &StructMetaType) || + (origin != NULL && PyType_IsSubtype(Py_TYPE(origin), &StructMetaType)) ) { out = typenode_collect_struct(state, t); } @@ -5598,7 +5598,7 @@ structmeta_collect_base(StructMetaInfo *info, MsgspecState *mod, PyObject *base) return -1; } - if (Py_TYPE(base) != &StructMetaType) { + if (!PyType_IsSubtype(Py_TYPE(base), &StructMetaType)) { if (((PyTypeObject *)base)->tp_dictoffset) { info->has_non_slots_bases = true; } @@ -5805,7 +5805,7 @@ structmeta_process_default(StructMetaInfo *info, PyObject *name) { if (default_val == NULL) return -1; } else if ( - (Py_TYPE(type) == &StructMetaType) && + (PyType_IsSubtype(Py_TYPE(type), &StructMetaType)) && ((StructMetaObject *)type)->frozen != OPT_TRUE ) { goto error_mutable_struct; @@ -6800,7 +6800,7 @@ StructInfo_Convert_lock_held(PyObject *obj) { PyObject *annotations = NULL; StructInfo *info = NULL; bool cache_set = false; - bool is_struct = Py_TYPE(obj) == &StructMetaType; + bool is_struct = PyType_IsSubtype(Py_TYPE(obj), &StructMetaType); /* Check for a cached StructInfo, and return if one exists */ if (MS_LIKELY(is_struct)) { @@ -6818,7 +6818,7 @@ StructInfo_Convert_lock_held(PyObject *obj) { } PyObject *origin = PyObject_GetAttr(obj, mod->str___origin__); if (origin == NULL) return NULL; - if (Py_TYPE(origin) != &StructMetaType) { + if (!PyType_IsSubtype(Py_TYPE(origin), &StructMetaType)) { Py_DECREF(origin); PyErr_SetString( PyExc_RuntimeError, "Expected __origin__ to be a Struct type" @@ -7276,7 +7276,7 @@ static PyTypeObject StructMetaType = { .tp_name = "msgspec._core.StructMeta", .tp_basicsize = sizeof(StructMetaObject), .tp_itemsize = 0, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_TYPE_SUBCLASS | Py_TPFLAGS_HAVE_GC | _Py_TPFLAGS_HAVE_VECTORCALL, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_TYPE_SUBCLASS | Py_TPFLAGS_HAVE_GC | _Py_TPFLAGS_HAVE_VECTORCALL | Py_TPFLAGS_BASETYPE, .tp_new = StructMeta_new, .tp_dealloc = (destructor) StructMeta_dealloc, .tp_clear = (inquiry) StructMeta_clear, @@ -7915,7 +7915,7 @@ struct_replace(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject if (!check_positional_nargs(nargs, 1, 1)) return NULL; PyObject *obj = args[0]; - if (Py_TYPE(Py_TYPE(obj)) != &StructMetaType) { + if (!PyType_IsSubtype(Py_TYPE(Py_TYPE(obj)), &StructMetaType)) { PyErr_SetString(PyExc_TypeError, "`struct` must be a `msgspec.Struct`"); return NULL; } @@ -7956,7 +7956,7 @@ struct_asdict(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { if (!check_positional_nargs(nargs, 1, 1)) return NULL; PyObject *obj = args[0]; - if (Py_TYPE(Py_TYPE(obj)) != &StructMetaType) { + if (!PyType_IsSubtype(Py_TYPE(Py_TYPE(obj)), &StructMetaType)) { PyErr_SetString(PyExc_TypeError, "`struct` must be a `msgspec.Struct`"); return NULL; } @@ -8014,7 +8014,7 @@ struct_astuple(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { if (!check_positional_nargs(nargs, 1, 1)) return NULL; PyObject *obj = args[0]; - if (Py_TYPE(Py_TYPE(obj)) != &StructMetaType) { + if (!PyType_IsSubtype(Py_TYPE(Py_TYPE(obj)), &StructMetaType)) { PyErr_SetString(PyExc_TypeError, "`struct` must be a `msgspec.Struct`"); return NULL; } @@ -8066,7 +8066,7 @@ struct_force_setattr(PyObject *self, PyObject *const *args, Py_ssize_t nargs) PyObject *obj = args[0]; PyObject *name = args[1]; PyObject *value = args[2]; - if (Py_TYPE(Py_TYPE(obj)) != &StructMetaType) { + if (!PyType_IsSubtype(Py_TYPE(Py_TYPE(obj)), &StructMetaType)) { PyErr_SetString(PyExc_TypeError, "`struct` must be a `msgspec.Struct`"); return NULL; } @@ -13245,7 +13245,7 @@ mpack_encode_uncommon(EncoderState *self, PyTypeObject *type, PyObject *obj) else if (type == &PyBool_Type) { return mpack_encode_bool(self, obj); } - else if (Py_TYPE(type) == &StructMetaType) { + else if (PyType_IsSubtype(Py_TYPE(type), &StructMetaType)) { return mpack_encode_struct(self, obj); } else if (type == &PyBytes_Type) { @@ -14352,7 +14352,7 @@ json_encode_uncommon(EncoderState *self, PyTypeObject *type, PyObject *obj) { else if (obj == Py_False) { return ms_write(self, "false", 5); } - else if (Py_TYPE(type) == &StructMetaType) { + else if (PyType_IsSubtype(Py_TYPE(type), &StructMetaType)) { return json_encode_struct(self, obj); } else if (PyTuple_Check(obj)) { @@ -16481,7 +16481,7 @@ msgspec_msgpack_decode(PyObject *self, PyObject *const *args, Py_ssize_t nargs, if (type == NULL || type == mod->typing_any) { state.type = &typenode_any; } - else if (Py_TYPE(type) == &StructMetaType) { + else if (PyType_IsSubtype(Py_TYPE(type), &StructMetaType)) { PyObject *info = StructInfo_Convert(type); if (info == NULL) return NULL; bool array_like = ((StructMetaObject *)type)->array_like == OPT_TRUE; @@ -19516,7 +19516,7 @@ msgspec_json_decode(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyO if (type == NULL || type == mod->typing_any) { state.type = &typenode_any; } - else if (Py_TYPE(type) == &StructMetaType) { + else if (PyType_IsSubtype(Py_TYPE(type), &StructMetaType)) { PyObject *info = StructInfo_Convert(type); if (info == NULL) return NULL; bool array_like = ((StructMetaObject *)type)->array_like == OPT_TRUE; @@ -20084,7 +20084,7 @@ to_builtins(ToBuiltinsState *self, PyObject *obj, bool is_key) { else if (PyDict_Check(obj)) { return to_builtins_dict(self, obj); } - else if (Py_TYPE(type) == &StructMetaType) { + else if (PyType_IsSubtype(Py_TYPE(type), &StructMetaType)) { return to_builtins_struct(self, obj, is_key); } else if (Py_TYPE(type) == self->mod->EnumMetaType) { @@ -21983,7 +21983,7 @@ msgspec_convert(PyObject *self, PyObject *args, PyObject *kwargs) state.dec_hook = dec_hook; /* Avoid allocating a new TypeNode for struct types */ - if (Py_TYPE(pytype) == &StructMetaType) { + if (PyType_IsSubtype(Py_TYPE(pytype), &StructMetaType)) { PyObject *info = StructInfo_Convert(pytype); if (info == NULL) return NULL; bool array_like = ((StructMetaObject *)pytype)->array_like == OPT_TRUE; @@ -22305,7 +22305,13 @@ PyInit__core(void) return NULL; Py_INCREF(&Unset_Type); if (PyModule_AddObject(m, "UnsetType", (PyObject *)&Unset_Type) < 0) + return NULL; + Py_INCREF((PyObject *)&StructMetaType); + if (PyModule_AddObject(m, "StructMeta", (PyObject *)&StructMetaType) < 0) { + Py_DECREF((PyObject *)&StructMetaType); + Py_DECREF(m); return NULL; + } st = msgspec_get_state(m); diff --git a/tests/test_struct_meta.py b/tests/test_struct_meta.py new file mode 100644 index 00000000..ebd9b0b1 --- /dev/null +++ b/tests/test_struct_meta.py @@ -0,0 +1,364 @@ +"""Tests for the exposed StructMeta metaclass.""" + +import pytest +import msgspec +from msgspec import Struct, StructMeta +from msgspec.structs import asdict, astuple, replace, force_setattr + + +def test_struct_meta_exists(): + """Test that StructMeta is properly exposed.""" + assert hasattr(msgspec, "StructMeta") + assert isinstance(Struct, StructMeta) + assert issubclass(StructMeta, type) + + +def test_struct_meta_direct_usage(): + """Test that StructMeta can be used directly as a metaclass.""" + + class CustomStruct(metaclass=StructMeta): + x: int + y: str + + # Verify the struct works as expected + instance = CustomStruct(x=1, y="test") + assert instance.x == 1 + assert instance.y == "test" + assert isinstance(instance, CustomStruct) + assert isinstance(CustomStruct, StructMeta) + + +def test_struct_meta_options(): + """Test that StructMeta properly handles struct options.""" + + class CustomStruct(metaclass=StructMeta, frozen=True): + x: int + + # Verify options were applied + instance = CustomStruct(x=1) + with pytest.raises(AttributeError): + instance.x = 2 # Should be frozen + + +def test_struct_meta_field_processing(): + """Test that StructMeta properly processes fields.""" + + class CustomStruct(metaclass=StructMeta): + x: int + y: str = "default" + + # Verify struct functionality + instance = CustomStruct(x=1) + assert instance.x == 1 + assert instance.y == "default" + + # Check struct metadata + assert hasattr(CustomStruct, "__struct_fields__") + assert "x" in CustomStruct.__struct_fields__ + assert "y" in CustomStruct.__struct_fields__ + + +def test_struct_meta_with_struct_base(): + """Test using StructMeta with Struct as a base class.""" + + class CustomStruct(Struct): + x: int + y: str + + # Verify the struct works as expected + instance = CustomStruct(x=1, y="test") + assert instance.x == 1 + assert instance.y == "test" + assert isinstance(instance, CustomStruct) + assert isinstance(CustomStruct, StructMeta) + + +def test_struct_meta_validation(): + """Test that StructMeta validation works.""" + # Should raise TypeError for invalid field name + with pytest.raises(TypeError): + + class InvalidStruct(metaclass=StructMeta): + __dict__: int # __dict__ is a reserved name + + +def test_struct_meta_with_options(): + """Test StructMeta with various options.""" + + class Point(metaclass=StructMeta, frozen=True, eq=True, order=True): + x: int + y: int + + p1 = Point(x=1, y=2) + p2 = Point(x=1, y=3) + + # Test frozen + with pytest.raises(AttributeError): + p1.x = 10 + + # Test eq - note that we need to compare fields manually + # since equality is based on identity by default + assert p1.x == Point(x=1, y=2).x and p1.y == Point(x=1, y=2).y + assert p1.x == p2.x and p1.y != p2.y + + # Test order - we can't directly compare instances + # but we can compare their field values + assert (p1.x, p1.y) < (p2.x, p2.y) + + +def test_struct_meta_inheritance(): + """Test that StructMeta can be inherited in Python code.""" + + class CustomMeta(StructMeta): + """A custom metaclass that inherits from StructMeta. + + This metaclass adds a kw_only_default parameter that can be used to + set the default kw_only value for all subclasses. + + When a class is created with this metaclass: + 1. If kw_only is explicitly specified, use that value + 2. If kw_only is not specified but kw_only_default is, use kw_only_default + 3. If neither is specified but a parent class has kw_only_default defined, + use the parent's kw_only_default + 4. Otherwise, default to False + """ + + # Class attribute to store kw_only_default settings for each class + _kw_only_default_settings = {} + + def __new__(mcls, name, bases, namespace, **kwargs): + # Check if kw_only is explicitly specified + kw_only_specified = "kw_only" in kwargs + + # Process kw_only_default parameter + kw_only_default = kwargs.pop("kw_only_default", None) + + # If kw_only_default is specified, store it + if kw_only_default is not None: + # Remember this setting for future subclasses + mcls._kw_only_default_settings[name] = kw_only_default + else: + # Check if any parent class has kw_only_default defined + for base in bases: + base_name = base.__name__ + if base_name in mcls._kw_only_default_settings: + # Use parent's kw_only_default + kw_only_default = mcls._kw_only_default_settings[base_name] + break + + # If kw_only is not specified but kw_only_default is available, use it + if not kw_only_specified and kw_only_default is not None: + kwargs["kw_only"] = kw_only_default + + # Create the class + return super().__new__(mcls, name, bases, namespace, **kwargs) + + # Test basic functionality - without kw_only_default + class SimpleModel(metaclass=CustomMeta): + x: int + y: str + + # Verify the class was created correctly + assert isinstance(SimpleModel, CustomMeta) + assert issubclass(CustomMeta, StructMeta) + + # Test creating an instance with positional arguments (should work) + instance = SimpleModel(1, "test") + assert instance.x == 1 + assert instance.y == "test" + + # Test setting kw_only_default=True + class KwOnlyBase(metaclass=CustomMeta, kw_only_default=True): + """Base class that sets kw_only_default=True""" + + # Test a simple child class, should inherit kw_only_default + class SimpleChild(KwOnlyBase): + x: int + + # Should only allow keyword arguments + with pytest.raises(TypeError): + SimpleChild(1) + + class BadFieldOrder(KwOnlyBase): + x: int = 0 + y: int + + BadFieldOrder(y=10) + + # Create instance with keyword arguments + child = SimpleChild(x=1) + assert child.x == 1 + + # Test overriding inherited kw_only_default + class NonKwOnlyChild(KwOnlyBase, kw_only=False): + x: int + + # Should allow positional arguments + non_kw_child = NonKwOnlyChild(1) + assert non_kw_child.x == 1 + + # Test independent class, not inheriting kw_only_default + class IndependentModel(metaclass=CustomMeta): + x: int + y: str + + # Should allow positional arguments + independent = IndependentModel(1, "test") + assert independent.x == 1 + assert independent.y == "test" + + # Print debug information + print( + f"KwOnlyBase in _kw_only_default_settings: {'KwOnlyBase' in CustomMeta._kw_only_default_settings}" + ) + print( + f"KwOnlyBase default: {CustomMeta._kw_only_default_settings.get('KwOnlyBase')}" + ) + print( + f"SimpleChild in _kw_only_default_settings: {'SimpleChild' in CustomMeta._kw_only_default_settings}" + ) + + # Test that kw_only_default values are correctly passed + assert "KwOnlyBase" in CustomMeta._kw_only_default_settings + assert CustomMeta._kw_only_default_settings["KwOnlyBase"] is True + + # Test asdict + d = asdict(independent) + assert d["x"] == 1 + assert d["y"] == "test" + + +def test_struct_meta_subclass_functions(): + """Test if structs created by StructMeta subclasses support various function operations.""" + + # Define a custom metaclass + class CustomMeta(StructMeta): + """Custom metaclass that inherits from StructMeta""" + + # Use the custom metaclass to create a struct class + class CustomStruct(metaclass=CustomMeta): + x: int + y: str + z: float = 3.14 + + # Create an instance + obj = CustomStruct(x=1, y="test") + assert obj.x == 1 + assert obj.y == "test" + assert obj.z == 3.14 + + # Test asdict function + d = asdict(obj) + assert isinstance(d, dict) + assert d["x"] == 1 + assert d["y"] == "test" + assert d["z"] == 3.14 + + # Test astuple function + t = astuple(obj) + assert isinstance(t, tuple) + assert t == (1, "test", 3.14) + + # Test replace function + obj2 = replace(obj, y="replaced") + assert obj2.x == 1 + assert obj2.y == "replaced" + assert obj2.z == 3.14 + + # Test force_setattr function + force_setattr(obj, "x", 100) + assert obj.x == 100 + + # Test nested structs + class NestedStruct(metaclass=CustomMeta): + inner: CustomStruct + name: str + + nested = NestedStruct(inner=obj, name="nested") + assert nested.inner.x == 100 + assert nested.inner.y == "test" + assert nested.name == "nested" + + # Test asdict with nested structs + nested_dict = asdict(nested) + assert isinstance(nested_dict, dict) + # Note: asdict doesn't recursively convert nested struct objects, so inner remains a CustomStruct object + assert isinstance(nested_dict["inner"], CustomStruct) + assert nested_dict["inner"].x == 100 + assert nested_dict["inner"].y == "test" + assert nested_dict["name"] == "nested" + + +def test_struct_meta_subclass_inheritance(): + """Test multi-level inheritance of StructMeta subclasses.""" + + # Define the first level custom metaclass + class BaseMeta(StructMeta): + """Base custom metaclass""" + + # Define the second level custom metaclass + class DerivedMeta(BaseMeta): + """Derived custom metaclass""" + + # Use the second level custom metaclass to create a struct class + class DerivedStruct(metaclass=DerivedMeta): + a: int + b: str + + # Create an instance + obj = DerivedStruct(a=42, b="derived") + assert obj.a == 42 + assert obj.b == "derived" + + # Test various functions + # asdict + d = asdict(obj) + assert d["a"] == 42 + assert d["b"] == "derived" + + # astuple + t = astuple(obj) + assert t == (42, "derived") + + # replace + obj2 = replace(obj, a=99) + assert obj2.a == 99 + assert obj2.b == "derived" + + +def test_struct_meta_subclass_with_encoder(): + """Test compatibility of structs created by StructMeta subclasses with encoders.""" + + # Define a custom metaclass + class EncoderMeta(StructMeta): + """Custom metaclass for testing encoders""" + + # Use the custom metaclass to create a struct class + class EncoderStruct(metaclass=EncoderMeta): + id: int + name: str + tags: list[str] = [] + + # Create an instance + obj = EncoderStruct(id=123, name="test") + + # Test JSON encoding and decoding + json_bytes = msgspec.json.encode(obj) + decoded = msgspec.json.decode(json_bytes, type=EncoderStruct) + + assert decoded.id == 123 + assert decoded.name == "test" + assert decoded.tags == [] + + # Test encoding and decoding with nested structs + class Container(metaclass=EncoderMeta): + item: EncoderStruct + count: int + + container = Container(item=obj, count=1) + json_bytes = msgspec.json.encode(container) + decoded = msgspec.json.decode(json_bytes, type=Container) + + assert decoded.count == 1 + assert decoded.item.id == 123 + assert decoded.item.name == "test"