From e464d5c6ba26ddce17b732e03119057e0b7303b3 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Mon, 14 Jul 2025 20:24:10 +0000 Subject: [PATCH 01/18] feat: proper weakref support --- mypyc/irbuild/vtable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/irbuild/vtable.py b/mypyc/irbuild/vtable.py index 2d4f7261e4ca..1b6c35a25643 100644 --- a/mypyc/irbuild/vtable.py +++ b/mypyc/irbuild/vtable.py @@ -14,7 +14,7 @@ def compute_vtable(cls: ClassIR) -> None: return if not cls.is_generated: - cls.has_dict = any(x.inherits_python for x in cls.mro) + cls.has_dict = cls.supports_weakref = any(x.inherits_python for x in cls.mro) for t in cls.mro[1:]: # Make sure all ancestors are processed first From 54cbf0419b9d9deac160aceca55c3d899a2020d1 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Mon, 14 Jul 2025 20:24:10 +0000 Subject: [PATCH 02/18] Update emitclass.py --- mypyc/codegen/emitclass.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 0c2d470104d0..8b99ae80af1b 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -313,6 +313,16 @@ def emit_line() -> None: if emitter.capi_version < (3, 12): fields["tp_dictoffset"] = base_size fields["tp_weaklistoffset"] = weak_offset + elif cl.supports_weakref: + # __weakref__ lives right after the struct + # TODO: It should get a member in the struct instead of doing this nonsense. + weak_offset = base_size + emitter.emit_lines( + f"PyMemberDef {members_name}[] = {{", + f'{{"__weakref__", T_OBJECT_EX, {base_size}, 0, NULL}},', + "{0}", + "};", + ) else: fields["tp_basicsize"] = base_size From 681923d7a7d179b9ae6ff9595415d37f0c0a574e Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Mon, 14 Jul 2025 20:24:10 +0000 Subject: [PATCH 03/18] Update class_ir.py --- mypyc/ir/class_ir.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mypyc/ir/class_ir.py b/mypyc/ir/class_ir.py index 561dc9d438c4..acf9a056c321 100644 --- a/mypyc/ir/class_ir.py +++ b/mypyc/ir/class_ir.py @@ -94,6 +94,7 @@ def __init__( is_abstract: bool = False, is_ext_class: bool = True, is_final_class: bool = False, + supports_weakref: bool = False ) -> None: self.name = name self.module_name = module_name @@ -102,6 +103,7 @@ def __init__( self.is_abstract = is_abstract self.is_ext_class = is_ext_class self.is_final_class = is_final_class + self.supports_weakref = supports_weakref # An augmented class has additional methods separate from what mypyc generates. # Right now the only one is dataclasses. self.is_augmented = False From 2b5fb1d7792a2ffbcce35f412805e47a9df5b642 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 14 Jul 2025 20:24:10 +0000 Subject: [PATCH 04/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mypyc/ir/class_ir.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/ir/class_ir.py b/mypyc/ir/class_ir.py index acf9a056c321..0fad0e51c327 100644 --- a/mypyc/ir/class_ir.py +++ b/mypyc/ir/class_ir.py @@ -94,7 +94,7 @@ def __init__( is_abstract: bool = False, is_ext_class: bool = True, is_final_class: bool = False, - supports_weakref: bool = False + supports_weakref: bool = False, ) -> None: self.name = name self.module_name = module_name From 54da50a6362e1bdd09b5a97c939a3bca24948cbe Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Mon, 14 Jul 2025 20:24:10 +0000 Subject: [PATCH 05/18] Update emitclass.py --- mypyc/codegen/emitclass.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 8b99ae80af1b..cda215a2134b 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -316,7 +316,6 @@ def emit_line() -> None: elif cl.supports_weakref: # __weakref__ lives right after the struct # TODO: It should get a member in the struct instead of doing this nonsense. - weak_offset = base_size emitter.emit_lines( f"PyMemberDef {members_name}[] = {{", f'{{"__weakref__", T_OBJECT_EX, {base_size}, 0, NULL}},', From 36c62dd0c7eaad01cf13a8a72c7306e21464b473 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Mon, 14 Jul 2025 20:24:10 +0000 Subject: [PATCH 06/18] Update emitclass.py --- mypyc/codegen/emitclass.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index cda215a2134b..4919df410155 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -322,6 +322,8 @@ def emit_line() -> None: "{0}", "};", ) + if emitter.capi_version < (3, 12): + fields["tp_weaklistoffset"] = base_size else: fields["tp_basicsize"] = base_size From b941f5b717d8fa02ee661978e808bb3b5ee514f9 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Mon, 14 Jul 2025 20:24:10 +0000 Subject: [PATCH 07/18] Update class_ir.py --- mypyc/ir/class_ir.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mypyc/ir/class_ir.py b/mypyc/ir/class_ir.py index 0fad0e51c327..636ae83b2994 100644 --- a/mypyc/ir/class_ir.py +++ b/mypyc/ir/class_ir.py @@ -368,6 +368,7 @@ def serialize(self) -> JsonDict: "is_generated": self.is_generated, "is_augmented": self.is_augmented, "is_final_class": self.is_final_class, + "supports_weakref": self.supports_weakref, "inherits_python": self.inherits_python, "has_dict": self.has_dict, "allow_interpreted_subclasses": self.allow_interpreted_subclasses, From 9ef6e1382d23b8260b850611fa9d3dae68822bc7 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Mon, 14 Jul 2025 20:24:10 +0000 Subject: [PATCH 08/18] feat: support_weakrefs=True in mypyc_attr --- mypyc/irbuild/prepare.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index 1d6117ab7b1e..4f854ae1b256 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -326,6 +326,9 @@ def prepare_class_def( if attrs.get("serializable") is True: # Supports copy.copy and pickle (including subclasses) ir._serializable = True + if attrs.get("support_weakrefs") is True: + # Supports weakrefs (including subclasses) + ir.support_weakrefs = True # Check for subclassing from builtin types for cls in info.mro: From 043d17fb14c29d1112132894f5e870b89082127c Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Mon, 14 Jul 2025 20:24:10 +0000 Subject: [PATCH 09/18] Update vtable.py --- mypyc/irbuild/vtable.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypyc/irbuild/vtable.py b/mypyc/irbuild/vtable.py index 1b6c35a25643..458824061787 100644 --- a/mypyc/irbuild/vtable.py +++ b/mypyc/irbuild/vtable.py @@ -14,7 +14,8 @@ def compute_vtable(cls: ClassIR) -> None: return if not cls.is_generated: - cls.has_dict = cls.supports_weakref = any(x.inherits_python for x in cls.mro) + cls.has_dict = any(x.inherits_python for x in cls.mro) + cls.supports_weakref = cls.supports_weakref or cls.has_dict for t in cls.mro[1:]: # Make sure all ancestors are processed first From 68e28bb1ddd9ba3792cb7ffac2eca00aa7a45184 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Mon, 14 Jul 2025 20:24:10 +0000 Subject: [PATCH 10/18] Update prepare.py --- mypyc/irbuild/prepare.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mypyc/irbuild/prepare.py b/mypyc/irbuild/prepare.py index 4f854ae1b256..dc13539aa47a 100644 --- a/mypyc/irbuild/prepare.py +++ b/mypyc/irbuild/prepare.py @@ -326,9 +326,9 @@ def prepare_class_def( if attrs.get("serializable") is True: # Supports copy.copy and pickle (including subclasses) ir._serializable = True - if attrs.get("support_weakrefs") is True: - # Supports weakrefs (including subclasses) - ir.support_weakrefs = True + if attrs.get("supports_weakref") is True: + # Has a tp_weakrefoffset slot allowing the creation of weak references (including subclasses) + ir.supports_weakref = True # Check for subclassing from builtin types for cls in info.mro: From 91cf813bfb43a2cc9562caf18de648022a9346a1 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Mon, 14 Jul 2025 20:24:10 +0000 Subject: [PATCH 11/18] fix: serialization segfault --- mypyc/ir/class_ir.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mypyc/ir/class_ir.py b/mypyc/ir/class_ir.py index 636ae83b2994..efb6f084c67a 100644 --- a/mypyc/ir/class_ir.py +++ b/mypyc/ir/class_ir.py @@ -94,7 +94,6 @@ def __init__( is_abstract: bool = False, is_ext_class: bool = True, is_final_class: bool = False, - supports_weakref: bool = False, ) -> None: self.name = name self.module_name = module_name @@ -103,7 +102,6 @@ def __init__( self.is_abstract = is_abstract self.is_ext_class = is_ext_class self.is_final_class = is_final_class - self.supports_weakref = supports_weakref # An augmented class has additional methods separate from what mypyc generates. # Right now the only one is dataclasses. self.is_augmented = False @@ -111,6 +109,8 @@ def __init__( self.inherits_python = False # Do instances of this class have __dict__? self.has_dict = False + # Do instances of this class have __weakref__? + self.supports_weakref = False # Do we allow interpreted subclasses? Derived from a mypyc_attr. self.allow_interpreted_subclasses = False # Does this class need getseters to be generated for its attributes? (getseters are also @@ -368,9 +368,9 @@ def serialize(self) -> JsonDict: "is_generated": self.is_generated, "is_augmented": self.is_augmented, "is_final_class": self.is_final_class, - "supports_weakref": self.supports_weakref, "inherits_python": self.inherits_python, "has_dict": self.has_dict, + "supports_weakref": self.supports_weakref, "allow_interpreted_subclasses": self.allow_interpreted_subclasses, "needs_getseters": self.needs_getseters, "_serializable": self._serializable, From 907f379338d7f2c69faf768bc4f69d2cbce32cac Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Mon, 14 Jul 2025 20:24:10 +0000 Subject: [PATCH 12/18] chore: add comment --- mypyc/irbuild/vtable.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mypyc/irbuild/vtable.py b/mypyc/irbuild/vtable.py index 458824061787..766b4086c594 100644 --- a/mypyc/irbuild/vtable.py +++ b/mypyc/irbuild/vtable.py @@ -15,6 +15,7 @@ def compute_vtable(cls: ClassIR) -> None: if not cls.is_generated: cls.has_dict = any(x.inherits_python for x in cls.mro) + # TODO: define more weakref triggers cls.supports_weakref = cls.supports_weakref or cls.has_dict for t in cls.mro[1:]: From 91b69b093504f6b08471c68be397d4a35248910a Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Mon, 14 Jul 2025 20:24:10 +0000 Subject: [PATCH 13/18] fix: deserialization discrepancy --- mypyc/ir/class_ir.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mypyc/ir/class_ir.py b/mypyc/ir/class_ir.py index efb6f084c67a..a1659f14980b 100644 --- a/mypyc/ir/class_ir.py +++ b/mypyc/ir/class_ir.py @@ -429,6 +429,7 @@ def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> ClassIR: ir.is_final_class = data["is_final_class"] ir.inherits_python = data["inherits_python"] ir.has_dict = data["has_dict"] + ir.supports_weakref = data["supports_weakref"] ir.allow_interpreted_subclasses = data["allow_interpreted_subclasses"] ir.needs_getseters = data["needs_getseters"] ir._serializable = data["_serializable"] From 0239833e323e459c6c30fbd1ea7760cca97a733f Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Tue, 13 May 2025 12:42:34 -0400 Subject: [PATCH 14/18] feat(test): test mypyc_attr(supports_weakref=True) --- mypyc/test-data/irbuild-classes.test | 42 ++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/mypyc/test-data/irbuild-classes.test b/mypyc/test-data/irbuild-classes.test index 1a2c237cc3c9..a8bbeb96d3c8 100644 --- a/mypyc/test-data/irbuild-classes.test +++ b/mypyc/test-data/irbuild-classes.test @@ -1408,3 +1408,45 @@ class TestOverload: def __mypyc_generator_helper__(self, x: Any) -> Any: return x + +[case testMypycAttrSupportsWeakref] +import weakref +from mypy_extensions import mypyc_attr + +@mypyc_attr(supports_weakref=True) +class WeakrefClass: + pass + +obj = WeakrefClass() +ref = weakref.ref(obj) +assert ref() is obj + +[case testMypycAttrSupportsWeakrefInheritance] +import weakref +from mypy_extensions import mypyc_attr + +@mypyc_attr(supports_weakref=True) +class WeakrefClass: + pass + +class WeakrefInheritor(WeakrefClass): + pass + +obj = WeakrefInheritor() +ref = weakref.ref(obj) +assert ref() is obj + +[case testMypycAttrSupportsWeakrefSubclass] +import weakref +from mypy_extensions import mypyc_attr + +class NativeClass: + pass + +@mypyc_attr(supports_weakref=True) +class WeakrefSubclass(NativeClass): + pass + +obj = WeakrefSubclass() +ref = weakref.ref(obj) +assert ref() is obj From 1e02502ae12695e7472f884c30718fddd74218d2 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Tue, 13 May 2025 13:05:59 -0400 Subject: [PATCH 15/18] fix: set Py_TPFLAGS_MANAGED_WEAKREF flag on python>=3.12 --- mypyc/codegen/emitclass.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 4919df410155..6a7054943796 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -323,6 +323,7 @@ def emit_line() -> None: "};", ) if emitter.capi_version < (3, 12): + # versions >= 3.12 do not define tp_weaklistoffset, but set Py_TPFLAGS_MANAGED_WEAKREF flag instead fields["tp_weaklistoffset"] = base_size else: fields["tp_basicsize"] = base_size @@ -384,6 +385,9 @@ def emit_line() -> None: fields["tp_call"] = "PyVectorcall_Call" if has_managed_dict(cl, emitter): flags.append("Py_TPFLAGS_MANAGED_DICT") + if cl.supports_weakref and emitter.capi_version >= (3, 12): + flags.append("Py_TPFLAGS_MANAGED_WEAKREF") + fields["tp_flags"] = " | ".join(flags) fields["tp_doc"] = native_class_doc_initializer(cl) From 82a69a5c9b278647336c582cfe0bcaa14a7fb400 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 13 May 2025 17:07:28 +0000 Subject: [PATCH 16/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- mypyc/codegen/emitclass.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 6a7054943796..1d659faaba80 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -387,7 +387,7 @@ def emit_line() -> None: flags.append("Py_TPFLAGS_MANAGED_DICT") if cl.supports_weakref and emitter.capi_version >= (3, 12): flags.append("Py_TPFLAGS_MANAGED_WEAKREF") - + fields["tp_flags"] = " | ".join(flags) fields["tp_doc"] = native_class_doc_initializer(cl) From 36597c713521a29114271559f6c5f656af5759c8 Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Tue, 13 May 2025 13:08:38 -0400 Subject: [PATCH 17/18] Update emitclass.py --- mypyc/codegen/emitclass.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 1d659faaba80..d097c6a4b082 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -323,7 +323,8 @@ def emit_line() -> None: "};", ) if emitter.capi_version < (3, 12): - # versions >= 3.12 do not define tp_weaklistoffset, but set Py_TPFLAGS_MANAGED_WEAKREF flag instead + # versions >= 3.12 set Py_TPFLAGS_MANAGED_WEAKREF flag instead + # https://docs.python.org/3.12/extending/newtypes.html#weak-reference-support fields["tp_weaklistoffset"] = base_size else: fields["tp_basicsize"] = base_size From 17285925e223002afe8629c1591ddd50096a525b Mon Sep 17 00:00:00 2001 From: BobTheBuidler <70677534+BobTheBuidler@users.noreply.github.com> Date: Tue, 13 May 2025 13:20:31 -0400 Subject: [PATCH 18/18] fix: handle weakrefs in tp_dealloc --- mypyc/codegen/emitclass.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index d097c6a4b082..b8877e262072 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -858,6 +858,13 @@ def generate_dealloc_for_class( emitter.emit_line("static void") emitter.emit_line(f"{dealloc_func_name}({cl.struct_name(emitter.names)} *self)") emitter.emit_line("{") + if cl.supports_weakref: + if emitter.capi_version < (3, 12): + emitter.emit_line("if (self->weakreflist != NULL) {") + emitter.emit_line("PyObject_ClearWeakRefs((PyObject *) self);") + emitter.emit_line("}") + else: + emitter.emit_line("PyObject_ClearWeakRefs((PyObject *) self);") if has_tp_finalize: emitter.emit_line("if (!PyObject_GC_IsFinalized((PyObject *)self)) {") emitter.emit_line("Py_TYPE(self)->tp_finalize((PyObject *)self);")