Merge remote-tracking branch 'upstream/master' into chore/st-sync-fixture-variance

Author: sterliakov

mypy/checker.py

@@ -80,6 +80,7 @@
     AssertStmt,
     AssignmentExpr,
     AssignmentStmt,
+    AwaitExpr,
     Block,
     BreakStmt,
     BytesExpr,
@@ -4924,7 +4925,11 @@ def check_return_stmt(self, s: ReturnStmt) -> None:
                 allow_none_func_call = is_lambda or declared_none_return or declared_any_return
 
                 # Return with a value.
-                if isinstance(s.expr, (CallExpr, ListExpr, TupleExpr, DictExpr, SetExpr, OpExpr)):
+                if (
+                    isinstance(s.expr, (CallExpr, ListExpr, TupleExpr, DictExpr, SetExpr, OpExpr))
+                    or isinstance(s.expr, AwaitExpr)
+                    and isinstance(s.expr.expr, CallExpr)
+                ):
                     # For expressions that (strongly) depend on type context (i.e. those that
                     # are handled like a function call), we allow fallback to empty type context
                     # in case of errors, this improves user experience in some cases,

mypy/test/helpers.py

@@ -233,6 +233,9 @@ def clean_up(a: list[str]) -> list[str]:
         for p in prefix, prefix.replace(os.sep, "/"):
             if p != "/" and p != "//" and p != "\\" and p != "\\\\":
                 ss = ss.replace(p, "")
+        # Replace memory address with zeros
+        if "at 0x" in ss:
+            ss = re.sub(r"(at 0x)\w+>", r"\g<1>000000000000>", ss)
         # Ignore spaces at end of line.
         ss = re.sub(" +$", "", ss)
         # Remove pwd from driver.py's path

mypy/typeanal.py

@@ -779,7 +779,7 @@ def try_analyze_special_unbound_type(self, t: UnboundType, fullname: str) -> Typ
             if self.api.type.has_base("builtins.type"):
                 self.fail("Self type cannot be used in a metaclass", t)
             if self.api.type.self_type is not None:
-                if self.api.type.is_final:
+                if self.api.type.is_final or self.api.type.is_enum and self.api.type.enum_members:
                     return fill_typevars(self.api.type)
                 return self.api.type.self_type.copy_modified(line=t.line, column=t.column)
             # TODO: verify this is unreachable and replace with an assert?

mypyc/codegen/emitclass.py

@@ -205,10 +205,6 @@ def generate_class_reuse(
     TODO: Generalize to support a free list with up to N objects.
     """
     assert cl.reuse_freed_instance
-
-    # The free list implementation doesn't support class hierarchies
-    assert cl.is_final_class or cl.children == []
-
     context = c_emitter.context
     name = cl.name_prefix(c_emitter.names) + "_free_instance"
     struct_name = cl.struct_name(c_emitter.names)
@@ -889,8 +885,21 @@ def generate_dealloc_for_class(
     emitter.emit_line(f"{dealloc_func_name}({cl.struct_name(emitter.names)} *self)")
     emitter.emit_line("{")
     if has_tp_finalize:
-        emitter.emit_line("if (!PyObject_GC_IsFinalized((PyObject *)self)) {")
-        emitter.emit_line("Py_TYPE(self)->tp_finalize((PyObject *)self);")
+        emitter.emit_line("PyObject *type, *value, *traceback;")
+        emitter.emit_line("PyErr_Fetch(&type, &value, &traceback);")
+        emitter.emit_line("int res = PyObject_CallFinalizerFromDealloc((PyObject *)self);")
+        # CPython interpreter uses PyErr_WriteUnraisable: https://docs.python.org/3/c-api/exceptions.html#c.PyErr_WriteUnraisable
+        # However, the message is slightly different due to the way mypyc compiles classes.
+        # CPython interpreter prints: Exception ignored in: <function F.__del__ at 0x100aed940>
+        # mypyc prints: Exception ignored in: <slot wrapper '__del__' of 'F' objects>
+        emitter.emit_line("if (PyErr_Occurred() != NULL) {")
+        # Don't untrack instance if error occurred
+        emitter.emit_line("PyErr_WriteUnraisable((PyObject *)self);")
+        emitter.emit_line("res = -1;")
+        emitter.emit_line("}")
+        emitter.emit_line("PyErr_Restore(type, value, traceback);")
+        emitter.emit_line("if (res < 0) {")
+        emitter.emit_line("goto done;")
         emitter.emit_line("}")
     emitter.emit_line("PyObject_GC_UnTrack(self);")
     if cl.reuse_freed_instance:
@@ -900,6 +909,7 @@ def generate_dealloc_for_class(
     emitter.emit_line(f"{clear_func_name}(self);")
     emitter.emit_line("Py_TYPE(self)->tp_free((PyObject *)self);")
     emitter.emit_line("CPy_TRASHCAN_END(self)")
+    emitter.emit_line("done: ;")
     emitter.emit_line("}")
 
 
@@ -930,30 +940,13 @@ def generate_finalize_for_class(
     emitter.emit_line("static void")
     emitter.emit_line(f"{finalize_func_name}(PyObject *self)")
     emitter.emit_line("{")
-    emitter.emit_line("PyObject *type, *value, *traceback;")
-    emitter.emit_line("PyErr_Fetch(&type, &value, &traceback);")
     emitter.emit_line(
         "{}{}{}(self);".format(
             emitter.get_group_prefix(del_method.decl),
             NATIVE_PREFIX,
             del_method.cname(emitter.names),
         )
     )
-    emitter.emit_line("if (PyErr_Occurred() != NULL) {")
-    emitter.emit_line('PyObject *del_str = PyUnicode_FromString("__del__");')
-    emitter.emit_line(
-        "PyObject *del_method = (del_str == NULL) ? NULL : _PyType_Lookup(Py_TYPE(self), del_str);"
-    )
-    # CPython interpreter uses PyErr_WriteUnraisable: https://docs.python.org/3/c-api/exceptions.html#c.PyErr_WriteUnraisable
-    # However, the message is slightly different due to the way mypyc compiles classes.
-    # CPython interpreter prints: Exception ignored in: <function F.__del__ at 0x100aed940>
-    # mypyc prints: Exception ignored in: <slot wrapper '__del__' of 'F' objects>
-    emitter.emit_line("PyErr_WriteUnraisable(del_method);")
-    emitter.emit_line("Py_XDECREF(del_method);")
-    emitter.emit_line("Py_XDECREF(del_str);")
-    emitter.emit_line("}")
-    # PyErr_Restore also clears exception raised in __del__.
-    emitter.emit_line("PyErr_Restore(type, value, traceback);")
     emitter.emit_line("}")
 
 

mypyc/irbuild/prepare.py

@@ -351,13 +351,29 @@ def prepare_class_def(
     ir = mapper.type_to_ir[cdef.info]
     info = cdef.info
 
-    attrs = get_mypyc_attrs(cdef)
+    attrs, attrs_lines = get_mypyc_attrs(cdef)
     if attrs.get("allow_interpreted_subclasses") is True:
         ir.allow_interpreted_subclasses = True
     if attrs.get("serializable") is True:
         # Supports copy.copy and pickle (including subclasses)
         ir._serializable = True
 
+    free_list_len = attrs.get("free_list_len")
+    if free_list_len is not None:
+        line = attrs_lines["free_list_len"]
+        if ir.is_trait:
+            errors.error('"free_list_len" can\'t be used with traits', path, line)
+        if ir.allow_interpreted_subclasses:
+            errors.error(
+                '"free_list_len" can\'t be used in a class that allows interpreted subclasses',
+                path,
+                line,
+            )
+        if free_list_len == 1:
+            ir.reuse_freed_instance = True
+        else:
+            errors.error(f'Unsupported value for "free_list_len": {free_list_len}', path, line)
+
     # Check for subclassing from builtin types
     for cls in info.mro:
         # Special case exceptions and dicts

mypyc/irbuild/util.py

@@ -96,6 +96,8 @@ def get_mypyc_attr_literal(e: Expression) -> Any:
         return False
     elif isinstance(e, RefExpr) and e.fullname == "builtins.None":
         return None
+    elif isinstance(e, IntExpr):
+        return e.value
     return NotImplemented
 
 
@@ -110,20 +112,23 @@ def get_mypyc_attr_call(d: Expression) -> CallExpr | None:
     return None
 
 
-def get_mypyc_attrs(stmt: ClassDef | Decorator) -> dict[str, Any]:
+def get_mypyc_attrs(stmt: ClassDef | Decorator) -> tuple[dict[str, Any], dict[str, int]]:
     """Collect all the mypyc_attr attributes on a class definition or a function."""
     attrs: dict[str, Any] = {}
+    lines: dict[str, int] = {}
     for dec in stmt.decorators:
         d = get_mypyc_attr_call(dec)
         if d:
             for name, arg in zip(d.arg_names, d.args):
                 if name is None:
                     if isinstance(arg, StrExpr):
                         attrs[arg.value] = True
+                        lines[arg.value] = d.line
                 else:
                     attrs[name] = get_mypyc_attr_literal(arg)
+                    lines[name] = d.line
 
-    return attrs
+    return attrs, lines
 
 
 def is_extension_class(path: str, cdef: ClassDef, errors: Errors) -> bool:

mypyc/test-data/irbuild-classes.test

@@ -1734,3 +1734,27 @@ class NonNative:
 class InheritsPython(dict):
     def __new__(cls) -> InheritsPython:
         return super().__new__(cls)  # E: super().__new__() not supported for classes inheriting from non-native classes
+
+[case testClassWithFreeList]
+from mypy_extensions import mypyc_attr, trait
+
+@mypyc_attr(free_list_len=1)
+class UsesFreeList:
+    pass
+
+@mypyc_attr(free_list_len=None)
+class NoFreeList:
+    pass
+
+@mypyc_attr(free_list_len=2)  # E: Unsupported value for "free_list_len": 2
+class FreeListError:
+    pass
+
+@trait
+@mypyc_attr(free_list_len=1)  # E: "free_list_len" can't be used with traits
+class NonNative:
+    pass
+
+@mypyc_attr(free_list_len=1, allow_interpreted_subclasses=True)  # E: "free_list_len" can't be used in a class that allows interpreted subclasses
+class InterpSub:
+    pass

mypyc/test-data/run-classes.test

@@ -3110,7 +3110,7 @@ f = native.F()
 del f
 
 [out]
-Exception ignored in: <slot wrapper '__del__' of 'F' objects>
+Exception ignored in: <native.F object at 0x000000000000>
 Traceback (most recent call last):
   File "native.py", line 5, in __del__
     raise Exception("e2")
@@ -3794,3 +3794,100 @@ assert t.native == 43
 assert t.generic == "{}"
 assert t.bitfield == 0x0C
 assert t.default == 10
+
+[case testPerTypeFreeList]
+from __future__ import annotations
+
+from mypy_extensions import mypyc_attr
+
+a = []
+
+@mypyc_attr(free_list_len=1)
+class Foo:
+    def __init__(self, x: int) -> None:
+        self.x = x
+        a.append(x)
+
+def test_alloc() -> None:
+    x: Foo | None
+    y: Foo | None
+
+    x = Foo(1)
+    assert x.x == 1
+    x = None
+
+    x = Foo(2)
+    assert x.x == 2
+    y = Foo(3)
+    assert x.x == 2
+    assert y.x == 3
+    x = None
+    y = None
+    assert a == [1, 2, 3]
+
+    x = Foo(4)
+    assert x.x == 4
+    y = Foo(5)
+    assert x.x == 4
+    assert y.x == 5
+
+@mypyc_attr(free_list_len=1)
+class Base:
+    def __init__(self, x: str) -> None:
+        self.x = x
+
+class Deriv(Base):
+    def __init__(self, x: str, y: str) -> None:
+        super().__init__(x)
+        self.y = y
+
+@mypyc_attr(free_list_len=1)
+class Deriv2(Base):
+    def __init__(self, x: str, y: str) -> None:
+        super().__init__(x)
+        self.y = y
+
+def test_inheritance() -> None:
+    x: Base | None
+    y: Base | None
+    x = Base('x' + str())
+    y = Base('y' + str())
+    y = None
+    d = Deriv('a' + str(), 'b' + str())
+    assert type(d) is Deriv
+    assert d.x == 'a'
+    assert d.y == 'b'
+    assert x.x == 'x'
+    y = Base('z' + str())
+    assert d.x == 'a'
+    assert d.y == 'b'
+    assert y.x == 'z'
+    x = None
+    y = None
+
+def test_inheritance_2() -> None:
+    x: Base | None
+    y: Base | None
+    d: Deriv2 | None
+    x = Base('x' + str())
+    y = Base('y' + str())
+    y = None
+    d = Deriv2('a' + str(), 'b' + str())
+    assert type(d) is Deriv2
+    assert d.x == 'a'
+    assert d.y == 'b'
+    assert x.x == 'x'
+    d = None
+    d = Deriv2('c' + str(), 'd' + str())
+    assert type(d) is Deriv2
+    assert d.x == 'c'
+    assert d.y == 'd'
+    assert x.x == 'x'
+    y = Base('z' + str())
+    assert type(y) is Base
+    assert d.x == 'c'
+    assert d.y == 'd'
+    assert y.x == 'z'
+    x = None
+    y = None
+    d = None

test-data/unit/check-inference-context.test

@@ -1582,3 +1582,5 @@ async def inner(c: Cls[T]) -> Optional[T]:
 
 async def outer(c: Cls[T]) -> Optional[T]:
     return await inner(c)
+[builtins fixtures/async_await.pyi]
+[typing fixtures/typing-async.pyi]

test-data/unit/check-selftype.test

@@ -2346,3 +2346,35 @@ gc: G[D2]
 reveal_type(gb.test())  # N: Revealed type is "typing.Sequence[__main__.D1]"
 reveal_type(gc.test())  # N: Revealed type is "builtins.list[__main__.D2]"
 [builtins fixtures/list.pyi]
+
+[case testEnumImplicitlyFinalForSelfType]
+from enum import Enum
+from typing import Self
+
+# This enum has members and so is implicitly final.
+# Foo and Self are interchangeable within the class.
+class Foo(Enum):
+    A = 1
+
+    @classmethod
+    def foo(cls) -> Self:
+        return Foo.A
+
+    @classmethod
+    def foo2(cls) -> Self:
+        return cls.bar()
+
+    @classmethod
+    def bar(cls) -> Foo:
+        ...
+
+# This enum is empty and should not be assignable to Self
+class Bar(Enum):
+    @classmethod
+    def foo(cls) -> Self:
+        return cls.bar()  # E: Incompatible return value type (got "Bar", expected "Self")
+
+    @classmethod
+    def bar(cls) -> Bar:
+        ...
+[builtins fixtures/classmethod.pyi]