Merge branch 'master' into patch-3

Author: BobTheBuidler

mypyc/codegen/emitclass.py

@@ -304,9 +304,6 @@ def emit_line() -> None:
         emit_line()
         generate_dealloc_for_class(cl, dealloc_name, clear_name, bool(del_method), emitter)
         emit_line()
-        if del_method:
-            generate_finalize_for_class(del_method, finalize_name, emitter)
-            emit_line()
 
         if cl.allow_interpreted_subclasses:
             shadow_vtable_name: str | None = generate_vtables(
@@ -317,6 +314,9 @@ def emit_line() -> None:
             shadow_vtable_name = None
         vtable_name = generate_vtables(cl, vtable_setup_name, vtable_name, emitter, shadow=False)
         emit_line()
+    if del_method:
+        generate_finalize_for_class(del_method, finalize_name, emitter)
+        emit_line()
     if needs_getseters:
         generate_getseter_declarations(cl, emitter)
         emit_line()

mypyc/codegen/emitmodule.py

@@ -306,7 +306,10 @@ def compile_ir_to_c(
         for source in sources
     }
 
-    names = NameGenerator([[source.module for source in sources] for sources, _ in groups])
+    names = NameGenerator(
+        [[source.module for source in sources] for sources, _ in groups],
+        separate=compiler_options.separate,
+    )
 
     # Generate C code for each compilation group. Each group will be
     # compiled into a separate extension module.

mypyc/irbuild/context.py

@@ -95,6 +95,11 @@ def curr_env_reg(self) -> Value:
         assert self._curr_env_reg is not None
         return self._curr_env_reg
 
+    def can_merge_generator_and_env_classes(self) -> bool:
+        # In simple cases we can place the environment into the generator class,
+        # instead of having two separate classes.
+        return self.is_generator and not self.is_nested and not self.contains_nested
+
 
 class ImplicitClass:
     """Contains information regarding implicitly generated classes.

mypyc/irbuild/env_class.py

@@ -58,7 +58,8 @@ class is generated, the function environment has not yet been
 
 def finalize_env_class(builder: IRBuilder) -> None:
     """Generate, instantiate, and set up the environment of an environment class."""
-    instantiate_env_class(builder)
+    if not builder.fn_info.can_merge_generator_and_env_classes():
+        instantiate_env_class(builder)
 
     # Iterate through the function arguments and replace local definitions (using registers)
     # that were previously added to the environment with references to the function's

mypyc/irbuild/function.py

@@ -243,7 +243,9 @@ def c() -> None:
     # are free in their nested functions. Generator functions need an environment class to
     # store a variable denoting the next instruction to be executed when the __next__ function
     # is called, along with all the variables inside the function itself.
-    if contains_nested or is_generator:
+    if contains_nested or (
+        is_generator and not builder.fn_info.can_merge_generator_and_env_classes()
+    ):
         setup_env_class(builder)
 
     if is_nested or in_non_ext:

mypyc/irbuild/generator.py

@@ -64,8 +64,14 @@ def gen_generator_func(
     setup_generator_class(builder)
     load_env_registers(builder)
     gen_arg_defaults(builder)
-    finalize_env_class(builder)
-    builder.add(Return(instantiate_generator_class(builder)))
+    if builder.fn_info.can_merge_generator_and_env_classes():
+        gen = instantiate_generator_class(builder)
+        builder.fn_info._curr_env_reg = gen
+        finalize_env_class(builder)
+    else:
+        finalize_env_class(builder)
+        gen = instantiate_generator_class(builder)
+    builder.add(Return(gen))
 
     args, _, blocks, ret_type, fn_info = builder.leave()
     func_ir, func_reg = gen_func_ir(args, blocks, fn_info)
@@ -122,30 +128,38 @@ def instantiate_generator_class(builder: IRBuilder) -> Value:
     fitem = builder.fn_info.fitem
     generator_reg = builder.add(Call(builder.fn_info.generator_class.ir.ctor, [], fitem.line))
 
-    # Get the current environment register. If the current function is nested, then the
-    # generator class gets instantiated from the callable class' '__call__' method, and hence
-    # we use the callable class' environment register. Otherwise, we use the original
-    # function's environment register.
-    if builder.fn_info.is_nested:
-        curr_env_reg = builder.fn_info.callable_class.curr_env_reg
+    if builder.fn_info.can_merge_generator_and_env_classes():
+        # Set the generator instance to the initial state (zero).
+        zero = Integer(0)
+        builder.add(SetAttr(generator_reg, NEXT_LABEL_ATTR_NAME, zero, fitem.line))
     else:
-        curr_env_reg = builder.fn_info.curr_env_reg
-
-    # Set the generator class' environment attribute to point at the environment class
-    # defined in the current scope.
-    builder.add(SetAttr(generator_reg, ENV_ATTR_NAME, curr_env_reg, fitem.line))
-
-    # Set the generator class' environment class' NEXT_LABEL_ATTR_NAME attribute to 0.
-    zero = Integer(0)
-    builder.add(SetAttr(curr_env_reg, NEXT_LABEL_ATTR_NAME, zero, fitem.line))
+        # Get the current environment register. If the current function is nested, then the
+        # generator class gets instantiated from the callable class' '__call__' method, and hence
+        # we use the callable class' environment register. Otherwise, we use the original
+        # function's environment register.
+        if builder.fn_info.is_nested:
+            curr_env_reg = builder.fn_info.callable_class.curr_env_reg
+        else:
+            curr_env_reg = builder.fn_info.curr_env_reg
+
+        # Set the generator class' environment attribute to point at the environment class
+        # defined in the current scope.
+        builder.add(SetAttr(generator_reg, ENV_ATTR_NAME, curr_env_reg, fitem.line))
+
+        # Set the generator instance's environment to the initial state (zero).
+        zero = Integer(0)
+        builder.add(SetAttr(curr_env_reg, NEXT_LABEL_ATTR_NAME, zero, fitem.line))
     return generator_reg
 
 
 def setup_generator_class(builder: IRBuilder) -> ClassIR:
     name = f"{builder.fn_info.namespaced_name()}_gen"
 
     generator_class_ir = ClassIR(name, builder.module_name, is_generated=True)
-    generator_class_ir.attributes[ENV_ATTR_NAME] = RInstance(builder.fn_info.env_class)
+    if builder.fn_info.can_merge_generator_and_env_classes():
+        builder.fn_info.env_class = generator_class_ir
+    else:
+        generator_class_ir.attributes[ENV_ATTR_NAME] = RInstance(builder.fn_info.env_class)
     generator_class_ir.mro = [generator_class_ir]
 
     builder.classes.append(generator_class_ir)
@@ -392,7 +406,10 @@ def setup_env_for_generator_class(builder: IRBuilder) -> None:
     cls.send_arg_reg = exc_arg
 
     cls.self_reg = builder.read(self_target, fitem.line)
-    cls.curr_env_reg = load_outer_env(builder, cls.self_reg, builder.symtables[-1])
+    if builder.fn_info.can_merge_generator_and_env_classes():
+        cls.curr_env_reg = cls.self_reg
+    else:
+        cls.curr_env_reg = load_outer_env(builder, cls.self_reg, builder.symtables[-1])
 
     # Define a variable representing the label to go to the next time
     # the '__next__' function of the generator is called, and add it

mypyc/namegen.py

@@ -34,20 +34,30 @@ class NameGenerator:
 
     The generated should be internal to a build and thus the mapping is
     arbitrary. Just generating names '1', '2', ... would be correct,
-    though not very usable.
+    though not very usable. The generated names may be visible in CPU
+    profiles and when debugging using native debuggers.
     """
 
-    def __init__(self, groups: Iterable[list[str]]) -> None:
+    def __init__(self, groups: Iterable[list[str]], *, separate: bool = False) -> None:
         """Initialize with a list of modules in each compilation group.
 
         The names of modules are used to shorten names referring to
         modules, for convenience. Arbitrary module
         names are supported for generated names, but uncompiled modules
         will use long names.
+
+        If separate is True, assume separate compilation. This implies
+        that we don't have knowledge of all sources that will be linked
+        together. In this case we won't trim module prefixes, since we
+        don't have enough information to determine common module prefixes.
         """
         self.module_map: dict[str, str] = {}
         for names in groups:
-            self.module_map.update(make_module_translation_map(names))
+            if not separate:
+                self.module_map.update(make_module_translation_map(names))
+            else:
+                for name in names:
+                    self.module_map[name] = name + "."
         self.translations: dict[tuple[str, str], str] = {}
         self.used_names: set[str] = set()
 

mypyc/test-data/run-async.test

@@ -561,3 +561,85 @@ def test_bool() -> None:
 
 [file asyncio/__init__.pyi]
 def run(x: object) -> object: ...
+
+[case testRunAsyncNestedFunctions]
+import asyncio
+from typing import cast, Iterator
+
+from testutil import assertRaises
+
+def normal_contains_async_def(x: int) -> int:
+    async def f(y: int) -> int:
+        return x + y
+
+    return 5 + cast(int, asyncio.run(f(6)))
+
+def test_def_contains_async_def() -> None:
+    assert normal_contains_async_def(3) == 14
+
+async def inc(x: int) -> int:
+    return x + 1
+
+async def async_def_contains_normal(x: int) -> int:
+    def nested(y: int, z: int) -> int:
+        return x + y + z
+
+    a = x
+    a += nested((await inc(3)), (await inc(4)))
+    return a
+
+def test_async_def_contains_normal() -> None:
+    assert normal_contains_async_def(2) == (2 + 2 + 4 + 5)
+
+async def async_def_contains_async_def(x: int) -> int:
+    async def f(y: int) -> int:
+        return (await inc(x)) + (await inc(y))
+
+    return (await f(1)) + (await f(2))
+
+def test_async_def_contains_async_def() -> None:
+    assert asyncio.run(async_def_contains_async_def(3)) == (3 + 1 + 1 + 1) + (3 + 1 + 2 + 1)
+
+async def async_def_contains_generator(x: int) -> tuple[int, int, int]:
+    def gen(y: int) -> Iterator[int]:
+        yield x + 1
+        yield x + y
+
+    it = gen(4)
+    res = x + 10, next(it), next(it)
+
+    with assertRaises(StopIteration):
+        next(it)
+
+    return res
+
+def test_async_def_contains_generator() -> None:
+    assert asyncio.run(async_def_contains_generator(3)) == (13, 4, 7)
+
+def generator_contains_async_def(x: int) -> Iterator[int]:
+    async def f(y: int) -> int:
+        return (await inc(x)) + (await inc(y))
+
+    yield cast(int, asyncio.run(f(2)))
+    yield cast(int, asyncio.run(f(3)))
+    yield x + 10
+
+def test_generator_contains_async_def() -> None:
+    assert list(generator_contains_async_def(5)) == [6 + 3, 6 + 4, 15]
+
+async def async_def_contains_two_nested_functions(x: int, y: int) -> tuple[int, int]:
+    def f(a: int) -> int:
+        return x + a
+
+    def g(b: int, c: int) -> int:
+        return y + b + c
+
+    return (await inc(f(3))), (await inc(g(4, 10)))
+
+def test_async_def_contains_two_nested_functions() -> None:
+    assert asyncio.run(async_def_contains_two_nested_functions(5, 7)) == (
+        (5 + 3 + 1), (7 + 4 + 10 + 1)
+    )
+
+[file asyncio/__init__.pyi]
+def run(x: object) -> object: ...

mypyc/test-data/run-classes.test

@@ -2754,6 +2754,21 @@ def test_function():
     assert(isinstance(d.fitem, ForwardDefinedClass))
     assert(isinstance(d.fitems, ForwardDefinedClass))
 
+[case testDelForDictSubclass-xfail]
+# The crash in issue mypy#19175 is fixed.
+# But, for classes that derive from built-in Python classes, user-defined __del__ method is not
+# being invoked.
+class DictSubclass(dict):
+    def __del__(self):
+        print("deleting DictSubclass...")
+
+[file driver.py]
+import native
+native.DictSubclass()
+
+[out]
+deleting DictSubclass...
+
 [case testDel]
 class A:
     def __del__(self):
@@ -2774,6 +2789,12 @@ class C(B):
 class D(A):
     pass
 
+# Just make sure that this class compiles (see issue mypy#19175). testDelForDictSubclass tests for
+# correct output.
+class NormDict(dict):
+    def __del__(self) -> None:
+        pass
+
 [file driver.py]
 import native
 native.C()

mypyc/test/test_namegen.py

@@ -52,3 +52,17 @@ def test_name_generator(self) -> None:
         assert g.private_name("foo", "C_x_y") == "foo___C_x_y"
         assert g.private_name("foo", "C_x_y") == "foo___C_x_y"
         assert g.private_name("foo", "___") == "foo______3_"
+
+        g = NameGenerator([["foo.zar"]])
+        assert g.private_name("foo.zar", "f") == "f"
+
+    def test_name_generator_with_separate(self) -> None:
+        g = NameGenerator([["foo", "foo.zar"]], separate=True)
+        assert g.private_name("foo", "f") == "foo___f"
+        assert g.private_name("foo", "C.x.y") == "foo___C___x___y"
+        assert g.private_name("foo.zar", "C.x.y") == "foo___zar___C___x___y"
+        assert g.private_name("foo", "C.x_y") == "foo___C___x_y"
+        assert g.private_name("foo", "___") == "foo______3_"
+
+        g = NameGenerator([["foo.zar"]], separate=True)
+        assert g.private_name("foo.zar", "f") == "foo___zar___f"