Merge branch 'master' into fix_variance_polymorphic_constructor

Author: randolf-scholz

docs/source/conf.py

@@ -278,9 +278,9 @@
 intersphinx_mapping = {
     "python": ("https://docs.python.org/3", None),
     "attrs": ("https://www.attrs.org/en/stable/", None),
-    "cython": ("https://docs.cython.org/en/latest", None),
+    "cython": ("https://cython.readthedocs.io/en/stable", None),
     "monkeytype": ("https://monkeytype.readthedocs.io/en/latest", None),
-    "setuptools": ("https://setuptools.readthedocs.io/en/latest", None),
+    "setuptools": ("https://setuptools.pypa.io/en/latest", None),
 }
 
 

misc/log_trace_check.py

@@ -0,0 +1,85 @@
+"""Compile mypy using mypyc with trace logging enabled, and collect a trace.
+
+The trace log can be used to analyze low-level performance bottlenecks.
+
+By default does a self check as the workload.
+
+This works on all supported platforms, unlike some of our other performance tools.
+"""
+
+from __future__ import annotations
+
+import argparse
+import glob
+import os
+import shutil
+import subprocess
+import sys
+import time
+
+from perf_compare import build_mypy, clone
+
+# Generated files, including binaries, go under this directory to avoid overwriting user state.
+TARGET_DIR = "mypy.log_trace.tmpdir"
+
+
+def perform_type_check(target_dir: str, code: str | None) -> None:
+    cache_dir = os.path.join(target_dir, ".mypy_cache")
+    if os.path.exists(cache_dir):
+        shutil.rmtree(cache_dir)
+    args = []
+    if code is None:
+        args.extend(["--config-file", "mypy_self_check.ini"])
+        for pat in "mypy/*.py", "mypy/*/*.py", "mypyc/*.py", "mypyc/test/*.py":
+            args.extend(glob.glob(pat))
+    else:
+        args.extend(["-c", code])
+    check_cmd = ["python", "-m", "mypy"] + args
+    t0 = time.time()
+    subprocess.run(check_cmd, cwd=target_dir, check=True)
+    elapsed = time.time() - t0
+    print(f"{elapsed:.2f}s elapsed")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Compile mypy and collect a trace log while type checking (by default, self check)."
+    )
+    parser.add_argument(
+        "--multi-file",
+        action="store_true",
+        help="compile mypy into one C file per module (to reduce RAM use during compilation)",
+    )
+    parser.add_argument(
+        "--skip-compile", action="store_true", help="use compiled mypy from previous run"
+    )
+    parser.add_argument(
+        "-c",
+        metavar="CODE",
+        default=None,
+        type=str,
+        help="type check Python code fragment instead of mypy self-check",
+    )
+    args = parser.parse_args()
+    multi_file: bool = args.multi_file
+    skip_compile: bool = args.skip_compile
+    code: str | None = args.c
+
+    target_dir = TARGET_DIR
+
+    if not skip_compile:
+        clone(target_dir, "HEAD")
+
+        print(f"Building mypy in {target_dir} with trace logging enabled...")
+        build_mypy(target_dir, multi_file, log_trace=True, opt_level="0")
+    elif not os.path.isdir(target_dir):
+        sys.exit("error: Can't find compile mypy from previous run -- can't use --skip-compile")
+
+    perform_type_check(target_dir, code)
+
+    trace_fnam = os.path.join(target_dir, "mypyc_trace.txt")
+    print(f"Generated event trace log in {trace_fnam}")
+
+
+if __name__ == "__main__":
+    main()

misc/perf_compare.py

@@ -37,13 +37,22 @@ def heading(s: str) -> None:
     print()
 
 
-def build_mypy(target_dir: str, multi_file: bool, *, cflags: str | None = None) -> None:
+def build_mypy(
+    target_dir: str,
+    multi_file: bool,
+    *,
+    cflags: str | None = None,
+    log_trace: bool = False,
+    opt_level: str = "2",
+) -> None:
     env = os.environ.copy()
     env["CC"] = "clang"
-    env["MYPYC_OPT_LEVEL"] = "2"
+    env["MYPYC_OPT_LEVEL"] = opt_level
     env["PYTHONHASHSEED"] = "1"
     if multi_file:
         env["MYPYC_MULTI_FILE"] = "1"
+    if log_trace:
+        env["MYPYC_LOG_TRACE"] = "1"
     if cflags is not None:
         env["CFLAGS"] = cflags
     cmd = [sys.executable, "setup.py", "--use-mypyc", "build_ext", "--inplace"]

mypy/checker.py

@@ -5577,6 +5577,8 @@ def infer_variable_types_from_type_maps(
                 previous_type, _, _ = self.check_lvalue(expr)
                 if previous_type is not None:
                     already_exists = True
+                    if isinstance(expr.node, Var) and expr.node.is_final:
+                        self.msg.cant_assign_to_final(expr.name, False, expr)
                     if self.check_subtype(
                         typ,
                         previous_type,

mypy/checkexpr.py

@@ -1671,6 +1671,10 @@ def check_call(
                 object_type,
                 original_type=callee,
             )
+        elif isinstance(callee, UninhabitedType):
+            ret = UninhabitedType()
+            ret.ambiguous = callee.ambiguous
+            return callee, ret
         else:
             return self.msg.not_callable(callee, context), AnyType(TypeOfAny.from_error)
 
@@ -1751,14 +1755,6 @@ def check_callable_call(
                         return AnyType(TypeOfAny.from_error), callee
                     seen_unpack = True
 
-        formal_to_actual = map_actuals_to_formals(
-            arg_kinds,
-            arg_names,
-            callee.arg_kinds,
-            callee.arg_names,
-            lambda i: self.accept(args[i]),
-        )
-
         # This is tricky: return type may contain its own type variables, like in
         # def [S] (S) -> def [T] (T) -> tuple[S, T], so we need to update their ids
         # to avoid possible id clashes if this call itself appears in a generic
@@ -1769,27 +1765,29 @@ def check_callable_call(
             freeze_all_type_vars(fresh_ret_type)
             callee = callee.copy_modified(ret_type=fresh_ret_type)
 
+        if callee.is_generic():
+            callee = freshen_function_type_vars(callee)
+            callee = self.infer_function_type_arguments_using_context(callee, context)
+
+        formal_to_actual = map_actuals_to_formals(
+            arg_kinds,
+            arg_names,
+            callee.arg_kinds,
+            callee.arg_names,
+            lambda i: self.accept(args[i]),
+        )
+
         if callee.is_generic():
             need_refresh = any(
                 isinstance(v, (ParamSpecType, TypeVarTupleType)) for v in callee.variables
             )
-            callee = freshen_function_type_vars(callee)
-            callee = self.infer_function_type_arguments_using_context(callee, context)
-            if need_refresh:
-                # Argument kinds etc. may have changed due to
-                # ParamSpec or TypeVarTuple variables being replaced with an arbitrary
-                # number of arguments; recalculate actual-to-formal map
-                formal_to_actual = map_actuals_to_formals(
-                    arg_kinds,
-                    arg_names,
-                    callee.arg_kinds,
-                    callee.arg_names,
-                    lambda i: self.accept(args[i]),
-                )
             callee = self.infer_function_type_arguments(
                 callee, args, arg_kinds, arg_names, formal_to_actual, need_refresh, context
             )
             if need_refresh:
+                # Argument kinds etc. may have changed due to
+                # ParamSpec or TypeVarTuple variables being replaced with an arbitrary
+                # number of arguments; recalculate actual-to-formal map
                 formal_to_actual = map_actuals_to_formals(
                     arg_kinds,
                     arg_names,
@@ -2254,6 +2252,11 @@ def infer_function_type_arguments_pass2(
             if isinstance(arg, (NoneType, UninhabitedType)) or has_erased_component(arg):
                 inferred_args[i] = None
         callee_type = self.apply_generic_arguments(callee_type, inferred_args, context)
+
+        if not callee_type.is_generic():
+            # Fast path, second pass can't give new information.
+            return callee_type, []
+
         if need_refresh:
             formal_to_actual = map_actuals_to_formals(
                 arg_kinds,
@@ -3465,7 +3468,7 @@ def visit_op_expr(self, e: OpExpr) -> Type:
             # It's actually a type expression X | Y.
             return self.accept(e.analyzed)
         if e.op == "and" or e.op == "or":
-            return self.check_boolean_op(e, e)
+            return self.check_boolean_op(e)
         if e.op == "*" and isinstance(e.left, ListExpr):
             # Expressions of form [...] * e get special type inference.
             return self.check_list_multiply(e)
@@ -4252,20 +4255,18 @@ def check_op(
                 context=context,
             )
 
-    def check_boolean_op(self, e: OpExpr, context: Context) -> Type:
+    def check_boolean_op(self, e: OpExpr) -> Type:
         """Type check a boolean operation ('and' or 'or')."""
 
         # A boolean operation can evaluate to either of the operands.
 
-        # We use the current type context to guide the type inference of of
+        # We use the current type context to guide the type inference of
         # the left operand. We also use the left operand type to guide the type
         # inference of the right operand so that expressions such as
         # '[1] or []' are inferred correctly.
         ctx = self.type_context[-1]
         left_type = self.accept(e.left, ctx)
-        expanded_left_type = try_expanding_sum_type_to_union(
-            self.accept(e.left, ctx), "builtins.bool"
-        )
+        expanded_left_type = try_expanding_sum_type_to_union(left_type, "builtins.bool")
 
         assert e.op in ("and", "or")  # Checked by visit_op_expr
 

mypy/checkmember.py

@@ -69,6 +69,7 @@
     TypeVarLikeType,
     TypeVarTupleType,
     TypeVarType,
+    UninhabitedType,
     UnionType,
     get_proper_type,
 )
@@ -268,6 +269,10 @@ def _analyze_member_access(
         if not mx.suppress_errors:
             mx.msg.deleted_as_rvalue(typ, mx.context)
         return AnyType(TypeOfAny.from_error)
+    elif isinstance(typ, UninhabitedType):
+        attr_type = UninhabitedType()
+        attr_type.ambiguous = typ.ambiguous
+        return attr_type
     return report_missing_attribute(mx.original_type, typ, name, mx)
 
 
@@ -951,7 +956,7 @@ def expand_and_bind_callable(
 ) -> Type:
     if not mx.preserve_type_var_ids:
         functype = freshen_all_functions_type_vars(functype)
-    typ = get_proper_type(expand_self_type(var, functype, mx.original_type))
+    typ = get_proper_type(expand_self_type(var, functype, mx.self_type))
     assert isinstance(typ, FunctionLike)
     if is_trivial_self:
         typ = bind_self_fast(typ, mx.self_type)

mypy/constraints.py

@@ -1110,7 +1110,10 @@ def visit_callable_type(self, template: CallableType) -> list[Constraint]:
                     # like U -> U, should be Callable[..., Any], but if U is a self-type, we can
                     # allow it to leak, to be later bound to self. A bunch of existing code
                     # depends on this old behaviour.
-                    and not any(tv.id.is_self() for tv in cactual.variables)
+                    and not (
+                        any(tv.id.is_self() for tv in cactual.variables)
+                        and template.is_ellipsis_args
+                    )
                 ):
                     # If the actual callable is generic, infer constraints in the opposite
                     # direction, and indicate to the solver there are extra type variables

mypy/expandtype.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from collections.abc import Iterable, Mapping, Sequence
+from collections.abc import Iterable, Mapping
 from typing import Final, TypeVar, cast, overload
 
 from mypy.nodes import ARG_STAR, FakeInfo, Var
@@ -209,7 +209,11 @@ def visit_erased_type(self, t: ErasedType) -> Type:
         return t
 
     def visit_instance(self, t: Instance) -> Type:
-        args = self.expand_types_with_unpack(list(t.args))
+        if len(t.args) == 0:
+            # TODO: Why do we need to create a copy here?
+            return t.copy_modified()
+
+        args = self.expand_type_tuple_with_unpack(t.args)
 
         if isinstance(t.type, FakeInfo):
             # The type checker expands function definitions and bodies
@@ -431,7 +435,7 @@ def visit_overloaded(self, t: Overloaded) -> Type:
             items.append(new_item)
         return Overloaded(items)
 
-    def expand_types_with_unpack(self, typs: Sequence[Type]) -> list[Type]:
+    def expand_type_list_with_unpack(self, typs: list[Type]) -> list[Type]:
         """Expands a list of types that has an unpack."""
         items: list[Type] = []
         for item in typs:
@@ -441,8 +445,19 @@ def expand_types_with_unpack(self, typs: Sequence[Type]) -> list[Type]:
                 items.append(item.accept(self))
         return items
 
+    def expand_type_tuple_with_unpack(self, typs: tuple[Type, ...]) -> list[Type]:
+        """Expands a tuple of types that has an unpack."""
+        # Micro-optimization: Specialized variant of expand_type_list_with_unpack
+        items: list[Type] = []
+        for item in typs:
+            if isinstance(item, UnpackType) and isinstance(item.type, TypeVarTupleType):
+                items.extend(self.expand_unpack(item))
+            else:
+                items.append(item.accept(self))
+        return items
+
     def visit_tuple_type(self, t: TupleType) -> Type:
-        items = self.expand_types_with_unpack(t.items)
+        items = self.expand_type_list_with_unpack(t.items)
         if len(items) == 1:
             # Normalize Tuple[*Tuple[X, ...]] -> Tuple[X, ...]
             item = items[0]
@@ -510,7 +525,7 @@ def visit_type_type(self, t: TypeType) -> Type:
     def visit_type_alias_type(self, t: TypeAliasType) -> Type:
         # Target of the type alias cannot contain type variables (not bound by the type
         # alias itself), so we just expand the arguments.
-        args = self.expand_types_with_unpack(t.args)
+        args = self.expand_type_list_with_unpack(t.args)
         # TODO: normalize if target is Tuple, and args are [*tuple[X, ...]]?
         return t.copy_modified(args=args)