Merge branch 'master' into fix_match_callable

Author: randolf-scholz

misc/log_trace_check.py

@@ -0,0 +1,85 @@
+"""Compile mypy using mypyc with trace logging enabled, and collect a trace.
+
+The trace log can be used to analyze low-level performance bottlenecks.
+
+By default does a self check as the workload.
+
+This works on all supported platforms, unlike some of our other performance tools.
+"""
+
+from __future__ import annotations
+
+import argparse
+import glob
+import os
+import shutil
+import subprocess
+import sys
+import time
+
+from perf_compare import build_mypy, clone
+
+# Generated files, including binaries, go under this directory to avoid overwriting user state.
+TARGET_DIR = "mypy.log_trace.tmpdir"
+
+
+def perform_type_check(target_dir: str, code: str | None) -> None:
+    cache_dir = os.path.join(target_dir, ".mypy_cache")
+    if os.path.exists(cache_dir):
+        shutil.rmtree(cache_dir)
+    args = []
+    if code is None:
+        args.extend(["--config-file", "mypy_self_check.ini"])
+        for pat in "mypy/*.py", "mypy/*/*.py", "mypyc/*.py", "mypyc/test/*.py":
+            args.extend(glob.glob(pat))
+    else:
+        args.extend(["-c", code])
+    check_cmd = ["python", "-m", "mypy"] + args
+    t0 = time.time()
+    subprocess.run(check_cmd, cwd=target_dir, check=True)
+    elapsed = time.time() - t0
+    print(f"{elapsed:.2f}s elapsed")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Compile mypy and collect a trace log while type checking (by default, self check)."
+    )
+    parser.add_argument(
+        "--multi-file",
+        action="store_true",
+        help="compile mypy into one C file per module (to reduce RAM use during compilation)",
+    )
+    parser.add_argument(
+        "--skip-compile", action="store_true", help="use compiled mypy from previous run"
+    )
+    parser.add_argument(
+        "-c",
+        metavar="CODE",
+        default=None,
+        type=str,
+        help="type check Python code fragment instead of mypy self-check",
+    )
+    args = parser.parse_args()
+    multi_file: bool = args.multi_file
+    skip_compile: bool = args.skip_compile
+    code: str | None = args.c
+
+    target_dir = TARGET_DIR
+
+    if not skip_compile:
+        clone(target_dir, "HEAD")
+
+        print(f"Building mypy in {target_dir} with trace logging enabled...")
+        build_mypy(target_dir, multi_file, log_trace=True, opt_level="0")
+    elif not os.path.isdir(target_dir):
+        sys.exit("error: Can't find compile mypy from previous run -- can't use --skip-compile")
+
+    perform_type_check(target_dir, code)
+
+    trace_fnam = os.path.join(target_dir, "mypyc_trace.txt")
+    print(f"Generated event trace log in {trace_fnam}")
+
+
+if __name__ == "__main__":
+    main()

misc/perf_compare.py

@@ -37,13 +37,22 @@ def heading(s: str) -> None:
     print()
 
 
-def build_mypy(target_dir: str, multi_file: bool, *, cflags: str | None = None) -> None:
+def build_mypy(
+    target_dir: str,
+    multi_file: bool,
+    *,
+    cflags: str | None = None,
+    log_trace: bool = False,
+    opt_level: str = "2",
+) -> None:
     env = os.environ.copy()
     env["CC"] = "clang"
-    env["MYPYC_OPT_LEVEL"] = "2"
+    env["MYPYC_OPT_LEVEL"] = opt_level
     env["PYTHONHASHSEED"] = "1"
     if multi_file:
         env["MYPYC_MULTI_FILE"] = "1"
+    if log_trace:
+        env["MYPYC_LOG_TRACE"] = "1"
     if cflags is not None:
         env["CFLAGS"] = cflags
     cmd = [sys.executable, "setup.py", "--use-mypyc", "build_ext", "--inplace"]

mypy/checkexpr.py

@@ -1671,6 +1671,10 @@ def check_call(
                 object_type,
                 original_type=callee,
             )
+        elif isinstance(callee, UninhabitedType):
+            ret = UninhabitedType()
+            ret.ambiguous = callee.ambiguous
+            return callee, ret
         else:
             return self.msg.not_callable(callee, context), AnyType(TypeOfAny.from_error)
 

mypy/checkmember.py

@@ -69,6 +69,7 @@
     TypeVarLikeType,
     TypeVarTupleType,
     TypeVarType,
+    UninhabitedType,
     UnionType,
     get_proper_type,
 )
@@ -268,6 +269,10 @@ def _analyze_member_access(
         if not mx.suppress_errors:
             mx.msg.deleted_as_rvalue(typ, mx.context)
         return AnyType(TypeOfAny.from_error)
+    elif isinstance(typ, UninhabitedType):
+        attr_type = UninhabitedType()
+        attr_type.ambiguous = typ.ambiguous
+        return attr_type
     return report_missing_attribute(mx.original_type, typ, name, mx)
 
 

mypy/expandtype.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from collections.abc import Iterable, Mapping, Sequence
+from collections.abc import Iterable, Mapping
 from typing import Final, TypeVar, cast, overload
 
 from mypy.nodes import ARG_STAR, FakeInfo, Var
@@ -209,7 +209,11 @@ def visit_erased_type(self, t: ErasedType) -> Type:
         return t
 
     def visit_instance(self, t: Instance) -> Type:
-        args = self.expand_types_with_unpack(list(t.args))
+        if len(t.args) == 0:
+            # TODO: Why do we need to create a copy here?
+            return t.copy_modified()
+
+        args = self.expand_type_tuple_with_unpack(t.args)
 
         if isinstance(t.type, FakeInfo):
             # The type checker expands function definitions and bodies
@@ -431,7 +435,7 @@ def visit_overloaded(self, t: Overloaded) -> Type:
             items.append(new_item)
         return Overloaded(items)
 
-    def expand_types_with_unpack(self, typs: Sequence[Type]) -> list[Type]:
+    def expand_type_list_with_unpack(self, typs: list[Type]) -> list[Type]:
         """Expands a list of types that has an unpack."""
         items: list[Type] = []
         for item in typs:
@@ -441,8 +445,19 @@ def expand_types_with_unpack(self, typs: Sequence[Type]) -> list[Type]:
                 items.append(item.accept(self))
         return items
 
+    def expand_type_tuple_with_unpack(self, typs: tuple[Type, ...]) -> list[Type]:
+        """Expands a tuple of types that has an unpack."""
+        # Micro-optimization: Specialized variant of expand_type_list_with_unpack
+        items: list[Type] = []
+        for item in typs:
+            if isinstance(item, UnpackType) and isinstance(item.type, TypeVarTupleType):
+                items.extend(self.expand_unpack(item))
+            else:
+                items.append(item.accept(self))
+        return items
+
     def visit_tuple_type(self, t: TupleType) -> Type:
-        items = self.expand_types_with_unpack(t.items)
+        items = self.expand_type_list_with_unpack(t.items)
         if len(items) == 1:
             # Normalize Tuple[*Tuple[X, ...]] -> Tuple[X, ...]
             item = items[0]
@@ -510,7 +525,7 @@ def visit_type_type(self, t: TypeType) -> Type:
     def visit_type_alias_type(self, t: TypeAliasType) -> Type:
         # Target of the type alias cannot contain type variables (not bound by the type
         # alias itself), so we just expand the arguments.
-        args = self.expand_types_with_unpack(t.args)
+        args = self.expand_type_list_with_unpack(t.args)
         # TODO: normalize if target is Tuple, and args are [*tuple[X, ...]]?
         return t.copy_modified(args=args)
 

mypy/indirection.py

@@ -32,11 +32,29 @@ def find_modules(self, typs: Iterable[types.Type]) -> set[str]:
         self.modules = set()
         self.seen_fullnames = set()
         self.seen_aliases = set()
-        self._visit(typs)
+        for typ in typs:
+            self._visit(typ)
         return self.modules
 
-    def _visit(self, typ_or_typs: types.Type | Iterable[types.Type]) -> None:
-        typs = [typ_or_typs] if isinstance(typ_or_typs, types.Type) else typ_or_typs
+    def _visit(self, typ: types.Type) -> None:
+        if isinstance(typ, types.TypeAliasType):
+            # Avoid infinite recursion for recursive type aliases.
+            if typ not in self.seen_aliases:
+                self.seen_aliases.add(typ)
+        typ.accept(self)
+
+    def _visit_type_tuple(self, typs: tuple[types.Type, ...]) -> None:
+        # Micro-optimization: Specialized version of _visit for lists
+        for typ in typs:
+            if isinstance(typ, types.TypeAliasType):
+                # Avoid infinite recursion for recursive type aliases.
+                if typ in self.seen_aliases:
+                    continue
+                self.seen_aliases.add(typ)
+            typ.accept(self)
+
+    def _visit_type_list(self, typs: list[types.Type]) -> None:
+        # Micro-optimization: Specialized version of _visit for tuples
         for typ in typs:
             if isinstance(typ, types.TypeAliasType):
                 # Avoid infinite recursion for recursive type aliases.
@@ -50,7 +68,7 @@ def _visit_module_name(self, module_name: str) -> None:
             self.modules.update(split_module_names(module_name))
 
     def visit_unbound_type(self, t: types.UnboundType) -> None:
-        self._visit(t.args)
+        self._visit_type_tuple(t.args)
 
     def visit_any(self, t: types.AnyType) -> None:
         pass
@@ -68,7 +86,7 @@ def visit_deleted_type(self, t: types.DeletedType) -> None:
         pass
 
     def visit_type_var(self, t: types.TypeVarType) -> None:
-        self._visit(t.values)
+        self._visit_type_list(t.values)
         self._visit(t.upper_bound)
         self._visit(t.default)
 
@@ -84,10 +102,10 @@ def visit_unpack_type(self, t: types.UnpackType) -> None:
         t.type.accept(self)
 
     def visit_parameters(self, t: types.Parameters) -> None:
-        self._visit(t.arg_types)
+        self._visit_type_list(t.arg_types)
 
     def visit_instance(self, t: types.Instance) -> None:
-        self._visit(t.args)
+        self._visit_type_tuple(t.args)
         if t.type:
             # Uses of a class depend on everything in the MRO,
             # as changes to classes in the MRO can add types to methods,
@@ -98,7 +116,7 @@ def visit_instance(self, t: types.Instance) -> None:
                 self._visit_module_name(t.type.metaclass_type.type.module_name)
 
     def visit_callable_type(self, t: types.CallableType) -> None:
-        self._visit(t.arg_types)
+        self._visit_type_list(t.arg_types)
         self._visit(t.ret_type)
         if t.definition is not None:
             fullname = t.definition.fullname
@@ -107,22 +125,22 @@ def visit_callable_type(self, t: types.CallableType) -> None:
                 self.seen_fullnames.add(fullname)
 
     def visit_overloaded(self, t: types.Overloaded) -> None:
-        self._visit(t.items)
+        self._visit_type_list(list(t.items))
         self._visit(t.fallback)
 
     def visit_tuple_type(self, t: types.TupleType) -> None:
-        self._visit(t.items)
+        self._visit_type_list(t.items)
         self._visit(t.partial_fallback)
 
     def visit_typeddict_type(self, t: types.TypedDictType) -> None:
-        self._visit(t.items.values())
+        self._visit_type_list(list(t.items.values()))
         self._visit(t.fallback)
 
     def visit_literal_type(self, t: types.LiteralType) -> None:
         self._visit(t.fallback)
 
     def visit_union_type(self, t: types.UnionType) -> None:
-        self._visit(t.items)
+        self._visit_type_list(t.items)
 
     def visit_partial_type(self, t: types.PartialType) -> None:
         pass

mypy/plugin.py

@@ -846,26 +846,51 @@ def get_additional_deps(self, file: MypyFile) -> list[tuple[int, str, int]]:
         return deps
 
     def get_type_analyze_hook(self, fullname: str) -> Callable[[AnalyzeTypeContext], Type] | None:
-        return self._find_hook(lambda plugin: plugin.get_type_analyze_hook(fullname))
+        # Micro-optimization: Inline iteration over plugins
+        for plugin in self._plugins:
+            hook = plugin.get_type_analyze_hook(fullname)
+            if hook is not None:
+                return hook
+        return None
 
     def get_function_signature_hook(
         self, fullname: str
     ) -> Callable[[FunctionSigContext], FunctionLike] | None:
-        return self._find_hook(lambda plugin: plugin.get_function_signature_hook(fullname))
+        # Micro-optimization: Inline iteration over plugins
+        for plugin in self._plugins:
+            hook = plugin.get_function_signature_hook(fullname)
+            if hook is not None:
+                return hook
+        return None
 
     def get_function_hook(self, fullname: str) -> Callable[[FunctionContext], Type] | None:
         return self._find_hook(lambda plugin: plugin.get_function_hook(fullname))
 
     def get_method_signature_hook(
         self, fullname: str
     ) -> Callable[[MethodSigContext], FunctionLike] | None:
-        return self._find_hook(lambda plugin: plugin.get_method_signature_hook(fullname))
+        # Micro-optimization: Inline iteration over plugins
+        for plugin in self._plugins:
+            hook = plugin.get_method_signature_hook(fullname)
+            if hook is not None:
+                return hook
+        return None
 
     def get_method_hook(self, fullname: str) -> Callable[[MethodContext], Type] | None:
-        return self._find_hook(lambda plugin: plugin.get_method_hook(fullname))
+        # Micro-optimization: Inline iteration over plugins
+        for plugin in self._plugins:
+            hook = plugin.get_method_hook(fullname)
+            if hook is not None:
+                return hook
+        return None
 
     def get_attribute_hook(self, fullname: str) -> Callable[[AttributeContext], Type] | None:
-        return self._find_hook(lambda plugin: plugin.get_attribute_hook(fullname))
+        # Micro-optimization: Inline iteration over plugins
+        for plugin in self._plugins:
+            hook = plugin.get_attribute_hook(fullname)
+            if hook is not None:
+                return hook
+        return None
 
     def get_class_attribute_hook(self, fullname: str) -> Callable[[AttributeContext], Type] | None:
         return self._find_hook(lambda plugin: plugin.get_class_attribute_hook(fullname))
@@ -897,6 +922,6 @@ def get_dynamic_class_hook(
     def _find_hook(self, lookup: Callable[[Plugin], T]) -> T | None:
         for plugin in self._plugins:
             hook = lookup(plugin)
-            if hook:
+            if hook is not None:
                 return hook
         return None

mypy/server/astmerge.py

@@ -345,13 +345,11 @@ def visit_type_alias(self, node: TypeAlias) -> None:
     def fixup(self, node: SN) -> SN:
         if node in self.replacements:
             new = self.replacements[node]
-            skip_slots: tuple[str, ...] = ()
             if isinstance(node, TypeInfo) and isinstance(new, TypeInfo):
                 # Special case: special_alias is not exposed in symbol tables, but may appear
                 # in external types (e.g. named tuples), so we need to update it manually.
-                skip_slots = ("special_alias",)
                 replace_object_state(new.special_alias, node.special_alias)
-            replace_object_state(new, node, skip_slots=skip_slots)
+            replace_object_state(new, node, skip_slots=_get_ignored_slots(new))
             return cast(SN, new)
         return node
 
@@ -556,9 +554,16 @@ def replace_nodes_in_symbol_table(
             if node.node in replacements:
                 new = replacements[node.node]
                 old = node.node
-                # Needed for TypeInfo, see comment in fixup() above.
-                replace_object_state(new, old, skip_slots=("special_alias",))
+                replace_object_state(new, old, skip_slots=_get_ignored_slots(new))
                 node.node = new
             if isinstance(node.node, (Var, TypeAlias)):
                 # Handle them here just in case these aren't exposed through the AST.
                 node.node.accept(NodeReplaceVisitor(replacements))
+
+
+def _get_ignored_slots(node: SymbolNode) -> tuple[str, ...]:
+    if isinstance(node, OverloadedFuncDef):
+        return ("setter",)
+    if isinstance(node, TypeInfo):
+        return ("special_alias",)
+    return ()