Merge branch 'master' into patch-3

Author: BobTheBuidler

docs/source/generics.rst

@@ -630,7 +630,7 @@ Let us illustrate this by few simple examples:
 
      my_circles: list[Circle] = []
      add_one(my_circles)     # This may appear safe, but...
-     my_circles[-1].rotate()  # ...this will fail, since my_circles[0] is now a Shape, not a Circle
+     my_circles[0].rotate()  # ...this will fail, since my_circles[0] is now a Shape, not a Circle
 
   Another example of invariant type is ``dict``. Most mutable containers
   are invariant.

misc/perf_compare.py

@@ -35,11 +35,15 @@ def heading(s: str) -> None:
     print()
 
 
-def build_mypy(target_dir: str) -> None:
+def build_mypy(target_dir: str, multi_file: bool, *, cflags: str | None = None) -> None:
     env = os.environ.copy()
     env["CC"] = "clang"
     env["MYPYC_OPT_LEVEL"] = "2"
     env["PYTHONHASHSEED"] = "1"
+    if multi_file:
+        env["MYPYC_MULTI_FILE"] = "1"
+    if cflags is not None:
+        env["CFLAGS"] = cflags
     cmd = [sys.executable, "setup.py", "--use-mypyc", "build_ext", "--inplace"]
     subprocess.run(cmd, env=env, check=True, cwd=target_dir)
 
@@ -110,6 +114,12 @@ def main() -> None:
         action="store_true",
         help="measure incremental run (fully cached)",
     )
+    parser.add_argument(
+        "--multi-file",
+        default=False,
+        action="store_true",
+        help="compile each mypy module to a separate C file (reduces RAM use)",
+    )
     parser.add_argument(
         "--dont-setup",
         default=False,
@@ -127,9 +137,9 @@ def main() -> None:
     parser.add_argument(
         "-j",
         metavar="N",
-        default=8,
+        default=4,
         type=int,
-        help="set maximum number of parallel builds (default=8)",
+        help="set maximum number of parallel builds (default=4) -- high numbers require a lot of RAM!",
     )
     parser.add_argument(
         "-r",
@@ -155,6 +165,7 @@ def main() -> None:
     args = parser.parse_args()
     incremental: bool = args.incremental
     dont_setup: bool = args.dont_setup
+    multi_file: bool = args.multi_file
     commits = args.commit
     num_runs: int = args.num_runs + 1
     max_workers: int = args.j
@@ -185,7 +196,9 @@ def main() -> None:
         print("(This will take a while...)")
 
         with ThreadPoolExecutor(max_workers=max_workers) as executor:
-            futures = [executor.submit(build_mypy, target_dir) for target_dir in target_dirs]
+            futures = [
+                executor.submit(build_mypy, target_dir, multi_file) for target_dir in target_dirs
+            ]
             for future in as_completed(futures):
                 future.result()
 

misc/profile_self_check.py

@@ -0,0 +1,129 @@
+"""Compile mypy using mypyc and profile self-check using perf.
+
+Notes:
+ - Only Linux is supported for now (TODO: add support for other profilers)
+ - The profile is collected at C level
+   - It includes C functions compiled by mypyc and CPython runtime functions
+ - The names of mypy functions are mangled to C names, but usually it's clear what they mean
+   - Generally CPyDef_ prefix for native functions and CPyPy_ prefix for wrapper functions
+ - It's important to compile CPython using special flags (see below) to get good results
+ - Generally use the latest Python feature release (or the most recent beta if supported by mypyc)
+ - The tool prints a command that can be used to analyze the profile afterwards
+
+You may need to adjust kernel parameters temporarily, e.g. this (note that this has security
+implications):
+
+  sudo sysctl kernel.perf_event_paranoid=-1
+
+This is the recommended way to configure CPython for profiling:
+
+  ./configure \
+      --enable-optimizations \
+      --with-lto \
+      CFLAGS="-O2 -g -fno-omit-frame-pointer"
+"""
+
+import argparse
+import glob
+import os
+import shutil
+import subprocess
+import sys
+import time
+
+from perf_compare import build_mypy, clone
+
+# Use these C compiler flags when compiling mypy (important). Note that it's strongly recommended
+# to also compile CPython using similar flags, but we don't enforce it in this script.
+CFLAGS = "-O2 -fno-omit-frame-pointer -g"
+
+# Generated files, including binaries, go under this directory to avoid overwriting user state.
+TARGET_DIR = "mypy.profile.tmpdir"
+
+
+def _profile_self_check(target_dir: str) -> None:
+    cache_dir = os.path.join(target_dir, ".mypy_cache")
+    if os.path.exists(cache_dir):
+        shutil.rmtree(cache_dir)
+    files = []
+    for pat in "mypy/*.py", "mypy/*/*.py", "mypyc/*.py", "mypyc/test/*.py":
+        files.extend(glob.glob(pat))
+    self_check_cmd = ["python", "-m", "mypy", "--config-file", "mypy_self_check.ini"] + files
+    cmdline = ["perf", "record", "-g"] + self_check_cmd
+    t0 = time.time()
+    subprocess.run(cmdline, cwd=target_dir, check=True)
+    elapsed = time.time() - t0
+    print(f"{elapsed:.2f}s elapsed")
+
+
+def profile_self_check(target_dir: str) -> None:
+    try:
+        _profile_self_check(target_dir)
+    except subprocess.CalledProcessError:
+        print("\nProfiling failed! You may missing some permissions.")
+        print("\nThis may help (note that it has security implications):")
+        print("  sudo sysctl kernel.perf_event_paranoid=-1")
+        sys.exit(1)
+
+
+def check_requirements() -> None:
+    if sys.platform != "linux":
+        # TODO: How to make this work on other platforms?
+        sys.exit("error: Only Linux is supported")
+
+    try:
+        subprocess.run(["perf", "-h"], capture_output=True)
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        print("error: The 'perf' profiler is not installed")
+        sys.exit(1)
+
+    try:
+        subprocess.run(["clang", "--version"], capture_output=True)
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        print("error: The clang compiler is not installed")
+        sys.exit(1)
+
+    if not os.path.isfile("mypy_self_check.ini"):
+        print("error: Run this in the mypy repository root")
+        sys.exit(1)
+
+
+def main() -> None:
+    check_requirements()
+
+    parser = argparse.ArgumentParser(
+        description="Compile mypy and profile self checking using 'perf'."
+    )
+    parser.add_argument(
+        "--multi-file",
+        action="store_true",
+        help="compile mypy into one C file per module (to reduce RAM use during compilation)",
+    )
+    parser.add_argument(
+        "--skip-compile", action="store_true", help="use compiled mypy from previous run"
+    )
+    args = parser.parse_args()
+    multi_file: bool = args.multi_file
+    skip_compile: bool = args.skip_compile
+
+    target_dir = TARGET_DIR
+
+    if not skip_compile:
+        clone(target_dir, "HEAD")
+
+        print(f"Building mypy in {target_dir}...")
+        build_mypy(target_dir, multi_file, cflags=CFLAGS)
+    elif not os.path.isdir(target_dir):
+        sys.exit("error: Can't find compile mypy from previous run -- can't use --skip-compile")
+
+    profile_self_check(target_dir)
+
+    print()
+    print('NOTE: Compile CPython using CFLAGS="-O2 -g -fno-omit-frame-pointer" for good results')
+    print()
+    print("CPU profile collected. You can now analyze the profile:")
+    print(f"  perf report -i {target_dir}/perf.data ")
+
+
+if __name__ == "__main__":
+    main()

mypy/checker.py

@@ -6185,6 +6185,7 @@ def find_isinstance_check_helper(
                                     self.lookup_type(expr),
                                     [TypeRange(node.callee.type_is, is_upper_bound=False)],
                                     expr,
+                                    consider_runtime_isinstance=False,
                                 ),
                             )
         elif isinstance(node, ComparisonExpr):
@@ -7612,11 +7613,19 @@ def conditional_types_with_intersection(
         type_ranges: list[TypeRange] | None,
         ctx: Context,
         default: None = None,
+        *,
+        consider_runtime_isinstance: bool = True,
     ) -> tuple[Type | None, Type | None]: ...
 
     @overload
     def conditional_types_with_intersection(
-        self, expr_type: Type, type_ranges: list[TypeRange] | None, ctx: Context, default: Type
+        self,
+        expr_type: Type,
+        type_ranges: list[TypeRange] | None,
+        ctx: Context,
+        default: Type,
+        *,
+        consider_runtime_isinstance: bool = True,
     ) -> tuple[Type, Type]: ...
 
     def conditional_types_with_intersection(
@@ -7625,8 +7634,15 @@ def conditional_types_with_intersection(
         type_ranges: list[TypeRange] | None,
         ctx: Context,
         default: Type | None = None,
+        *,
+        consider_runtime_isinstance: bool = True,
     ) -> tuple[Type | None, Type | None]:
-        initial_types = conditional_types(expr_type, type_ranges, default)
+        initial_types = conditional_types(
+            expr_type,
+            type_ranges,
+            default,
+            consider_runtime_isinstance=consider_runtime_isinstance,
+        )
         # For some reason, doing "yes_map, no_map = conditional_types_to_typemaps(...)"
         # doesn't work: mypyc will decide that 'yes_map' is of type None if we try.
         yes_type: Type | None = initial_types[0]
@@ -7938,18 +7954,30 @@ def visit_type_var(self, t: TypeVarType) -> None:
 
 @overload
 def conditional_types(
-    current_type: Type, proposed_type_ranges: list[TypeRange] | None, default: None = None
+    current_type: Type,
+    proposed_type_ranges: list[TypeRange] | None,
+    default: None = None,
+    *,
+    consider_runtime_isinstance: bool = True,
 ) -> tuple[Type | None, Type | None]: ...
 
 
 @overload
 def conditional_types(
-    current_type: Type, proposed_type_ranges: list[TypeRange] | None, default: Type
+    current_type: Type,
+    proposed_type_ranges: list[TypeRange] | None,
+    default: Type,
+    *,
+    consider_runtime_isinstance: bool = True,
 ) -> tuple[Type, Type]: ...
 
 
 def conditional_types(
-    current_type: Type, proposed_type_ranges: list[TypeRange] | None, default: Type | None = None
+    current_type: Type,
+    proposed_type_ranges: list[TypeRange] | None,
+    default: Type | None = None,
+    *,
+    consider_runtime_isinstance: bool = True,
 ) -> tuple[Type | None, Type | None]:
     """Takes in the current type and a proposed type of an expression.
 
@@ -7991,7 +8019,11 @@ def conditional_types(
                     if not type_range.is_upper_bound
                 ]
             )
-            remaining_type = restrict_subtype_away(current_type, proposed_precise_type)
+            remaining_type = restrict_subtype_away(
+                current_type,
+                proposed_precise_type,
+                consider_runtime_isinstance=consider_runtime_isinstance,
+            )
             return proposed_type, remaining_type
     else:
         # An isinstance check, but we don't understand the type

mypy/checkexpr.py

@@ -318,6 +318,8 @@ class ExpressionChecker(ExpressionVisitor[Type], ExpressionCheckerSharedApi):
     strfrm_checker: StringFormatterChecker
     plugin: Plugin
 
+    _arg_infer_context_cache: ArgumentInferContext | None
+
     def __init__(
         self,
         chk: mypy.checker.TypeChecker,
@@ -352,6 +354,8 @@ def __init__(
         self.is_callee = False
         type_state.infer_polymorphic = not self.chk.options.old_type_inference
 
+        self._arg_infer_context_cache = None
+
     def reset(self) -> None:
         self.resolved_type = {}
 
@@ -2277,9 +2281,11 @@ def infer_function_type_arguments_pass2(
         return callee_type, inferred_args
 
     def argument_infer_context(self) -> ArgumentInferContext:
-        return ArgumentInferContext(
-            self.chk.named_type("typing.Mapping"), self.chk.named_type("typing.Iterable")
-        )
+        if self._arg_infer_context_cache is None:
+            self._arg_infer_context_cache = ArgumentInferContext(
+                self.chk.named_type("typing.Mapping"), self.chk.named_type("typing.Iterable")
+            )
+        return self._arg_infer_context_cache
 
     def get_arg_infer_passes(
         self,

mypy/subtypes.py

@@ -2073,7 +2073,7 @@ def try_restrict_literal_union(t: UnionType, s: Type) -> list[Type] | None:
     return new_items
 
 
-def restrict_subtype_away(t: Type, s: Type) -> Type:
+def restrict_subtype_away(t: Type, s: Type, *, consider_runtime_isinstance: bool = True) -> Type:
     """Return t minus s for runtime type assertions.
 
     If we can't determine a precise result, return a supertype of the
@@ -2087,16 +2087,27 @@ def restrict_subtype_away(t: Type, s: Type) -> Type:
         new_items = try_restrict_literal_union(p_t, s)
         if new_items is None:
             new_items = [
-                restrict_subtype_away(item, s)
+                restrict_subtype_away(
+                    item, s, consider_runtime_isinstance=consider_runtime_isinstance
+                )
                 for item in p_t.relevant_items()
-                if (isinstance(get_proper_type(item), AnyType) or not covers_at_runtime(item, s))
             ]
-        return UnionType.make_union(new_items)
+        return UnionType.make_union(
+            [item for item in new_items if not isinstance(get_proper_type(item), UninhabitedType)]
+        )
     elif isinstance(p_t, TypeVarType):
         return p_t.copy_modified(upper_bound=restrict_subtype_away(p_t.upper_bound, s))
-    elif covers_at_runtime(t, s):
-        return UninhabitedType()
+
+    if consider_runtime_isinstance:
+        if covers_at_runtime(t, s):
+            return UninhabitedType()
+        else:
+            return t
     else:
+        if is_proper_subtype(t, s, ignore_promotions=True):
+            return UninhabitedType()
+        if is_proper_subtype(t, s, ignore_promotions=True, erase_instances=True):
+            return UninhabitedType()
         return t
 
 
@@ -2106,7 +2117,8 @@ def covers_at_runtime(item: Type, supertype: Type) -> bool:
     supertype = get_proper_type(supertype)
 
     # Since runtime type checks will ignore type arguments, erase the types.
-    supertype = erase_type(supertype)
+    if not (isinstance(supertype, CallableType) and supertype.is_type_obj()):
+        supertype = erase_type(supertype)
     if is_proper_subtype(
         erase_type(item), supertype, ignore_promotions=True, erase_instances=True
     ):

mypy/suggestions.py

@@ -852,7 +852,7 @@ def visit_instance(self, t: Instance) -> str:
         if self.module:
             parts = obj.split(".")  # need to split the object part if it is a nested class
             tree = self.graph[self.module].tree
-            if tree and parts[0] in tree.names:
+            if tree and parts[0] in tree.names and mod not in tree.names:
                 mod = self.module
 
         if (mod, obj) == ("builtins", "tuple"):

mypy/typeanal.py

@@ -1109,8 +1109,8 @@ def visit_callable_type(
                 variables = t.variables
             else:
                 variables, _ = self.bind_function_type_variables(t, t)
-            type_guard = self.anal_type_guard(t.ret_type)
-            type_is = self.anal_type_is(t.ret_type)
+            type_guard = self.anal_type_guard(t.ret_type) if t.type_guard is None else t.type_guard
+            type_is = self.anal_type_is(t.ret_type) if t.type_is is None else t.type_is
 
             arg_kinds = t.arg_kinds
             arg_types = []