Merge branch 'master' into bugfix/st-synthetic-named-expr-in-match

Author: sterliakov

misc/perf_compare.py

@@ -35,13 +35,15 @@ def heading(s: str) -> None:
     print()
 
 
-def build_mypy(target_dir: str, multi_file: bool) -> None:
+def build_mypy(target_dir: str, multi_file: bool, *, cflags: str | None = None) -> None:
     env = os.environ.copy()
     env["CC"] = "clang"
     env["MYPYC_OPT_LEVEL"] = "2"
     env["PYTHONHASHSEED"] = "1"
     if multi_file:
         env["MYPYC_MULTI_FILE"] = "1"
+    if cflags is not None:
+        env["CFLAGS"] = cflags
     cmd = [sys.executable, "setup.py", "--use-mypyc", "build_ext", "--inplace"]
     subprocess.run(cmd, env=env, check=True, cwd=target_dir)
 

misc/profile_self_check.py

@@ -0,0 +1,129 @@
+"""Compile mypy using mypyc and profile self-check using perf.
+
+Notes:
+ - Only Linux is supported for now (TODO: add support for other profilers)
+ - The profile is collected at C level
+   - It includes C functions compiled by mypyc and CPython runtime functions
+ - The names of mypy functions are mangled to C names, but usually it's clear what they mean
+   - Generally CPyDef_ prefix for native functions and CPyPy_ prefix for wrapper functions
+ - It's important to compile CPython using special flags (see below) to get good results
+ - Generally use the latest Python feature release (or the most recent beta if supported by mypyc)
+ - The tool prints a command that can be used to analyze the profile afterwards
+
+You may need to adjust kernel parameters temporarily, e.g. this (note that this has security
+implications):
+
+  sudo sysctl kernel.perf_event_paranoid=-1
+
+This is the recommended way to configure CPython for profiling:
+
+  ./configure \
+      --enable-optimizations \
+      --with-lto \
+      CFLAGS="-O2 -g -fno-omit-frame-pointer"
+"""
+
+import argparse
+import glob
+import os
+import shutil
+import subprocess
+import sys
+import time
+
+from perf_compare import build_mypy, clone
+
+# Use these C compiler flags when compiling mypy (important). Note that it's strongly recommended
+# to also compile CPython using similar flags, but we don't enforce it in this script.
+CFLAGS = "-O2 -fno-omit-frame-pointer -g"
+
+# Generated files, including binaries, go under this directory to avoid overwriting user state.
+TARGET_DIR = "mypy.profile.tmpdir"
+
+
+def _profile_self_check(target_dir: str) -> None:
+    cache_dir = os.path.join(target_dir, ".mypy_cache")
+    if os.path.exists(cache_dir):
+        shutil.rmtree(cache_dir)
+    files = []
+    for pat in "mypy/*.py", "mypy/*/*.py", "mypyc/*.py", "mypyc/test/*.py":
+        files.extend(glob.glob(pat))
+    self_check_cmd = ["python", "-m", "mypy", "--config-file", "mypy_self_check.ini"] + files
+    cmdline = ["perf", "record", "-g"] + self_check_cmd
+    t0 = time.time()
+    subprocess.run(cmdline, cwd=target_dir, check=True)
+    elapsed = time.time() - t0
+    print(f"{elapsed:.2f}s elapsed")
+
+
+def profile_self_check(target_dir: str) -> None:
+    try:
+        _profile_self_check(target_dir)
+    except subprocess.CalledProcessError:
+        print("\nProfiling failed! You may missing some permissions.")
+        print("\nThis may help (note that it has security implications):")
+        print("  sudo sysctl kernel.perf_event_paranoid=-1")
+        sys.exit(1)
+
+
+def check_requirements() -> None:
+    if sys.platform != "linux":
+        # TODO: How to make this work on other platforms?
+        sys.exit("error: Only Linux is supported")
+
+    try:
+        subprocess.run(["perf", "-h"], capture_output=True)
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        print("error: The 'perf' profiler is not installed")
+        sys.exit(1)
+
+    try:
+        subprocess.run(["clang", "--version"], capture_output=True)
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        print("error: The clang compiler is not installed")
+        sys.exit(1)
+
+    if not os.path.isfile("mypy_self_check.ini"):
+        print("error: Run this in the mypy repository root")
+        sys.exit(1)
+
+
+def main() -> None:
+    check_requirements()
+
+    parser = argparse.ArgumentParser(
+        description="Compile mypy and profile self checking using 'perf'."
+    )
+    parser.add_argument(
+        "--multi-file",
+        action="store_true",
+        help="compile mypy into one C file per module (to reduce RAM use during compilation)",
+    )
+    parser.add_argument(
+        "--skip-compile", action="store_true", help="use compiled mypy from previous run"
+    )
+    args = parser.parse_args()
+    multi_file: bool = args.multi_file
+    skip_compile: bool = args.skip_compile
+
+    target_dir = TARGET_DIR
+
+    if not skip_compile:
+        clone(target_dir, "HEAD")
+
+        print(f"Building mypy in {target_dir}...")
+        build_mypy(target_dir, multi_file, cflags=CFLAGS)
+    elif not os.path.isdir(target_dir):
+        sys.exit("error: Can't find compile mypy from previous run -- can't use --skip-compile")
+
+    profile_self_check(target_dir)
+
+    print()
+    print('NOTE: Compile CPython using CFLAGS="-O2 -g -fno-omit-frame-pointer" for good results')
+    print()
+    print("CPU profile collected. You can now analyze the profile:")
+    print(f"  perf report -i {target_dir}/perf.data ")
+
+
+if __name__ == "__main__":
+    main()

mypy/subtypes.py

@@ -2117,7 +2117,8 @@ def covers_at_runtime(item: Type, supertype: Type) -> bool:
     supertype = get_proper_type(supertype)
 
     # Since runtime type checks will ignore type arguments, erase the types.
-    supertype = erase_type(supertype)
+    if not (isinstance(supertype, FunctionLike) and supertype.is_type_obj()):
+        supertype = erase_type(supertype)
     if is_proper_subtype(
         erase_type(item), supertype, ignore_promotions=True, erase_instances=True
     ):

mypy/typeanal.py

@@ -1109,8 +1109,8 @@ def visit_callable_type(
                 variables = t.variables
             else:
                 variables, _ = self.bind_function_type_variables(t, t)
-            type_guard = self.anal_type_guard(t.ret_type)
-            type_is = self.anal_type_is(t.ret_type)
+            type_guard = self.anal_type_guard(t.ret_type) if t.type_guard is None else t.type_guard
+            type_is = self.anal_type_is(t.ret_type) if t.type_is is None else t.type_is
 
             arg_kinds = t.arg_kinds
             arg_types = []

test-data/unit/check-python310.test

@@ -2684,6 +2684,28 @@ def f(t: T) -> None:
             reveal_type(k)  # N: Revealed type is "tuple[builtins.int, fallback=__main__.K]"
 [builtins fixtures/tuple.pyi]
 
+[case testMatchTypeObjectTypeVar]
+# flags: --warn-unreachable
+from typing import TypeVar
+import b
+
+T_Choice = TypeVar("T_Choice", bound=b.One | b.Two)
+
+def switch(choice: type[T_Choice]) -> None:
+    match choice:
+        case b.One:
+            reveal_type(choice)  # N: Revealed type is "def () -> b.One"
+        case b.Two:
+            reveal_type(choice)  # N: Revealed type is "def () -> b.Two"
+        case _:
+            reveal_type(choice)  # N: Revealed type is "type[T_Choice`-1]"
+
+[file b.py]
+class One: ...
+class Two: ...
+
+[builtins fixtures/tuple.pyi]
+
 [case testNewRedefineMatchBasics]
 # flags: --allow-redefinition-new --local-partial-types
 

test-data/unit/check-typeguard.test

@@ -813,3 +813,20 @@ raw_target: object
 if isinstance(raw_target, type) and is_xlike(raw_target):
     reveal_type(raw_target)  # N: Revealed type is "type[__main__.X]"
 [builtins fixtures/tuple.pyi]
+
+[case testTypeGuardWithDefer]
+from typing import Union
+from typing_extensions import TypeGuard
+
+class A: ...
+class B: ...
+
+def is_a(x: object) -> TypeGuard[A]:
+    return defer_not_defined()  # E: Name "defer_not_defined" is not defined
+
+def main(x: Union[A, B]) -> None:
+    if is_a(x):
+        reveal_type(x)  # N: Revealed type is "__main__.A"
+    else:
+        reveal_type(x)  # N: Revealed type is "Union[__main__.A, __main__.B]"
+[builtins fixtures/tuple.pyi]

test-data/unit/check-typeis.test

@@ -933,3 +933,20 @@ def main(f: Callable[[], int | Awaitable[int]]) -> None:
     else:
         reveal_type(f)  # N: Revealed type is "def () -> Union[builtins.int, typing.Awaitable[builtins.int]]"
 [builtins fixtures/tuple.pyi]
+
+[case testTypeIsWithDefer]
+from typing import Union
+from typing_extensions import TypeIs
+
+class A: ...
+class B: ...
+
+def is_a(x: object) -> TypeIs[A]:
+    return defer_not_defined()  # E: Name "defer_not_defined" is not defined
+
+def main(x: Union[A, B]) -> None:
+    if is_a(x):
+        reveal_type(x)  # N: Revealed type is "__main__.A"
+    else:
+        reveal_type(x)  # N: Revealed type is "__main__.B"
+[builtins fixtures/tuple.pyi]