Merge branch 'master' into fix/fix_narrow_promotions_in_unions

Author: ilevkivskyi

docs/source/error_code_list.rst

@@ -215,6 +215,35 @@ You can use :py:class:`~collections.abc.Callable` as the type for callable objec
         for x in objs:
             f(x)
 
+.. _code-metaclass:
+
+Check the validity of a class's metaclass [metaclass]
+-----------------------------------------------------
+
+Mypy checks whether the metaclass of a class is valid. The metaclass
+must be a subclass of ``type``. Further, the class hierarchy must yield
+a consistent metaclass. For more details, see the
+`Python documentation <https://docs.python.org/3.13/reference/datamodel.html#determining-the-appropriate-metaclass>`_
+
+Note that mypy's metaclass checking is limited and may produce false-positives.
+See also :ref:`limitations`.
+
+Example with an error:
+
+.. code-block:: python
+
+    class GoodMeta(type):
+        pass
+
+    class BadMeta:
+        pass
+
+    class A1(metaclass=GoodMeta):  # OK
+        pass
+
+    class A2(metaclass=BadMeta):  # Error:  Metaclasses not inheriting from "type" are not supported  [metaclass]
+        pass
+
 .. _code-var-annotated:
 
 Require annotation if variable type is unclear [var-annotated]

docs/source/generics.rst

@@ -630,7 +630,7 @@ Let us illustrate this by few simple examples:
 
      my_circles: list[Circle] = []
      add_one(my_circles)     # This may appear safe, but...
-     my_circles[-1].rotate()  # ...this will fail, since my_circles[0] is now a Shape, not a Circle
+     my_circles[0].rotate()  # ...this will fail, since my_circles[0] is now a Shape, not a Circle
 
   Another example of invariant type is ``dict``. Most mutable containers
   are invariant.

docs/source/metaclasses.rst

@@ -90,3 +90,28 @@ so it's better not to combine metaclasses and class hierarchies:
 * ``Self`` is not allowed as annotation in metaclasses as per `PEP 673`_.
 
 .. _PEP 673: https://peps.python.org/pep-0673/#valid-locations-for-self
+
+For some builtin types, mypy may think their metaclass is :py:class:`abc.ABCMeta`
+even if it is :py:class:`type` at runtime. In those cases, you can either:
+
+* use :py:class:`abc.ABCMeta` instead of :py:class:`type` as the
+  superclass of your metaclass if that works in your use-case
+* mute the error with ``# type: ignore[metaclass]``
+
+.. code-block:: python
+
+    import abc
+
+    assert type(tuple) is type  # metaclass of tuple is type at runtime
+
+    # The problem:
+    class M0(type): pass
+    class A0(tuple, metaclass=M0): pass  # Mypy Error: metaclass conflict
+
+    # Option 1: use ABCMeta instead of type
+    class M1(abc.ABCMeta): pass
+    class A1(tuple, metaclass=M1): pass
+
+    # Option 2: mute the error
+    class M2(type): pass
+    class A2(tuple, metaclass=M2): pass  # type: ignore[metaclass]

misc/perf_compare.py

@@ -35,11 +35,15 @@ def heading(s: str) -> None:
     print()
 
 
-def build_mypy(target_dir: str) -> None:
+def build_mypy(target_dir: str, multi_file: bool, *, cflags: str | None = None) -> None:
     env = os.environ.copy()
     env["CC"] = "clang"
     env["MYPYC_OPT_LEVEL"] = "2"
     env["PYTHONHASHSEED"] = "1"
+    if multi_file:
+        env["MYPYC_MULTI_FILE"] = "1"
+    if cflags is not None:
+        env["CFLAGS"] = cflags
     cmd = [sys.executable, "setup.py", "--use-mypyc", "build_ext", "--inplace"]
     subprocess.run(cmd, env=env, check=True, cwd=target_dir)
 
@@ -110,6 +114,12 @@ def main() -> None:
         action="store_true",
         help="measure incremental run (fully cached)",
     )
+    parser.add_argument(
+        "--multi-file",
+        default=False,
+        action="store_true",
+        help="compile each mypy module to a separate C file (reduces RAM use)",
+    )
     parser.add_argument(
         "--dont-setup",
         default=False,
@@ -127,9 +137,9 @@ def main() -> None:
     parser.add_argument(
         "-j",
         metavar="N",
-        default=8,
+        default=4,
         type=int,
-        help="set maximum number of parallel builds (default=8)",
+        help="set maximum number of parallel builds (default=4) -- high numbers require a lot of RAM!",
     )
     parser.add_argument(
         "-r",
@@ -155,6 +165,7 @@ def main() -> None:
     args = parser.parse_args()
     incremental: bool = args.incremental
     dont_setup: bool = args.dont_setup
+    multi_file: bool = args.multi_file
     commits = args.commit
     num_runs: int = args.num_runs + 1
     max_workers: int = args.j
@@ -185,7 +196,9 @@ def main() -> None:
         print("(This will take a while...)")
 
         with ThreadPoolExecutor(max_workers=max_workers) as executor:
-            futures = [executor.submit(build_mypy, target_dir) for target_dir in target_dirs]
+            futures = [
+                executor.submit(build_mypy, target_dir, multi_file) for target_dir in target_dirs
+            ]
             for future in as_completed(futures):
                 future.result()
 

misc/profile_check.py

@@ -0,0 +1,145 @@
+"""Compile mypy using mypyc and profile type checking using perf.
+
+By default does a self check.
+
+Notes:
+ - Only Linux is supported for now (TODO: add support for other profilers)
+ - The profile is collected at C level
+   - It includes C functions compiled by mypyc and CPython runtime functions
+ - The names of mypy functions are mangled to C names, but usually it's clear what they mean
+   - Generally CPyDef_ prefix for native functions and CPyPy_ prefix for wrapper functions
+ - It's important to compile CPython using special flags (see below) to get good results
+ - Generally use the latest Python feature release (or the most recent beta if supported by mypyc)
+ - The tool prints a command that can be used to analyze the profile afterwards
+
+You may need to adjust kernel parameters temporarily, e.g. this (note that this has security
+implications):
+
+  sudo sysctl kernel.perf_event_paranoid=-1
+
+This is the recommended way to configure CPython for profiling:
+
+  ./configure \
+      --enable-optimizations \
+      --with-lto \
+      CFLAGS="-O2 -g -fno-omit-frame-pointer"
+"""
+
+from __future__ import annotations
+
+import argparse
+import glob
+import os
+import shutil
+import subprocess
+import sys
+import time
+
+from perf_compare import build_mypy, clone
+
+# Use these C compiler flags when compiling mypy (important). Note that it's strongly recommended
+# to also compile CPython using similar flags, but we don't enforce it in this script.
+CFLAGS = "-O2 -fno-omit-frame-pointer -g"
+
+# Generated files, including binaries, go under this directory to avoid overwriting user state.
+TARGET_DIR = "mypy.profile.tmpdir"
+
+
+def _profile_type_check(target_dir: str, code: str | None) -> None:
+    cache_dir = os.path.join(target_dir, ".mypy_cache")
+    if os.path.exists(cache_dir):
+        shutil.rmtree(cache_dir)
+    args = []
+    if code is None:
+        args.extend(["--config-file", "mypy_self_check.ini"])
+        for pat in "mypy/*.py", "mypy/*/*.py", "mypyc/*.py", "mypyc/test/*.py":
+            args.extend(glob.glob(pat))
+    else:
+        args.extend(["-c", code])
+    check_cmd = ["python", "-m", "mypy"] + args
+    cmdline = ["perf", "record", "-g"] + check_cmd
+    t0 = time.time()
+    subprocess.run(cmdline, cwd=target_dir, check=True)
+    elapsed = time.time() - t0
+    print(f"{elapsed:.2f}s elapsed")
+
+
+def profile_type_check(target_dir: str, code: str | None) -> None:
+    try:
+        _profile_type_check(target_dir, code)
+    except subprocess.CalledProcessError:
+        print("\nProfiling failed! You may missing some permissions.")
+        print("\nThis may help (note that it has security implications):")
+        print("  sudo sysctl kernel.perf_event_paranoid=-1")
+        sys.exit(1)
+
+
+def check_requirements() -> None:
+    if sys.platform != "linux":
+        # TODO: How to make this work on other platforms?
+        sys.exit("error: Only Linux is supported")
+
+    try:
+        subprocess.run(["perf", "-h"], capture_output=True)
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        print("error: The 'perf' profiler is not installed")
+        sys.exit(1)
+
+    try:
+        subprocess.run(["clang", "--version"], capture_output=True)
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        print("error: The clang compiler is not installed")
+        sys.exit(1)
+
+    if not os.path.isfile("mypy_self_check.ini"):
+        print("error: Run this in the mypy repository root")
+        sys.exit(1)
+
+
+def main() -> None:
+    check_requirements()
+
+    parser = argparse.ArgumentParser(
+        description="Compile mypy and profile type checking using 'perf' (by default, self check)."
+    )
+    parser.add_argument(
+        "--multi-file",
+        action="store_true",
+        help="compile mypy into one C file per module (to reduce RAM use during compilation)",
+    )
+    parser.add_argument(
+        "--skip-compile", action="store_true", help="use compiled mypy from previous run"
+    )
+    parser.add_argument(
+        "-c",
+        metavar="CODE",
+        default=None,
+        type=str,
+        help="profile type checking Python code fragment instead of mypy self-check",
+    )
+    args = parser.parse_args()
+    multi_file: bool = args.multi_file
+    skip_compile: bool = args.skip_compile
+    code: str | None = args.c
+
+    target_dir = TARGET_DIR
+
+    if not skip_compile:
+        clone(target_dir, "HEAD")
+
+        print(f"Building mypy in {target_dir}...")
+        build_mypy(target_dir, multi_file, cflags=CFLAGS)
+    elif not os.path.isdir(target_dir):
+        sys.exit("error: Can't find compile mypy from previous run -- can't use --skip-compile")
+
+    profile_type_check(target_dir, code)
+
+    print()
+    print('NOTE: Compile CPython using CFLAGS="-O2 -g -fno-omit-frame-pointer" for good results')
+    print()
+    print("CPU profile collected. You can now analyze the profile:")
+    print(f"  perf report -i {target_dir}/perf.data ")
+
+
+if __name__ == "__main__":
+    main()

misc/upload-pypi.py

@@ -108,7 +108,7 @@ def tmp_twine() -> Iterator[Path]:
 def upload_dist(dist: Path, dry_run: bool = True) -> None:
     with tmp_twine() as twine:
         files = [item for item in dist.iterdir() if item_ok_for_pypi(item.name)]
-        cmd: list[Any] = [twine, "upload"]
+        cmd: list[Any] = [twine, "upload", "--skip-existing"]
         cmd += files
         if dry_run:
             print("[dry run] " + " ".join(map(str, cmd)))