Merge remote-tracking branch 'upstream/master' into fix-nested-deferral

Author: ilevkivskyi

.github/workflows/test.yml

@@ -33,7 +33,7 @@ jobs:
         # the oldest and newest supported Python versions
         - name: Test suite with py39-ubuntu, mypyc-compiled
           python: '3.9'
-          os: ubuntu-22.04-arm
+          os: ubuntu-24.04-arm
           toxenv: py
           tox_extra_args: "-n 4"
           test_mypyc: true
@@ -44,31 +44,31 @@ jobs:
           tox_extra_args: "-n 4"
         - name: Test suite with py310-ubuntu
           python: '3.10'
-          os: ubuntu-22.04-arm
+          os: ubuntu-24.04-arm
           toxenv: py
           tox_extra_args: "-n 4"
         - name: Test suite with py311-ubuntu, mypyc-compiled
           python: '3.11'
-          os: ubuntu-22.04-arm
+          os: ubuntu-24.04-arm
           toxenv: py
           tox_extra_args: "-n 4"
           test_mypyc: true
         - name: Test suite with py312-ubuntu, mypyc-compiled
           python: '3.12'
-          os: ubuntu-22.04-arm
+          os: ubuntu-24.04-arm
           toxenv: py
           tox_extra_args: "-n 4"
           test_mypyc: true
         - name: Test suite with py313-ubuntu, mypyc-compiled
           python: '3.13'
-          os: ubuntu-22.04-arm
+          os: ubuntu-24.04-arm
           toxenv: py
           tox_extra_args: "-n 4"
           test_mypyc: true
 
         # - name: Test suite with py314-dev-ubuntu
         #   python: '3.14-dev'
-        #   os: ubuntu-22.04-arm
+        #   os: ubuntu-24.04-arm
         #   toxenv: py
         #   tox_extra_args: "-n 4"
         #   allow_failure: true

mypy/semanal_main.py

@@ -26,7 +26,9 @@
 
 from __future__ import annotations
 
+from collections.abc import Iterator
 from contextlib import nullcontext
+from itertools import groupby
 from typing import TYPE_CHECKING, Callable, Final, Optional, Union
 from typing_extensions import TypeAlias as _TypeAlias
 
@@ -232,26 +234,66 @@ def process_top_levels(graph: Graph, scc: list[str], patches: Patches) -> None:
         final_iteration = not any_progress
 
 
+def order_by_subclassing(targets: list[FullTargetInfo]) -> Iterator[FullTargetInfo]:
+    """Make sure that superclass methods are always processed before subclass methods.
+
+    This algorithm is not very optimal, but it is simple and should work well for lists
+    that are already almost correctly ordered.
+    """
+
+    # First, group the targets by their TypeInfo (since targets are sorted by line,
+    # we know that each TypeInfo will appear as group key only once).
+    grouped = [(k, list(g)) for k, g in groupby(targets, key=lambda x: x[3])]
+    remaining_infos = {info for info, _ in grouped if info is not None}
+
+    next_group = 0
+    while grouped:
+        if next_group >= len(grouped):
+            # This should never happen, if there is an MRO cycle, it should be reported
+            # and fixed during top-level processing.
+            raise ValueError("Cannot order method targets by MRO")
+        next_info, group = grouped[next_group]
+        if next_info is None:
+            # Trivial case, not methods but functions, process them straight away.
+            yield from group
+            grouped.pop(next_group)
+            continue
+        if any(parent in remaining_infos for parent in next_info.mro[1:]):
+            # We cannot process this method group yet, try a next one.
+            next_group += 1
+            continue
+        yield from group
+        grouped.pop(next_group)
+        remaining_infos.discard(next_info)
+        # Each time after processing a method group we should retry from start,
+        # since there may be some groups that are not blocked on parents anymore.
+        next_group = 0
+
+
 def process_functions(graph: Graph, scc: list[str], patches: Patches) -> None:
     # Process functions.
+    all_targets = []
     for module in scc:
         tree = graph[module].tree
         assert tree is not None
-        analyzer = graph[module].manager.semantic_analyzer
         # In principle, functions can be processed in arbitrary order,
         # but _methods_ must be processed in the order they are defined,
         # because some features (most notably partial types) depend on
         # order of definitions on self.
         #
         # There can be multiple generated methods per line. Use target
-        # name as the second sort key to get a repeatable sort order on
-        # Python 3.5, which doesn't preserve dictionary order.
+        # name as the second sort key to get a repeatable sort order.
         targets = sorted(get_all_leaf_targets(tree), key=lambda x: (x[1].line, x[0]))
-        for target, node, active_type in targets:
-            assert isinstance(node, (FuncDef, OverloadedFuncDef, Decorator))
-            process_top_level_function(
-                analyzer, graph[module], module, target, node, active_type, patches
-            )
+        all_targets.extend(
+            [(module, target, node, active_type) for target, node, active_type in targets]
+        )
+
+    for module, target, node, active_type in order_by_subclassing(all_targets):
+        analyzer = graph[module].manager.semantic_analyzer
+        assert isinstance(node, (FuncDef, OverloadedFuncDef, Decorator))
+        process_top_level_function(
+            analyzer, graph[module], module, target, node, active_type, patches
+        )
 
 
 def process_top_level_function(
@@ -308,6 +350,11 @@ def process_top_level_function(
     str, Union[MypyFile, FuncDef, OverloadedFuncDef, Decorator], Optional[TypeInfo]
 ]
 
+# Same as above but includes module as first item.
+FullTargetInfo: _TypeAlias = tuple[
+    str, str, Union[MypyFile, FuncDef, OverloadedFuncDef, Decorator], Optional[TypeInfo]
+]
+
 
 def get_all_leaf_targets(file: MypyFile) -> list[TargetInfo]:
     """Return all leaf targets in a symbol table (module-level and methods)."""

mypy/stubdoc.py

@@ -21,7 +21,7 @@
 Sig: _TypeAlias = tuple[str, str]
 
 
-_TYPE_RE: Final = re.compile(r"^[a-zA-Z_][\w\[\], .\"\']*(\.[a-zA-Z_][\w\[\], ]*)*$")
+_TYPE_RE: Final = re.compile(r"^[a-zA-Z_][\w\[\], .\"\'|]*(\.[a-zA-Z_][\w\[\], ]*)*$")
 _ARG_NAME_RE: Final = re.compile(r"\**[A-Za-z_][A-Za-z0-9_]*$")
 
 

mypy/test/teststubgen.py

@@ -1405,6 +1405,9 @@ def test_is_valid_type(self) -> None:
         assert is_valid_type("Literal[True]")
         assert is_valid_type("Literal[Color.RED]")
         assert is_valid_type("Literal[None]")
+        assert is_valid_type("str | int")
+        assert is_valid_type("dict[str, int] | int")
+        assert is_valid_type("tuple[str, ...]")
         assert is_valid_type(
             'Literal[26, 0x1A, "hello world", b"hello world", u"hello world", True, Color.RED, None]'
         )

mypy/typeshed/stdlib/_frozen_importlib_external.pyi

@@ -26,8 +26,8 @@ else:
 
 MAGIC_NUMBER: bytes
 
-def cache_from_source(path: str, debug_override: bool | None = None, *, optimization: Any | None = None) -> str: ...
-def source_from_cache(path: str) -> str: ...
+def cache_from_source(path: StrPath, debug_override: bool | None = None, *, optimization: Any | None = None) -> str: ...
+def source_from_cache(path: StrPath) -> str: ...
 def decode_source(source_bytes: ReadableBuffer) -> str: ...
 def spec_from_file_location(
     name: str,

mypy/typeshed/stdlib/codecs.pyi

@@ -3,8 +3,8 @@ from _codecs import *
 from _typeshed import ReadableBuffer
 from abc import abstractmethod
 from collections.abc import Callable, Generator, Iterable
-from typing import Any, BinaryIO, ClassVar, Final, Literal, Protocol, TextIO
-from typing_extensions import Self
+from typing import Any, BinaryIO, ClassVar, Final, Literal, Protocol, TextIO, overload
+from typing_extensions import Self, TypeAlias
 
 __all__ = [
     "register",
@@ -58,6 +58,21 @@ BOM32_LE: Final = b"\xff\xfe"
 BOM64_BE: Final = b"\x00\x00\xfe\xff"
 BOM64_LE: Final = b"\xff\xfe\x00\x00"
 
+_BufferedEncoding: TypeAlias = Literal[
+    "idna",
+    "raw-unicode-escape",
+    "unicode-escape",
+    "utf-16",
+    "utf-16-be",
+    "utf-16-le",
+    "utf-32",
+    "utf-32-be",
+    "utf-32-le",
+    "utf-7",
+    "utf-8",
+    "utf-8-sig",
+]
+
 class _WritableStream(Protocol):
     def write(self, data: bytes, /) -> object: ...
     def seek(self, offset: int, whence: int, /) -> object: ...
@@ -94,6 +109,9 @@ class _IncrementalEncoder(Protocol):
 class _IncrementalDecoder(Protocol):
     def __call__(self, errors: str = ...) -> IncrementalDecoder: ...
 
+class _BufferedIncrementalDecoder(Protocol):
+    def __call__(self, errors: str = ...) -> BufferedIncrementalDecoder: ...
+
 class CodecInfo(tuple[_Encoder, _Decoder, _StreamReader, _StreamWriter]):
     _is_text_encoding: bool
     @property
@@ -125,6 +143,9 @@ class CodecInfo(tuple[_Encoder, _Decoder, _StreamReader, _StreamWriter]):
 def getencoder(encoding: str) -> _Encoder: ...
 def getdecoder(encoding: str) -> _Decoder: ...
 def getincrementalencoder(encoding: str) -> _IncrementalEncoder: ...
+@overload
+def getincrementaldecoder(encoding: _BufferedEncoding) -> _BufferedIncrementalDecoder: ...
+@overload
 def getincrementaldecoder(encoding: str) -> _IncrementalDecoder: ...
 def getreader(encoding: str) -> _StreamReader: ...
 def getwriter(encoding: str) -> _StreamWriter: ...

mypy/typeshed/stdlib/compileall.pyi

@@ -25,7 +25,7 @@ if sys.version_info >= (3, 10):
         prependdir: StrPath | None = None,
         limit_sl_dest: StrPath | None = None,
         hardlink_dupes: bool = False,
-    ) -> int: ...
+    ) -> bool: ...
     def compile_file(
         fullname: StrPath,
         ddir: StrPath | None = None,
@@ -40,7 +40,7 @@ if sys.version_info >= (3, 10):
         prependdir: StrPath | None = None,
         limit_sl_dest: StrPath | None = None,
         hardlink_dupes: bool = False,
-    ) -> int: ...
+    ) -> bool: ...
 
 elif sys.version_info >= (3, 9):
     def compile_dir(
@@ -59,7 +59,7 @@ elif sys.version_info >= (3, 9):
         prependdir: StrPath | None = None,
         limit_sl_dest: StrPath | None = None,
         hardlink_dupes: bool = False,
-    ) -> int: ...
+    ) -> bool: ...
     def compile_file(
         fullname: StrPath,
         ddir: StrPath | None = None,
@@ -74,7 +74,7 @@ elif sys.version_info >= (3, 9):
         prependdir: StrPath | None = None,
         limit_sl_dest: StrPath | None = None,
         hardlink_dupes: bool = False,
-    ) -> int: ...
+    ) -> bool: ...
 
 else:
     def compile_dir(
@@ -88,7 +88,7 @@ else:
         optimize: int = -1,
         workers: int = 1,
         invalidation_mode: PycInvalidationMode | None = None,
-    ) -> int: ...
+    ) -> bool: ...
     def compile_file(
         fullname: StrPath,
         ddir: StrPath | None = None,
@@ -98,7 +98,7 @@ else:
         legacy: bool = False,
         optimize: int = -1,
         invalidation_mode: PycInvalidationMode | None = None,
-    ) -> int: ...
+    ) -> bool: ...
 
 def compile_path(
     skip_curdir: bool = ...,
@@ -108,4 +108,4 @@ def compile_path(
     legacy: bool = False,
     optimize: int = -1,
     invalidation_mode: PycInvalidationMode | None = None,
-) -> int: ...
+) -> bool: ...

mypy/typeshed/stdlib/email/__init__.pyi

@@ -1,7 +1,7 @@
 from collections.abc import Callable
 from email.message import Message
-from email.policy import Policy
-from typing import IO
+from email.policy import Policy, _MessageT
+from typing import IO, overload
 from typing_extensions import TypeAlias
 
 # At runtime, listing submodules in __all__ without them being imported is
@@ -31,7 +31,29 @@ __all__ = [  # noqa: F822  # Undefined names in __all__
 _ParamType: TypeAlias = str | tuple[str | None, str | None, str]  # noqa: Y047
 _ParamsType: TypeAlias = str | None | tuple[str, str | None, str]  # noqa: Y047
 
-def message_from_string(s: str, _class: Callable[[], Message] = ..., *, policy: Policy = ...) -> Message: ...
-def message_from_bytes(s: bytes | bytearray, _class: Callable[[], Message] = ..., *, policy: Policy = ...) -> Message: ...
-def message_from_file(fp: IO[str], _class: Callable[[], Message] = ..., *, policy: Policy = ...) -> Message: ...
-def message_from_binary_file(fp: IO[bytes], _class: Callable[[], Message] = ..., *, policy: Policy = ...) -> Message: ...
+@overload
+def message_from_string(s: str) -> Message: ...
+@overload
+def message_from_string(s: str, _class: Callable[[], _MessageT]) -> _MessageT: ...
+@overload
+def message_from_string(s: str, _class: Callable[[], _MessageT] = ..., *, policy: Policy[_MessageT]) -> _MessageT: ...
+@overload
+def message_from_bytes(s: bytes | bytearray) -> Message: ...
+@overload
+def message_from_bytes(s: bytes | bytearray, _class: Callable[[], _MessageT]) -> _MessageT: ...
+@overload
+def message_from_bytes(
+    s: bytes | bytearray, _class: Callable[[], _MessageT] = ..., *, policy: Policy[_MessageT]
+) -> _MessageT: ...
+@overload
+def message_from_file(fp: IO[str]) -> Message: ...
+@overload
+def message_from_file(fp: IO[str], _class: Callable[[], _MessageT]) -> _MessageT: ...
+@overload
+def message_from_file(fp: IO[str], _class: Callable[[], _MessageT] = ..., *, policy: Policy[_MessageT]) -> _MessageT: ...
+@overload
+def message_from_binary_file(fp: IO[bytes]) -> Message: ...
+@overload
+def message_from_binary_file(fp: IO[bytes], _class: Callable[[], _MessageT]) -> _MessageT: ...
+@overload
+def message_from_binary_file(fp: IO[bytes], _class: Callable[[], _MessageT] = ..., *, policy: Policy[_MessageT]) -> _MessageT: ...

mypy/typeshed/stdlib/email/mime/message.pyi

@@ -1,8 +1,7 @@
-from email.message import Message
 from email.mime.nonmultipart import MIMENonMultipart
-from email.policy import Policy
+from email.policy import Policy, _MessageT
 
 __all__ = ["MIMEMessage"]
 
 class MIMEMessage(MIMENonMultipart):
-    def __init__(self, _msg: Message, _subtype: str = "rfc822", *, policy: Policy | None = None) -> None: ...
+    def __init__(self, _msg: _MessageT, _subtype: str = "rfc822", *, policy: Policy[_MessageT] | None = None) -> None: ...

mypy/typeshed/stdlib/email/mime/multipart.pyi

@@ -1,8 +1,7 @@
 from collections.abc import Sequence
 from email import _ParamsType
-from email.message import Message
 from email.mime.base import MIMEBase
-from email.policy import Policy
+from email.policy import Policy, _MessageT
 
 __all__ = ["MIMEMultipart"]
 
@@ -11,8 +10,8 @@ class MIMEMultipart(MIMEBase):
         self,
         _subtype: str = "mixed",
         boundary: str | None = None,
-        _subparts: Sequence[Message] | None = None,
+        _subparts: Sequence[_MessageT] | None = None,
         *,
-        policy: Policy | None = None,
+        policy: Policy[_MessageT] | None = None,
         **_params: _ParamsType,
     ) -> None: ...