Merge branch 'master' into pos_only-in-stubgen

Author: InvincibleRMC

misc/profile_self_check.py renamed to misc/profile_check.py

@@ -1,4 +1,6 @@
-"""Compile mypy using mypyc and profile self-check using perf.
+"""Compile mypy using mypyc and profile type checking using perf.
+
+By default does a self check.
 
 Notes:
  - Only Linux is supported for now (TODO: add support for other profilers)
@@ -23,6 +25,8 @@
       CFLAGS="-O2 -g -fno-omit-frame-pointer"
 """
 
+from __future__ import annotations
+
 import argparse
 import glob
 import os
@@ -41,24 +45,28 @@
 TARGET_DIR = "mypy.profile.tmpdir"
 
 
-def _profile_self_check(target_dir: str) -> None:
+def _profile_type_check(target_dir: str, code: str | None) -> None:
     cache_dir = os.path.join(target_dir, ".mypy_cache")
     if os.path.exists(cache_dir):
         shutil.rmtree(cache_dir)
-    files = []
-    for pat in "mypy/*.py", "mypy/*/*.py", "mypyc/*.py", "mypyc/test/*.py":
-        files.extend(glob.glob(pat))
-    self_check_cmd = ["python", "-m", "mypy", "--config-file", "mypy_self_check.ini"] + files
-    cmdline = ["perf", "record", "-g"] + self_check_cmd
+    args = []
+    if code is None:
+        args.extend(["--config-file", "mypy_self_check.ini"])
+        for pat in "mypy/*.py", "mypy/*/*.py", "mypyc/*.py", "mypyc/test/*.py":
+            args.extend(glob.glob(pat))
+    else:
+        args.extend(["-c", code])
+    check_cmd = ["python", "-m", "mypy"] + args
+    cmdline = ["perf", "record", "-g"] + check_cmd
     t0 = time.time()
     subprocess.run(cmdline, cwd=target_dir, check=True)
     elapsed = time.time() - t0
     print(f"{elapsed:.2f}s elapsed")
 
 
-def profile_self_check(target_dir: str) -> None:
+def profile_type_check(target_dir: str, code: str | None) -> None:
     try:
-        _profile_self_check(target_dir)
+        _profile_type_check(target_dir, code)
     except subprocess.CalledProcessError:
         print("\nProfiling failed! You may missing some permissions.")
         print("\nThis may help (note that it has security implications):")
@@ -92,7 +100,7 @@ def main() -> None:
     check_requirements()
 
     parser = argparse.ArgumentParser(
-        description="Compile mypy and profile self checking using 'perf'."
+        description="Compile mypy and profile type checking using 'perf' (by default, self check)."
     )
     parser.add_argument(
         "--multi-file",
@@ -102,9 +110,17 @@ def main() -> None:
     parser.add_argument(
         "--skip-compile", action="store_true", help="use compiled mypy from previous run"
     )
+    parser.add_argument(
+        "-c",
+        metavar="CODE",
+        default=None,
+        type=str,
+        help="profile type checking Python code fragment instead of mypy self-check",
+    )
     args = parser.parse_args()
     multi_file: bool = args.multi_file
     skip_compile: bool = args.skip_compile
+    code: str | None = args.c
 
     target_dir = TARGET_DIR
 
@@ -116,7 +132,7 @@ def main() -> None:
     elif not os.path.isdir(target_dir):
         sys.exit("error: Can't find compile mypy from previous run -- can't use --skip-compile")
 
-    profile_self_check(target_dir)
+    profile_type_check(target_dir, code)
 
     print()
     print('NOTE: Compile CPython using CFLAGS="-O2 -g -fno-omit-frame-pointer" for good results')

mypy/checker.py

@@ -618,7 +618,7 @@ def accept_loop(
                     if on_enter_body is not None:
                         on_enter_body()
 
-                    with IterationErrorWatcher(self.msg.errors, iter_errors) as watcher:
+                    with IterationErrorWatcher(self.msg.errors, iter_errors):
                         self.accept(body)
 
                 partials_new = sum(len(pts.map) for pts in self.partial_types)
@@ -641,10 +641,7 @@ def accept_loop(
                 if iter == 20:
                     raise RuntimeError("Too many iterations when checking a loop")
 
-            for error_info in watcher.yield_error_infos():
-                self.msg.fail(*error_info[:2], code=error_info[2])
-            for note_info in watcher.yield_note_infos(self.options):
-                self.note(*note_info)
+            self.msg.iteration_dependent_errors(iter_errors)
 
             # If exit_condition is set, assume it must be False on exit from the loop:
             if exit_condition:
@@ -3041,7 +3038,7 @@ def is_noop_for_reachability(self, s: Statement) -> bool:
             if isinstance(s.expr, EllipsisExpr):
                 return True
             elif isinstance(s.expr, CallExpr):
-                with self.expr_checker.msg.filter_errors():
+                with self.expr_checker.msg.filter_errors(filter_revealed_type=True):
                     typ = get_proper_type(
                         self.expr_checker.accept(
                             s.expr, allow_none_return=True, always_allow_any=True
@@ -4969,7 +4966,7 @@ def visit_try_stmt(self, s: TryStmt) -> None:
             if s.finally_body:
                 # First we check finally_body is type safe on all abnormal exit paths
                 iter_errors = IterationDependentErrors()
-                with IterationErrorWatcher(self.msg.errors, iter_errors) as watcher:
+                with IterationErrorWatcher(self.msg.errors, iter_errors):
                     self.accept(s.finally_body)
 
         if s.finally_body:
@@ -4986,13 +4983,9 @@ def visit_try_stmt(self, s: TryStmt) -> None:
             # that follows the try statement.)
             assert iter_errors is not None
             if not self.binder.is_unreachable():
-                with IterationErrorWatcher(self.msg.errors, iter_errors) as watcher:
+                with IterationErrorWatcher(self.msg.errors, iter_errors):
                     self.accept(s.finally_body)
-
-            for error_info in watcher.yield_error_infos():
-                self.msg.fail(*error_info[:2], code=error_info[2])
-            for note_info in watcher.yield_note_infos(self.options):
-                self.msg.note(*note_info)
+            self.msg.iteration_dependent_errors(iter_errors)
 
     def visit_try_without_finally(self, s: TryStmt, try_frame: bool) -> None:
         """Type check a try statement, ignoring the finally block.

mypy/errors.py

@@ -15,6 +15,7 @@
 from mypy.nodes import Context
 from mypy.options import Options
 from mypy.scope import Scope
+from mypy.types import Type
 from mypy.util import DEFAULT_SOURCE_OFFSET, is_typeshed_file
 from mypy.version import __version__ as mypy_version
 
@@ -166,18 +167,24 @@ class ErrorWatcher:
     out by one of the ErrorWatcher instances.
     """
 
+    # public attribute for the special treatment of `reveal_type` by
+    # `MessageBuilder.reveal_type`:
+    filter_revealed_type: bool
+
     def __init__(
         self,
         errors: Errors,
         *,
         filter_errors: bool | Callable[[str, ErrorInfo], bool] = False,
         save_filtered_errors: bool = False,
         filter_deprecated: bool = False,
+        filter_revealed_type: bool = False,
     ) -> None:
         self.errors = errors
         self._has_new_errors = False
         self._filter = filter_errors
         self._filter_deprecated = filter_deprecated
+        self.filter_revealed_type = filter_revealed_type
         self._filtered: list[ErrorInfo] | None = [] if save_filtered_errors else None
 
     def __enter__(self) -> Self:
@@ -236,15 +243,41 @@ class IterationDependentErrors:
     # the error report occurs but really all unreachable lines.
     unreachable_lines: list[set[int]]
 
-    # One set of revealed types for each `reveal_type` statement.  Each created set can
-    # grow during the iteration.  Meaning of the tuple items: function_or_member, line,
-    # column, end_line, end_column:
-    revealed_types: dict[tuple[str | None, int, int, int, int], set[str]]
+    # One list of revealed types for each `reveal_type` statement.  Each created list
+    # can grow during the iteration.  Meaning of the tuple items: line, column,
+    # end_line, end_column:
+    revealed_types: dict[tuple[int, int, int | None, int | None], list[Type]]
 
     def __init__(self) -> None:
         self.uselessness_errors = []
         self.unreachable_lines = []
-        self.revealed_types = defaultdict(set)
+        self.revealed_types = defaultdict(list)
+
+    def yield_uselessness_error_infos(self) -> Iterator[tuple[str, Context, ErrorCode]]:
+        """Report only those `unreachable`, `redundant-expr`, and `redundant-casts`
+        errors that could not be ruled out in any iteration step."""
+
+        persistent_uselessness_errors = set()
+        for candidate in set(chain(*self.uselessness_errors)):
+            if all(
+                (candidate in errors) or (candidate[2] in lines)
+                for errors, lines in zip(self.uselessness_errors, self.unreachable_lines)
+            ):
+                persistent_uselessness_errors.add(candidate)
+        for error_info in persistent_uselessness_errors:
+            context = Context(line=error_info[2], column=error_info[3])
+            context.end_line = error_info[4]
+            context.end_column = error_info[5]
+            yield error_info[1], context, error_info[0]
+
+    def yield_revealed_type_infos(self) -> Iterator[tuple[list[Type], Context]]:
+        """Yield all types revealed in at least one iteration step."""
+
+        for note_info, types in self.revealed_types.items():
+            context = Context(line=note_info[0], column=note_info[1])
+            context.end_line = note_info[2]
+            context.end_column = note_info[3]
+            yield types, context
 
 
 class IterationErrorWatcher(ErrorWatcher):
@@ -287,53 +320,8 @@ def on_error(self, file: str, info: ErrorInfo) -> bool:
                 iter_errors.unreachable_lines[-1].update(range(info.line, info.end_line + 1))
             return True
 
-        if info.code == codes.MISC and info.message.startswith("Revealed type is "):
-            key = info.function_or_member, info.line, info.column, info.end_line, info.end_column
-            types = info.message.split('"')[1]
-            if types.startswith("Union["):
-                iter_errors.revealed_types[key].update(types[6:-1].split(", "))
-            else:
-                iter_errors.revealed_types[key].add(types)
-            return True
-
         return super().on_error(file, info)
 
-    def yield_error_infos(self) -> Iterator[tuple[str, Context, ErrorCode]]:
-        """Report only those `unreachable`, `redundant-expr`, and `redundant-casts`
-        errors that could not be ruled out in any iteration step."""
-
-        persistent_uselessness_errors = set()
-        iter_errors = self.iteration_dependent_errors
-        for candidate in set(chain(*iter_errors.uselessness_errors)):
-            if all(
-                (candidate in errors) or (candidate[2] in lines)
-                for errors, lines in zip(
-                    iter_errors.uselessness_errors, iter_errors.unreachable_lines
-                )
-            ):
-                persistent_uselessness_errors.add(candidate)
-        for error_info in persistent_uselessness_errors:
-            context = Context(line=error_info[2], column=error_info[3])
-            context.end_line = error_info[4]
-            context.end_column = error_info[5]
-            yield error_info[1], context, error_info[0]
-
-    def yield_note_infos(self, options: Options) -> Iterator[tuple[str, Context]]:
-        """Yield all types revealed in at least one iteration step."""
-
-        for note_info, types in self.iteration_dependent_errors.revealed_types.items():
-            sorted_ = sorted(types, key=lambda typ: typ.lower())
-            if len(types) == 1:
-                revealed = sorted_[0]
-            elif options.use_or_syntax():
-                revealed = " | ".join(sorted_)
-            else:
-                revealed = f"Union[{', '.join(sorted_)}]"
-            context = Context(line=note_info[1], column=note_info[2])
-            context.end_line = note_info[3]
-            context.end_column = note_info[4]
-            yield f'Revealed type is "{revealed}"', context
-
 
 class Errors:
     """Container for compile errors.
@@ -596,18 +584,19 @@ def _add_error_info(self, file: str, info: ErrorInfo) -> None:
         if info.code in (IMPORT, IMPORT_UNTYPED, IMPORT_NOT_FOUND):
             self.seen_import_error = True
 
+    def get_watchers(self) -> Iterator[ErrorWatcher]:
+        """Yield the `ErrorWatcher` stack from top to bottom."""
+        i = len(self._watchers)
+        while i > 0:
+            i -= 1
+            yield self._watchers[i]
+
     def _filter_error(self, file: str, info: ErrorInfo) -> bool:
         """
         process ErrorWatcher stack from top to bottom,
         stopping early if error needs to be filtered out
         """
-        i = len(self._watchers)
-        while i > 0:
-            i -= 1
-            w = self._watchers[i]
-            if w.on_error(file, info):
-                return True
-        return False
+        return any(w.on_error(file, info) for w in self.get_watchers())
 
     def add_error_info(self, info: ErrorInfo) -> None:
         file, lines = info.origin

mypy/messages.py

@@ -23,7 +23,13 @@
 from mypy import errorcodes as codes, message_registry
 from mypy.erasetype import erase_type
 from mypy.errorcodes import ErrorCode
-from mypy.errors import ErrorInfo, Errors, ErrorWatcher
+from mypy.errors import (
+    ErrorInfo,
+    Errors,
+    ErrorWatcher,
+    IterationDependentErrors,
+    IterationErrorWatcher,
+)
 from mypy.nodes import (
     ARG_NAMED,
     ARG_NAMED_OPT,
@@ -188,12 +194,14 @@ def filter_errors(
         filter_errors: bool | Callable[[str, ErrorInfo], bool] = True,
         save_filtered_errors: bool = False,
         filter_deprecated: bool = False,
+        filter_revealed_type: bool = False,
     ) -> ErrorWatcher:
         return ErrorWatcher(
             self.errors,
             filter_errors=filter_errors,
             save_filtered_errors=save_filtered_errors,
             filter_deprecated=filter_deprecated,
+            filter_revealed_type=filter_revealed_type,
         )
 
     def add_errors(self, errors: list[ErrorInfo]) -> None:
@@ -1738,6 +1746,24 @@ def invalid_signature_for_special_method(
         )
 
     def reveal_type(self, typ: Type, context: Context) -> None:
+
+        # Search for an error watcher that modifies the "normal" behaviour (we do not
+        # rely on the normal `ErrorWatcher` filtering approach because we might need to
+        # collect the original types for a later unionised response):
+        for watcher in self.errors.get_watchers():
+            # The `reveal_type` statement should be ignored:
+            if watcher.filter_revealed_type:
+                return
+            # The `reveal_type` statement might be visited iteratively due to being
+            # placed in a loop or so. Hence, we collect the respective types of
+            # individual iterations so that we can report them all in one step later:
+            if isinstance(watcher, IterationErrorWatcher):
+                watcher.iteration_dependent_errors.revealed_types[
+                    (context.line, context.column, context.end_line, context.end_column)
+                ].append(typ)
+                return
+
+        # Nothing special here; just create the note:
         visitor = TypeStrVisitor(options=self.options)
         self.note(f'Revealed type is "{typ.accept(visitor)}"', context)
 
@@ -2481,6 +2507,12 @@ def match_statement_inexhaustive_match(self, typ: Type, context: Context) -> Non
             code=codes.EXHAUSTIVE_MATCH,
         )
 
+    def iteration_dependent_errors(self, iter_errors: IterationDependentErrors) -> None:
+        for error_info in iter_errors.yield_uselessness_error_infos():
+            self.fail(*error_info[:2], code=error_info[2])
+        for types, context in iter_errors.yield_revealed_type_infos():
+            self.reveal_type(mypy.typeops.make_simplified_union(types), context)
+
 
 def quote_type_string(type_string: str) -> str:
     """Quotes a type representation for use in messages."""

mypy/plugins/constants.py

@@ -0,0 +1,20 @@
+"""Constant definitions for plugins kept here to help with import cycles."""
+
+from typing import Final
+
+from mypy.semanal_enum import ENUM_BASES
+
+SINGLEDISPATCH_TYPE: Final = "functools._SingleDispatchCallable"
+SINGLEDISPATCH_REGISTER_METHOD: Final = f"{SINGLEDISPATCH_TYPE}.register"
+SINGLEDISPATCH_CALLABLE_CALL_METHOD: Final = f"{SINGLEDISPATCH_TYPE}.__call__"
+SINGLEDISPATCH_REGISTER_RETURN_CLASS: Final = "_SingleDispatchRegisterCallable"
+SINGLEDISPATCH_REGISTER_CALLABLE_CALL_METHOD: Final = (
+    f"functools.{SINGLEDISPATCH_REGISTER_RETURN_CLASS}.__call__"
+)
+
+ENUM_NAME_ACCESS: Final = {f"{prefix}.name" for prefix in ENUM_BASES} | {
+    f"{prefix}._name_" for prefix in ENUM_BASES
+}
+ENUM_VALUE_ACCESS: Final = {f"{prefix}.value" for prefix in ENUM_BASES} | {
+    f"{prefix}._value_" for prefix in ENUM_BASES
+}