Merge branch 'master' into coveruntime

Author: hauntsaninja

docs/source/generics.rst

@@ -630,7 +630,7 @@ Let us illustrate this by few simple examples:
 
      my_circles: list[Circle] = []
      add_one(my_circles)     # This may appear safe, but...
-     my_circles[-1].rotate()  # ...this will fail, since my_circles[0] is now a Shape, not a Circle
+     my_circles[0].rotate()  # ...this will fail, since my_circles[0] is now a Shape, not a Circle
 
   Another example of invariant type is ``dict``. Most mutable containers
   are invariant.

misc/perf_compare.py

@@ -35,11 +35,13 @@ def heading(s: str) -> None:
     print()
 
 
-def build_mypy(target_dir: str) -> None:
+def build_mypy(target_dir: str, multi_file: bool) -> None:
     env = os.environ.copy()
     env["CC"] = "clang"
     env["MYPYC_OPT_LEVEL"] = "2"
     env["PYTHONHASHSEED"] = "1"
+    if multi_file:
+        env["MYPYC_MULTI_FILE"] = "1"
     cmd = [sys.executable, "setup.py", "--use-mypyc", "build_ext", "--inplace"]
     subprocess.run(cmd, env=env, check=True, cwd=target_dir)
 
@@ -110,6 +112,12 @@ def main() -> None:
         action="store_true",
         help="measure incremental run (fully cached)",
     )
+    parser.add_argument(
+        "--multi-file",
+        default=False,
+        action="store_true",
+        help="compile each mypy module to a separate C file (reduces RAM use)",
+    )
     parser.add_argument(
         "--dont-setup",
         default=False,
@@ -127,9 +135,9 @@ def main() -> None:
     parser.add_argument(
         "-j",
         metavar="N",
-        default=8,
+        default=4,
         type=int,
-        help="set maximum number of parallel builds (default=8)",
+        help="set maximum number of parallel builds (default=4) -- high numbers require a lot of RAM!",
     )
     parser.add_argument(
         "-r",
@@ -155,6 +163,7 @@ def main() -> None:
     args = parser.parse_args()
     incremental: bool = args.incremental
     dont_setup: bool = args.dont_setup
+    multi_file: bool = args.multi_file
     commits = args.commit
     num_runs: int = args.num_runs + 1
     max_workers: int = args.j
@@ -185,7 +194,9 @@ def main() -> None:
         print("(This will take a while...)")
 
         with ThreadPoolExecutor(max_workers=max_workers) as executor:
-            futures = [executor.submit(build_mypy, target_dir) for target_dir in target_dirs]
+            futures = [
+                executor.submit(build_mypy, target_dir, multi_file) for target_dir in target_dirs
+            ]
             for future in as_completed(futures):
                 future.result()
 

mypy/checker.py

@@ -107,6 +107,7 @@
     OperatorAssignmentStmt,
     OpExpr,
     OverloadedFuncDef,
+    OverloadPart,
     PassStmt,
     PromoteExpr,
     RaiseStmt,
@@ -407,6 +408,11 @@ def __init__(
         # argument through various `checker` and `checkmember` functions.
         self._is_final_def = False
 
+        # Track when we enter an overload implementation. Some checks should not be applied
+        # to the implementation signature when specific overloads are available.
+        # Use `enter_overload_impl` to modify.
+        self.overload_impl_stack: list[OverloadPart] = []
+
         # This flag is set when we run type-check or attribute access check for the purpose
         # of giving a note on possibly missing "await". It is used to avoid infinite recursion.
         self.checking_missing_await = False
@@ -709,7 +715,8 @@ def _visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None:
         if num_abstract not in (0, len(defn.items)):
             self.fail(message_registry.INCONSISTENT_ABSTRACT_OVERLOAD, defn)
         if defn.impl:
-            defn.impl.accept(self)
+            with self.enter_overload_impl(defn.impl):
+                defn.impl.accept(self)
         if not defn.is_property:
             self.check_overlapping_overloads(defn)
             if defn.type is None:
@@ -752,6 +759,14 @@ def _visit_overloaded_func_def(self, defn: OverloadedFuncDef) -> None:
             self.check_explicit_override_decorator(defn, found_method_base_classes, defn.impl)
             self.check_inplace_operator_method(defn)
 
+    @contextmanager
+    def enter_overload_impl(self, impl: OverloadPart) -> Iterator[None]:
+        self.overload_impl_stack.append(impl)
+        try:
+            yield
+        finally:
+            assert self.overload_impl_stack.pop() == impl
+
     def extract_callable_type(self, inner_type: Type | None, ctx: Context) -> CallableType | None:
         """Get type as seen by an overload item caller."""
         inner_type = get_proper_type(inner_type)
@@ -1278,7 +1293,11 @@ def check_func_def(
                     )
 
                 if name:  # Special method names
-                    if defn.info and self.is_reverse_op_method(name):
+                    if (
+                        defn.info
+                        and self.is_reverse_op_method(name)
+                        and defn not in self.overload_impl_stack
+                    ):
                         self.check_reverse_op_method(item, typ, name, defn)
                     elif name in ("__getattr__", "__getattribute__"):
                         self.check_getattr_method(typ, defn, name)
@@ -6166,6 +6185,7 @@ def find_isinstance_check_helper(
                                     self.lookup_type(expr),
                                     [TypeRange(node.callee.type_is, is_upper_bound=False)],
                                     expr,
+                                    consider_runtime_isinstance=False,
                                 ),
                             )
         elif isinstance(node, ComparisonExpr):
@@ -7593,11 +7613,19 @@ def conditional_types_with_intersection(
         type_ranges: list[TypeRange] | None,
         ctx: Context,
         default: None = None,
+        *,
+        consider_runtime_isinstance: bool = True,
     ) -> tuple[Type | None, Type | None]: ...
 
     @overload
     def conditional_types_with_intersection(
-        self, expr_type: Type, type_ranges: list[TypeRange] | None, ctx: Context, default: Type
+        self,
+        expr_type: Type,
+        type_ranges: list[TypeRange] | None,
+        ctx: Context,
+        default: Type,
+        *,
+        consider_runtime_isinstance: bool = True,
     ) -> tuple[Type, Type]: ...
 
     def conditional_types_with_intersection(
@@ -7606,8 +7634,15 @@ def conditional_types_with_intersection(
         type_ranges: list[TypeRange] | None,
         ctx: Context,
         default: Type | None = None,
+        *,
+        consider_runtime_isinstance: bool = True,
     ) -> tuple[Type | None, Type | None]:
-        initial_types = conditional_types(expr_type, type_ranges, default)
+        initial_types = conditional_types(
+            expr_type,
+            type_ranges,
+            default,
+            consider_runtime_isinstance=consider_runtime_isinstance,
+        )
         # For some reason, doing "yes_map, no_map = conditional_types_to_typemaps(...)"
         # doesn't work: mypyc will decide that 'yes_map' is of type None if we try.
         yes_type: Type | None = initial_types[0]
@@ -7919,18 +7954,30 @@ def visit_type_var(self, t: TypeVarType) -> None:
 
 @overload
 def conditional_types(
-    current_type: Type, proposed_type_ranges: list[TypeRange] | None, default: None = None
+    current_type: Type,
+    proposed_type_ranges: list[TypeRange] | None,
+    default: None = None,
+    *,
+    consider_runtime_isinstance: bool = True,
 ) -> tuple[Type | None, Type | None]: ...
 
 
 @overload
 def conditional_types(
-    current_type: Type, proposed_type_ranges: list[TypeRange] | None, default: Type
+    current_type: Type,
+    proposed_type_ranges: list[TypeRange] | None,
+    default: Type,
+    *,
+    consider_runtime_isinstance: bool = True,
 ) -> tuple[Type, Type]: ...
 
 
 def conditional_types(
-    current_type: Type, proposed_type_ranges: list[TypeRange] | None, default: Type | None = None
+    current_type: Type,
+    proposed_type_ranges: list[TypeRange] | None,
+    default: Type | None = None,
+    *,
+    consider_runtime_isinstance: bool = True,
 ) -> tuple[Type | None, Type | None]:
     """Takes in the current type and a proposed type of an expression.
 
@@ -7972,7 +8019,11 @@ def conditional_types(
                     if not type_range.is_upper_bound
                 ]
             )
-            remaining_type = restrict_subtype_away(current_type, proposed_precise_type)
+            remaining_type = restrict_subtype_away(
+                current_type,
+                proposed_precise_type,
+                consider_runtime_isinstance=consider_runtime_isinstance,
+            )
             return proposed_type, remaining_type
     else:
         # An isinstance check, but we don't understand the type

mypy/checkexpr.py

@@ -318,6 +318,8 @@ class ExpressionChecker(ExpressionVisitor[Type], ExpressionCheckerSharedApi):
     strfrm_checker: StringFormatterChecker
     plugin: Plugin
 
+    _arg_infer_context_cache: ArgumentInferContext | None
+
     def __init__(
         self,
         chk: mypy.checker.TypeChecker,
@@ -352,6 +354,8 @@ def __init__(
         self.is_callee = False
         type_state.infer_polymorphic = not self.chk.options.old_type_inference
 
+        self._arg_infer_context_cache = None
+
     def reset(self) -> None:
         self.resolved_type = {}
 
@@ -2277,9 +2281,11 @@ def infer_function_type_arguments_pass2(
         return callee_type, inferred_args
 
     def argument_infer_context(self) -> ArgumentInferContext:
-        return ArgumentInferContext(
-            self.chk.named_type("typing.Mapping"), self.chk.named_type("typing.Iterable")
-        )
+        if self._arg_infer_context_cache is None:
+            self._arg_infer_context_cache = ArgumentInferContext(
+                self.chk.named_type("typing.Mapping"), self.chk.named_type("typing.Iterable")
+            )
+        return self._arg_infer_context_cache
 
     def get_arg_infer_passes(
         self,

mypy/errors.py

@@ -848,6 +848,8 @@ def generate_unused_ignore_errors(self, file: str) -> None:
                 code=codes.UNUSED_IGNORE,
                 blocker=False,
                 only_once=False,
+                origin=(self.file, [line]),
+                target=self.target_module,
             )
             self._add_error_info(file, info)
 
@@ -899,6 +901,8 @@ def generate_ignore_without_code_errors(
                 code=codes.IGNORE_WITHOUT_CODE,
                 blocker=False,
                 only_once=False,
+                origin=(self.file, [line]),
+                target=self.target_module,
             )
             self._add_error_info(file, info)
 

mypy/server/update.py

@@ -668,6 +668,8 @@ def restore(ids: list[str]) -> None:
     state.type_check_first_pass()
     state.type_check_second_pass()
     state.detect_possibly_undefined_vars()
+    state.generate_unused_ignore_notes()
+    state.generate_ignore_without_code_notes()
     t2 = time.time()
     state.finish_passes()
     t3 = time.time()

mypy/stubtest.py

@@ -2062,7 +2062,7 @@ def warning_callback(msg: str) -> None:
             if args.generate_allowlist:
                 generated_allowlist.add(error.object_desc)
                 continue
-            print(error.get_description(concise=args.concise))
+            safe_print(error.get_description(concise=args.concise))
             error_count += 1
 
     # Print unused allowlist entries
@@ -2102,6 +2102,19 @@ def warning_callback(msg: str) -> None:
     return exit_code
 
 
+def safe_print(text: str) -> None:
+    """Print a text replacing chars not representable in stdout encoding."""
+    # If `sys.stdout` encoding is not the same as out (usually UTF8) string,
+    # if may cause painful crashes. I don't want to reconfigure `sys.stdout`
+    # to do `errors = "replace"` as that sounds scary.
+    out_encoding = sys.stdout.encoding
+    if out_encoding is not None:
+        # Can be None if stdout is replaced (including our own tests). This should be
+        # safe to omit if the actual stream doesn't care about encoding.
+        text = text.encode(out_encoding, errors="replace").decode(out_encoding, errors="replace")
+    print(text)
+
+
 def parse_options(args: list[str]) -> _Arguments:
     parser = argparse.ArgumentParser(
         description="Compares stubs to objects introspected from the runtime."

mypy/subtypes.py

@@ -2073,7 +2073,7 @@ def try_restrict_literal_union(t: UnionType, s: Type) -> list[Type] | None:
     return new_items
 
 
-def restrict_subtype_away(t: Type, s: Type) -> Type:
+def restrict_subtype_away(t: Type, s: Type, *, consider_runtime_isinstance: bool = True) -> Type:
     """Return t minus s for runtime type assertions.
 
     If we can't determine a precise result, return a supertype of the
@@ -2087,16 +2087,27 @@ def restrict_subtype_away(t: Type, s: Type) -> Type:
         new_items = try_restrict_literal_union(p_t, s)
         if new_items is None:
             new_items = [
-                restrict_subtype_away(item, s)
+                restrict_subtype_away(
+                    item, s, consider_runtime_isinstance=consider_runtime_isinstance
+                )
                 for item in p_t.relevant_items()
-                if (isinstance(get_proper_type(item), AnyType) or not covers_at_runtime(item, s))
             ]
-        return UnionType.make_union(new_items)
+        return UnionType.make_union(
+            [item for item in new_items if not isinstance(get_proper_type(item), UninhabitedType)]
+        )
     elif isinstance(p_t, TypeVarType):
         return p_t.copy_modified(upper_bound=restrict_subtype_away(p_t.upper_bound, s))
-    elif covers_at_runtime(t, s):
-        return UninhabitedType()
+
+    if consider_runtime_isinstance:
+        if covers_at_runtime(t, s):
+            return UninhabitedType()
+        else:
+            return t
     else:
+        if is_proper_subtype(t, s, ignore_promotions=True):
+            return UninhabitedType()
+        if is_proper_subtype(t, s, ignore_promotions=True, erase_instances=True):
+            return UninhabitedType()
         return t
 
 

mypy/suggestions.py

@@ -852,7 +852,7 @@ def visit_instance(self, t: Instance) -> str:
         if self.module:
             parts = obj.split(".")  # need to split the object part if it is a nested class
             tree = self.graph[self.module].tree
-            if tree and parts[0] in tree.names:
+            if tree and parts[0] in tree.names and mod not in tree.names:
                 mod = self.module
 
         if (mod, obj) == ("builtins", "tuple"):