Merge branch 'master' into for-map

Author: BobTheBuidler

misc/typeshed_patches/0001-Remove-use-of-LiteralString-in-builtins-13743.patch

@@ -1,4 +1,4 @@
-From e6995c91231e1915eba43a29a22dd4cbfaf9e08e Mon Sep 17 00:00:00 2001
+From 805d7fc06a8bee350959512e0908a18a87b7f8c2 Mon Sep 17 00:00:00 2001
 From: Shantanu <[email protected]>
 Date: Mon, 26 Sep 2022 12:55:07 -0700
 Subject: [PATCH] Remove use of LiteralString in builtins (#13743)
@@ -8,7 +8,7 @@ Subject: [PATCH] Remove use of LiteralString in builtins (#13743)
  1 file changed, 1 insertion(+), 99 deletions(-)
 
 diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi
-index 00728f42d..ea77a730f 100644
+index c7ab95482..3e93da36e 100644
 --- a/mypy/typeshed/stdlib/builtins.pyi
 +++ b/mypy/typeshed/stdlib/builtins.pyi
 @@ -63,7 +63,6 @@ from typing import (  # noqa: Y022,UP035
@@ -19,7 +19,7 @@ index 00728f42d..ea77a730f 100644
      ParamSpec,
      Self,
      TypeAlias,
-@@ -453,31 +452,16 @@ class str(Sequence[str]):
+@@ -468,31 +467,16 @@ class str(Sequence[str]):
      def __new__(cls, object: object = ...) -> Self: ...
      @overload
      def __new__(cls, object: ReadableBuffer, encoding: str = ..., errors: str = ...) -> Self: ...
@@ -51,7 +51,7 @@ index 00728f42d..ea77a730f 100644
      def format(self, *args: object, **kwargs: object) -> str: ...
      def format_map(self, mapping: _FormatMapMapping, /) -> str: ...
      def index(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ...
-@@ -493,98 +477,34 @@ class str(Sequence[str]):
+@@ -508,98 +492,34 @@ class str(Sequence[str]):
      def isspace(self) -> bool: ...
      def istitle(self) -> bool: ...
      def isupper(self) -> bool: ...
@@ -150,7 +150,7 @@ index 00728f42d..ea77a730f 100644
      def zfill(self, width: SupportsIndex, /) -> str: ...  # type: ignore[misc]
      @staticmethod
      @overload
-@@ -595,39 +515,21 @@ class str(Sequence[str]):
+@@ -610,39 +530,21 @@ class str(Sequence[str]):
      @staticmethod
      @overload
      def maketrans(x: str, y: str, z: str, /) -> dict[int, int | None]: ...
@@ -190,7 +190,7 @@ index 00728f42d..ea77a730f 100644
 -    @overload
      def __rmul__(self, value: SupportsIndex, /) -> str: ...  # type: ignore[misc]
      def __getnewargs__(self) -> tuple[str]: ...
- 
+     def __format__(self, format_spec: str, /) -> str: ...
 -- 
-2.49.0
+2.50.1
 

mypy/build.py

@@ -116,6 +116,8 @@
     "abc",
 }
 
+# We are careful now, we can increase this in future if safe/useful.
+MAX_GC_FREEZE_CYCLES = 1
 
 Graph: _TypeAlias = dict[str, "State"]
 
@@ -707,6 +709,8 @@ def __init__(
         # new file can be processed O(n**2) times. This cache
         # avoids most of this redundant work.
         self.ast_cache: dict[str, tuple[MypyFile, list[ErrorInfo]]] = {}
+        # Number of times we used GC optimization hack for fresh SCCs.
+        self.gc_freeze_cycles = 0
 
     def dump_stats(self) -> None:
         if self.options.dump_build_stats:
@@ -3326,8 +3330,29 @@ def process_graph(graph: Graph, manager: BuildManager) -> None:
                 #
                 # TODO: see if it's possible to determine if we need to process only a
                 # _subset_ of the past SCCs instead of having to process them all.
+                if (
+                    platform.python_implementation() == "CPython"
+                    and manager.gc_freeze_cycles < MAX_GC_FREEZE_CYCLES
+                ):
+                    # When deserializing cache we create huge amount of new objects, so even
+                    # with our generous GC thresholds, GC is still doing a lot of pointless
+                    # work searching for garbage. So, we temporarily disable it when
+                    # processing fresh SCCs, and then move all the new objects to the oldest
+                    # generation with the freeze()/unfreeze() trick below. This is arguably
+                    # a hack, but it gives huge performance wins for large third-party
+                    # libraries, like torch.
+                    gc.collect()
+                    gc.disable()
                 for prev_scc in fresh_scc_queue:
                     process_fresh_modules(graph, prev_scc, manager)
+                if (
+                    platform.python_implementation() == "CPython"
+                    and manager.gc_freeze_cycles < MAX_GC_FREEZE_CYCLES
+                ):
+                    manager.gc_freeze_cycles += 1
+                    gc.freeze()
+                    gc.unfreeze()
+                    gc.enable()
                 fresh_scc_queue = []
             size = len(scc)
             if size == 1:

mypy/checker.py

@@ -171,6 +171,7 @@
 from mypy.typeanal import check_for_explicit_any, has_any_from_unimported_type, make_optional_type
 from mypy.typeops import (
     bind_self,
+    can_have_shared_disjoint_base,
     coerce_to_literal,
     custom_special_method,
     erase_def_to_union_or_bound,
@@ -2658,6 +2659,8 @@ def visit_class_def(self, defn: ClassDef) -> None:
         for base in typ.mro[1:]:
             if base.is_final:
                 self.fail(message_registry.CANNOT_INHERIT_FROM_FINAL.format(base.name), defn)
+        if not can_have_shared_disjoint_base(typ.bases):
+            self.fail(message_registry.INCOMPATIBLE_DISJOINT_BASES.format(typ.name), defn)
         with self.tscope.class_scope(defn.info), self.enter_partial_types(is_class=True):
             old_binder = self.binder
             self.binder = ConditionalTypeBinder(self.options)
@@ -5502,7 +5505,7 @@ def visit_with_stmt(self, s: WithStmt) -> None:
             self.accept(s.body)
 
     def check_untyped_after_decorator(self, typ: Type, func: FuncDef) -> None:
-        if not self.options.disallow_any_decorated or self.is_stub:
+        if not self.options.disallow_any_decorated or self.is_stub or self.current_node_deferred:
             return
 
         if mypy.checkexpr.has_any_type(typ):
@@ -5826,6 +5829,10 @@ def _make_fake_typeinfo_and_full_name(
             format_type_distinctly(*base_classes, options=self.options, bare=True), "and"
         )
 
+        if not can_have_shared_disjoint_base(base_classes):
+            errors.append((pretty_names_list, "have distinct disjoint bases"))
+            return None
+
         new_errors = []
         for base in base_classes:
             if base.type.is_final:

mypy/checkmember.py

@@ -1484,13 +1484,7 @@ def analyze_decorator_or_funcbase_access(
     if isinstance(defn, Decorator):
         return analyze_var(name, defn.var, itype, mx)
     typ = function_type(defn, mx.chk.named_type("builtins.function"))
-    is_trivial_self = False
-    if isinstance(defn, Decorator):
-        # Use fast path if there are trivial decorators like @classmethod or @property
-        is_trivial_self = defn.func.is_trivial_self and not defn.decorators
-    elif isinstance(defn, (FuncDef, OverloadedFuncDef)):
-        is_trivial_self = defn.is_trivial_self
-    if is_trivial_self:
+    if isinstance(defn, (FuncDef, OverloadedFuncDef)) and defn.is_trivial_self:
         return bind_self_fast(typ, mx.self_type)
     typ = check_self_arg(typ, mx.self_type, defn.is_class, mx.context, name, mx.msg)
     return bind_self(typ, original_type=mx.self_type, is_classmethod=defn.is_class)

mypy/message_registry.py

@@ -239,6 +239,9 @@ def with_additional_msg(self, info: str) -> ErrorMessage:
 )
 CANNOT_MAKE_DELETABLE_FINAL: Final = ErrorMessage("Deletable attribute cannot be final")
 
+# Disjoint bases
+INCOMPATIBLE_DISJOINT_BASES: Final = ErrorMessage('Class "{}" has incompatible disjoint bases')
+
 # Enum
 ENUM_MEMBERS_ATTR_WILL_BE_OVERRIDDEN: Final = ErrorMessage(
     'Assigned "__members__" will be overridden by "Enum" internally'

mypy/nodes.py

@@ -3004,6 +3004,7 @@ class is generic then it will be a type constructor of higher kind.
         "_mro_refs",
         "bad_mro",
         "is_final",
+        "is_disjoint_base",
         "declared_metaclass",
         "metaclass_type",
         "names",
@@ -3055,6 +3056,7 @@ class is generic then it will be a type constructor of higher kind.
     _mro_refs: list[str] | None
     bad_mro: bool  # Could not construct full MRO
     is_final: bool
+    is_disjoint_base: bool
 
     declared_metaclass: mypy.types.Instance | None
     metaclass_type: mypy.types.Instance | None
@@ -3209,6 +3211,7 @@ class is generic then it will be a type constructor of higher kind.
         "is_protocol",
         "runtime_protocol",
         "is_final",
+        "is_disjoint_base",
         "is_intersection",
     ]
 
@@ -3241,6 +3244,7 @@ def __init__(self, names: SymbolTable, defn: ClassDef, module_name: str) -> None
         self.type_var_tuple_suffix: int | None = None
         self.add_type_vars()
         self.is_final = False
+        self.is_disjoint_base = False
         self.is_enum = False
         self.fallback_to_any = False
         self.meta_fallback_to_any = False

mypy/semanal.py

@@ -254,6 +254,7 @@
     ASSERT_TYPE_NAMES,
     DATACLASS_TRANSFORM_NAMES,
     DEPRECATED_TYPE_NAMES,
+    DISJOINT_BASE_DECORATOR_NAMES,
     FINAL_DECORATOR_NAMES,
     FINAL_TYPE_NAMES,
     IMPORTED_REVEAL_TYPE_NAMES,
@@ -1059,7 +1060,7 @@ def remove_unpack_kwargs(self, defn: FuncDef, typ: CallableType) -> CallableType
         # It is OK for TypedDict to have a key named 'kwargs'.
         overlap.discard(typ.arg_names[-1])
         if overlap:
-            overlapped = ", ".join([f'"{name}"' for name in overlap])
+            overlapped = ", ".join([f'"{name}"' for name in sorted(filter(None, overlap))])
             self.fail(f"Overlap between argument names and ** TypedDict items: {overlapped}", defn)
             new_arg_types = typ.arg_types[:-1] + [AnyType(TypeOfAny.from_error)]
             return typ.copy_modified(arg_types=new_arg_types)
@@ -2188,6 +2189,8 @@ def analyze_class_decorator_common(
         """
         if refers_to_fullname(decorator, FINAL_DECORATOR_NAMES):
             info.is_final = True
+        elif refers_to_fullname(decorator, DISJOINT_BASE_DECORATOR_NAMES):
+            info.is_disjoint_base = True
         elif refers_to_fullname(decorator, TYPE_CHECK_ONLY_NAMES):
             info.is_type_check_only = True
         elif (deprecated := self.get_deprecated(decorator)) is not None:
@@ -5332,13 +5335,15 @@ def visit_expression_stmt(self, s: ExpressionStmt) -> None:
         s.expr.accept(self)
 
     def visit_return_stmt(self, s: ReturnStmt) -> None:
+        old = self.statement
         self.statement = s
         if not self.is_func_scope():
             self.fail('"return" outside function', s)
         if self.return_stmt_inside_except_star_block:
             self.fail('"return" not allowed in except* block', s, serious=True)
         if s.expr:
             s.expr.accept(self)
+        self.statement = old
 
     def visit_raise_stmt(self, s: RaiseStmt) -> None:
         self.statement = s

mypy/stubtest.py

@@ -17,6 +17,7 @@
 import os
 import pkgutil
 import re
+import struct
 import symtable
 import sys
 import traceback
@@ -466,6 +467,67 @@ class SubClass(runtime):  # type: ignore[misc]
         )
 
 
+SIZEOF_PYOBJECT = struct.calcsize("P")
+
+
+def _shape_differs(t1: type[object], t2: type[object]) -> bool:
+    """Check whether two types differ in shape.
+
+    Mirrors the shape_differs() function in typeobject.c in CPython."""
+    if sys.version_info >= (3, 12):
+        return t1.__basicsize__ != t2.__basicsize__ or t1.__itemsize__ != t2.__itemsize__
+    else:
+        # CPython had more complicated logic before 3.12:
+        # https://github.com/python/cpython/blob/f3c6f882cddc8dc30320d2e73edf019e201394fc/Objects/typeobject.c#L2224
+        # We attempt to mirror it here well enough to support the most common cases.
+        if t1.__itemsize__ or t2.__itemsize__:
+            return t1.__basicsize__ != t2.__basicsize__ or t1.__itemsize__ != t2.__itemsize__
+        t_size = t1.__basicsize__
+        if not t2.__weakrefoffset__ and t1.__weakrefoffset__ + SIZEOF_PYOBJECT == t_size:
+            t_size -= SIZEOF_PYOBJECT
+        if not t2.__dictoffset__ and t1.__dictoffset__ + SIZEOF_PYOBJECT == t_size:
+            t_size -= SIZEOF_PYOBJECT
+        if not t2.__weakrefoffset__ and t2.__weakrefoffset__ == t_size:
+            t_size -= SIZEOF_PYOBJECT
+        return t_size != t2.__basicsize__
+
+
+def _is_disjoint_base(typ: type[object]) -> bool:
+    """Return whether a type is a disjoint base at runtime, mirroring CPython's logic in typeobject.c.
+
+    See PEP 800."""
+    if typ is object:
+        return True
+    base = typ.__base__
+    assert base is not None, f"Type {typ} has no base"
+    return _shape_differs(typ, base)
+
+
+def _verify_disjoint_base(
+    stub: nodes.TypeInfo, runtime: type[object], object_path: list[str]
+) -> Iterator[Error]:
+    # If it's final, doesn't matter whether it's a disjoint base or not
+    if stub.is_final:
+        return
+    is_disjoint_runtime = _is_disjoint_base(runtime)
+    if is_disjoint_runtime and not stub.is_disjoint_base:
+        yield Error(
+            object_path,
+            "is a disjoint base at runtime, but isn't marked with @disjoint_base in the stub",
+            stub,
+            runtime,
+            stub_desc=repr(stub),
+        )
+    elif not is_disjoint_runtime and stub.is_disjoint_base:
+        yield Error(
+            object_path,
+            "is marked with @disjoint_base in the stub, but isn't a disjoint base at runtime",
+            stub,
+            runtime,
+            stub_desc=repr(stub),
+        )
+
+
 def _verify_metaclass(
     stub: nodes.TypeInfo, runtime: type[Any], object_path: list[str], *, is_runtime_typeddict: bool
 ) -> Iterator[Error]:
@@ -534,6 +596,7 @@ def verify_typeinfo(
         return
 
     yield from _verify_final(stub, runtime, object_path)
+    yield from _verify_disjoint_base(stub, runtime, object_path)
     is_runtime_typeddict = stub.typeddict_type is not None and is_typeddict(runtime)
     yield from _verify_metaclass(
         stub, runtime, object_path, is_runtime_typeddict=is_runtime_typeddict

mypy/test/testsolve.py

@@ -64,12 +64,14 @@ def test_multiple_variables(self) -> None:
         )
 
     def test_no_constraints_for_var(self) -> None:
-        self.assert_solve([self.fx.t], [], [self.fx.uninhabited])
-        self.assert_solve([self.fx.t, self.fx.s], [], [self.fx.uninhabited, self.fx.uninhabited])
+        self.assert_solve([self.fx.t], [], [self.fx.a_uninhabited])
+        self.assert_solve(
+            [self.fx.t, self.fx.s], [], [self.fx.a_uninhabited, self.fx.a_uninhabited]
+        )
         self.assert_solve(
             [self.fx.t, self.fx.s],
             [self.supc(self.fx.s, self.fx.a)],
-            [self.fx.uninhabited, self.fx.a],
+            [self.fx.a_uninhabited, self.fx.a],
         )
 
     def test_simple_constraints_with_dynamic_type(self) -> None:
@@ -116,7 +118,7 @@ def test_poly_no_constraints(self) -> None:
         self.assert_solve(
             [self.fx.t, self.fx.u],
             [],
-            [self.fx.uninhabited, self.fx.uninhabited],
+            [self.fx.a_uninhabited, self.fx.a_uninhabited],
             allow_polymorphic=True,
         )
 
@@ -152,7 +154,7 @@ def test_poly_free_pair_with_bounds_uninhabited(self) -> None:
         self.assert_solve(
             [self.fx.ub, self.fx.uc],
             [self.subc(self.fx.ub, self.fx.uc)],
-            [self.fx.uninhabited, self.fx.uninhabited],
+            [self.fx.a_uninhabited, self.fx.a_uninhabited],
             [],
             allow_polymorphic=True,
         )