Merge remote-tracking branch 'upstream/master' into stubtover

Author: hauntsaninja

.pre-commit-config.yaml

@@ -11,7 +11,7 @@ repos:
       - id: black
         exclude: '^(test-data/)'
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.8.6
+    rev: v0.9.10
     hooks:
       - id: ruff
         args: [--exit-non-zero-on-fix]

docs/source/type_narrowing.rst

@@ -8,6 +8,15 @@ techniques which are supported by mypy.
 
 Type narrowing is when you convince a type checker that a broader type is actually more specific, for instance, that an object of type ``Shape`` is actually of the narrower type ``Square``.
 
+The following type narrowing techniques are available:
+
+- :ref:`type-narrowing-expressions`
+- :ref:`casts`
+- :ref:`type-guards`
+- :ref:`typeis`
+
+
+.. _type-narrowing-expressions:
 
 Type narrowing expressions
 --------------------------
@@ -356,40 +365,6 @@ What happens here?
 
   The same will work with ``isinstance(x := a, float)`` as well.
 
-Limitations
------------
-
-Mypy's analysis is limited to individual symbols and it will not track
-relationships between symbols. For example, in the following code
-it's easy to deduce that if :code:`a` is None then :code:`b` must not be,
-therefore :code:`a or b` will always be an instance of :code:`C`,
-but Mypy will not be able to tell that:
-
-.. code-block:: python
-
-    class C:
-        pass
-
-    def f(a: C | None, b: C | None) -> C:
-        if a is not None or b is not None:
-            return a or b  # Incompatible return value type (got "C | None", expected "C")
-        return C()
-
-Tracking these sort of cross-variable conditions in a type checker would add significant complexity
-and performance overhead.
-
-You can use an ``assert`` to convince the type checker, override it with a :ref:`cast <casts>`
-or rewrite the function to be slightly more verbose:
-
-.. code-block:: python
-
-    def f(a: C | None, b: C | None) -> C:
-        if a is not None:
-            return a
-        elif b is not None:
-            return b
-        return C()
-
 
 .. _typeis:
 
@@ -555,3 +530,38 @@ You can use the assignment expression operator ``:=`` with ``TypeIs`` to create
             reveal_type(x)  # Revealed type is 'float'
             # x is narrowed to float in this block
             print(x + 1.0)
+
+
+Limitations
+-----------
+
+Mypy's analysis is limited to individual symbols and it will not track
+relationships between symbols. For example, in the following code
+it's easy to deduce that if :code:`a` is None then :code:`b` must not be,
+therefore :code:`a or b` will always be an instance of :code:`C`,
+but Mypy will not be able to tell that:
+
+.. code-block:: python
+
+    class C:
+        pass
+
+    def f(a: C | None, b: C | None) -> C:
+        if a is not None or b is not None:
+            return a or b  # Incompatible return value type (got "C | None", expected "C")
+        return C()
+
+Tracking these sort of cross-variable conditions in a type checker would add significant complexity
+and performance overhead.
+
+You can use an ``assert`` to convince the type checker, override it with a :ref:`cast <casts>`
+or rewrite the function to be slightly more verbose:
+
+.. code-block:: python
+
+    def f(a: C | None, b: C | None) -> C:
+        if a is not None:
+            return a
+        elif b is not None:
+            return b
+        return C()

mypy/binder.py

@@ -7,8 +7,9 @@
 from typing_extensions import TypeAlias as _TypeAlias
 
 from mypy.erasetype import remove_instance_last_known_values
-from mypy.literals import Key, literal, literal_hash, subkeys
+from mypy.literals import Key, extract_var_from_literal_hash, literal, literal_hash, subkeys
 from mypy.nodes import Expression, IndexExpr, MemberExpr, NameExpr, RefExpr, TypeInfo, Var
+from mypy.options import Options
 from mypy.subtypes import is_same_type, is_subtype
 from mypy.typeops import make_simplified_union
 from mypy.types import (
@@ -39,6 +40,7 @@ class CurrentType(NamedTuple):
 
 class Frame:
     """A Frame represents a specific point in the execution of a program.
+
     It carries information about the current types of expressions at
     that point, arising either from assignments to those expressions
     or the result of isinstance checks and other type narrowing
@@ -97,7 +99,7 @@ class A:
     # This maps an expression to a list of bound types for every item in the union type.
     type_assignments: Assigns | None = None
 
-    def __init__(self) -> None:
+    def __init__(self, options: Options) -> None:
         # Each frame gets an increasing, distinct id.
         self.next_id = 1
 
@@ -131,6 +133,11 @@ def __init__(self) -> None:
         self.break_frames: list[int] = []
         self.continue_frames: list[int] = []
 
+        # If True, initial assignment to a simple variable (e.g. "x", but not "x.y")
+        # is added to the binder. This allows more precise narrowing and more
+        # flexible inference of variable types (--allow-redefinition-new).
+        self.bind_all = options.allow_redefinition_new
+
     def _get_id(self) -> int:
         self.next_id += 1
         return self.next_id
@@ -226,12 +233,20 @@ def update_from_options(self, frames: list[Frame]) -> bool:
         for key in keys:
             current_value = self._get(key)
             resulting_values = [f.types.get(key, current_value) for f in frames]
-            if any(x is None for x in resulting_values):
+            # Keys can be narrowed using two different semantics. The new semantics
+            # is enabled for plain variables when bind_all is true, and it allows
+            # variable types to be widened using subsequent assignments. This is
+            # tricky to support for instance attributes (primarily due to deferrals),
+            # so we don't use it for them.
+            old_semantics = not self.bind_all or extract_var_from_literal_hash(key) is None
+            if old_semantics and any(x is None for x in resulting_values):
                 # We didn't know anything about key before
                 # (current_value must be None), and we still don't
                 # know anything about key in at least one possible frame.
                 continue
 
+            resulting_values = [x for x in resulting_values if x is not None]
+
             if all_reachable and all(
                 x is not None and not x.from_assignment for x in resulting_values
             ):
@@ -278,7 +293,11 @@ def update_from_options(self, frames: list[Frame]) -> bool:
                     # still equivalent to such type).
                     if isinstance(type, UnionType):
                         type = collapse_variadic_union(type)
-                    if isinstance(type, ProperType) and isinstance(type, UnionType):
+                    if (
+                        old_semantics
+                        and isinstance(type, ProperType)
+                        and isinstance(type, UnionType)
+                    ):
                         # Simplify away any extra Any's that were added to the declared
                         # type when popping a frame.
                         simplified = UnionType.make_union(

mypy/build.py

@@ -2240,8 +2240,10 @@ def semantic_analysis_pass1(self) -> None:
         # TODO: Do this while constructing the AST?
         self.tree.names = SymbolTable()
         if not self.tree.is_stub:
-            # Always perform some low-key variable renaming
-            self.tree.accept(LimitedVariableRenameVisitor())
+            if not self.options.allow_redefinition_new:
+                # Perform some low-key variable renaming when assignments can't
+                # widen inferred types
+                self.tree.accept(LimitedVariableRenameVisitor())
             if options.allow_redefinition:
                 # Perform more renaming across the AST to allow variable redefinitions
                 self.tree.accept(VariableRenameVisitor())