Incorporate more fixes

Author: ilevkivskyi

mypy/build.py

@@ -47,16 +47,7 @@
 from mypy.graph_utils import prepare_sccs, strongly_connected_components, topsort
 from mypy.indirection import TypeIndirectionVisitor
 from mypy.messages import MessageBuilder
-from mypy.nodes import (
-    Decorator,
-    Import,
-    ImportAll,
-    ImportBase,
-    ImportFrom,
-    MypyFile,
-    OverloadedFuncDef,
-    SymbolTable,
-)
+from mypy.nodes import Import, ImportAll, ImportBase, ImportFrom, MypyFile, SymbolTable
 from mypy.partially_defined import PossiblyUndefinedVariableVisitor
 from mypy.semanal import SemanticAnalyzer
 from mypy.semanal_pass1 import SemanticAnalyzerPreAnalysis
@@ -1773,12 +1764,13 @@ def delete_cache(id: str, path: str, manager: BuildManager) -> None:
 stale so that they need to be re-parsed as well.
 
 Note on indirect dependencies: normally dependencies are determined from
-imports, but since our type interfaces are "opaque" (i.e. symbol tables can
-contain types identified by name), these are not enough.  We *must* also
-add "indirect" dependencies from types to their definitions.  For this
-purpose, after we finished processing a module, we travers its type map and
-symbol tables, and for each type we find (transitively) on which opaque/named
-types it depends.
+imports, but since our interfaces are "opaque" (i.e. symbol tables can
+contain cross-references as well as types identified by name), these are not
+enough.  We *must* also add "indirect" dependencies from symbols and types to
+their definitions.  For this purpose, we record all accessed symbols during
+semantic analysis, and after we finished processing a module, we traverse its
+type map, and for each type we find (transitively) on which named types it
+depends.
 
 Import cycles
 -------------
@@ -2416,22 +2408,14 @@ def finish_passes(self) -> None:
 
             # We should always patch indirect dependencies, even in full (non-incremental) builds,
             # because the cache still may be written, and it must be correct.
-            all_types = set(self.type_map().values())
-            for _, sym, _ in self.tree.local_definitions():
-                if sym.type is not None:
-                    all_types.add(sym.type)
-                # Special case: settable properties may have two types.
-                if isinstance(sym.node, OverloadedFuncDef) and sym.node.is_property:
-                    assert isinstance(first_node := sym.node.items[0], Decorator)
-                    if first_node.var.setter_type:
-                        all_types.add(first_node.var.setter_type)
-            # Using mod_alias_deps is unfortunate but needed, since it is highly impractical
-            # (and practically impossible) to avoid all get_proper_type() calls. For example,
-            # TypeInfo.bases and metaclass, *args and **kwargs, Overloaded.items, and trivial
-            # aliases like Text = str, etc. all currently forced to proper types. Thus, we need
-            # to record the original definitions as they are first seen in semanal.py.
             self._patch_indirect_dependencies(
-                self.type_checker().module_refs | self.tree.mod_alias_deps, all_types
+                # Two possible sources of indirect dependencies:
+                # * Symbols not directly imported in this module but accessed via an attribute
+                #   or via a re-export (vast majority of these recorded in semantic analysis).
+                # * For each expression type we need to record definitions of type components
+                #   since "meaning" of the type may be updated when definitions are updated.
+                self.tree.module_refs | self.type_checker().module_refs,
+                set(self.type_map().values()),
             )
 
             if self.options.dump_inference_stats:

mypy/checker.py

@@ -378,11 +378,9 @@ class TypeChecker(NodeVisitor[None], TypeCheckerSharedApi):
     inferred_attribute_types: dict[Var, Type] | None = None
     # Don't infer partial None types if we are processing assignment from Union
     no_partial_types: bool = False
-
-    # The set of all dependencies (suppressed or not) that this module accesses, either
-    # directly or indirectly.
+    # Extra module references not detected during semantic analysis (these are rare cases
+    # e.g. access to class-level import via instance).
     module_refs: set[str]
-
     # A map from variable nodes to a snapshot of the frame ids of the
     # frames that were active when the variable was declared. This can
     # be used to determine nearest common ancestor frame of a variable's

mypy/checkexpr.py

@@ -198,7 +198,6 @@
 )
 from mypy.typestate import type_state
 from mypy.typevars import fill_typevars
-from mypy.util import split_module_names
 from mypy.visitor import ExpressionVisitor
 
 # Type of callback user for checking individual function arguments. See
@@ -248,36 +247,6 @@ def allow_fast_container_literal(t: Type) -> bool:
     )
 
 
-def extract_refexpr_names(expr: RefExpr, output: set[str]) -> None:
-    """Recursively extracts all module references from a reference expression.
-
-    Note that currently, the only two subclasses of RefExpr are NameExpr and
-    MemberExpr."""
-    while isinstance(expr.node, MypyFile) or expr.fullname:
-        if isinstance(expr.node, MypyFile) and expr.fullname:
-            # If it's None, something's wrong (perhaps due to an
-            # import cycle or a suppressed error).  For now we just
-            # skip it.
-            output.add(expr.fullname)
-
-        if isinstance(expr, NameExpr):
-            is_suppressed_import = isinstance(expr.node, Var) and expr.node.is_suppressed_import
-            if isinstance(expr.node, TypeInfo):
-                # Reference to a class or a nested class
-                output.update(split_module_names(expr.node.module_name))
-            elif "." in expr.fullname and not is_suppressed_import:
-                # Everything else (that is not a silenced import within a class)
-                output.add(expr.fullname.rsplit(".", 1)[0])
-            break
-        elif isinstance(expr, MemberExpr):
-            if isinstance(expr.expr, RefExpr):
-                expr = expr.expr
-            else:
-                break
-        else:
-            raise AssertionError(f"Unknown RefExpr subclass: {type(expr)}")
-
-
 class Finished(Exception):
     """Raised if we can terminate overload argument check early (no match)."""
 
@@ -370,7 +339,6 @@ def visit_name_expr(self, e: NameExpr) -> Type:
 
         It can be of any kind: local, member or global.
         """
-        extract_refexpr_names(e, self.chk.module_refs)
         result = self.analyze_ref_expr(e)
         narrowed = self.narrow_type_from_binder(e, result)
         self.chk.check_deprecated(e.node, e)
@@ -3344,7 +3312,6 @@ def check_union_call(
 
     def visit_member_expr(self, e: MemberExpr, is_lvalue: bool = False) -> Type:
         """Visit member expression (of form e.id)."""
-        extract_refexpr_names(e, self.chk.module_refs)
         result = self.analyze_ordinary_member_access(e, is_lvalue)
         narrowed = self.narrow_type_from_binder(e, result)
         self.chk.warn_deprecated(e.node, e)

mypy/checkmember.py

@@ -543,6 +543,8 @@ def analyze_member_var_access(
     if isinstance(v, FuncDef):
         assert False, "Did not expect a function"
     if isinstance(v, MypyFile):
+        # Special case: accessing module on instances is allowed, but will not
+        # be recorded by semantic analyzer.
         mx.chk.module_refs.add(v.fullname)
 
     if isinstance(vv, (TypeInfo, TypeAlias, MypyFile, TypeVarLikeExpr)):

mypy/indirection.py

@@ -7,16 +7,6 @@
 from mypy.util import split_module_names
 
 
-def extract_module_names(type_name: str | None) -> list[str]:
-    """Returns the module names of a fully qualified type name."""
-    if type_name is not None:
-        # Discard the first one, which is just the qualified name of the type
-        possible_module_names = split_module_names(type_name)
-        return possible_module_names[1:]
-    else:
-        return []
-
-
 class TypeIndirectionVisitor(TypeVisitor[None]):
     """Returns all module references within a particular type."""
 
@@ -25,12 +15,9 @@ def __init__(self) -> None:
         self.modules: set[str] = set()
         # User to avoid infinite recursion with recursive types
         self.seen_types: set[types.TypeAliasType | types.Instance] = set()
-        # Used to avoid redundant work
-        self.seen_fullnames: set[str] = set()
 
     def find_modules(self, typs: Iterable[types.Type]) -> set[str]:
         self.modules = set()
-        self.seen_fullnames = set()
         self.seen_types = set()
         for typ in typs:
             self._visit(typ)
@@ -145,11 +132,7 @@ def visit_instance(self, t: types.Instance) -> None:
     def visit_callable_type(self, t: types.CallableType) -> None:
         self._visit_type_list(t.arg_types)
         self._visit(t.ret_type)
-        if t.definition is not None:
-            fullname = t.definition.fullname
-            if fullname not in self.seen_fullnames:
-                self.modules.update(extract_module_names(t.definition.fullname))
-                self.seen_fullnames.add(fullname)
+        self._visit_type_tuple(t.variables)
 
     def visit_overloaded(self, t: types.Overloaded) -> None:
         for item in t.items:

mypy/nodes.py

@@ -288,7 +288,7 @@ class MypyFile(SymbolNode):
         "path",
         "defs",
         "alias_deps",
-        "mod_alias_deps",
+        "module_refs",
         "is_bom",
         "names",
         "imports",
@@ -312,8 +312,9 @@ class MypyFile(SymbolNode):
     defs: list[Statement]
     # Type alias dependencies as mapping from target to set of alias full names
     alias_deps: defaultdict[str, set[str]]
-    # Same as above but for coarse-grained dependencies (i.e. modules instead of full names)
-    mod_alias_deps: set[str]
+    # The set of all dependencies (suppressed or not) that this module accesses, either
+    # directly or indirectly.
+    module_refs: set[str]
     # Is there a UTF-8 BOM at the start?
     is_bom: bool
     names: SymbolTable
@@ -354,7 +355,7 @@ def __init__(
         self.imports = imports
         self.is_bom = is_bom
         self.alias_deps = defaultdict(set)
-        self.mod_alias_deps = set()
+        self.module_refs = set()
         self.plugin_deps = {}
         if ignored_lines:
             self.ignored_lines = ignored_lines

mypy/semanal.py

@@ -305,7 +305,14 @@
 )
 from mypy.types_utils import is_invalid_recursive_alias, store_argument_type
 from mypy.typevars import fill_typevars
-from mypy.util import correct_relative_import, is_dunder, module_prefix, unmangle, unnamed_function
+from mypy.util import (
+    correct_relative_import,
+    is_dunder,
+    module_prefix,
+    split_module_names,
+    unmangle,
+    unnamed_function,
+)
 from mypy.visitor import NodeVisitor
 
 T = TypeVar("T")
@@ -2808,7 +2815,6 @@ def get_declared_metaclass(
                 and not sym.node.alias_tvars
             ):
                 target = get_proper_type(sym.node.target)
-                self.add_type_alias_deps({(sym.node.module, sym.node.fullname)})
                 if isinstance(target, Instance):
                     metaclass_info = target.type
 
@@ -3887,15 +3893,15 @@ def analyze_alias(
         declared_type_vars: TypeVarLikeList | None = None,
         all_declared_type_params_names: list[str] | None = None,
         python_3_12_type_alias: bool = False,
-    ) -> tuple[Type | None, list[TypeVarLikeType], set[tuple[str, str]], bool]:
+    ) -> tuple[Type | None, list[TypeVarLikeType], set[str], bool]:
         """Check if 'rvalue' is a valid type allowed for aliasing (e.g. not a type variable).
 
         If yes, return the corresponding type, a list of type variables for generic aliases,
         a set of names the alias depends on, and True if the original type has empty tuple index.
         An example for the dependencies:
             A = int
             B = str
-            analyze_alias(dict[A, B])[2] == {('mod', 'mod.A'), ('mod', 'mod.B')}
+            analyze_alias(dict[A, B])[2] == {'__main__.A', '__main__.B'}
         """
         dynamic = bool(self.function_stack and self.function_stack[-1].is_dynamic())
         global_scope = not self.type and not self.function_stack
@@ -4040,7 +4046,7 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool:
         if self.is_none_alias(rvalue):
             res = NoneType()
             alias_tvars: list[TypeVarLikeType] = []
-            depends_on: set[tuple[str, str]] = set()
+            depends_on: set[str] = set()
             empty_tuple_index = False
         else:
             tag = self.track_incomplete_refs()
@@ -5929,6 +5935,8 @@ def visit_member_expr(self, expr: MemberExpr) -> None:
                 if isinstance(sym.node, PlaceholderNode):
                     self.process_placeholder(expr.name, "attribute", expr)
                     return
+                if sym.node is not None:
+                    self.record_imported_symbol(sym.node)
                 expr.kind = sym.kind
                 expr.fullname = sym.fullname or ""
                 expr.node = sym.node
@@ -5959,8 +5967,7 @@ def visit_member_expr(self, expr: MemberExpr) -> None:
             if type_info:
                 n = type_info.names.get(expr.name)
                 if n is not None and isinstance(n.node, (MypyFile, TypeInfo, TypeAlias)):
-                    if not n:
-                        return
+                    self.record_imported_symbol(n.node)
                     expr.kind = n.kind
                     expr.fullname = n.fullname or ""
                     expr.node = n.node
@@ -6280,6 +6287,24 @@ def visit_class_pattern(self, p: ClassPattern) -> None:
 
     def lookup(
         self, name: str, ctx: Context, suppress_errors: bool = False
+    ) -> SymbolTableNode | None:
+        node = self._lookup(name, ctx, suppress_errors)
+        if node is not None and node.node is not None:
+            # This call is unfortunate from performance point of view, but
+            # needed for rare cases like e.g. testIncrementalChangingAlias.
+            self.record_imported_symbol(node.node)
+        return node
+
+    def record_imported_symbol(self, node: SymbolNode) -> None:
+        fullname = node.fullname
+        if not isinstance(node, MypyFile):
+            while fullname not in self.modules and "." in fullname:
+                fullname = fullname.rsplit(".")[0]
+        if fullname != self.cur_mod_id and fullname not in self.cur_mod_node.module_refs:
+            self.cur_mod_node.module_refs.update(split_module_names(fullname))
+
+    def _lookup(
+        self, name: str, ctx: Context, suppress_errors: bool = False
     ) -> SymbolTableNode | None:
         """Look up an unqualified (no dots) name in all active namespaces.
 
@@ -6488,6 +6513,8 @@ def lookup_qualified(
                         self.name_not_defined(name, ctx, namespace=namespace)
                     return None
                 sym = nextsym
+        if sym is not None and sym.node is not None:
+            self.record_imported_symbol(sym.node)
         return sym
 
     def lookup_type_node(self, expr: Expression) -> SymbolTableNode | None:
@@ -7526,21 +7553,18 @@ def add_plugin_dependency(self, trigger: str, target: str | None = None) -> None
         self.cur_mod_node.plugin_deps.setdefault(trigger, set()).add(target)
 
     def add_type_alias_deps(
-        self, aliases_used: Collection[tuple[str, str]], target: str | None = None
+        self, aliases_used: Collection[str], target: str | None = None
     ) -> None:
         """Add full names of type aliases on which the current node depends.
 
         This is used by fine-grained incremental mode to re-check the corresponding nodes.
-        If `target` is None, then the target node used will be the current scope. For
-        coarse-grained mode, add just the module names where aliases are defined.
+        If `target` is None, then the target node used will be the current scope.
         """
         if not aliases_used:
             return
         if target is None:
             target = self.scope.current_target()
-        for mod, fn in aliases_used:
-            self.cur_mod_node.alias_deps[target].add(fn)
-            self.cur_mod_node.mod_alias_deps.add(mod)
+        self.cur_mod_node.alias_deps[target].update(aliases_used)
 
     def is_mangled_global(self, name: str) -> bool:
         # A global is mangled if there exists at least one renamed variant.

mypy/typeanal.py

@@ -155,11 +155,11 @@ def analyze_type_alias(
     allowed_alias_tvars: list[TypeVarLikeType] | None = None,
     alias_type_params_names: list[str] | None = None,
     python_3_12_type_alias: bool = False,
-) -> tuple[Type, set[tuple[str, str]]]:
+) -> tuple[Type, set[str]]:
     """Analyze r.h.s. of a (potential) type alias definition.
 
     If `node` is valid as a type alias rvalue, return the resulting type and a set of
-    module and full names of type aliases it depends on (directly or indirectly).
+    full names of type aliases it depends on (directly or indirectly).
     'node' must have been semantically analyzed.
     """
     analyzer = TypeAnalyser(
@@ -263,9 +263,8 @@ def __init__(
         self.options = options
         self.cur_mod_node = cur_mod_node
         self.is_typeshed_stub = is_typeshed_stub
-        # Names of type aliases encountered while analysing a type will be collected here
-        # (each tuple in the set is (module_name, fullname)).
-        self.aliases_used: set[tuple[str, str]] = set()
+        # Names of type aliases encountered while analysing a type will be collected here.
+        self.aliases_used: set[str] = set()
         self.prohibit_self_type = prohibit_self_type
         # Set when we analyze TypedDicts or NamedTuples, since they are special:
         self.prohibit_special_class_field_types = prohibit_special_class_field_types
@@ -458,7 +457,7 @@ def visit_unbound_type_nonoptional(self, t: UnboundType, defining_literal: bool)
             if special is not None:
                 return special
             if isinstance(node, TypeAlias):
-                self.aliases_used.add((node.module, fullname))
+                self.aliases_used.add(fullname)
                 an_args = self.anal_array(
                     t.args,
                     allow_param_spec=True,