Finish main part

Author: ilevkivskyi

mypy/build.py

@@ -48,14 +48,14 @@
 from mypy.indirection import TypeIndirectionVisitor
 from mypy.messages import MessageBuilder
 from mypy.nodes import (
+    Decorator,
     Import,
     ImportAll,
     ImportBase,
     ImportFrom,
     MypyFile,
+    OverloadedFuncDef,
     SymbolTable,
-    TypeAlias,
-    TypeInfo,
 )
 from mypy.partially_defined import PossiblyUndefinedVariableVisitor
 from mypy.semanal import SemanticAnalyzer
@@ -1767,26 +1767,23 @@ def delete_cache(id: str, path: str, manager: BuildManager) -> None:
 
 For single nodes, processing is simple.  If the node was cached, we
 deserialize the cache data and fix up cross-references.  Otherwise, we
-do semantic analysis followed by type checking.  We also handle (c)
-above; if a module has valid cache data *but* any of its
-dependencies was processed from source, then the module should be
-processed from source.
-
-A relatively simple optimization (outside SCCs) we might do in the
-future is as follows: if a node's cache data is valid, but one or more
-of its dependencies are out of date so we have to re-parse the node
-from source, once we have fully type-checked the node, we can decide
-whether its symbol table actually changed compared to the cache data
-(by reading the cache data and comparing it to the data we would be
-writing).  If there is no change we can declare the node up to date,
-and any node that depends (and for which we have cached data, and
-whose other dependencies are up to date) on it won't need to be
-re-parsed from source.
+do semantic analysis followed by type checking.  Once we (re-)processed
+an SCC we check whether its interface (symbol table) is still fresh
+(matches previous cached value). If it is not, we consider dependent SCCs
+stale so that they need to be re-parsed as well.
+
+Note on indirect dependencies: normally dependencies are determined from
+imports, but since our type interfaces are "opaque" (i.e. symbol tables can
+contain types identified by name), these are not enough.  We *must* also
+add "indirect" dependencies from types to their definitions.  For this
+purpose, after we finished processing a module, we travers its type map and
+symbol tables, and for each type we find (transitively) on which opaque/named
+types it depends.
 
 Import cycles
 -------------
 
-Finally we have to decide how to handle (c), import cycles.  Here
+Finally we have to decide how to handle (b), import cycles.  Here
 we'll need a modified version of the original state machine
 (build.py), but we only need to do this per SCC, and we won't have to
 deal with changes to the list of nodes while we're processing it.
@@ -2419,23 +2416,23 @@ def finish_passes(self) -> None:
 
             # We should always patch indirect dependencies, even in full (non-incremental) builds,
             # because the cache still may be written, and it must be correct.
-            # TODO: find a more robust way to traverse *all* relevant types?
-            all_types = list(self.type_map().values())
+            all_types = set(self.type_map().values())
             for _, sym, _ in self.tree.local_definitions():
                 if sym.type is not None:
-                    all_types.append(sym.type)
-                if isinstance(sym.node, TypeAlias):
-                    all_types.append(sym.node.target)
-                if isinstance(sym.node, TypeInfo):
-                    # TypeInfo symbols have some extra relevant types.
-                    all_types.extend(sym.node.bases)
-                    if sym.node.metaclass_type:
-                        all_types.append(sym.node.metaclass_type)
-                    if sym.node.typeddict_type:
-                        all_types.append(sym.node.typeddict_type)
-                    if sym.node.tuple_type:
-                        all_types.append(sym.node.tuple_type)
-            self._patch_indirect_dependencies(self.type_checker().module_refs, all_types)
+                    all_types.add(sym.type)
+                # Special case: settable properties may have two types.
+                if isinstance(sym.node, OverloadedFuncDef) and sym.node.is_property:
+                    assert isinstance(first_node := sym.node.items[0], Decorator)
+                    if first_node.var.setter_type:
+                        all_types.add(first_node.var.setter_type)
+            # Using mod_alias_deps is unfortunate but needed, since it is highly impractical
+            # (and practically impossible) to avoid all get_proper_type() calls. For example,
+            # TypeInfo.bases and metaclass, *args and **kwargs, Overloaded.items, and trivial
+            # aliases like Text = str, etc. all currently forced to proper types. Thus, we need
+            # to record the original definitions as they are first seen in semanal.py.
+            self._patch_indirect_dependencies(
+                self.type_checker().module_refs | self.tree.mod_alias_deps, all_types
+            )
 
             if self.options.dump_inference_stats:
                 dump_type_stats(
@@ -2464,7 +2461,7 @@ def free_state(self) -> None:
             self._type_checker.reset()
             self._type_checker = None
 
-    def _patch_indirect_dependencies(self, module_refs: set[str], types: list[Type]) -> None:
+    def _patch_indirect_dependencies(self, module_refs: set[str], types: set[Type]) -> None:
         assert None not in types
         valid = self.valid_references()
 
@@ -3298,7 +3295,10 @@ def process_graph(graph: Graph, manager: BuildManager) -> None:
         for id in scc:
             deps.update(graph[id].dependencies)
         deps -= ascc
-        stale_deps = {id for id in deps if id in graph and not graph[id].is_interface_fresh()}
+        # Note: if a dependency is not in graph anymore, it should be considered interface-stale.
+        # This is important to trigger any relevant updates from indirect dependencies that were
+        # removed in load_graph().
+        stale_deps = {id for id in deps if id not in graph or not graph[id].is_interface_fresh()}
         fresh = fresh and not stale_deps
         undeps = set()
         if fresh:

mypy/indirection.py

@@ -23,44 +23,58 @@ class TypeIndirectionVisitor(TypeVisitor[None]):
     def __init__(self) -> None:
         # Module references are collected here
         self.modules: set[str] = set()
-        # User to avoid infinite recursion with recursive type aliases
-        self.seen_aliases: set[types.TypeAliasType] = set()
+        # User to avoid infinite recursion with recursive types
+        self.seen_types: set[types.TypeAliasType | types.Instance] = set()
         # Used to avoid redundant work
         self.seen_fullnames: set[str] = set()
 
     def find_modules(self, typs: Iterable[types.Type]) -> set[str]:
         self.modules = set()
         self.seen_fullnames = set()
-        self.seen_aliases = set()
+        self.seen_types = set()
         for typ in typs:
             self._visit(typ)
         return self.modules
 
     def _visit(self, typ: types.Type) -> None:
-        if isinstance(typ, types.TypeAliasType):
-            # Avoid infinite recursion for recursive type aliases.
-            if typ not in self.seen_aliases:
-                self.seen_aliases.add(typ)
+        # Note: instances are needed for `class str(Sequence[str]): ...`
+        if (
+            isinstance(typ, types.TypeAliasType)
+            or isinstance(typ, types.ProperType)
+            and isinstance(typ, types.Instance)
+        ):
+            # Avoid infinite recursion for recursive types.
+            if typ in self.seen_types:
+                return
+            self.seen_types.add(typ)
         typ.accept(self)
 
     def _visit_type_tuple(self, typs: tuple[types.Type, ...]) -> None:
         # Micro-optimization: Specialized version of _visit for lists
         for typ in typs:
-            if isinstance(typ, types.TypeAliasType):
-                # Avoid infinite recursion for recursive type aliases.
-                if typ in self.seen_aliases:
+            if (
+                isinstance(typ, types.TypeAliasType)
+                or isinstance(typ, types.ProperType)
+                and isinstance(typ, types.Instance)
+            ):
+                # Avoid infinite recursion for recursive types.
+                if typ in self.seen_types:
                     continue
-                self.seen_aliases.add(typ)
+                self.seen_types.add(typ)
             typ.accept(self)
 
     def _visit_type_list(self, typs: list[types.Type]) -> None:
         # Micro-optimization: Specialized version of _visit for tuples
         for typ in typs:
-            if isinstance(typ, types.TypeAliasType):
-                # Avoid infinite recursion for recursive type aliases.
-                if typ in self.seen_aliases:
+            if (
+                isinstance(typ, types.TypeAliasType)
+                or isinstance(typ, types.ProperType)
+                and isinstance(typ, types.Instance)
+            ):
+                # Avoid infinite recursion for recursive types.
+                if typ in self.seen_types:
                     continue
-                self.seen_aliases.add(typ)
+                self.seen_types.add(typ)
             typ.accept(self)
 
     def _visit_module_name(self, module_name: str) -> None:
@@ -105,15 +119,27 @@ def visit_parameters(self, t: types.Parameters) -> None:
         self._visit_type_list(t.arg_types)
 
     def visit_instance(self, t: types.Instance) -> None:
+        # Instance is named, record its definition and continue digging into
+        # components that constitute semantic meaning of this type: bases, metaclass,
+        # tuple type, and typeddict type.
+        # Note: we cannot simply record the MRO, in case an intermediate base contains
+        # a reference to type alias, this affects meaning of map_instance_to_supertype(),
+        # see e.g. testDoubleReexportGenericUpdated.
         self._visit_type_tuple(t.args)
         if t.type:
-            # Uses of a class depend on everything in the MRO,
-            # as changes to classes in the MRO can add types to methods,
-            # change property types, change the MRO itself, etc.
+            # Important optimization: instead of simply recording the definition and
+            # recursing into bases, record the MRO and only traverse generic bases.
             for s in t.type.mro:
                 self._visit_module_name(s.module_name)
-            if t.type.metaclass_type is not None:
-                self._visit_module_name(t.type.metaclass_type.type.module_name)
+                for base in s.bases:
+                    if base.args:
+                        self._visit_type_tuple(base.args)
+            if t.type.metaclass_type:
+                self._visit(t.type.metaclass_type)
+            if t.type.typeddict_type:
+                self._visit(t.type.typeddict_type)
+            if t.type.tuple_type:
+                self._visit(t.type.tuple_type)
 
     def visit_callable_type(self, t: types.CallableType) -> None:
         self._visit_type_list(t.arg_types)
@@ -125,7 +151,8 @@ def visit_callable_type(self, t: types.CallableType) -> None:
                 self.seen_fullnames.add(fullname)
 
     def visit_overloaded(self, t: types.Overloaded) -> None:
-        self._visit_type_list(list(t.items))
+        for item in t.items:
+            self._visit(item)
         self._visit(t.fallback)
 
     def visit_tuple_type(self, t: types.TupleType) -> None:
@@ -149,6 +176,8 @@ def visit_type_type(self, t: types.TypeType) -> None:
         self._visit(t.item)
 
     def visit_type_alias_type(self, t: types.TypeAliasType) -> None:
+        # Type alias is named, record its definition and continue digging into
+        # components that constitute semantic meaning of this type: target and args.
         if t.alias:
             self._visit_module_name(t.alias.module)
             self._visit(t.alias.target)

mypy/nodes.py

@@ -288,6 +288,7 @@ class MypyFile(SymbolNode):
         "path",
         "defs",
         "alias_deps",
+        "mod_alias_deps",
         "is_bom",
         "names",
         "imports",
@@ -311,6 +312,8 @@ class MypyFile(SymbolNode):
     defs: list[Statement]
     # Type alias dependencies as mapping from target to set of alias full names
     alias_deps: defaultdict[str, set[str]]
+    # Same as above but for coarse-grained dependencies (i.e. modules instead of full names)
+    mod_alias_deps: set[str]
     # Is there a UTF-8 BOM at the start?
     is_bom: bool
     names: SymbolTable
@@ -351,6 +354,7 @@ def __init__(
         self.imports = imports
         self.is_bom = is_bom
         self.alias_deps = defaultdict(set)
+        self.mod_alias_deps = set()
         self.plugin_deps = {}
         if ignored_lines:
             self.ignored_lines = ignored_lines

mypy/semanal.py

@@ -2808,6 +2808,7 @@ def get_declared_metaclass(
                 and not sym.node.alias_tvars
             ):
                 target = get_proper_type(sym.node.target)
+                self.add_type_alias_deps({(sym.node.module, sym.node.fullname)})
                 if isinstance(target, Instance):
                     metaclass_info = target.type
 
@@ -3886,16 +3887,15 @@ def analyze_alias(
         declared_type_vars: TypeVarLikeList | None = None,
         all_declared_type_params_names: list[str] | None = None,
         python_3_12_type_alias: bool = False,
-    ) -> tuple[Type | None, list[TypeVarLikeType], set[str], list[str], bool]:
+    ) -> tuple[Type | None, list[TypeVarLikeType], set[tuple[str, str]], bool]:
         """Check if 'rvalue' is a valid type allowed for aliasing (e.g. not a type variable).
 
-        If yes, return the corresponding type, a list of
-        qualified type variable names for generic aliases, a set of names the alias depends on,
-        and a list of type variables if the alias is generic.
-        A schematic example for the dependencies:
+        If yes, return the corresponding type, a list of type variables for generic aliases,
+        a set of names the alias depends on, and True if the original type has empty tuple index.
+        An example for the dependencies:
             A = int
             B = str
-            analyze_alias(Dict[A, B])[2] == {'__main__.A', '__main__.B'}
+            analyze_alias(dict[A, B])[2] == {('mod', 'mod.A'), ('mod', 'mod.B')}
         """
         dynamic = bool(self.function_stack and self.function_stack[-1].is_dynamic())
         global_scope = not self.type and not self.function_stack
@@ -3907,10 +3907,9 @@ def analyze_alias(
             self.fail(
                 "Invalid type alias: expression is not a valid type", rvalue, code=codes.VALID_TYPE
             )
-            return None, [], set(), [], False
+            return None, [], set(), False
 
         found_type_vars = self.find_type_var_likes(typ)
-        tvar_defs: list[TypeVarLikeType] = []
         namespace = self.qualified_name(name)
         alias_type_vars = found_type_vars if declared_type_vars is None else declared_type_vars
         with self.tvar_scope_frame(self.tvar_scope.class_frame(namespace)):
@@ -3946,9 +3945,8 @@ def analyze_alias(
                 variadic = True
             new_tvar_defs.append(td)
 
-        qualified_tvars = [node.fullname for _name, node in alias_type_vars]
         empty_tuple_index = typ.empty_tuple_index if isinstance(typ, UnboundType) else False
-        return analyzed, new_tvar_defs, depends_on, qualified_tvars, empty_tuple_index
+        return analyzed, new_tvar_defs, depends_on, empty_tuple_index
 
     def is_pep_613(self, s: AssignmentStmt) -> bool:
         if s.unanalyzed_type is not None and isinstance(s.unanalyzed_type, UnboundType):
@@ -4042,12 +4040,11 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool:
         if self.is_none_alias(rvalue):
             res = NoneType()
             alias_tvars: list[TypeVarLikeType] = []
-            depends_on: set[str] = set()
-            qualified_tvars: list[str] = []
+            depends_on: set[tuple[str, str]] = set()
             empty_tuple_index = False
         else:
             tag = self.track_incomplete_refs()
-            res, alias_tvars, depends_on, qualified_tvars, empty_tuple_index = self.analyze_alias(
+            res, alias_tvars, depends_on, empty_tuple_index = self.analyze_alias(
                 lvalue.name,
                 rvalue,
                 allow_placeholder=True,
@@ -4071,12 +4068,6 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> bool:
                 self.mark_incomplete(lvalue.name, rvalue, becomes_typeinfo=True)
                 return True
         self.add_type_alias_deps(depends_on)
-        # In addition to the aliases used, we add deps on unbound
-        # type variables, since they are erased from target type.
-        self.add_type_alias_deps(qualified_tvars)
-        # The above are only direct deps on other aliases.
-        # For subscripted aliases, type deps from expansion are added in deps.py
-        # (because the type is stored).
         check_for_explicit_any(res, self.options, self.is_typeshed_stub_file, self.msg, context=s)
         # When this type alias gets "inlined", the Any is not explicit anymore,
         # so we need to replace it with non-explicit Anys.
@@ -5579,7 +5570,7 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None:
                 return
 
             tag = self.track_incomplete_refs()
-            res, alias_tvars, depends_on, qualified_tvars, empty_tuple_index = self.analyze_alias(
+            res, alias_tvars, depends_on, empty_tuple_index = self.analyze_alias(
                 s.name.name,
                 s.value.expr(),
                 allow_placeholder=True,
@@ -5608,12 +5599,6 @@ def visit_type_alias_stmt(self, s: TypeAliasStmt) -> None:
                 return
 
             self.add_type_alias_deps(depends_on)
-            # In addition to the aliases used, we add deps on unbound
-            # type variables, since they are erased from target type.
-            self.add_type_alias_deps(qualified_tvars)
-            # The above are only direct deps on other aliases.
-            # For subscripted aliases, type deps from expansion are added in deps.py
-            # (because the type is stored).
             check_for_explicit_any(
                 res, self.options, self.is_typeshed_stub_file, self.msg, context=s
             )
@@ -7541,20 +7526,21 @@ def add_plugin_dependency(self, trigger: str, target: str | None = None) -> None
         self.cur_mod_node.plugin_deps.setdefault(trigger, set()).add(target)
 
     def add_type_alias_deps(
-        self, aliases_used: Collection[str], target: str | None = None
+        self, aliases_used: Collection[tuple[str, str]], target: str | None = None
     ) -> None:
         """Add full names of type aliases on which the current node depends.
 
         This is used by fine-grained incremental mode to re-check the corresponding nodes.
-        If `target` is None, then the target node used will be the current scope.
+        If `target` is None, then the target node used will be the current scope. For
+        coarse-grained mode, add just the module names where aliases are defined.
         """
         if not aliases_used:
-            # A basic optimization to avoid adding targets with no dependencies to
-            # the `alias_deps` dict.
             return
         if target is None:
             target = self.scope.current_target()
-        self.cur_mod_node.alias_deps[target].update(aliases_used)
+        for mod, fn in aliases_used:
+            self.cur_mod_node.alias_deps[target].add(fn)
+            self.cur_mod_node.mod_alias_deps.add(mod)
 
     def is_mangled_global(self, name: str) -> bool:
         # A global is mangled if there exists at least one renamed variant.