Merge branch 'master' into patch-6

Author: BobTheBuidler

.github/workflows/docs.yml

@@ -12,6 +12,9 @@ on:
     # so it's important to do the docs build on all PRs touching mypy/errorcodes.py
     # in case somebody's adding a new error code without any docs
     - 'mypy/errorcodes.py'
+    # Part of the documentation is automatically generated from the options
+    # definitions in mypy/main.py
+    - 'mypy/main.py'
     - 'mypyc/doc/**'
     - '**/*.rst'
     - '**/*.md'

CHANGELOG.md

@@ -15,7 +15,6 @@
 
 import collections
 import contextlib
-import errno
 import gc
 import json
 import os
@@ -337,6 +336,7 @@ class CacheMeta(NamedTuple):
     dep_lines: list[int]
     dep_hashes: dict[str, str]
     interface_hash: str  # hash representing the public interface
+    error_lines: list[str]
     version_id: str  # mypy version for cache invalidation
     ignore_all: bool  # if errors were ignored
     plugin_data: Any  # config data from plugins
@@ -376,6 +376,7 @@ def cache_meta_from_dict(meta: dict[str, Any], data_json: str) -> CacheMeta:
         meta.get("dep_lines", []),
         meta.get("dep_hashes", {}),
         meta.get("interface_hash", ""),
+        meta.get("error_lines", []),
         meta.get("version_id", sentinel),
         meta.get("ignore_all", True),
         meta.get("plugin_data", None),
@@ -1502,6 +1503,7 @@ def validate_meta(
                 "dep_lines": meta.dep_lines,
                 "dep_hashes": meta.dep_hashes,
                 "interface_hash": meta.interface_hash,
+                "error_lines": meta.error_lines,
                 "version_id": manager.version_id,
                 "ignore_all": meta.ignore_all,
                 "plugin_data": meta.plugin_data,
@@ -1678,28 +1680,6 @@ def write_cache_meta(
     return cache_meta_from_dict(meta, data_json)
 
 
-def delete_cache(id: str, path: str, manager: BuildManager) -> None:
-    """Delete cache files for a module.
-
-    The cache files for a module are deleted when mypy finds errors there.
-    This avoids inconsistent states with cache files from different mypy runs,
-    see #4043 for an example.
-    """
-    # We don't delete .deps files on errors, since the dependencies
-    # are mostly generated from other files and the metadata is
-    # tracked separately.
-    meta_path, data_path, _ = get_cache_names(id, path, manager.options)
-    cache_paths = [meta_path, data_path]
-    manager.log(f"Deleting {id} {path} {' '.join(x for x in cache_paths if x)}")
-
-    for filename in cache_paths:
-        try:
-            manager.metastore.remove(filename)
-        except OSError as e:
-            if e.errno != errno.ENOENT:
-                manager.log(f"Error deleting cache file {filename}: {e.strerror}")
-
-
 """Dependency manager.
 
 Design
@@ -1875,6 +1855,9 @@ class State:
     # Map from dependency id to its last observed interface hash
     dep_hashes: dict[str, str] = {}
 
+    # List of errors reported for this file last time.
+    error_lines: list[str] = []
+
     # Parent package, its parent, etc.
     ancestors: list[str] | None = None
 
@@ -1896,9 +1879,6 @@ class State:
     # Whether to ignore all errors
     ignore_all = False
 
-    # Whether the module has an error or any of its dependencies have one.
-    transitive_error = False
-
     # Errors reported before semantic analysis, to allow fine-grained
     # mode to keep reporting them.
     early_errors: list[ErrorInfo]
@@ -2000,6 +1980,7 @@ def __init__(
             assert len(all_deps) == len(self.meta.dep_lines)
             self.dep_line_map = {id: line for id, line in zip(all_deps, self.meta.dep_lines)}
             self.dep_hashes = self.meta.dep_hashes
+            self.error_lines = self.meta.error_lines
             if temporary:
                 self.load_tree(temporary=True)
             if not manager.use_fine_grained_cache():
@@ -2517,11 +2498,6 @@ def write_cache(self) -> tuple[dict[str, Any], str, str] | None:
                     print(f"Error serializing {self.id}", file=self.manager.stdout)
                     raise  # Propagate to display traceback
             return None
-        is_errors = self.transitive_error
-        if is_errors:
-            delete_cache(self.id, self.path, self.manager)
-            self.meta = None
-            return None
         dep_prios = self.dependency_priorities()
         dep_lines = self.dependency_lines()
         assert self.source_hash is not None
@@ -3303,34 +3279,7 @@ def process_graph(graph: Graph, manager: BuildManager) -> None:
             if undeps:
                 fresh = False
         if fresh:
-            # All cache files are fresh.  Check that no dependency's
-            # cache file is newer than any scc node's cache file.
-            oldest_in_scc = min(graph[id].xmeta.data_mtime for id in scc)
-            viable = {id for id in stale_deps if graph[id].meta is not None}
-            newest_in_deps = (
-                0 if not viable else max(graph[dep].xmeta.data_mtime for dep in viable)
-            )
-            if manager.options.verbosity >= 3:  # Dump all mtimes for extreme debugging.
-                all_ids = sorted(ascc | viable, key=lambda id: graph[id].xmeta.data_mtime)
-                for id in all_ids:
-                    if id in scc:
-                        if graph[id].xmeta.data_mtime < newest_in_deps:
-                            key = "*id:"
-                        else:
-                            key = "id:"
-                    else:
-                        if graph[id].xmeta.data_mtime > oldest_in_scc:
-                            key = "+dep:"
-                        else:
-                            key = "dep:"
-                    manager.trace(" %5s %.0f %s" % (key, graph[id].xmeta.data_mtime, id))
-            # If equal, give the benefit of the doubt, due to 1-sec time granularity
-            # (on some platforms).
-            if oldest_in_scc < newest_in_deps:
-                fresh = False
-                fresh_msg = f"out of date by {newest_in_deps - oldest_in_scc:.0f} seconds"
-            else:
-                fresh_msg = "fresh"
+            fresh_msg = "fresh"
         elif undeps:
             fresh_msg = f"stale due to changed suppression ({' '.join(sorted(undeps))})"
         elif stale_scc:
@@ -3342,15 +3291,14 @@ def process_graph(graph: Graph, manager: BuildManager) -> None:
         else:
             fresh_msg = f"stale due to deps ({' '.join(sorted(stale_deps))})"
 
-        # Initialize transitive_error for all SCC members from union
-        # of transitive_error of dependencies.
-        if any(graph[dep].transitive_error for dep in deps if dep in graph):
-            for id in scc:
-                graph[id].transitive_error = True
-
         scc_str = " ".join(scc)
         if fresh:
             manager.trace(f"Queuing {fresh_msg} SCC ({scc_str})")
+            for id in scc:
+                if graph[id].error_lines:
+                    manager.flush_errors(
+                        manager.errors.simplify_path(graph[id].xpath), graph[id].error_lines, False
+                    )
             fresh_scc_queue.append(scc)
         else:
             if fresh_scc_queue:
@@ -3362,11 +3310,6 @@ def process_graph(graph: Graph, manager: BuildManager) -> None:
                 # single fresh SCC. This is intentional -- we don't need those modules
                 # loaded if there are no more stale SCCs to be rechecked.
                 #
-                # Also note we shouldn't have to worry about transitive_error here,
-                # since modules with transitive errors aren't written to the cache,
-                # and if any dependencies were changed, this SCC would be stale.
-                # (Also, in quick_and_dirty mode we don't care about transitive errors.)
-                #
                 # TODO: see if it's possible to determine if we need to process only a
                 # _subset_ of the past SCCs instead of having to process them all.
                 if (
@@ -3518,16 +3461,17 @@ def process_stale_scc(graph: Graph, scc: list[str], manager: BuildManager) -> No
     for id in stale:
         graph[id].generate_unused_ignore_notes()
         graph[id].generate_ignore_without_code_notes()
-    if any(manager.errors.is_errors_for_file(graph[id].xpath) for id in stale):
-        for id in stale:
-            graph[id].transitive_error = True
+
+    # Flush errors, and write cache in two phases: first data files, then meta files.
     meta_tuples = {}
+    errors_by_id = {}
     for id in stale:
         if graph[id].xpath not in manager.errors.ignored_files:
             errors = manager.errors.file_messages(
                 graph[id].xpath, formatter=manager.error_formatter
             )
             manager.flush_errors(manager.errors.simplify_path(graph[id].xpath), errors, False)
+            errors_by_id[id] = errors
         meta_tuples[id] = graph[id].write_cache()
         graph[id].mark_as_rechecked()
     for id in stale:
@@ -3539,6 +3483,7 @@ def process_stale_scc(graph: Graph, scc: list[str], manager: BuildManager) -> No
         meta["dep_hashes"] = {
             dep: graph[dep].interface_hash for dep in graph[id].dependencies if dep in graph
         }
+        meta["error_lines"] = errors_by_id.get(id, [])
         graph[id].meta = write_cache_meta(meta, manager, meta_json, data_json)
 
 

mypy/build.py

@@ -1163,22 +1163,22 @@ def add_invertible_flag(
         "--skip-c-gen", dest="mypyc_skip_c_generation", action="store_true", help=argparse.SUPPRESS
     )
 
-    other_group = parser.add_argument_group(title="Miscellaneous")
-    other_group.add_argument("--quickstart-file", help=argparse.SUPPRESS)
-    other_group.add_argument("--junit-xml", help="Write junit.xml to the given file")
+    misc_group = parser.add_argument_group(title="Miscellaneous")
+    misc_group.add_argument("--quickstart-file", help=argparse.SUPPRESS)
+    misc_group.add_argument("--junit-xml", help="Write junit.xml to the given file")
     imports_group.add_argument(
         "--junit-format",
         choices=["global", "per_file"],
         default="global",
         help="If --junit-xml is set, specifies format. global: single test with all errors; per_file: one test entry per file with failures",
     )
-    other_group.add_argument(
+    misc_group.add_argument(
         "--find-occurrences",
         metavar="CLASS.MEMBER",
         dest="special-opts:find_occurrences",
         help="Print out all usages of a class member (experimental)",
     )
-    other_group.add_argument(
+    misc_group.add_argument(
         "--scripts-are-modules",
         action="store_true",
         help="Script x becomes module x instead of __main__",
@@ -1189,7 +1189,7 @@ def add_invertible_flag(
         default=False,
         strict_flag=False,
         help="Install detected missing library stub packages using pip",
-        group=other_group,
+        group=misc_group,
     )
     add_invertible_flag(
         "--non-interactive",
@@ -1199,19 +1199,12 @@ def add_invertible_flag(
             "Install stubs without asking for confirmation and hide "
             + "errors, with --install-types"
         ),
-        group=other_group,
+        group=misc_group,
         inverse="--interactive",
     )
 
     if server_options:
-        # TODO: This flag is superfluous; remove after a short transition (2018-03-16)
-        other_group.add_argument(
-            "--experimental",
-            action="store_true",
-            dest="fine_grained_incremental",
-            help="Enable fine-grained incremental mode",
-        )
-        other_group.add_argument(
+        misc_group.add_argument(
             "--use-fine-grained-cache",
             action="store_true",
             help="Use the cache in fine-grained incremental mode",

mypy/main.py

@@ -1053,7 +1053,10 @@ def get_kind(arg_name: str) -> nodes.ArgKind:
 
 
 def _verify_signature(
-    stub: Signature[nodes.Argument], runtime: Signature[inspect.Parameter], function_name: str
+    stub: Signature[nodes.Argument],
+    runtime: Signature[inspect.Parameter],
+    function_name: str,
+    warn_runtime_is_object_init: bool = False,
 ) -> Iterator[str]:
     # Check positional arguments match up
     for stub_arg, runtime_arg in zip(stub.pos, runtime.pos):
@@ -1098,6 +1101,8 @@ def _verify_signature(
                     msg = f'runtime does not have parameter "{stub_arg.variable.name}"'
                     if runtime.varkw is not None:
                         msg += ". Maybe you forgot to make it keyword-only in the stub?"
+                    elif warn_runtime_is_object_init:
+                        msg += ". You may need to write stubs for __new__ instead of __init__."
                     yield msg
                 else:
                     yield f'stub parameter "{stub_arg.variable.name}" is not keyword-only'
@@ -1137,7 +1142,11 @@ def _verify_signature(
                 if arg not in {runtime_arg.name for runtime_arg in runtime.pos[len(stub.pos) :]}:
                     yield f'runtime parameter "{arg}" is not keyword-only'
             else:
-                yield f'runtime does not have parameter "{arg}"'
+                msg = f'runtime does not have parameter "{arg}"'
+                if warn_runtime_is_object_init:
+                    msg += ". You may need to write stubs for __new__ instead of __init__."
+                yield msg
+
     for arg in sorted(set(runtime.kwonly) - set(stub.kwonly)):
         if arg in {stub_arg.variable.name for stub_arg in stub.pos}:
             # Don't report this if we've reported it before
@@ -1223,7 +1232,12 @@ def verify_funcitem(
     if not signature:
         return
 
-    for message in _verify_signature(stub_sig, runtime_sig, function_name=stub.name):
+    for message in _verify_signature(
+        stub_sig,
+        runtime_sig,
+        function_name=stub.name,
+        warn_runtime_is_object_init=runtime is object.__init__,
+    ):
         yield Error(
             object_path,
             "is inconsistent, " + message,
@@ -1333,7 +1347,12 @@ def verify_overloadedfuncdef(
     stub_sig = Signature.from_overloadedfuncdef(stub)
     runtime_sig = Signature.from_inspect_signature(signature)
 
-    for message in _verify_signature(stub_sig, runtime_sig, function_name=stub.name):
+    for message in _verify_signature(
+        stub_sig,
+        runtime_sig,
+        function_name=stub.name,
+        warn_runtime_is_object_init=runtime is object.__init__,
+    ):
         # TODO: This is a little hacky, but the addition here is super useful
         if "has a default value of type" in message:
             message += (