Merge branch 'master' into test-range

Author: BobTheBuidler

.github/workflows/docs.yml

@@ -12,6 +12,9 @@ on:
     # so it's important to do the docs build on all PRs touching mypy/errorcodes.py
     # in case somebody's adding a new error code without any docs
     - 'mypy/errorcodes.py'
+    # Part of the documentation is automatically generated from the options
+    # definitions in mypy/main.py
+    - 'mypy/main.py'
     - 'mypyc/doc/**'
     - '**/*.rst'
     - '**/*.md'

.github/workflows/mypy_primer.yml

@@ -67,6 +67,7 @@ jobs:
             --debug \
             --additional-flags="--debug-serialize" \
             --output concise \
+            --mypy-install-librt \
             | tee diff_${{ matrix.shard-index }}.txt
           ) || [ $? -eq 1 ]
       - if: ${{ matrix.shard-index == 0 }}

CHANGELOG.md

@@ -1255,12 +1255,18 @@ Miscellaneous
    stub packages were found, they are installed and then another run
    is performed.
 
-.. option:: --junit-xml JUNIT_XML
+.. option:: --junit-xml JUNIT_XML_OUTPUT_FILE
 
     Causes mypy to generate a JUnit XML test result document with
     type checking results. This can make it easier to integrate mypy
     with continuous integration (CI) tools.
 
+.. option:: --junit-format {global,per_file}
+
+    If --junit-xml is set, specifies format.
+    global (default): single test with all errors;
+    per_file: one test entry per file with failures.
+
 .. option:: --find-occurrences CLASS.MEMBER
 
     This flag will make mypy print out all usages of a class member

docs/source/command_line.rst

@@ -1153,6 +1153,15 @@ These options may only be set in the global section (``[mypy]``).
     type checking results. This can make it easier to integrate mypy
     with continuous integration (CI) tools.
 
+.. confval:: junit_format
+
+    :type: string
+    :default: ``global``
+
+    If junit_xml is set, specifies format.
+    global (default): single test with all errors;
+    per_file: one test entry per file with failures.
+
 .. confval:: scripts_are_modules
 
     :type: boolean

docs/source/config_file.rst

@@ -41,7 +41,7 @@ def extract(chunks: Iterable[JsonDict]) -> Iterable[JsonDict]:
             if isinstance(chunk, dict):
                 yield chunk
                 yield from extract(chunk.values())
-            elif isinstance(chunk, list):
+            elif isinstance(chunk, list):  # type: ignore[unreachable] #TODO: is this actually unreachable, or are our types wrong?
                 yield from extract(chunk)
 
     yield from extract([chunk.data for chunk in chunks])
@@ -93,7 +93,7 @@ def compress(chunk: JsonDict) -> JsonDict:
     def helper(chunk: JsonDict) -> JsonDict:
         nonlocal counter
         if not isinstance(chunk, dict):
-            return chunk
+            return chunk  # type: ignore[unreachable] #TODO: is this actually unreachable, or are our types wrong?
 
         if len(chunk) <= 2:
             return chunk
@@ -124,7 +124,7 @@ def decompress(chunk: JsonDict) -> JsonDict:
 
     def helper(chunk: JsonDict) -> JsonDict:
         if not isinstance(chunk, dict):
-            return chunk
+            return chunk  # type: ignore[unreachable] #TODO: is this actually unreachable, or are our types wrong?
         if ".id" in chunk:
             return cache[chunk[".id"]]
 

misc/analyze_cache.py

@@ -78,22 +78,22 @@ def check_requirements() -> None:
     if sys.platform != "linux":
         # TODO: How to make this work on other platforms?
         sys.exit("error: Only Linux is supported")
-
-    try:
-        subprocess.run(["perf", "-h"], capture_output=True)
-    except (subprocess.CalledProcessError, FileNotFoundError):
-        print("error: The 'perf' profiler is not installed")
-        sys.exit(1)
-
-    try:
-        subprocess.run(["clang", "--version"], capture_output=True)
-    except (subprocess.CalledProcessError, FileNotFoundError):
-        print("error: The clang compiler is not installed")
-        sys.exit(1)
-
-    if not os.path.isfile("mypy_self_check.ini"):
-        print("error: Run this in the mypy repository root")
-        sys.exit(1)
+    else:  # fun fact/todo: we have to use else here, because of https://github.com/python/mypy/issues/10773
+        try:
+            subprocess.run(["perf", "-h"], capture_output=True)
+        except (subprocess.CalledProcessError, FileNotFoundError):
+            print("error: The 'perf' profiler is not installed")
+            sys.exit(1)
+
+        try:
+            subprocess.run(["clang", "--version"], capture_output=True)
+        except (subprocess.CalledProcessError, FileNotFoundError):
+            print("error: The clang compiler is not installed")
+            sys.exit(1)
+
+        if not os.path.isfile("mypy_self_check.ini"):
+            print("error: Run this in the mypy repository root")
+            sys.exit(1)
 
 
 def main() -> None:

misc/profile_check.py

@@ -15,7 +15,6 @@
 
 import collections
 import contextlib
-import errno
 import gc
 import json
 import os
@@ -337,6 +336,7 @@ class CacheMeta(NamedTuple):
     dep_lines: list[int]
     dep_hashes: dict[str, str]
     interface_hash: str  # hash representing the public interface
+    error_lines: list[str]
     version_id: str  # mypy version for cache invalidation
     ignore_all: bool  # if errors were ignored
     plugin_data: Any  # config data from plugins
@@ -376,6 +376,7 @@ def cache_meta_from_dict(meta: dict[str, Any], data_json: str) -> CacheMeta:
         meta.get("dep_lines", []),
         meta.get("dep_hashes", {}),
         meta.get("interface_hash", ""),
+        meta.get("error_lines", []),
         meta.get("version_id", sentinel),
         meta.get("ignore_all", True),
         meta.get("plugin_data", None),
@@ -1502,6 +1503,7 @@ def validate_meta(
                 "dep_lines": meta.dep_lines,
                 "dep_hashes": meta.dep_hashes,
                 "interface_hash": meta.interface_hash,
+                "error_lines": meta.error_lines,
                 "version_id": manager.version_id,
                 "ignore_all": meta.ignore_all,
                 "plugin_data": meta.plugin_data,
@@ -1678,28 +1680,6 @@ def write_cache_meta(
     return cache_meta_from_dict(meta, data_json)
 
 
-def delete_cache(id: str, path: str, manager: BuildManager) -> None:
-    """Delete cache files for a module.
-
-    The cache files for a module are deleted when mypy finds errors there.
-    This avoids inconsistent states with cache files from different mypy runs,
-    see #4043 for an example.
-    """
-    # We don't delete .deps files on errors, since the dependencies
-    # are mostly generated from other files and the metadata is
-    # tracked separately.
-    meta_path, data_path, _ = get_cache_names(id, path, manager.options)
-    cache_paths = [meta_path, data_path]
-    manager.log(f"Deleting {id} {path} {' '.join(x for x in cache_paths if x)}")
-
-    for filename in cache_paths:
-        try:
-            manager.metastore.remove(filename)
-        except OSError as e:
-            if e.errno != errno.ENOENT:
-                manager.log(f"Error deleting cache file {filename}: {e.strerror}")
-
-
 """Dependency manager.
 
 Design
@@ -1875,6 +1855,9 @@ class State:
     # Map from dependency id to its last observed interface hash
     dep_hashes: dict[str, str] = {}
 
+    # List of errors reported for this file last time.
+    error_lines: list[str] = []
+
     # Parent package, its parent, etc.
     ancestors: list[str] | None = None
 
@@ -1896,9 +1879,6 @@ class State:
     # Whether to ignore all errors
     ignore_all = False
 
-    # Whether the module has an error or any of its dependencies have one.
-    transitive_error = False
-
     # Errors reported before semantic analysis, to allow fine-grained
     # mode to keep reporting them.
     early_errors: list[ErrorInfo]
@@ -2000,6 +1980,7 @@ def __init__(
             assert len(all_deps) == len(self.meta.dep_lines)
             self.dep_line_map = {id: line for id, line in zip(all_deps, self.meta.dep_lines)}
             self.dep_hashes = self.meta.dep_hashes
+            self.error_lines = self.meta.error_lines
             if temporary:
                 self.load_tree(temporary=True)
             if not manager.use_fine_grained_cache():
@@ -2517,11 +2498,6 @@ def write_cache(self) -> tuple[dict[str, Any], str, str] | None:
                     print(f"Error serializing {self.id}", file=self.manager.stdout)
                     raise  # Propagate to display traceback
             return None
-        is_errors = self.transitive_error
-        if is_errors:
-            delete_cache(self.id, self.path, self.manager)
-            self.meta = None
-            return None
         dep_prios = self.dependency_priorities()
         dep_lines = self.dependency_lines()
         assert self.source_hash is not None
@@ -3303,34 +3279,7 @@ def process_graph(graph: Graph, manager: BuildManager) -> None:
             if undeps:
                 fresh = False
         if fresh:
-            # All cache files are fresh.  Check that no dependency's
-            # cache file is newer than any scc node's cache file.
-            oldest_in_scc = min(graph[id].xmeta.data_mtime for id in scc)
-            viable = {id for id in stale_deps if graph[id].meta is not None}
-            newest_in_deps = (
-                0 if not viable else max(graph[dep].xmeta.data_mtime for dep in viable)
-            )
-            if manager.options.verbosity >= 3:  # Dump all mtimes for extreme debugging.
-                all_ids = sorted(ascc | viable, key=lambda id: graph[id].xmeta.data_mtime)
-                for id in all_ids:
-                    if id in scc:
-                        if graph[id].xmeta.data_mtime < newest_in_deps:
-                            key = "*id:"
-                        else:
-                            key = "id:"
-                    else:
-                        if graph[id].xmeta.data_mtime > oldest_in_scc:
-                            key = "+dep:"
-                        else:
-                            key = "dep:"
-                    manager.trace(" %5s %.0f %s" % (key, graph[id].xmeta.data_mtime, id))
-            # If equal, give the benefit of the doubt, due to 1-sec time granularity
-            # (on some platforms).
-            if oldest_in_scc < newest_in_deps:
-                fresh = False
-                fresh_msg = f"out of date by {newest_in_deps - oldest_in_scc:.0f} seconds"
-            else:
-                fresh_msg = "fresh"
+            fresh_msg = "fresh"
         elif undeps:
             fresh_msg = f"stale due to changed suppression ({' '.join(sorted(undeps))})"
         elif stale_scc:
@@ -3342,15 +3291,14 @@ def process_graph(graph: Graph, manager: BuildManager) -> None:
         else:
             fresh_msg = f"stale due to deps ({' '.join(sorted(stale_deps))})"
 
-        # Initialize transitive_error for all SCC members from union
-        # of transitive_error of dependencies.
-        if any(graph[dep].transitive_error for dep in deps if dep in graph):
-            for id in scc:
-                graph[id].transitive_error = True
-
         scc_str = " ".join(scc)
         if fresh:
             manager.trace(f"Queuing {fresh_msg} SCC ({scc_str})")
+            for id in scc:
+                if graph[id].error_lines:
+                    manager.flush_errors(
+                        manager.errors.simplify_path(graph[id].xpath), graph[id].error_lines, False
+                    )
             fresh_scc_queue.append(scc)
         else:
             if fresh_scc_queue:
@@ -3362,11 +3310,6 @@ def process_graph(graph: Graph, manager: BuildManager) -> None:
                 # single fresh SCC. This is intentional -- we don't need those modules
                 # loaded if there are no more stale SCCs to be rechecked.
                 #
-                # Also note we shouldn't have to worry about transitive_error here,
-                # since modules with transitive errors aren't written to the cache,
-                # and if any dependencies were changed, this SCC would be stale.
-                # (Also, in quick_and_dirty mode we don't care about transitive errors.)
-                #
                 # TODO: see if it's possible to determine if we need to process only a
                 # _subset_ of the past SCCs instead of having to process them all.
                 if (
@@ -3518,16 +3461,17 @@ def process_stale_scc(graph: Graph, scc: list[str], manager: BuildManager) -> No
     for id in stale:
         graph[id].generate_unused_ignore_notes()
         graph[id].generate_ignore_without_code_notes()
-    if any(manager.errors.is_errors_for_file(graph[id].xpath) for id in stale):
-        for id in stale:
-            graph[id].transitive_error = True
+
+    # Flush errors, and write cache in two phases: first data files, then meta files.
     meta_tuples = {}
+    errors_by_id = {}
     for id in stale:
         if graph[id].xpath not in manager.errors.ignored_files:
             errors = manager.errors.file_messages(
                 graph[id].xpath, formatter=manager.error_formatter
             )
             manager.flush_errors(manager.errors.simplify_path(graph[id].xpath), errors, False)
+            errors_by_id[id] = errors
         meta_tuples[id] = graph[id].write_cache()
         graph[id].mark_as_rechecked()
     for id in stale:
@@ -3539,11 +3483,12 @@ def process_stale_scc(graph: Graph, scc: list[str], manager: BuildManager) -> No
         meta["dep_hashes"] = {
             dep: graph[dep].interface_hash for dep in graph[id].dependencies if dep in graph
         }
+        meta["error_lines"] = errors_by_id.get(id, [])
         graph[id].meta = write_cache_meta(meta, manager, meta_json, data_json)
 
 
 def sorted_components(
-    graph: Graph, vertices: AbstractSet[str] | None = None, pri_max: int = PRI_ALL
+    graph: Graph, vertices: AbstractSet[str] | None = None, pri_max: int = PRI_INDIRECT
 ) -> list[AbstractSet[str]]:
     """Return the graph's SCCs, topologically sorted by dependencies.