Serialize raw errors in cache metas

Author: ilevkivskyi

mypy/build.py

@@ -43,7 +43,14 @@
 from librt.internal import cache_version
 
 import mypy.semanal_main
-from mypy.cache import CACHE_VERSION, CacheMeta, ReadBuffer, WriteBuffer, write_json
+from mypy.cache import (
+    CACHE_VERSION,
+    CacheMeta,
+    ReadBuffer,
+    SerializedError,
+    WriteBuffer,
+    write_json,
+)
 from mypy.checker import TypeChecker
 from mypy.defaults import (
     WORKER_CONNECTION_TIMEOUT,
@@ -52,7 +59,7 @@
     WORKER_START_TIMEOUT,
 )
 from mypy.error_formatter import OUTPUT_CHOICES, ErrorFormatter
-from mypy.errors import CompileError, ErrorInfo, Errors, report_internal_error
+from mypy.errors import CompileError, ErrorInfo, Errors, ErrorTuple, report_internal_error
 from mypy.graph_utils import prepare_sccs, strongly_connected_components, topsort
 from mypy.indirection import TypeIndirectionVisitor
 from mypy.ipc import BadStatus, IPCClient, read_status, ready_to_read, receive, send
@@ -2046,7 +2053,7 @@ class State:
     dep_hashes: dict[str, bytes] = {}
 
     # List of errors reported for this file last time.
-    error_lines: list[str] = []
+    error_lines: list[SerializedError] = []
 
     # Parent package, its parent, etc.
     ancestors: list[str] | None = None
@@ -3511,9 +3518,13 @@ def find_stale_sccs(
                 scc = order_ascc_ex(graph, ascc)
             for id in scc:
                 if graph[id].error_lines:
-                    manager.flush_errors(
-                        manager.errors.simplify_path(graph[id].xpath), graph[id].error_lines, False
+                    path = manager.errors.simplify_path(graph[id].xpath)
+                    formatted = manager.errors.format_messages(
+                        path,
+                        deserialize_codes(graph[id].error_lines),
+                        formatter=manager.error_formatter,
                     )
+                    manager.flush_errors(path, formatted, False)
             fresh_sccs.append(ascc)
         else:
             size = len(ascc.mod_ids)
@@ -3759,21 +3770,24 @@ def process_stale_scc(
     # Flush errors, and write cache in two phases: first data files, then meta files.
     meta_tuples = {}
     errors_by_id = {}
+    formatted_by_id = {}
     for id in stale:
         if graph[id].xpath not in manager.errors.ignored_files:
-            errors = manager.errors.file_messages(
-                graph[id].xpath, formatter=manager.error_formatter
+            errors = manager.errors.file_messages(graph[id].xpath)
+            formatted = manager.errors.format_messages(
+                graph[id].xpath, errors, formatter=manager.error_formatter
             )
-            manager.flush_errors(manager.errors.simplify_path(graph[id].xpath), errors, False)
+            manager.flush_errors(manager.errors.simplify_path(graph[id].xpath), formatted, False)
             errors_by_id[id] = errors
+            formatted_by_id[id] = formatted
         meta_tuples[id] = graph[id].write_cache()
     for id in stale:
         meta_tuple = meta_tuples[id]
         if meta_tuple is None:
             continue
         meta, meta_file = meta_tuple
         meta.dep_hashes = [graph[dep].interface_hash for dep in graph[id].dependencies]
-        meta.error_lines = errors_by_id.get(id, [])
+        meta.error_lines = serialize_codes(errors_by_id.get(id, []))
         write_cache_meta(meta, manager, meta_file)
     manager.done_sccs.add(ascc.id)
     manager.add_stats(
@@ -3785,7 +3799,7 @@ def process_stale_scc(
     )
     scc_result = {}
     for id in scc:
-        scc_result[id] = graph[id].interface_hash.hex(), errors_by_id.get(id, [])
+        scc_result[id] = graph[id].interface_hash.hex(), formatted_by_id.get(id, [])
     return scc_result
 
 
@@ -3932,3 +3946,26 @@ def sccs_to_bytes(sccs: list[SCC]) -> bytes:
     buf = WriteBuffer()
     write_json(buf, {"sccs": scc_tuples})
     return buf.getvalue()
+
+
+def serialize_codes(errs: list[ErrorTuple]) -> list[SerializedError]:
+    return [
+        (path, line, column, end_line, end_column, severity, message, code.code if code else None)
+        for path, line, column, end_line, end_column, severity, message, code in errs
+    ]
+
+
+def deserialize_codes(errs: list[SerializedError]) -> list[ErrorTuple]:
+    return [
+        (
+            path,
+            line,
+            column,
+            end_line,
+            end_column,
+            severity,
+            message,
+            codes.error_codes.get(code) if code else None,
+        )
+        for path, line, column, end_line, end_column, severity, message, code in errs
+    ]

mypy/cache.py

@@ -69,7 +69,9 @@
 from mypy_extensions import u8
 
 # High-level cache layout format
-CACHE_VERSION: Final = 0
+CACHE_VERSION: Final = 1
+
+SerializedError: _TypeAlias = tuple[str | None, int, int, int, int, str, str, str | None]
 
 
 class CacheMeta:
@@ -92,7 +94,7 @@ def __init__(
         dep_lines: list[int],
         dep_hashes: list[bytes],
         interface_hash: bytes,
-        error_lines: list[str],
+        error_lines: list[SerializedError],
         version_id: str,
         ignore_all: bool,
         plugin_data: Any,
@@ -157,7 +159,7 @@ def deserialize(cls, meta: dict[str, Any], data_file: str) -> CacheMeta | None:
                 dep_lines=meta["dep_lines"],
                 dep_hashes=[bytes.fromhex(dep) for dep in meta["dep_hashes"]],
                 interface_hash=bytes.fromhex(meta["interface_hash"]),
-                error_lines=meta["error_lines"],
+                error_lines=[tuple(err) for err in meta["error_lines"]],
                 version_id=meta["version_id"],
                 ignore_all=meta["ignore_all"],
                 plugin_data=meta["plugin_data"],
@@ -179,7 +181,7 @@ def write(self, data: WriteBuffer) -> None:
         write_int_list(data, self.dep_lines)
         write_bytes_list(data, self.dep_hashes)
         write_bytes(data, self.interface_hash)
-        write_str_list(data, self.error_lines)
+        write_errors(data, self.error_lines)
         write_str(data, self.version_id)
         write_bool(data, self.ignore_all)
         # Plugin data may be not a dictionary, so we use
@@ -204,7 +206,7 @@ def read(cls, data: ReadBuffer, data_file: str) -> CacheMeta | None:
                 dep_lines=read_int_list(data),
                 dep_hashes=read_bytes_list(data),
                 interface_hash=read_bytes(data),
-                error_lines=read_str_list(data),
+                error_lines=read_errors(data),
                 version_id=read_str(data),
                 ignore_all=read_bool(data),
                 plugin_data=read_json_value(data),
@@ -231,6 +233,7 @@ def read(cls, data: ReadBuffer, data_file: str) -> CacheMeta | None:
 LIST_INT: Final[Tag] = 21
 LIST_STR: Final[Tag] = 22
 LIST_BYTES: Final[Tag] = 23
+TUPLE_GEN: Final[Tag] = 24
 DICT_STR_GEN: Final[Tag] = 30
 
 # Misc classes.
@@ -391,12 +394,11 @@ def write_str_opt_list(data: WriteBuffer, value: list[str | None]) -> None:
 
 
 Value: _TypeAlias = None | int | str | bool
-JsonValue: _TypeAlias = Value | list["JsonValue"] | dict[str, "JsonValue"]
 
-# Currently tuples are used by mypyc plugin. They will be normalized to
-# JSON lists after a roundtrip.
-JsonValueEx: _TypeAlias = (
-    Value | list["JsonValueEx"] | dict[str, "JsonValueEx"] | tuple["JsonValueEx", ...]
+# Our JSON format is somewhat non-standard as we distinguish lists and tuples.
+# This is convenient for some internal things, like mypyc plugin and error serialization.
+JsonValue: _TypeAlias = (
+    Value | list["JsonValue"] | dict[str, "JsonValue"] | tuple["JsonValue", ...]
 )
 
 
@@ -415,13 +417,16 @@ def read_json_value(data: ReadBuffer) -> JsonValue:
     if tag == LIST_GEN:
         size = read_int_bare(data)
         return [read_json_value(data) for _ in range(size)]
+    if tag == TUPLE_GEN:
+        size = read_int_bare(data)
+        return tuple(read_json_value(data) for _ in range(size))
     if tag == DICT_STR_GEN:
         size = read_int_bare(data)
         return {read_str_bare(data): read_json_value(data) for _ in range(size)}
     assert False, f"Invalid JSON tag: {tag}"
 
 
-def write_json_value(data: WriteBuffer, value: JsonValueEx) -> None:
+def write_json_value(data: WriteBuffer, value: JsonValue) -> None:
     if value is None:
         write_tag(data, LITERAL_NONE)
     elif isinstance(value, bool):
@@ -432,11 +437,16 @@ def write_json_value(data: WriteBuffer, value: JsonValueEx) -> None:
     elif isinstance(value, str):
         write_tag(data, LITERAL_STR)
         write_str_bare(data, value)
-    elif isinstance(value, (list, tuple)):
+    elif isinstance(value, list):
         write_tag(data, LIST_GEN)
         write_int_bare(data, len(value))
         for val in value:
             write_json_value(data, val)
+    elif isinstance(value, tuple):
+        write_tag(data, TUPLE_GEN)
+        write_int_bare(data, len(value))
+        for val in value:
+            write_json_value(data, val)
     elif isinstance(value, dict):
         write_tag(data, DICT_STR_GEN)
         write_int_bare(data, len(value))
@@ -461,3 +471,38 @@ def write_json(data: WriteBuffer, value: dict[str, Any]) -> None:
     for key in sorted(value):
         write_str_bare(data, key)
         write_json_value(data, value[key])
+
+
+def write_errors(data: WriteBuffer, errs: list[SerializedError]) -> None:
+    write_tag(data, LIST_GEN)
+    write_int_bare(data, len(errs))
+    for path, line, column, end_line, end_column, severity, message, code in errs:
+        write_tag(data, TUPLE_GEN)
+        write_str_opt(data, path)
+        write_int(data, line)
+        write_int(data, column)
+        write_int(data, end_line)
+        write_int(data, end_column)
+        write_str(data, severity)
+        write_str(data, message)
+        write_str_opt(data, code)
+
+
+def read_errors(data: ReadBuffer) -> list[SerializedError]:
+    assert read_tag(data) == LIST_GEN
+    result = []
+    for _ in range(read_int_bare(data)):
+        assert read_tag(data) == TUPLE_GEN
+        result.append(
+            (
+                read_str_opt(data),
+                read_int(data),
+                read_int(data),
+                read_int(data),
+                read_int(data),
+                read_str(data),
+                read_str(data),
+                read_str_opt(data),
+            )
+        )
+    return result

mypy/errors.py

@@ -951,7 +951,7 @@ def raise_error(self, use_stdout: bool = True) -> NoReturn:
             self.new_messages(), use_stdout=use_stdout, module_with_blocker=self.blocker_module()
         )
 
-    def format_messages(
+    def format_messages_default(
         self, error_tuples: list[ErrorTuple], source_lines: list[str] | None
     ) -> list[str]:
         """Return a string list that represents the error messages.
@@ -1009,24 +1009,28 @@ def format_messages(
                     a.append(" " * (DEFAULT_SOURCE_OFFSET + column) + marker)
         return a
 
-    def file_messages(self, path: str, formatter: ErrorFormatter | None = None) -> list[str]:
-        """Return a string list of new error messages from a given file.
-
-        Use a form suitable for displaying to the user.
-        """
+    def file_messages(self, path: str) -> list[ErrorTuple]:
+        """Return an error tuple list of new error messages from a given file."""
         if path not in self.error_info_map:
             return []
 
         error_info = self.error_info_map[path]
         error_info = [info for info in error_info if not info.hidden]
         error_info = self.remove_duplicates(self.sort_messages(error_info))
-        error_tuples = self.render_messages(error_info)
+        return self.render_messages(error_info)
 
+    def format_messages(
+        self, path: str, error_tuples: list[ErrorTuple], formatter: ErrorFormatter | None = None
+    ) -> list[str]:
+        """Return a string list of new error messages from a given file.
+
+        Use a form suitable for displaying to the user.
+        """
+        self.flushed_files.add(path)
         if formatter is not None:
             errors = create_errors(error_tuples)
             return [formatter.report_error(err) for err in errors]
 
-        self.flushed_files.add(path)
         source_lines = None
         if self.options.pretty and self.read_source:
             # Find shadow file mapping and read source lines if a shadow file exists for the given path.
@@ -1036,7 +1040,7 @@ def file_messages(self, path: str, formatter: ErrorFormatter | None = None) -> l
                 source_lines = self.read_source(mapped_path)
             else:
                 source_lines = self.read_source(path)
-        return self.format_messages(error_tuples, source_lines)
+        return self.format_messages_default(error_tuples, source_lines)
 
     def find_shadow_file_mapping(self, path: str) -> str | None:
         """Return the shadow file path for a given source file path or None."""
@@ -1058,7 +1062,8 @@ def new_messages(self) -> list[str]:
         msgs = []
         for path in self.error_info_map.keys():
             if path not in self.flushed_files:
-                msgs.extend(self.file_messages(path))
+                error_tuples = self.file_messages(path)
+                msgs.extend(self.format_messages(path, error_tuples))
         return msgs
 
     def targets(self) -> set[str]:

test-data/unit/check-incremental.test

@@ -7626,3 +7626,13 @@ y = 1
 class C: ...
 [out2]
 tmp/m.py:2: note: Revealed type is "def () -> other.C"
+
+[case testOutputFormatterIncremental]
+# flags2: --output json
+def wrong() -> int:
+    if wrong():
+        return 0
+[out]
+main:2: error: Missing return statement
+[out2]
+{"file": "main", "line": 2, "column": 0, "message": "Missing return statement", "hint": null, "code": "return", "severity": "error"}