Add PoC implementation

Author: ilevkivskyi

mypy/build.py

@@ -26,6 +26,7 @@
 import time
 import types
 from collections.abc import Iterator, Mapping, Sequence, Set as AbstractSet
+from io import BytesIO
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -1139,6 +1140,17 @@ def read_deps_cache(manager: BuildManager, graph: Graph) -> dict[str, FgDepMeta]
     return module_deps_metas
 
 
+def _load_ff_file(file: str, manager: BuildManager, log_error: str) -> bytes | None:
+    t0 = time.time()
+    try:
+        data = manager.metastore.read(file)
+    except OSError:
+        manager.log(log_error + file)
+        return None
+    manager.add_stats(metastore_read_time=time.time() - t0)
+    return data
+
+
 def _load_json_file(
     file: str, manager: BuildManager, log_success: str, log_error: str
 ) -> dict[str, Any] | None:
@@ -1259,7 +1271,11 @@ def get_cache_names(id: str, path: str, options: Options) -> tuple[str, str, str
     deps_json = None
     if options.cache_fine_grained:
         deps_json = prefix + ".deps.json"
-    return (prefix + ".meta.json", prefix + ".data.json", deps_json)
+    if options.fixed_format_cache:
+        data_suffix = ".data.ff"
+    else:
+        data_suffix = ".data.json"
+    return (prefix + ".meta.json", prefix + data_suffix, deps_json)
 
 
 def find_cache_meta(id: str, path: str, manager: BuildManager) -> CacheMeta | None:
@@ -1559,8 +1575,13 @@ def write_cache(
         tree.path = path
 
     # Serialize data and analyze interface
-    data = tree.serialize()
-    data_bytes = json_dumps(data, manager.options.debug_cache)
+    if manager.options.fixed_format_cache:
+        data_io = BytesIO()
+        tree.write(data_io)
+        data_bytes = data_io.getvalue()
+    else:
+        data = tree.serialize()
+        data_bytes = json_dumps(data, manager.options.debug_cache)
     interface_hash = hash_digest(data_bytes)
 
     plugin_data = manager.plugin.report_config_data(ReportConfigContext(id, path, is_check=False))
@@ -2085,15 +2106,23 @@ def load_tree(self, temporary: bool = False) -> None:
             self.meta is not None
         ), "Internal error: this method must be called only for cached modules"
 
-        data = _load_json_file(
-            self.meta.data_json, self.manager, "Load tree ", "Could not load tree: "
-        )
+        data: bytes | dict[str, Any] | None
+        if self.options.fixed_format_cache:
+            data = _load_ff_file(self.meta.data_json, self.manager, "Could not load tree: ")
+        else:
+            data = _load_json_file(
+                self.meta.data_json, self.manager, "Load tree ", "Could not load tree: "
+            )
         if data is None:
             return
 
         t0 = time.time()
         # TODO: Assert data file wasn't changed.
-        self.tree = MypyFile.deserialize(data)
+        if isinstance(data, bytes):
+            data_io = BytesIO(data)
+            self.tree = MypyFile.read(data_io)
+        else:
+            self.tree = MypyFile.deserialize(data)
         t1 = time.time()
         self.manager.add_stats(deserialize_time=t1 - t0)
         if not temporary:
@@ -2481,7 +2510,11 @@ def write_cache(self) -> None:
         ):
             if self.options.debug_serialize:
                 try:
-                    self.tree.serialize()
+                    if self.manager.options.fixed_format_cache:
+                        data = BytesIO()
+                        self.tree.write(data)
+                    else:
+                        self.tree.serialize()
                 except Exception:
                     print(f"Error serializing {self.id}", file=self.manager.stdout)
                     raise  # Propagate to display traceback
@@ -3410,6 +3443,7 @@ def process_fresh_modules(graph: Graph, modules: list[str], manager: BuildManage
     for id in modules:
         graph[id].fix_cross_refs()
     t2 = time.time()
+    # touch 22 again
     manager.add_stats(process_fresh_time=t2 - t0, load_tree_time=t1 - t0)
 
 

mypy/cache.py

@@ -0,0 +1,167 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
+from io import BytesIO
+from typing import Final
+
+INT_SIZE: Final = 2
+LONG_INT_SIZE: Final = 10
+FLOAT_LEN: Final = 32
+
+
+def read_int(data: BytesIO) -> int:
+    return int.from_bytes(data.read(INT_SIZE), "big", signed=True)
+
+
+def write_int(data: BytesIO, value: int) -> None:
+    data.write(value.to_bytes(INT_SIZE, "big", signed=True))
+
+
+def read_long_int(data: BytesIO) -> int:
+    return int.from_bytes(data.read(LONG_INT_SIZE), "big", signed=True)
+
+
+def write_long_int(data: BytesIO, value: int) -> None:
+    data.write(value.to_bytes(LONG_INT_SIZE, "big", signed=True))
+
+
+def read_str(data: BytesIO) -> str:
+    size = read_int(data)
+    encoded = data.read(size)
+    return encoded.decode()
+
+
+def write_str(data: BytesIO, value: str) -> None:
+    encoded = value.encode()
+    size = len(encoded)
+    write_int(data, size)
+    data.write(encoded)
+
+
+def read_bool(data: BytesIO) -> bool:
+    return data.read(1) == b"\xff"
+
+
+def write_bool(data: BytesIO, value: bool) -> None:
+    data.write(b"\xff" if value else b"\x00")
+
+
+def read_float(data: BytesIO) -> float:
+    value_str = data.read(FLOAT_LEN).decode()
+    return float(value_str)
+
+
+def write_float(data: BytesIO, value: float) -> None:
+    value_str = str(value)
+    value_str = "0" * (FLOAT_LEN - len(value_str)) + value_str
+    data.write(value_str.encode())
+
+
+LITERAL_INT: Final = 1
+LITERAL_STR: Final = 2
+LITERAL_BOOL: Final = 3
+LITERAL_FLOAT: Final = 4
+LITERAL_COMPLEX: Final = 5
+LITERAL_NONE: Final = 6
+
+
+def read_literal(data: BytesIO, marker: int) -> int | str | bool | float:
+    if marker == LITERAL_INT:
+        return read_long_int(data)
+    elif marker == LITERAL_STR:
+        return read_str(data)
+    elif marker == LITERAL_BOOL:
+        return read_bool(data)
+    elif marker == LITERAL_FLOAT:
+        return read_float(data)
+    assert False, f"Unknown literal marker {marker}"
+
+
+def write_literal(data: BytesIO, value: int | str | bool | float | complex | None) -> None:
+    if isinstance(value, bool):
+        write_int(data, LITERAL_BOOL)
+        write_bool(data, value)
+    elif isinstance(value, int):
+        write_int(data, LITERAL_INT)
+        write_long_int(data, value)
+    elif isinstance(value, str):
+        write_int(data, LITERAL_STR)
+        write_str(data, value)
+    elif isinstance(value, float):
+        write_int(data, LITERAL_FLOAT)
+        write_float(data, value)
+    elif isinstance(value, complex):
+        write_int(data, LITERAL_COMPLEX)
+        write_float(data, value.real)
+        write_float(data, value.imag)
+    else:
+        write_int(data, LITERAL_NONE)
+
+
+OPT_NO: Final = 0
+OPT_YES: Final = 1
+
+
+def read_int_opt(data: BytesIO) -> int | None:
+    marker = read_int(data)
+    if marker == OPT_YES:
+        return read_int(data)
+    assert marker == OPT_NO
+    return None
+
+
+def write_int_opt(data: BytesIO, value: int | None) -> None:
+    if value is not None:
+        write_int(data, OPT_YES)
+        write_int(data, value)
+    else:
+        write_int(data, OPT_NO)
+
+
+def read_str_opt(data: BytesIO) -> str | None:
+    marker = read_int(data)
+    if marker == OPT_YES:
+        return read_str(data)
+    assert marker == OPT_NO
+    return None
+
+
+def write_str_opt(data: BytesIO, value: str | None) -> None:
+    if value is not None:
+        write_int(data, OPT_YES)
+        write_str(data, value)
+    else:
+        write_int(data, OPT_NO)
+
+
+def read_int_list(data: BytesIO) -> list[int]:
+    size = read_int(data)
+    return [read_int(data) for _ in range(size)]
+
+
+def write_int_list(data: BytesIO, value: list[int]) -> None:
+    write_int(data, len(value))
+    for item in value:
+        write_int(data, item)
+
+
+def read_str_list(data: BytesIO) -> list[str]:
+    size = read_int(data)
+    return [read_str(data) for _ in range(size)]
+
+
+def write_str_list(data: BytesIO, value: Sequence[str]) -> None:
+    write_int(data, len(value))
+    for item in value:
+        write_str(data, item)
+
+
+def read_str_opt_list(data: BytesIO) -> list[str | None]:
+    size = read_int(data)
+    return [read_str_opt(data) for _ in range(size)]
+
+
+def write_str_opt_list(data: BytesIO, value: list[str | None]) -> None:
+    write_int(data, len(value))
+    for item in value:
+        write_str_opt(data, item)

mypy/fixup.py

@@ -97,6 +97,8 @@ def visit_type_info(self, info: TypeInfo) -> None:
                 info.declared_metaclass.accept(self.type_fixer)
             if info.metaclass_type:
                 info.metaclass_type.accept(self.type_fixer)
+            if info.self_type:
+                info.self_type.accept(self.type_fixer)
             if info.alt_promote:
                 info.alt_promote.accept(self.type_fixer)
                 instance = Instance(info, [])

mypy/main.py

@@ -1056,6 +1056,11 @@ def add_invertible_flag(
         action="store_true",
         help="Include fine-grained dependency information in the cache for the mypy daemon",
     )
+    incremental_group.add_argument(
+        "--fixed-format-cache",
+        action="store_true",
+        help="Use experimental binary fixed format cache",
+    )
     incremental_group.add_argument(
         "--skip-version-check",
         action="store_true",