Merge remote-tracking branch 'upstream/master' into bugfix/gh-19576-enum-comparison-overlap

Author: sterliakov

docs/requirements-docs.txt

@@ -1,3 +1,4 @@
+-r ../mypy-requirements.txt
 sphinx>=8.1.0
 furo>=2022.3.4
 myst-parser>=4.0.0

misc/typeshed_patches/0001-Remove-use-of-LiteralString-in-builtins-13743.patch

@@ -1,4 +1,4 @@
-From e6995c91231e1915eba43a29a22dd4cbfaf9e08e Mon Sep 17 00:00:00 2001
+From 805d7fc06a8bee350959512e0908a18a87b7f8c2 Mon Sep 17 00:00:00 2001
 From: Shantanu <[email protected]>
 Date: Mon, 26 Sep 2022 12:55:07 -0700
 Subject: [PATCH] Remove use of LiteralString in builtins (#13743)
@@ -8,7 +8,7 @@ Subject: [PATCH] Remove use of LiteralString in builtins (#13743)
  1 file changed, 1 insertion(+), 99 deletions(-)
 
 diff --git a/mypy/typeshed/stdlib/builtins.pyi b/mypy/typeshed/stdlib/builtins.pyi
-index 00728f42d..ea77a730f 100644
+index c7ab95482..3e93da36e 100644
 --- a/mypy/typeshed/stdlib/builtins.pyi
 +++ b/mypy/typeshed/stdlib/builtins.pyi
 @@ -63,7 +63,6 @@ from typing import (  # noqa: Y022,UP035
@@ -19,7 +19,7 @@ index 00728f42d..ea77a730f 100644
      ParamSpec,
      Self,
      TypeAlias,
-@@ -453,31 +452,16 @@ class str(Sequence[str]):
+@@ -468,31 +467,16 @@ class str(Sequence[str]):
      def __new__(cls, object: object = ...) -> Self: ...
      @overload
      def __new__(cls, object: ReadableBuffer, encoding: str = ..., errors: str = ...) -> Self: ...
@@ -51,7 +51,7 @@ index 00728f42d..ea77a730f 100644
      def format(self, *args: object, **kwargs: object) -> str: ...
      def format_map(self, mapping: _FormatMapMapping, /) -> str: ...
      def index(self, sub: str, start: SupportsIndex | None = ..., end: SupportsIndex | None = ..., /) -> int: ...
-@@ -493,98 +477,34 @@ class str(Sequence[str]):
+@@ -508,98 +492,34 @@ class str(Sequence[str]):
      def isspace(self) -> bool: ...
      def istitle(self) -> bool: ...
      def isupper(self) -> bool: ...
@@ -150,7 +150,7 @@ index 00728f42d..ea77a730f 100644
      def zfill(self, width: SupportsIndex, /) -> str: ...  # type: ignore[misc]
      @staticmethod
      @overload
-@@ -595,39 +515,21 @@ class str(Sequence[str]):
+@@ -610,39 +530,21 @@ class str(Sequence[str]):
      @staticmethod
      @overload
      def maketrans(x: str, y: str, z: str, /) -> dict[int, int | None]: ...
@@ -190,7 +190,7 @@ index 00728f42d..ea77a730f 100644
 -    @overload
      def __rmul__(self, value: SupportsIndex, /) -> str: ...  # type: ignore[misc]
      def __getnewargs__(self) -> tuple[str]: ...
- 
+     def __format__(self, format_spec: str, /) -> str: ...
 -- 
-2.49.0
+2.50.1
 

mypy/build.py

@@ -40,6 +40,7 @@
 from typing_extensions import TypeAlias as _TypeAlias
 
 import mypy.semanal_main
+from mypy.cache import Buffer
 from mypy.checker import TypeChecker
 from mypy.error_formatter import OUTPUT_CHOICES, ErrorFormatter
 from mypy.errors import CompileError, ErrorInfo, Errors, report_internal_error
@@ -116,6 +117,8 @@
     "abc",
 }
 
+# We are careful now, we can increase this in future if safe/useful.
+MAX_GC_FREEZE_CYCLES = 1
 
 Graph: _TypeAlias = dict[str, "State"]
 
@@ -707,6 +710,8 @@ def __init__(
         # new file can be processed O(n**2) times. This cache
         # avoids most of this redundant work.
         self.ast_cache: dict[str, tuple[MypyFile, list[ErrorInfo]]] = {}
+        # Number of times we used GC optimization hack for fresh SCCs.
+        self.gc_freeze_cycles = 0
 
     def dump_stats(self) -> None:
         if self.options.dump_build_stats:
@@ -1139,6 +1144,17 @@ def read_deps_cache(manager: BuildManager, graph: Graph) -> dict[str, FgDepMeta]
     return module_deps_metas
 
 
+def _load_ff_file(file: str, manager: BuildManager, log_error: str) -> bytes | None:
+    t0 = time.time()
+    try:
+        data = manager.metastore.read(file)
+    except OSError:
+        manager.log(log_error + file)
+        return None
+    manager.add_stats(metastore_read_time=time.time() - t0)
+    return data
+
+
 def _load_json_file(
     file: str, manager: BuildManager, log_success: str, log_error: str
 ) -> dict[str, Any] | None:
@@ -1259,7 +1275,11 @@ def get_cache_names(id: str, path: str, options: Options) -> tuple[str, str, str
     deps_json = None
     if options.cache_fine_grained:
         deps_json = prefix + ".deps.json"
-    return (prefix + ".meta.json", prefix + ".data.json", deps_json)
+    if options.fixed_format_cache:
+        data_suffix = ".data.ff"
+    else:
+        data_suffix = ".data.json"
+    return (prefix + ".meta.json", prefix + data_suffix, deps_json)
 
 
 def find_cache_meta(id: str, path: str, manager: BuildManager) -> CacheMeta | None:
@@ -1559,8 +1579,13 @@ def write_cache(
         tree.path = path
 
     # Serialize data and analyze interface
-    data = tree.serialize()
-    data_bytes = json_dumps(data, manager.options.debug_cache)
+    if manager.options.fixed_format_cache:
+        data_io = Buffer()
+        tree.write(data_io)
+        data_bytes = data_io.getvalue()
+    else:
+        data = tree.serialize()
+        data_bytes = json_dumps(data, manager.options.debug_cache)
     interface_hash = hash_digest(data_bytes)
 
     plugin_data = manager.plugin.report_config_data(ReportConfigContext(id, path, is_check=False))
@@ -2085,15 +2110,23 @@ def load_tree(self, temporary: bool = False) -> None:
             self.meta is not None
         ), "Internal error: this method must be called only for cached modules"
 
-        data = _load_json_file(
-            self.meta.data_json, self.manager, "Load tree ", "Could not load tree: "
-        )
+        data: bytes | dict[str, Any] | None
+        if self.options.fixed_format_cache:
+            data = _load_ff_file(self.meta.data_json, self.manager, "Could not load tree: ")
+        else:
+            data = _load_json_file(
+                self.meta.data_json, self.manager, "Load tree ", "Could not load tree: "
+            )
         if data is None:
             return
 
         t0 = time.time()
         # TODO: Assert data file wasn't changed.
-        self.tree = MypyFile.deserialize(data)
+        if isinstance(data, bytes):
+            data_io = Buffer(data)
+            self.tree = MypyFile.read(data_io)
+        else:
+            self.tree = MypyFile.deserialize(data)
         t1 = time.time()
         self.manager.add_stats(deserialize_time=t1 - t0)
         if not temporary:
@@ -2481,7 +2514,11 @@ def write_cache(self) -> None:
         ):
             if self.options.debug_serialize:
                 try:
-                    self.tree.serialize()
+                    if self.manager.options.fixed_format_cache:
+                        data = Buffer()
+                        self.tree.write(data)
+                    else:
+                        self.tree.serialize()
                 except Exception:
                     print(f"Error serializing {self.id}", file=self.manager.stdout)
                     raise  # Propagate to display traceback
@@ -3326,8 +3363,29 @@ def process_graph(graph: Graph, manager: BuildManager) -> None:
                 #
                 # TODO: see if it's possible to determine if we need to process only a
                 # _subset_ of the past SCCs instead of having to process them all.
+                if (
+                    platform.python_implementation() == "CPython"
+                    and manager.gc_freeze_cycles < MAX_GC_FREEZE_CYCLES
+                ):
+                    # When deserializing cache we create huge amount of new objects, so even
+                    # with our generous GC thresholds, GC is still doing a lot of pointless
+                    # work searching for garbage. So, we temporarily disable it when
+                    # processing fresh SCCs, and then move all the new objects to the oldest
+                    # generation with the freeze()/unfreeze() trick below. This is arguably
+                    # a hack, but it gives huge performance wins for large third-party
+                    # libraries, like torch.
+                    gc.collect()
+                    gc.disable()
                 for prev_scc in fresh_scc_queue:
                     process_fresh_modules(graph, prev_scc, manager)
+                if (
+                    platform.python_implementation() == "CPython"
+                    and manager.gc_freeze_cycles < MAX_GC_FREEZE_CYCLES
+                ):
+                    manager.gc_freeze_cycles += 1
+                    gc.freeze()
+                    gc.unfreeze()
+                    gc.enable()
                 fresh_scc_queue = []
             size = len(scc)
             if size == 1:

mypy/cache.py

@@ -0,0 +1,166 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import TYPE_CHECKING, Final
+
+from mypy_extensions import u8
+
+try:
+    from native_internal import (
+        Buffer as Buffer,
+        read_bool as read_bool,
+        read_float as read_float,
+        read_int as read_int,
+        read_str as read_str,
+        read_tag as read_tag,
+        write_bool as write_bool,
+        write_float as write_float,
+        write_int as write_int,
+        write_str as write_str,
+        write_tag as write_tag,
+    )
+except ImportError:
+    # TODO: temporary, remove this after we publish mypy-native on PyPI.
+    if not TYPE_CHECKING:
+
+        class Buffer:
+            def __init__(self, source: bytes = b"") -> None:
+                raise NotImplementedError
+
+            def getvalue(self) -> bytes:
+                raise NotImplementedError
+
+        def read_int(data: Buffer) -> int:
+            raise NotImplementedError
+
+        def write_int(data: Buffer, value: int) -> None:
+            raise NotImplementedError
+
+        def read_tag(data: Buffer) -> u8:
+            raise NotImplementedError
+
+        def write_tag(data: Buffer, value: u8) -> None:
+            raise NotImplementedError
+
+        def read_str(data: Buffer) -> str:
+            raise NotImplementedError
+
+        def write_str(data: Buffer, value: str) -> None:
+            raise NotImplementedError
+
+        def read_bool(data: Buffer) -> bool:
+            raise NotImplementedError
+
+        def write_bool(data: Buffer, value: bool) -> None:
+            raise NotImplementedError
+
+        def read_float(data: Buffer) -> float:
+            raise NotImplementedError
+
+        def write_float(data: Buffer, value: float) -> None:
+            raise NotImplementedError
+
+
+# Always use this type alias to refer to type tags.
+Tag = u8
+
+LITERAL_INT: Final[Tag] = 1
+LITERAL_STR: Final[Tag] = 2
+LITERAL_BOOL: Final[Tag] = 3
+LITERAL_FLOAT: Final[Tag] = 4
+LITERAL_COMPLEX: Final[Tag] = 5
+LITERAL_NONE: Final[Tag] = 6
+
+
+def read_literal(data: Buffer, tag: Tag) -> int | str | bool | float:
+    if tag == LITERAL_INT:
+        return read_int(data)
+    elif tag == LITERAL_STR:
+        return read_str(data)
+    elif tag == LITERAL_BOOL:
+        return read_bool(data)
+    elif tag == LITERAL_FLOAT:
+        return read_float(data)
+    assert False, f"Unknown literal tag {tag}"
+
+
+def write_literal(data: Buffer, value: int | str | bool | float | complex | None) -> None:
+    if isinstance(value, bool):
+        write_tag(data, LITERAL_BOOL)
+        write_bool(data, value)
+    elif isinstance(value, int):
+        write_tag(data, LITERAL_INT)
+        write_int(data, value)
+    elif isinstance(value, str):
+        write_tag(data, LITERAL_STR)
+        write_str(data, value)
+    elif isinstance(value, float):
+        write_tag(data, LITERAL_FLOAT)
+        write_float(data, value)
+    elif isinstance(value, complex):
+        write_tag(data, LITERAL_COMPLEX)
+        write_float(data, value.real)
+        write_float(data, value.imag)
+    else:
+        write_tag(data, LITERAL_NONE)
+
+
+def read_int_opt(data: Buffer) -> int | None:
+    if read_bool(data):
+        return read_int(data)
+    return None
+
+
+def write_int_opt(data: Buffer, value: int | None) -> None:
+    if value is not None:
+        write_bool(data, True)
+        write_int(data, value)
+    else:
+        write_bool(data, False)
+
+
+def read_str_opt(data: Buffer) -> str | None:
+    if read_bool(data):
+        return read_str(data)
+    return None
+
+
+def write_str_opt(data: Buffer, value: str | None) -> None:
+    if value is not None:
+        write_bool(data, True)
+        write_str(data, value)
+    else:
+        write_bool(data, False)
+
+
+def read_int_list(data: Buffer) -> list[int]:
+    size = read_int(data)
+    return [read_int(data) for _ in range(size)]
+
+
+def write_int_list(data: Buffer, value: list[int]) -> None:
+    write_int(data, len(value))
+    for item in value:
+        write_int(data, item)
+
+
+def read_str_list(data: Buffer) -> list[str]:
+    size = read_int(data)
+    return [read_str(data) for _ in range(size)]
+
+
+def write_str_list(data: Buffer, value: Sequence[str]) -> None:
+    write_int(data, len(value))
+    for item in value:
+        write_str(data, item)
+
+
+def read_str_opt_list(data: Buffer) -> list[str | None]:
+    size = read_int(data)
+    return [read_str_opt(data) for _ in range(size)]
+
+
+def write_str_opt_list(data: Buffer, value: list[str | None]) -> None:
+    write_int(data, len(value))
+    for item in value:
+        write_str_opt(data, item)