Fix --sqlite-cache

Author: ilevkivskyi

mypy/build.py

@@ -696,6 +696,7 @@ def __init__(
         stdout: TextIO,
         stderr: TextIO,
         error_formatter: ErrorFormatter | None = None,
+        parallel_worker: bool = False,
     ) -> None:
         self.stats: dict[str, Any] = {}  # Values are ints or floats
         self.stdout = stdout
@@ -769,7 +770,7 @@ def __init__(
                 ]
             )
 
-        self.metastore = create_metastore(options)
+        self.metastore = create_metastore(options, parallel_worker)
 
         # a mapping from source files to their corresponding shadow files
         # for efficient lookup
@@ -818,7 +819,7 @@ def __init__(
         # A global adding order for SCC queue, see comment above.
         self.queue_order: int = 0
         # Is this an instance used by a parallel worker?
-        self.parallel_worker = False
+        self.parallel_worker = parallel_worker
 
     def dump_stats(self) -> None:
         if self.options.dump_build_stats:
@@ -1397,10 +1398,13 @@ def exclude_from_backups(target_dir: str) -> None:
         pass
 
 
-def create_metastore(options: Options) -> MetadataStore:
+def create_metastore(options: Options, parallel_worker: bool = False) -> MetadataStore:
     """Create the appropriate metadata store."""
     if options.sqlite_cache:
-        mds: MetadataStore = SqliteMetadataStore(_cache_dir_prefix(options))
+        # We use this flag in both coordinator and workers to seep up commits,
+        # see mypy.metastore.connect_db() for details.
+        sync_off = options.num_workers > 0 or parallel_worker
+        mds: MetadataStore = SqliteMetadataStore(_cache_dir_prefix(options), sync_off=sync_off)
     else:
         mds = FilesystemMetadataStore(_cache_dir_prefix(options))
     return mds

mypy/build_worker/worker.py

@@ -135,6 +135,8 @@ def serve(server: IPCServer, ctx: ServerContext) -> None:
         t0 = time.time()
         try:
             result = process_stale_scc(graph, scc, manager)
+            # We must commit after each SCC, otherwise we break --sqlite-cache.
+            manager.metastore.commit()
         except CompileError as e:
             blocker = {
                 "messages": e.messages,
@@ -170,7 +172,7 @@ def flush_errors(filename: str | None, new_messages: list[str], is_serious: bool
         # We never flush errors in the worker, we send them back to coordinator.
         pass
 
-    manager = BuildManager(
+    return BuildManager(
         data_dir,
         search_paths,
         ignore_prefix=os.getcwd(),
@@ -186,9 +188,8 @@ def flush_errors(filename: str | None, new_messages: list[str], is_serious: bool
         fscache=ctx.fscache,
         stdout=sys.stdout,
         stderr=sys.stderr,
+        parallel_worker=True,
     )
-    manager.parallel_worker = True
-    return manager
 
 
 def console_entry() -> None:

mypy/metastore.py

@@ -145,16 +145,21 @@ def list_all(self) -> Iterable[str]:
 """
 
 
-def connect_db(db_file: str) -> sqlite3.Connection:
+def connect_db(db_file: str, sync_off: bool = False) -> sqlite3.Connection:
     import sqlite3.dbapi2
 
     db = sqlite3.dbapi2.connect(db_file)
+    if sync_off:
+        # This is a bit unfortunate (as we may get corrupt cache after e.g. Ctrl + C),
+        # but without this flag, commits are *very* slow, especially when using HDDs,
+        # see https://www.sqlite.org/faq.html#q19 for details.
+        db.execute("PRAGMA synchronous=OFF")
     db.executescript(SCHEMA)
     return db
 
 
 class SqliteMetadataStore(MetadataStore):
-    def __init__(self, cache_dir_prefix: str) -> None:
+    def __init__(self, cache_dir_prefix: str, sync_off: bool = False) -> None:
         # We check startswith instead of equality because the version
         # will have already been appended by the time the cache dir is
         # passed here.
@@ -163,7 +168,7 @@ def __init__(self, cache_dir_prefix: str) -> None:
             return
 
         os.makedirs(cache_dir_prefix, exist_ok=True)
-        self.db = connect_db(os.path.join(cache_dir_prefix, "cache.db"))
+        self.db = connect_db(os.path.join(cache_dir_prefix, "cache.db"), sync_off=sync_off)
 
     def _query(self, name: str, field: str) -> Any:
         # Raises FileNotFound for consistency with the file system version