switch better-sqlite3 --> node:sqlite, which shrinks size and removes a native dependency

Author: williamstein

src/packages/backend/conat/persist.ts

@@ -9,15 +9,15 @@ additional persist server:
 */
 
 import "./index";
-import betterSqlite3 from "better-sqlite3";
+import sqlite from "node:sqlite";
 import { initContext } from "@cocalc/conat/persist/context";
 const { zstdCompressSync, zstdDecompressSync } = require("node:zlib");
 import { syncFiles } from "@cocalc/backend/data";
 import ensureContainingDirectoryExists from "@cocalc/backend/misc/ensure-containing-directory-exists";
 import { statSync, copyFileSync } from "node:fs";
 
 initContext({
-  betterSqlite3,
+  sqlite,
   compress: zstdCompressSync,
   decompress: zstdDecompressSync,
   syncFiles,

src/packages/backend/conat/test/persist/backup-archive.test.ts

@@ -15,7 +15,7 @@ import { pathExists } from "fs-extra";
 import { join } from "path";
 import * as fs from "fs/promises";
 import { messageData } from "@cocalc/conat/core/client";
-import sqlite from "better-sqlite3";
+import { DatabaseSync as sqlite } from "node:sqlite";
 import { openPaths } from "@cocalc/conat/persist/storage";
 
 beforeAll(async () => {

src/packages/backend/package.json

@@ -45,7 +45,6 @@
     "@cocalc/util": "workspace:*",
     "@isaacs/ttlcache": "^1.4.1",
     "awaiting": "^3.0.0",
-    "better-sqlite3": "^12.4.1",
     "chokidar": "^4.0.3",
     "debug": "^4.4.0",
     "fs-extra": "^11.3.1",
@@ -68,6 +67,6 @@
   "devDependencies": {
     "@types/debug": "^4.1.12",
     "@types/jest": "^30.0.0",
-    "@types/node": "^18.16.14"
+    "@types/node": "^22.10.1"
   }
 }

src/packages/backend/tcp/enable-messaging-protocol.ts

@@ -11,7 +11,6 @@ Enable two new functions write_mesg and recv_mesg on a TCP socket.
 
 import { Buffer } from "node:buffer";
 import { Socket } from "node:net";
-
 import getLogger from "@cocalc/backend/logger";
 import { error } from "@cocalc/util/message";
 import { from_json_socket, to_json_socket, trunc } from "@cocalc/util/misc";
@@ -147,7 +146,7 @@ export default function enable(socket: CoCalcSocket, desc: string = "") {
         return;
       } else {
         socket.write(length);
-        socket.write(data, cb);
+        socket.write(data, "utf8", cb as any);
       }
     };
 

src/packages/conat/package.json

@@ -29,17 +29,9 @@
     "test": "pnpm exec jest",
     "depcheck": "pnpx depcheck --ignores events,bufferutil,utf-8-validate"
   },
-  "files": [
-    "dist/**",
-    "README.md",
-    "package.json"
-  ],
+  "files": ["dist/**", "README.md", "package.json"],
   "author": "SageMath, Inc.",
-  "keywords": [
-    "utilities",
-    "conat",
-    "cocalc"
-  ],
+  "keywords": ["utilities", "conat", "cocalc"],
   "license": "SEE LICENSE.md",
   "dependencies": {
     "@cocalc/comm": "workspace:*",
@@ -63,9 +55,8 @@
     "utf-8-validate": "^6.0.5"
   },
   "devDependencies": {
-    "@types/better-sqlite3": "^7.6.13",
     "@types/lodash": "^4.17.20",
-    "@types/node": "^18.16.14"
+    "@types/node": "^22.10.1"
   },
   "repository": {
     "type": "git",

src/packages/conat/persist/context.ts

@@ -5,11 +5,11 @@ not from a browser.    Making this explicit helps clarify the dependence
 on the backend and make the code more unit testable.
 */
 
-import type BetterSqlite3 from "better-sqlite3";
-type Database = BetterSqlite3.Database;
-export { type Database };
+import type { DatabaseSync } from "node:sqlite";
+export type Database = DatabaseSync;
 
-let betterSqlite3: any = null;
+let sqliteModule: { DatabaseSync: new (...args: any[]) => DatabaseSync } | null =
+  null;
 
 export let compress: (data: Buffer) => Buffer = () => {
   throw Error("must initialize persist context");
@@ -41,7 +41,7 @@ export let copyFileSync = (_src: string, _desc: string): void => {
 };
 
 export function initContext(opts: {
-  betterSqlite3;
+  sqlite: { DatabaseSync: new (...args: any[]) => DatabaseSync };
   compress: (Buffer) => Buffer;
   decompress: (Buffer) => Buffer;
   syncFiles: {
@@ -54,7 +54,7 @@ export function initContext(opts: {
   statSync: (path: string) => { mtimeMs: number };
   copyFileSync: (src: string, desc: string) => void;
 }) {
-  betterSqlite3 = opts.betterSqlite3;
+  sqliteModule = opts.sqlite;
   compress = opts.compress;
   decompress = opts.decompress;
   syncFiles = opts.syncFiles;
@@ -64,10 +64,10 @@ export function initContext(opts: {
 }
 
 export function createDatabase(...args): Database {
-  if (betterSqlite3 == null) {
+  if (sqliteModule == null) {
     throw Error(
-      "conat/persist must be initialized with the better-sqlite3 module -- import from backend/conat/persist instead",
+      "conat/persist must be initialized with node:sqlite -- import from backend/conat/persist instead",
     );
   }
-  return new betterSqlite3(...args);
+  return new sqliteModule.DatabaseSync(...args);
 }

src/packages/conat/persist/storage.ts

@@ -21,7 +21,7 @@ less than 1 read/write per second for each.  Thus memory is critical, and
 supporting at least 1000 writes/second is what we need.
 Fortunately, this implementation can do ~50,000+ writes per second and read
 over 500,000 per second.  Yes, it blocks the main thread, but by using
-better-sqlite3, we get speed increases over async code, so this is worth it.
+sync node:sqlite, we get speed increases over async code, so this is worth it.
 
 
 COMPRESSION:
@@ -69,6 +69,7 @@ import {
   decompress,
   statSync,
   copyFileSync,
+  ensureContainingDirectoryExists,
 } from "./context";
 import type { JSONValue } from "@cocalc/util/types";
 import { EventEmitter } from "events";
@@ -252,6 +253,7 @@ export interface StorageOptions {
 export class PersistentStream extends EventEmitter {
   private readonly options: StorageOptions;
   private readonly db: Database;
+  private readonly dbPath?: string;
   private readonly msgIDs = new TTL({ ttl: 2 * 60 * 1000 });
   private conf: Configuration;
   private throttledBackup?;
@@ -266,6 +268,9 @@ export class PersistentStream extends EventEmitter {
     const location = this.options.ephemeral
       ? ":memory:"
       : this.options.path + ".db";
+    if (location !== ":memory:") {
+      this.dbPath = location;
+    }
     this.initArchive();
     this.db = createDatabase(location);
     this.initSchema();
@@ -362,18 +367,36 @@ export class PersistentStream extends EventEmitter {
     if (path === undefined && !this.options.archive) {
       return;
     }
-    path = (path ?? this.options.archive) + ".db";
+    if (!this.dbPath) {
+      return;
+    }
+    const dest = (path ?? this.options.archive) + ".db";
     //console.log("backup", { path });
     try {
-      await this.db.backup(path);
+      await ensureContainingDirectoryExists(dest);
+      copyFileSync(this.dbPath, dest);
     } catch (err) {
       if (!process.env.COCALC_TEST_MODE) {
         console.log(err);
       }
-      logger.debug("WARNING: error creating a backup", path, err);
+      logger.debug("WARNING: error creating a backup", dest, err);
     }
   });
 
+  private runTransaction = <T>(fn: () => T): T => {
+    this.db.exec("BEGIN");
+    try {
+      const result = fn();
+      this.db.exec("COMMIT");
+      return result;
+    } catch (err) {
+      try {
+        this.db.exec("ROLLBACK");
+      } catch {}
+      throw err;
+    }
+  };
+
   private compress = (
     raw: Buffer,
   ): { raw: Buffer; compress: CompressionAlgorithm } => {
@@ -442,30 +465,25 @@ export class PersistentStream extends EventEmitter {
 
     this.enforceLimits(size);
 
-    const tx = this.db.transaction(
-      (time, compress, encoding, raw, headers, key, size, ttl) => {
-        if (key !== undefined) {
-          // insert with key -- delete all previous messages, as they will
-          // never be needed again and waste space.
-          this.db.prepare("DELETE FROM messages WHERE key = ?").run(key);
-        }
-        return this.db
-          .prepare(
-            "INSERT INTO messages(time, compress, encoding, raw, headers, key, size, ttl) VALUES (?, ?, ?, ?, ?, ?, ?, ?)  RETURNING seq",
-          )
-          .get(time / 1000, compress, encoding, raw, headers, key, size, ttl);
-      },
-    );
-    const row = tx(
-      time,
-      compressedRaw.compress,
-      encoding,
-      compressedRaw.raw,
-      serializedHeaders,
-      key,
-      size,
-      ttl,
-    );
+    const row = this.runTransaction(() => {
+      if (key !== undefined) {
+        this.db.prepare("DELETE FROM messages WHERE key = ?").run(key);
+      }
+      return this.db
+        .prepare(
+          "INSERT INTO messages(time, compress, encoding, raw, headers, key, size, ttl) VALUES (?, ?, ?, ?, ?, ?, ?, ?)  RETURNING seq",
+        )
+        .get(
+          time / 1000,
+          compressedRaw.compress,
+          encoding,
+          compressedRaw.raw,
+          serializedHeaders,
+          key ?? null,
+          size,
+          ttl ?? null,
+        );
+    });
     const seq = Number((row as any).seq);
     // lastInsertRowid - is a bigint from sqlite, but we won't hit that limit
     this.emit("change", {
@@ -572,12 +590,11 @@ export class PersistentStream extends EventEmitter {
       this.db.prepare("DELETE FROM messages WHERE seq=?").run(seq);
     } else if (seqs0) {
       const statement = this.db.prepare("DELETE FROM messages WHERE seq=?");
-      const transaction = this.db.transaction((seqs) => {
-        for (const s of seqs) {
+      this.runTransaction(() => {
+        for (const s of seqs0) {
           statement.run(s);
         }
       });
-      transaction(seqs0);
       seqs = seqs0;
     }
     this.emit("change", { op: "delete", seqs });

src/packages/lite/package.json

@@ -17,7 +17,7 @@
     "build:bundle": "pnpm build:static && cd sea && ./build-bundle.sh",
     "build:tarball": "pnpm build:bundle && cd build && tar Jcvf bundle.tar.xz bundle",
     "run:bundle": "node ./build/bundle/bundle/index.js",
-    "sea": "cd sea && ./build-sea.sh",
+    "sea": "pnpm build:tarball && cd sea && ./build-sea.sh",
     "build:all": "pnpm build && pnpm build:tarball && pnpm sea",
     "app": "pnpm exec cocalc-lite",
     "app-electron": "pnpm electron-rebuild && electron ."

src/packages/lite/sea/build-bundle.sh

@@ -10,7 +10,7 @@
 # packages/lite/bin/start.js (which calls @cocalc/lite/main),
 # and copies the static frontend assets.
 #
-# Native addons copied by ncc (e.g. zeromq, better-sqlite3) are preserved.
+# Native addons copied by ncc (e.g. zeromq, node-pty) are preserved.
 # Additional assets can be copied after this script if needed.
 
 set -euo pipefail

src/packages/package.json

@@ -34,6 +34,6 @@
       "axios@<1.12.0": "^1.12.0",
       "tmp@<0.2.4": "^0.2.4"
     },
-    "onlyBuiltDependencies": ["better-sqlite3", "node-pty"]
+    "onlyBuiltDependencies": ["node-pty"]
   }
 }