Address #746 and a small SQLite optimization.

Explicitly set the `temp_store` pragma to `file` instead of `memory`
after deciding to place temporary files in `/data` to keep RAM usage
down while addressing #746.

Added a helper function to automatically "flatten" transactions
when you don't need SAVEPOINTs to avoid unnecessary temporary files.

Signed-off-by: Bea <[email protected]>

Author: enbea

Dockerfile

@@ -23,6 +23,8 @@ COPY --from=build-stage /tmp/src/package.json /
 
 ENV NODE_ENV=production
 ENV NODE_CONFIG_DIR=/data/config
+# Set SQLite's temporary directory. See #746 for context.
+ENV SQLITE_TMPDIR=/data
 
 CMD ["bot"]
 ENTRYPOINT ["./draupnir-entrypoint.sh"]

src/backingstore/better-sqlite3/BetterSqliteStore.ts

@@ -14,7 +14,7 @@ import { Logger } from "matrix-protection-suite";
 const log = new Logger("BetterSqliteStore");
 
 export function sqliteV0Schema(db: Database) {
-  // we have to prepare and run them seperatley becasue prepare checks if the
+  // we have to prepare and run them separately because `prepare` checks if the
   // table exists.
   const createTable = db.transaction(() => {
     db.prepare(
@@ -176,3 +176,20 @@ export abstract class BetterSqliteStore {
     throw Error("Couldn't fetch schema version");
   }
 }
+
+/**
+ * Wraps `fn` in a transaction without creating nested SAVEPOINTs.
+ * Reduces the likelihood of temporary file creation.
+ * Initially investigated as part of #746.
+ *
+ * See https://www.sqlite.org/lang_savepoint.html
+ * See https://github.com/WiseLibs/better-sqlite3/blob/master/docs/api.md#transactionfunction---function
+ */
+export function flatTransaction<Arguments extends unknown[], Result>(
+  db: Database,
+  fn: (...args: Arguments) => Result
+) {
+  const t = db.transaction(fn);
+  return (...args: Arguments): Result =>
+    db.inTransaction ? fn(...args) : t(...args);
+}

src/backingstore/better-sqlite3/SqliteRoomStateBackingStore.ts

@@ -15,7 +15,7 @@ import {
   StateEvent,
   isError,
 } from "matrix-protection-suite";
-import { BetterSqliteStore } from "./BetterSqliteStore";
+import { BetterSqliteStore, flatTransaction } from "./BetterSqliteStore";
 import { jsonReviver } from "../../utils";
 import { StringRoomID } from "@the-draupnir-project/matrix-basic-types";
 
@@ -64,6 +64,7 @@ export class SqliteRoomStateBackingStore
     );
     this.db.pragma("journal_mode = WAL");
     this.db.pragma("foreign_keys = ON");
+    this.db.pragma("temp_store = file"); // Avoid unnecessary memory usage.
     this.ensureSchema();
   }
 
@@ -85,8 +86,8 @@ export class SqliteRoomStateBackingStore
         JSON.stringify(event),
       ];
     };
-    // i don't understand why the library makes us do this but ok.
-    const replace = this.db.transaction((events: StateEvent[]) => {
+    // `flatTransaction` optimizes away unnecessary temporary files.
+    const replace = flatTransaction(this.db, (events: StateEvent[]) => {
       for (const event of events) {
         replaceStatement.run(createValue(event));
       }