Support upstream better-sqlite3

Author: simolus3

packages/common/rollup.config.mjs

@@ -38,8 +38,8 @@ export default (commandLineArgs) => {
           ReadableStream: ['web-streams-polyfill/ponyfill', 'ReadableStream'],
           // Used by can-ndjson-stream
           TextDecoder: ['text-encoding', 'TextDecoder']
-        }),
-        terser({ sourceMap })
+        })
+        //terser({ sourceMap })
       ],
       // This makes life easier
       external: [

packages/node/package.json

@@ -56,10 +56,15 @@
   },
   "homepage": "https://docs.powersync.com/",
   "peerDependencies": {
-    "@powersync/common": "workspace:^1.40.0"
+    "@powersync/common": "workspace:^1.40.0",
+    "better-sqlite3": "12.x"
+  },
+  "peerDependenciesMeta": {
+    "better-sqlite3": {
+      "optional": true
+    }
   },
   "dependencies": {
-    "@powersync/better-sqlite3": "^0.2.0",
     "@powersync/common": "workspace:*",
     "async-lock": "^1.4.0",
     "bson": "^6.6.0",
@@ -70,6 +75,7 @@
     "@powersync/drizzle-driver": "workspace:*",
     "@types/async-lock": "^1.4.0",
     "@types/node": "^24.2.0",
+    "better-sqlite3": "^12.2.0",
     "drizzle-orm": "^0.35.2",
     "rollup": "4.14.3",
     "typescript": "^5.5.3",

packages/node/rollup.config.mjs

@@ -3,6 +3,13 @@ import dts from 'rollup-plugin-dts';
 const plugin = () => {
   return {
     name: 'mark-as-commonjs',
+    async resolveId(source, importer, options) {
+      if (importer && source.indexOf('modules.js')) {
+        return await this.resolve(source.replace('modules.js', 'modules_commonjs.js'), importer, options);
+      } else {
+        return await this.resolve(source, importer, options);
+      }
+    },
     resolveImportMeta: (property) => {
       if (property == 'isBundlingToCommonJs') {
         return 'true';

packages/node/src/db/AsyncDatabase.ts

@@ -23,9 +23,4 @@ export interface AsyncDatabase {
   executeRaw: (query: string, params: any[]) => Promise<any[][]>;
   executeBatch: (query: string, params: any[][]) => Promise<ProxiedQueryResult>;
   close: () => Promise<void>;
-  // Collect table updates made since the last call to collectCommittedUpdates.
-  // This happens on the worker because we otherwise get race conditions when wrapping
-  // callbacks to invoke on the main thread (we need a guarantee that collectCommittedUpdates
-  // contains entries immediately after calling COMMIT).
-  collectCommittedUpdates: () => Promise<string[]>;
 }

packages/node/src/db/BetterSqliteWorker.ts

@@ -1,45 +1,18 @@
-import * as Comlink from 'comlink';
-import BetterSQLite3Database, { Database } from '@powersync/better-sqlite3';
-import { AsyncDatabase, AsyncDatabaseOpener, AsyncDatabaseOpenOptions } from './AsyncDatabase.js';
+import type { Database } from 'better-sqlite3';
+import { AsyncDatabase, AsyncDatabaseOpenOptions } from './AsyncDatabase.js';
 import { PowerSyncWorkerOptions } from './SqliteWorker.js';
 import { threadId } from 'node:worker_threads';
+import { dynamicImport } from '../utils/modules.js';
 
 class BlockingAsyncDatabase implements AsyncDatabase {
   private readonly db: Database;
 
-  private readonly uncommittedUpdatedTables = new Set<string>();
-  private readonly committedUpdatedTables = new Set<string>();
-
   constructor(db: Database) {
     this.db = db;
 
     db.function('node_thread_id', () => threadId);
   }
 
-  collectCommittedUpdates() {
-    const resolved = Promise.resolve([...this.committedUpdatedTables]);
-    this.committedUpdatedTables.clear();
-    return resolved;
-  }
-
-  installUpdateHooks() {
-    this.db.updateHook((_op: string, _dbName: string, tableName: string, _rowid: bigint) => {
-      this.uncommittedUpdatedTables.add(tableName);
-    });
-
-    this.db.commitHook(() => {
-      for (const tableName of this.uncommittedUpdatedTables) {
-        this.committedUpdatedTables.add(tableName);
-      }
-      this.uncommittedUpdatedTables.clear();
-      return true;
-    });
-
-    this.db.rollbackHook(() => {
-      this.uncommittedUpdatedTables.clear();
-    });
-  }
-
   async close() {
     this.db.close();
   }
@@ -90,7 +63,8 @@ class BlockingAsyncDatabase implements AsyncDatabase {
   }
 }
 
-export async function openDatabase(worker: PowerSyncWorkerOptions, options: AsyncDatabaseOpenOptions) {
+export async function openDatabase(worker: PowerSyncWorkerOptions, options: AsyncDatabaseOpenOptions, pkg: string) {
+  const BetterSQLite3Database: typeof Database = (await dynamicImport(pkg)).default;
   const baseDB = new BetterSQLite3Database(options.path);
   baseDB.pragma('journal_mode = WAL');
   baseDB.loadExtension(worker.extensionPath(), 'sqlite3_powersync_init');
@@ -99,6 +73,5 @@ export async function openDatabase(worker: PowerSyncWorkerOptions, options: Asyn
   }
 
   const asyncDb = new BlockingAsyncDatabase(baseDB);
-  asyncDb.installUpdateHooks();
   return asyncDb;
 }

packages/node/src/db/NodeSqliteWorker.ts

@@ -1,9 +1,9 @@
 import { threadId } from 'node:worker_threads';
 import type { DatabaseSync } from 'node:sqlite';
 
-import * as Comlink from 'comlink';
-import { AsyncDatabase, AsyncDatabaseOpener, AsyncDatabaseOpenOptions } from './AsyncDatabase.js';
+import { AsyncDatabase, AsyncDatabaseOpenOptions } from './AsyncDatabase.js';
 import { PowerSyncWorkerOptions } from './SqliteWorker.js';
+import { dynamicImport } from '../utils/modules.js';
 
 class BlockingNodeDatabase implements AsyncDatabase {
   private readonly db: DatabaseSync;
@@ -12,16 +12,6 @@ class BlockingNodeDatabase implements AsyncDatabase {
     this.db = db;
 
     db.function('node_thread_id', () => threadId);
-    if (write) {
-      db.exec("SELECT powersync_update_hooks('install');");
-    }
-  }
-
-  async collectCommittedUpdates() {
-    const stmt = this.db.prepare("SELECT powersync_update_hooks('get') AS r;");
-    const row = stmt.get()!;
-
-    return JSON.parse(row['r'] as string) as string[];
   }
 
   async close() {
@@ -64,7 +54,7 @@ class BlockingNodeDatabase implements AsyncDatabase {
 export async function openDatabase(worker: PowerSyncWorkerOptions, options: AsyncDatabaseOpenOptions) {
   // NOTE: We want to import node:sqlite dynamically, to avoid bundlers unconditionally requiring node:sqlite in the
   // end, since that would make us incompatible with older Node.JS versions.
-  const { DatabaseSync } = await import('node:sqlite');
+  const { DatabaseSync } = await dynamicImport('node:sqlite');
 
   const baseDB = new DatabaseSync(options.path, { allowExtension: true });
   baseDB.exec('pragma journal_mode = WAL');

packages/node/src/db/PowerSyncDatabase.ts

@@ -17,7 +17,7 @@ import {
 import { NodeCustomConnectionOptions, NodeRemote } from '../sync/stream/NodeRemote.js';
 import { NodeStreamingSyncImplementation } from '../sync/stream/NodeStreamingSyncImplementation.js';
 
-import { BetterSQLite3DBAdapter } from './BetterSQLite3DBAdapter.js';
+import { WorkerConnectionPool } from './WorkerConnectionPool.js';
 import { NodeSQLOpenOptions } from './options.js';
 
 export type NodePowerSyncDatabaseOptions = PowerSyncDatabaseOptions & {
@@ -54,14 +54,14 @@ export class PowerSyncDatabase extends AbstractPowerSyncDatabase {
   }
 
   async _initialize(): Promise<void> {
-    await (this.database as BetterSQLite3DBAdapter).initialize();
+    await (this.database as WorkerConnectionPool).initialize();
   }
 
   /**
    * Opens a DBAdapter using better-sqlite3 as the default SQLite open factory.
    */
   protected openDBAdapter(options: PowerSyncDatabaseOptionsWithSettings): DBAdapter {
-    return new BetterSQLite3DBAdapter(options.database);
+    return new WorkerConnectionPool(options.database);
   }
 
   protected generateBucketStorageAdapter(): BucketStorageAdapter {

packages/node/src/db/SqliteWorker.ts

@@ -6,6 +6,7 @@ import url from 'node:url';
 import { openDatabase as openBetterSqliteDatabase } from './BetterSqliteWorker.js';
 import { openDatabase as openNodeDatabase } from './NodeSqliteWorker.js';
 import { AsyncDatabase, AsyncDatabaseOpener, AsyncDatabaseOpenOptions } from './AsyncDatabase.js';
+import { isBundledToCommonJs } from '../utils/modules.js';
 
 export interface PowerSyncWorkerOptions {
   /**
@@ -19,7 +20,7 @@ export interface PowerSyncWorkerOptions {
 export function startPowerSyncWorker(options?: Partial<PowerSyncWorkerOptions>) {
   const resolvedOptions: PowerSyncWorkerOptions = {
     extensionPath() {
-      const isCommonJsModule = import.meta.isBundlingToCommonJs ?? false;
+      const isCommonJsModule = isBundledToCommonJs;
 
       const platform = OS.platform();
       let extensionPath: string;
@@ -58,11 +59,12 @@ class DatabaseOpenHelper implements AsyncDatabaseOpener {
   async open(options: AsyncDatabaseOpenOptions): Promise<AsyncDatabase> {
     let database: AsyncDatabase;
 
-    switch (options.implementation) {
+    const implementation = options.implementation;
+    switch (implementation.type) {
       case 'better-sqlite3':
-        database = await openBetterSqliteDatabase(this.options, options);
+        database = await openBetterSqliteDatabase(this.options, options, implementation.package ?? 'better-sqlite3');
         break;
-      case 'node':
+      case 'node:sqlite':
         database = await openNodeDatabase(this.options, options);
         break;
       default:

packages/node/src/db/BetterSQLite3DBAdapter.ts renamed to packages/node/src/db/WorkerConnectionPool.ts

@@ -17,7 +17,8 @@ import { Remote } from 'comlink';
 import { AsyncResource } from 'node:async_hooks';
 import { AsyncDatabase, AsyncDatabaseOpener } from './AsyncDatabase.js';
 import { RemoteConnection } from './RemoteConnection.js';
-import { NodeSQLOpenOptions } from './options.js';
+import { NodeDatabaseImplementation, NodeSQLOpenOptions } from './options.js';
+import { isBundledToCommonJs } from '../utils/modules.js';
 
 export type BetterSQLite3LockContext = LockContext & {
   executeBatch(query: string, params?: any[][]): Promise<QueryResult>;
@@ -27,10 +28,15 @@ export type BetterSQLite3Transaction = Transaction & BetterSQLite3LockContext;
 
 const READ_CONNECTIONS = 5;
 
+const defaultDatabaseImplementation: NodeDatabaseImplementation = {
+  type: 'better-sqlite3',
+  package: 'better-sqlite3'
+};
+
 /**
  * Adapter for better-sqlite3
  */
-export class BetterSQLite3DBAdapter extends BaseObserver<DBAdapterListener> implements DBAdapter {
+export class WorkerConnectionPool extends BaseObserver<DBAdapterListener> implements DBAdapter {
   private readonly options: NodeSQLOpenOptions;
   public readonly name: string;
 
@@ -73,7 +79,7 @@ export class BetterSQLite3DBAdapter extends BaseObserver<DBAdapterListener> impl
     }
 
     const openWorker = async (isWriter: boolean) => {
-      const isCommonJsModule = import.meta.isBundlingToCommonJs ?? false;
+      const isCommonJsModule = isBundledToCommonJs;
       let worker: Worker;
       const workerName = isWriter ? `write ${dbFilePath}` : `read ${dbFilePath}`;
 
@@ -120,8 +126,12 @@ export class BetterSQLite3DBAdapter extends BaseObserver<DBAdapterListener> impl
       const database = (await comlink.open({
         path: dbFilePath,
         isWriter,
-        implementation: this.options.implementation ?? 'better-sqlite3'
+        implementation: this.options.implementation ?? defaultDatabaseImplementation
       })) as Remote<AsyncDatabase>;
+      if (isWriter) {
+        await database.execute("SELECT powersync_update_hooks('install');", []);
+      }
+
       return new RemoteConnection(worker, comlink, database);
     };
 
@@ -192,7 +202,11 @@ export class BetterSQLite3DBAdapter extends BaseObserver<DBAdapterListener> impl
         try {
           return await fn(this.writeConnection);
         } finally {
-          const updates = await this.writeConnection.database.collectCommittedUpdates();
+          const serializedUpdates = await this.writeConnection.database.executeRaw(
+            "SELECT powersync_update_hooks('get');",
+            []
+          );
+          const updates = JSON.parse(serializedUpdates[0][0] as string) as string[];
 
           if (updates.length > 0) {
             const event: BatchedUpdateNotification = {

packages/node/src/db/options.ts

@@ -3,7 +3,30 @@ import { SQLOpenOptions } from '@powersync/common';
 
 export type WorkerOpener = (...args: ConstructorParameters<typeof Worker>) => InstanceType<typeof Worker>;
 
-export type NodeDatabaseImplementation = 'better-sqlite3' | 'node';
+/**
+ * Use the [better-sqlite3](https://github.com/WiseLibs/better-sqlite3) package as a SQLite driver for PowerSync.
+ */
+export interface BetterSqlite3Options {
+  type: 'better-sqlite3';
+  /**
+   * The package import to resolve for better-sqlite3.
+   *
+   * While this defaults to `better-sqlite3`, this allows using forked better-sqlite3 packages, such as those used for
+   * encryption.
+   */
+  package?: string;
+}
+
+/**
+ * Use the experimental `node:sqlite` interface as a SQLite driver for PowerSync.
+ *
+ * Note that this option is not currently tested and highly unstable.
+ */
+export interface NodeSqliteOptions {
+  type: 'node:sqlite';
+}
+
+export type NodeDatabaseImplementation = BetterSqlite3Options | NodeSqliteOptions;
 
 /**
  * The {@link SQLOpenOptions} available across all PowerSync SDKs for JavaScript extended with