Node: Support using node:sqlite and encryption

.changeset/hungry-spiders-perform.md

@@ -0,0 +1,5 @@
+---
+'@powersync/node': minor
+---
+
+Experimental support for integrating with node:sqlite.

.changeset/khaki-years-design.md

@@ -0,0 +1,5 @@
+---
+'@powersync/node': minor
+---
+
+Support custom better-sqlite3 forks (see an example for encryption in the README).

.changeset/wet-cooks-dress.md

@@ -0,0 +1,12 @@
+---
+'@powersync/node': minor
+---
+
+Use upstream better-sqlite3 dependency instead of the PowerSync fork.
+
+After upgrading:
+
+1. Ensure you no longer depend on the `@powersync/better-sqlite3` package: `npm uninstall @powersync/better-sqlite3`.
+2. Unlike in older versions, the upstream `better-sqlite3` dependency is marked as optional since custom forks
+   are supported too.
+   Use `npm install better-sqlite3` to install it.

demos/example-electron-node/config.ts

@@ -1,6 +1,5 @@
 import OS from 'node:os';
 import path from 'node:path';
-import { createRequire } from 'node:module';
 
 import type { ForgeConfig } from '@electron-forge/shared-types';
 import { MakerSquirrel } from '@electron-forge/maker-squirrel';
@@ -14,7 +13,7 @@ import * as dotenv from 'dotenv';
 import type IForkTsCheckerWebpackPlugin from 'fork-ts-checker-webpack-plugin';
 import type ICopyPlugin from 'copy-webpack-plugin';
 
-dotenv.config({path: '.env.local'});
+dotenv.config({ path: '.env.local' });
 
 const ForkTsCheckerWebpackPlugin: typeof IForkTsCheckerWebpackPlugin = require('fork-ts-checker-webpack-plugin');
 const CopyPlugin: typeof ICopyPlugin = require('copy-webpack-plugin');
@@ -78,25 +77,27 @@ const mainConfig: Configuration = {
   entry: './src/main/index.ts',
   // Put your normal webpack config below here
   module: {
-    rules: defaultWebpackRules(),
+    rules: defaultWebpackRules()
   },
   plugins: [
     ...webpackPlugins,
     new CopyPlugin({
-      patterns: [{
-        from: path.resolve(require.resolve('@powersync/node/package.json'), `../lib/${extensionPath}`),
-        to: path.join('powersync', extensionPath),
-      }],
+      patterns: [
+        {
+          from: path.resolve(require.resolve('@powersync/node/package.json'), `../lib/${extensionPath}`),
+          to: path.join('powersync', extensionPath)
+        }
+      ]
     }),
     new DefinePluginImpl({
       POWERSYNC_URL: JSON.stringify(process.env.POWERSYNC_URL),
-      POWERSYNC_TOKEN: JSON.stringify(process.env.POWERSYNC_TOKEN),
-    }),
+      POWERSYNC_TOKEN: JSON.stringify(process.env.POWERSYNC_TOKEN)
+    })
   ],
   resolve: {
     extensions: ['.js', '.ts', '.jsx', '.tsx', '.css', '.json']
   },
-  target: "electron-main",
+  target: 'electron-main'
 };
 
 const rendererConfig: Configuration = {
@@ -107,7 +108,7 @@ const rendererConfig: Configuration = {
         test: /\.css$/,
         use: [{ loader: 'style-loader' }, { loader: 'css-loader' }]
       }
-    ],
+    ]
   },
   plugins: webpackPlugins,
   resolve: {
@@ -119,10 +120,10 @@ const config: ForgeConfig = {
   packagerConfig: {
     asar: {
       unpack: '**/{.**,**}/**/powersync/*'
-    },
+    }
   },
   rebuildConfig: {
-    force: true,
+    force: true
   },
   makers: [
     new MakerSquirrel(),
@@ -142,7 +143,7 @@ const config: ForgeConfig = {
             html: './src/render/index.html',
             js: './src/render/main.ts',
             preload: {
-              js: './src/render/preload.ts',
+              js: './src/render/preload.ts'
             }
           }
         ]

demos/example-electron-node/package.json

@@ -23,11 +23,12 @@
     "@electron-forge/maker-zip": "^7.7.0",
     "@electron-forge/plugin-auto-unpack-natives": "^7.7.0",
     "@electron-forge/plugin-webpack": "^7.7.0",
+    "@types/better-sqlite3": "^7.6.13",
     "@vercel/webpack-asset-relocator-loader": "1.7.3",
     "copy-webpack-plugin": "^13.0.0",
     "css-loader": "^6.11.0",
     "dotenv": "^16.4.7",
-    "electron": "30.0.2",
+    "electron": "37.0.0",
     "electron-rebuild": "^3.2.9",
     "fork-ts-checker-webpack-plugin": "^9.0.2",
     "node-loader": "^2.1.0",
@@ -40,6 +41,7 @@
   },
   "dependencies": {
     "@powersync/node": "workspace:*",
+    "better-sqlite3": "^12.2.0",
     "electron-squirrel-startup": "^1.0.1"
   }
 }

demos/example-electron-node/src/main/worker.ts

@@ -1,5 +1,6 @@
 import * as path from 'node:path';
 import OS from 'node:os';
+import Database from 'better-sqlite3';
 
 import { startPowerSyncWorker } from '@powersync/node/worker.js';
 
@@ -28,4 +29,8 @@ function resolvePowerSyncCoreExtension() {
   return libraryPath;
 }
 
-startPowerSyncWorker({ extensionPath: resolvePowerSyncCoreExtension });
+async function resolveBetterSqlite3() {
+  return Database;
+}
+
+startPowerSyncWorker({ extensionPath: resolvePowerSyncCoreExtension, loadBetterSqlite3: resolveBetterSqlite3 });

demos/example-node/.env

@@ -1,4 +1,5 @@
 BACKEND=http://localhost:6060
 SYNC_SERVICE=http://localhost:8080
 POWERSYNC_TOKEN=
-POWERSYNC_DEBUG=1
+POWERSYNC_DEBUG=1
+ENCRYPTION_KEY=

demos/example-node/README.md

@@ -15,6 +15,13 @@ Results from the query are printed every time it changes. Try:
 1. Updating a row in the backend database and see changes reflected in the running client.
 2. Enter `add('my list')` and see the new list show up in the backend database.
 
+## Encryption
+
+This demo can use encrypted databases with the `better-sqlite3-multiple-ciphers` package.
+To test encryption, set the `ENCRYPTION_KEY` in `.env` to a non-empty value.
+
+## References
+
 For more details, see the documentation for [the PowerSync node package](https://docs.powersync.com/client-sdk-references/node) and check other examples:
 
 - [example-electron-node](../example-electron-node/): An Electron example that runs PowerSync in the main process using the Node.js SDK.

demos/example-node/package.json

@@ -11,10 +11,13 @@
   },
   "dependencies": {
     "@powersync/node": "workspace:*",
+    "better-sqlite3": "^12.2.0",
+    "better-sqlite3-multiple-ciphers": "^12.2.0",
     "dotenv": "^16.4.7",
     "undici": "^7.11.0"
   },
   "devDependencies": {
+    "@types/better-sqlite3": "^7.6.13",
     "ts-node": "^10.9.2",
     "typescript": "^5.8.2"
   }

demos/example-node/src/encryption.worker.ts

@@ -0,0 +1,12 @@
+// This worker uses bindings to sqlite3 multiple ciphers instead of the original better-sqlite3 worker.
+//
+// It is used in main.ts only when an encryption key is set.
+import Database from 'better-sqlite3-multiple-ciphers';
+
+import { startPowerSyncWorker } from '@powersync/node/worker.js';
+
+async function resolveBetterSqlite3() {
+  return Database;
+}
+
+startPowerSyncWorker({ loadBetterSqlite3: resolveBetterSqlite3 });

demos/example-node/src/main.ts

@@ -1,5 +1,6 @@
 import { once } from 'node:events';
 import repl_factory from 'node:repl';
+import { Worker } from 'node:worker_threads';
 
 import {
   createBaseLogger,
@@ -11,11 +12,14 @@ import {
 import { exit } from 'node:process';
 import { AppSchema, DemoConnector } from './powersync.js';
 import { enableUncidiDiagnostics } from './UndiciDiagnostics.js';
+import { WorkerOpener } from 'node_modules/@powersync/node/src/db/options.js';
+import { LockContext } from 'node_modules/@powersync/node/dist/bundle.cjs';
 
 const main = async () => {
   const baseLogger = createBaseLogger();
   const logger = createLogger('PowerSyncDemo');
   const debug = process.env.POWERSYNC_DEBUG == '1';
+  const encryptionKey = process.env.ENCRYPTION_KEY ?? '';
   baseLogger.useDefaults({ defaultLevel: debug ? logger.TRACE : logger.WARN });
 
   // Enable detailed request/response logging for debugging purposes.
@@ -30,10 +34,27 @@ const main = async () => {
     return;
   }
 
+  let customWorker: WorkerOpener | undefined;
+  if (encryptionKey.length) {
+    customWorker = (_, options) => {
+      return new Worker(new URL('./encryption.worker.js', import.meta.url), options);
+    };
+  }
+
   const db = new PowerSyncDatabase({
     schema: AppSchema,
     database: {
-      dbFilename: 'test.db'
+      dbFilename: 'test.db',
+      openWorker: customWorker,
+      initializeConnection: async (db) => {
+        if (encryptionKey.length) {
+          const escapedKey = encryptionKey.replace("'", "''");
+          await db.execute(`pragma key = '${escapedKey}'`);
+        }
+
+        // Make sure the database is readable, this fails early if the key is wrong.
+        await db.execute('pragma user_version');
+      }
     },
     logger
   });

packages/node/README.md

@@ -15,12 +15,12 @@ See a summary of features [here](https://docs.powersync.com/client-sdk-reference
 
 The `@powersync/node` package is currently in a Beta release.
 
-# Installation
+## Installation
 
-## Install Package
+### Install Package
 
 ```bash
-npm install @powersync/node
+npm install @powersync/node better-sqlite3
 ```
 
 Both `@powersync/node` and the `better-sqlite3` packages have install scripts that need to run to compile
@@ -47,38 +47,119 @@ To resolve this, either:
 - Upgrade `node-gyp` to version 10 or later.
 - Install Python [setuptools](https://pypi.org/project/setuptools/), which includes `distutils`.
 
-# Getting Started
+#### Package better-sqlite3 not found errors
+
+This package does not import `better-sqlite3` statically (with unconditional `require()` or static `import` statements).
+Instead, to allow users to use `node:sqlite` instead of that package, a dynamic `require()` / `import` expression is used.
+This may prevent bundlers from detecting that `better-sqlite3` is used by this package.
+
+To fix this, ensure you have a dependency on `better-sqlite3` (and, if you're using TypeScript, a dev-dependency on
+`@types/better-sqlite3`).
+
+In your project, create a `PowerSync.worker.ts` file with the following contents:
+
+```TypeScript
+import Database from 'better-sqlite3';
+
+import { startPowerSyncWorker } from '@powersync/node/worker.js';
+
+async function resolveBetterSqlite3() {
+  return Database;
+}
+
+startPowerSyncWorker({ loadBetterSqlite3: resolveBetterSqlite3 });
+```
+
+Finally, when you open the `PowerSyncDatabase`, instruct PowerSync to use your custom worker:
+
+```TypeScript
+const db = new PowerSyncDatabase({
+  schema: AppSchema,
+  database: {
+    dbFilename: 'test.db',
+    openWorker: (_, options) => {
+      return new Worker(new URL('./PowerSync.worker.js', import.meta.url), options);
+    }
+  },
+  logger
+});
+```
+
+## Getting Started
 
 The [Node.js SDK reference](https://docs.powersync.com/client-sdk-references/node)
 contains everything you need to know to get started implementing PowerSync in your project.
 
-# Examples
+## Examples
 
 A simple example using `@powersync/node` is available in the [`demos/example-node/`](../demos/example-node) directory.
 
-# Proxy Support
+## Proxy Support
 
 This SDK supports HTTP, HTTPS, and WebSocket proxies via environment variables.
 
-## HTTP Connection Method
+### HTTP Connection Method
 
 Internally we probe the http environment variables and apply it to fetch requests ([undici](https://www.npmjs.com/package/undici/v/5.6.0))
 
 - Set the `HTTPS_PROXY` or `HTTP_PROXY` environment variable to automatically route HTTP requests through a proxy.
 
-## WEB Socket Connection Method
+### WEB Socket Connection Method
 
 Internally the [proxy-agent](https://www.npmjs.com/package/proxy-agent) dependency for WebSocket proxies, which has its own internal code for automatically picking up the appropriate environment variables:
 
 - Set the `WS_PROXY` or `WSS_PROXY` environment variable to route the webocket connections through a proxy.
 
-# Found a bug or need help?
+## Encryption
+
+This package can be used with the [`better-sqlite3-multiple-ciphers`](https://www.npmjs.com/package/better-sqlite3-multiple-ciphers) fork of `better-sqlite3` for encryption.
+
+This requires a custom worker loading the forked package:
+
+```TypeScript
+// encryption.worker.ts
+import Database from 'better-sqlite3-multiple-ciphers';
+
+import { startPowerSyncWorker } from '@powersync/node/worker.js';
+
+async function resolveBetterSqlite3() {
+  return Database;
+}
+
+startPowerSyncWorker({ loadBetterSqlite3: resolveBetterSqlite3 });
+```
+
+Then, when opening the database, use that custom worker:
+
+```TypeScript
+const db = new PowerSyncDatabase({
+  schema: AppSchema,
+  database: {
+    dbFilename: 'test.db',
+    openWorker: (_, options) => {
+      return new Worker(new URL('./PowerSync.worker.js', import.meta.url), options);
+    },
+    initializeConnection: async (db) => {
+      if (encryptionKey.length) {
+        const escapedKey = encryptionKey.replace("'", "''");
+        await db.execute(`pragma key = '${escapedKey}'`);
+      }
+
+      // Make sure the database is readable, this fails early if the key is wrong.
+      await db.execute('pragma user_version');
+    }
+  },
+  logger
+});
+```
+
+## Found a bug or need help?
 
 - Join our [Discord server](https://discord.gg/powersync) where you can browse topics from our community, ask questions, share feedback, or just say hello :)
 - Please open a [GitHub issue](https://github.com/powersync-ja/powersync-js/issues) when you come across a bug.
 - Have feedback or an idea? [Submit an idea](https://roadmap.powersync.com/tabs/5-roadmap/submit-idea) via our public roadmap or [schedule a chat](https://calendly.com/powersync-product/powersync-chat) with someone from our product team.
 
-# Thanks
+## Thanks
 
 The PowerSync Node.js SDK relies on the work contributors and maintainers have put into the upstream better-sqlite3 package.
 In particular, we'd like to thank [@spinda](https://github.com/spinda) for contributing support for update, commit and rollback hooks!