diff --git a/examples/drizzle/src/db/schema.ts b/examples/drizzle/src/db/schema.ts deleted file mode 100644 index f24058ac8d..0000000000 --- a/examples/drizzle/src/db/schema.ts +++ /dev/null @@ -1,9 +0,0 @@ -// import { int, sqliteTable, text } from "@rivetkit/db/drizzle"; - -// export const usersTable = sqliteTable("users_table", { -// id: int().primaryKey({ autoIncrement: true }), -// name: text().notNull(), -// age: int().notNull(), -// email: text().notNull().unique(), -// email2: text().notNull().unique(), -// }); diff --git a/examples/drizzle/src/registry.ts b/examples/drizzle/src/registry.ts deleted file mode 100644 index cabe7f8891..0000000000 --- a/examples/drizzle/src/registry.ts +++ /dev/null @@ -1,22 +0,0 @@ -// import { actor, setup } from "rivetkit"; -// import { db } from "@rivetkit/db/drizzle"; -// import * as schema from "./db/schema"; -// import migrations from "../drizzle/migrations"; - -// export const counter = actor({ -// db: db({ schema, migrations }), -// state: { -// count: 0, -// }, -// actions: { -// increment: (c, x: number) => { -// // createState or state fix fix fix -// c.db.c.state.count += x; -// return c.state.count; -// }, -// }, -// }); - -// export const registry = setup({ -// use: { counter }, -// }); diff --git a/examples/drizzle/src/server.ts b/examples/drizzle/src/server.ts deleted file mode 100644 index 5be165d642..0000000000 --- a/examples/drizzle/src/server.ts +++ /dev/null @@ -1,7 +0,0 @@ -// import { registry } from "./registry"; -// import { createMemoryDriver } from "@rivetkit/memory"; -// import { serve } from "@rivetkit/nodejs"; - -// serve(registry, { -// driver: createMemoryDriver(), -// }); diff --git a/examples/drizzle/.env.sample b/examples/sqlite-drizzle/.env.sample similarity index 100% rename from examples/drizzle/.env.sample rename to examples/sqlite-drizzle/.env.sample diff --git a/examples/drizzle/README.md b/examples/sqlite-drizzle/README.md similarity index 100% rename from examples/drizzle/README.md rename to examples/sqlite-drizzle/README.md diff --git a/examples/drizzle/drizzle.config.ts b/examples/sqlite-drizzle/drizzle.config.ts similarity index 61% rename from examples/drizzle/drizzle.config.ts rename to examples/sqlite-drizzle/drizzle.config.ts index cbb8b521e0..bbc1d1b3c8 100644 --- a/examples/drizzle/drizzle.config.ts +++ b/examples/sqlite-drizzle/drizzle.config.ts @@ -1,4 +1,4 @@ -import { defineConfig } from "@rivetkit/db/drizzle"; +import { defineConfig } from "rivetkit/db/drizzle"; export default defineConfig({ out: "./drizzle", diff --git a/examples/drizzle/drizzle/0000_wonderful_iron_patriot.sql b/examples/sqlite-drizzle/drizzle/0000_wonderful_iron_patriot.sql similarity index 100% rename from examples/drizzle/drizzle/0000_wonderful_iron_patriot.sql rename to examples/sqlite-drizzle/drizzle/0000_wonderful_iron_patriot.sql diff --git a/examples/drizzle/drizzle/meta/0000_snapshot.json b/examples/sqlite-drizzle/drizzle/meta/0000_snapshot.json similarity index 100% rename from examples/drizzle/drizzle/meta/0000_snapshot.json rename to examples/sqlite-drizzle/drizzle/meta/0000_snapshot.json diff --git a/examples/drizzle/drizzle/meta/_journal.json b/examples/sqlite-drizzle/drizzle/meta/_journal.json similarity index 100% rename from examples/drizzle/drizzle/meta/_journal.json rename to examples/sqlite-drizzle/drizzle/meta/_journal.json diff --git a/examples/drizzle/drizzle/migrations.js b/examples/sqlite-drizzle/drizzle/migrations.js similarity index 100% rename from examples/drizzle/drizzle/migrations.js rename to examples/sqlite-drizzle/drizzle/migrations.js diff --git a/examples/drizzle/package.json b/examples/sqlite-drizzle/package.json similarity index 84% rename from examples/drizzle/package.json rename to examples/sqlite-drizzle/package.json index 15baec4417..e1bfbfcb76 100644 --- a/examples/drizzle/package.json +++ b/examples/sqlite-drizzle/package.json @@ -1,5 +1,5 @@ { - "name": "example-sqlite", + "name": "example-sqlite-drizzle", "version": "2.0.21", "private": true, "type": "module", @@ -14,7 +14,8 @@ "typescript": "^5.5.2" }, "dependencies": { - "@rivetkit/db": "workspace:*", + "@hono/node-server": "^1.18.2", + "@hono/node-ws": "^1.1.1", "drizzle-kit": "^0.31.2", "drizzle-orm": "^0.44.2" }, diff --git a/examples/sqlite-drizzle/src/db/schema.ts b/examples/sqlite-drizzle/src/db/schema.ts new file mode 100644 index 0000000000..7cd1034466 --- /dev/null +++ b/examples/sqlite-drizzle/src/db/schema.ts @@ -0,0 +1,9 @@ +import { int, sqliteTable, text } from "rivetkit/db/drizzle"; + +export const usersTable = sqliteTable("users_table", { + id: int().primaryKey({ autoIncrement: true }), + name: text().notNull(), + age: int().notNull(), + email: text().notNull().unique(), + email2: text().notNull().unique(), +}); diff --git a/examples/sqlite-drizzle/src/registry.ts b/examples/sqlite-drizzle/src/registry.ts new file mode 100644 index 0000000000..5c2a8ace2e --- /dev/null +++ b/examples/sqlite-drizzle/src/registry.ts @@ -0,0 +1,21 @@ +import { actor, setup } from "rivetkit"; +import { db } from "rivetkit/db/drizzle"; +import * as schema from "./db/schema"; +import migrations from "../drizzle/migrations"; + +export const counter = actor({ + db: db({ schema, migrations }), + state: { + count: 0, + }, + actions: { + increment: (c, x: number) => { + c.state.count += x; + return c.state.count; + }, + }, +}); + +export const registry = setup({ + use: { counter }, +}); diff --git a/examples/sqlite-drizzle/src/server.ts b/examples/sqlite-drizzle/src/server.ts new file mode 100644 index 0000000000..aa0ee6ed61 --- /dev/null +++ b/examples/sqlite-drizzle/src/server.ts @@ -0,0 +1,3 @@ +import { registry } from "./registry"; + +registry.start(); diff --git a/examples/drizzle/tsconfig.json b/examples/sqlite-drizzle/tsconfig.json similarity index 100% rename from examples/drizzle/tsconfig.json rename to examples/sqlite-drizzle/tsconfig.json diff --git a/examples/drizzle/turbo.json b/examples/sqlite-drizzle/turbo.json similarity index 100% rename from examples/drizzle/turbo.json rename to examples/sqlite-drizzle/turbo.json diff --git a/examples/sqlite-raw/README.md b/examples/sqlite-raw/README.md new file mode 100644 index 0000000000..c441ac6674 --- /dev/null +++ b/examples/sqlite-raw/README.md @@ -0,0 +1,49 @@ +# SQLite Raw Example + +This example demonstrates using the raw SQLite driver with RivetKit actors. + +## Features + +- **Raw SQLite API**: Direct SQL access using `@rivetkit/db/raw` +- **Migrations on Wake**: Uses `onMigrate` to create tables on actor wake +- **Todo List**: Simple CRUD operations with raw SQL queries + +## Running the Example + +```bash +pnpm install +pnpm dev +``` + +## Usage + +The example creates a `todoList` actor with the following actions: + +- `addTodo(title: string)` - Add a new todo +- `getTodos()` - Get all todos +- `toggleTodo(id: number)` - Toggle todo completion status +- `deleteTodo(id: number)` - Delete a todo + +## Code Structure + +- `src/registry.ts` - Actor definition with database configuration +- `src/server.ts` - Server entry point + +## Database + +The database uses the KV-backed SQLite VFS, which stores data in a key-value store. The schema is created using raw SQL in the `onMigrate` hook: + +```typescript +db: db({ + onMigrate: async (db) => { + await db.execute(` + CREATE TABLE IF NOT EXISTS todos ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + title TEXT NOT NULL, + completed INTEGER DEFAULT 0, + created_at INTEGER NOT NULL + ) + `); + }, +}) +``` diff --git a/examples/sqlite-raw/package.json b/examples/sqlite-raw/package.json new file mode 100644 index 0000000000..095ed9f82d --- /dev/null +++ b/examples/sqlite-raw/package.json @@ -0,0 +1,31 @@ +{ + "name": "example-sqlite-raw", + "version": "2.0.21", + "private": true, + "type": "module", + "scripts": { + "dev": "tsx --watch src/server.ts", + "check-types": "tsc --noEmit", + "build": "tsc", + "client": "tsx scripts/client.ts" + }, + "devDependencies": { + "@types/node": "^22.13.9", + "rivetkit": "workspace:*", + "tsx": "^3.12.7", + "typescript": "^5.5.2" + }, + "dependencies": {}, + "stableVersion": "0.8.0", + "template": { + "technologies": [ + "sqlite", + "typescript" + ], + "tags": [ + "database" + ], + "noFrontend": true + }, + "license": "MIT" +} diff --git a/examples/sqlite-raw/scripts/client.ts b/examples/sqlite-raw/scripts/client.ts new file mode 100644 index 0000000000..2d93a93075 --- /dev/null +++ b/examples/sqlite-raw/scripts/client.ts @@ -0,0 +1,61 @@ +import { createClient } from "rivetkit/client"; +import type { registry } from "../src/registry.js"; + +// Get endpoint from environment variable or default to localhost +const endpoint = process.env.RIVETKIT_ENDPOINT ?? "http://localhost:8080"; +console.log("šŸ”— Using endpoint:", endpoint); + +// Create RivetKit client +const client = createClient(endpoint); + +async function main() { + console.log("šŸš€ SQLite Raw Database Demo"); + + try { + // Create todo list instance + const todoList = client.todoList.getOrCreate("my-todos"); + + // Add some todos + console.log("\nšŸ“ Adding todos..."); + const todo1 = await todoList.addTodo("Buy groceries"); + console.log("Added:", todo1); + + const todo2 = await todoList.addTodo("Write documentation"); + console.log("Added:", todo2); + + const todo3 = await todoList.addTodo("Review pull requests"); + console.log("Added:", todo3); + + // Get all todos + console.log("\nšŸ“‹ Getting all todos..."); + const todos = await todoList.getTodos(); + console.log("Todos:", todos); + + // Toggle a todo (assuming first todo has id 1) + console.log("\nāœ… Toggling first todo..."); + const toggled = await todoList.toggleTodo(1); + console.log("Toggled:", toggled); + + // Get todos again to see the change + console.log("\nšŸ“‹ Getting all todos after toggle..."); + const todosAfterToggle = await todoList.getTodos(); + console.log("Todos:", todosAfterToggle); + + // Delete a todo (assuming second todo has id 2) + console.log("\nšŸ—‘ļø Deleting second todo..."); + const deleted = await todoList.deleteTodo(2); + console.log("Deleted:", deleted); + + // Get todos one more time + console.log("\nšŸ“‹ Final todos list..."); + const finalTodos = await todoList.getTodos(); + console.log("Todos:", finalTodos); + + console.log("\nāœ… Demo completed!"); + } catch (error) { + console.error("āŒ Error:", error); + process.exit(1); + } +} + +main().catch(console.error); diff --git a/examples/sqlite-raw/src/registry.ts b/examples/sqlite-raw/src/registry.ts new file mode 100644 index 0000000000..720ac4ab17 --- /dev/null +++ b/examples/sqlite-raw/src/registry.ts @@ -0,0 +1,49 @@ +import { actor, setup } from "rivetkit"; +import { db } from "rivetkit/db"; + +export const todoList = actor({ + db: db({ + onMigrate: async (db) => { + // Run migrations on wake + await db.execute(` + CREATE TABLE IF NOT EXISTS todos ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + title TEXT NOT NULL, + completed INTEGER DEFAULT 0, + created_at INTEGER NOT NULL + ) + `); + }, + }), + actions: { + addTodo: async (c, title: string) => { + const createdAt = Date.now(); + await c.db.execute( + "INSERT INTO todos (title, created_at) VALUES (?, ?)", + title, + createdAt, + ); + return { title, createdAt }; + }, + getTodos: async (c) => { + const rows = await c.db.execute("SELECT * FROM todos ORDER BY created_at DESC"); + return rows; + }, + toggleTodo: async (c, id: number) => { + await c.db.execute( + "UPDATE todos SET completed = NOT completed WHERE id = ?", + id, + ); + const rows = await c.db.execute("SELECT * FROM todos WHERE id = ?", id); + return rows[0]; + }, + deleteTodo: async (c, id: number) => { + await c.db.execute("DELETE FROM todos WHERE id = ?", id); + return { id }; + }, + }, +}); + +export const registry = setup({ + use: { todoList }, +}); diff --git a/examples/sqlite-raw/src/server.ts b/examples/sqlite-raw/src/server.ts new file mode 100644 index 0000000000..aa0ee6ed61 --- /dev/null +++ b/examples/sqlite-raw/src/server.ts @@ -0,0 +1,3 @@ +import { registry } from "./registry"; + +registry.start(); diff --git a/rivetkit-typescript/packages/db/tsconfig.json b/examples/sqlite-raw/tsconfig.json similarity index 61% rename from rivetkit-typescript/packages/db/tsconfig.json rename to examples/sqlite-raw/tsconfig.json index b1efe75212..b55b7d3a11 100644 --- a/rivetkit-typescript/packages/db/tsconfig.json +++ b/examples/sqlite-raw/tsconfig.json @@ -1,6 +1,7 @@ { - "extends": "../../../tsconfig.base.json", + "extends": "../../tsconfig.base.json", "compilerOptions": { + "baseUrl": ".", "paths": { "@/*": ["./src/*"] } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1f7c3f6e8a..c3593f4df0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -737,31 +737,6 @@ importers: specifier: ^24.5.2 version: 24.7.1 - examples/drizzle: - dependencies: - '@rivetkit/db': - specifier: workspace:* - version: link:../../rivetkit-typescript/packages/db - drizzle-kit: - specifier: ^0.31.2 - version: 0.31.5 - drizzle-orm: - specifier: ^0.44.2 - version: 0.44.6(@cloudflare/workers-types@4.20251014.0)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@11.10.0)(bun-types@1.3.0(@types/react@19.2.2))(kysely@0.28.8) - devDependencies: - '@types/node': - specifier: ^22.13.9 - version: 22.18.1 - rivetkit: - specifier: workspace:* - version: link:../../rivetkit-typescript/packages/rivetkit - tsx: - specifier: ^3.12.7 - version: 3.14.0 - typescript: - specifier: ^5.5.2 - version: 5.9.2 - examples/elysia: dependencies: '@rivetkit/react': @@ -1411,6 +1386,49 @@ importers: specifier: ^3.1.1 version: 3.2.4(@types/debug@4.1.12)(@types/node@22.19.1)(less@4.4.1)(lightningcss@1.30.2)(sass@1.93.2)(stylus@0.62.0)(terser@5.44.0) + examples/sqlite-drizzle: + dependencies: + '@hono/node-server': + specifier: ^1.18.2 + version: 1.19.1(hono@4.11.3) + '@hono/node-ws': + specifier: ^1.1.1 + version: 1.2.0(@hono/node-server@1.19.1(hono@4.11.3))(hono@4.11.3) + drizzle-kit: + specifier: ^0.31.2 + version: 0.31.5 + drizzle-orm: + specifier: ^0.44.2 + version: 0.44.6(@cloudflare/workers-types@4.20251014.0)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@11.10.0)(bun-types@1.3.0(@types/react@19.2.2))(kysely@0.28.8) + devDependencies: + '@types/node': + specifier: ^22.13.9 + version: 22.19.1 + rivetkit: + specifier: workspace:* + version: link:../../rivetkit-typescript/packages/rivetkit + tsx: + specifier: ^3.12.7 + version: 3.14.0 + typescript: + specifier: ^5.5.2 + version: 5.9.3 + + examples/sqlite-raw: + devDependencies: + '@types/node': + specifier: ^22.13.9 + version: 22.19.1 + rivetkit: + specifier: workspace:* + version: link:../../rivetkit-typescript/packages/rivetkit + tsx: + specifier: ^3.12.7 + version: 3.14.0 + typescript: + specifier: ^5.5.2 + version: 5.9.3 + examples/state: dependencies: rivetkit: @@ -2314,9 +2332,18 @@ importers: '@rivetkit/virtual-websocket': specifier: workspace:* version: link:../virtual-websocket + better-sqlite3: + specifier: ^11.0.0 + version: 11.10.0 cbor-x: specifier: ^1.6.0 version: 1.6.0 + drizzle-kit: + specifier: ^0.31.2 + version: 0.31.5 + drizzle-orm: + specifier: ^0.44.2 + version: 0.44.6(@cloudflare/workers-types@4.20251014.0)(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(better-sqlite3@11.10.0)(bun-types@1.3.0(@types/react@19.2.2))(kysely@0.28.8) get-port: specifier: ^7.1.0 version: 7.1.0 @@ -2341,6 +2368,9 @@ importers: vbare: specifier: ^0.0.4 version: 0.0.4 + wa-sqlite: + specifier: ^1.0.0 + version: 1.0.0 zod: specifier: ^4.1.0 version: 4.1.13 @@ -2357,6 +2387,9 @@ importers: '@hono/node-ws': specifier: ^1.1.1 version: 1.2.0(@hono/node-server@1.19.1(hono@4.9.8))(hono@4.9.8) + '@types/better-sqlite3': + specifier: ^7.6.13 + version: 7.6.13 '@types/invariant': specifier: ^2 version: 2.2.37 @@ -2369,6 +2402,9 @@ importers: '@vitest/ui': specifier: 3.1.1 version: 3.1.1(vitest@3.2.4) + cli-table3: + specifier: ^0.6.5 + version: 0.6.5 commander: specifier: ^12.1.0 version: 12.1.0 @@ -3637,6 +3673,10 @@ packages: '@coinbase/wallet-sdk@4.3.0': resolution: {integrity: sha512-T3+SNmiCw4HzDm4we9wCHCxlP0pqCiwKe4sOwPH3YAK2KSKjxPRydKu6UQJrdONFVLG7ujXvbd/6ZqmvJb8rkw==} + '@colors/colors@1.5.0': + resolution: {integrity: sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==} + engines: {node: '>=0.1.90'} + '@corex/deepmerge@4.0.43': resolution: {integrity: sha512-N8uEMrMPL0cu/bdboEWpQYb/0i2K5Qn8eCsxzOmxSggJbbQte7ljMRoXm917AbntqTGOzdTu+vP3KOOzoC70HQ==} @@ -8611,6 +8651,10 @@ packages: resolution: {integrity: sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==} engines: {node: '>=6'} + cli-table3@0.6.5: + resolution: {integrity: sha512-+W/5efTR7y5HRD7gACw9yQjqMVvEMLBHmboM/kPWam+H+Hmyrgjh6YncVKK122YZkXrLudzTuAukUw9FnMf7IQ==} + engines: {node: 10.* || >= 12.*} + cli-width@4.1.0: resolution: {integrity: sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ==} engines: {node: '>= 12'} @@ -14325,6 +14369,9 @@ packages: w3c-keyname@2.2.8: resolution: {integrity: sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ==} + wa-sqlite@1.0.0: + resolution: {integrity: sha512-Kyybo5/BaJp76z7gDWGk2J6Hthl4NIPsE+swgraEjy3IY6r5zIR02wAs1OJH4XtJp1y3puj3Onp5eMGS0z7nUA==} + walker@1.0.8: resolution: {integrity: sha512-ts/8E8l5b7kY0vlWLewOkDXMmPdLcVV4GmOQLyxuSswIJsweeFZtAsMF7k1Nszz+TYBQrlYRmzOnr398y1JemQ==} @@ -15880,6 +15927,9 @@ snapshots: eventemitter3: 5.0.1 preact: 10.27.2 + '@colors/colors@1.5.0': + optional: true + '@corex/deepmerge@4.0.43': {} '@cspotcode/source-map-support@0.8.1': @@ -16975,10 +17025,23 @@ snapshots: '@hey-api/client-fetch@0.5.7': {} + '@hono/node-server@1.19.1(hono@4.11.3)': + dependencies: + hono: 4.11.3 + '@hono/node-server@1.19.1(hono@4.9.8)': dependencies: hono: 4.9.8 + '@hono/node-ws@1.2.0(@hono/node-server@1.19.1(hono@4.11.3))(hono@4.11.3)': + dependencies: + '@hono/node-server': 1.19.1(hono@4.11.3) + hono: 4.11.3 + ws: 8.18.3 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + '@hono/node-ws@1.2.0(@hono/node-server@1.19.1(hono@4.9.8))(hono@4.9.8)': dependencies: '@hono/node-server': 1.19.1(hono@4.9.8) @@ -21496,6 +21559,12 @@ snapshots: cli-spinners@2.9.2: {} + cli-table3@0.6.5: + dependencies: + string-width: 4.2.3 + optionalDependencies: + '@colors/colors': 1.5.0 + cli-width@4.1.0: {} client-only@0.0.1: {} @@ -29082,6 +29151,8 @@ snapshots: w3c-keyname@2.2.8: {} + wa-sqlite@1.0.0: {} + walker@1.0.8: dependencies: makeerror: 1.0.12 diff --git a/rivetkit-typescript/packages/db/README.md b/rivetkit-typescript/packages/db/README.md deleted file mode 100644 index 07b6ff3d3a..0000000000 --- a/rivetkit-typescript/packages/db/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# RivetKit Database - -_Lightweight Libraries for Backends_ - -[Learn More ā†’](https://github.com/rivet-dev/rivetkit) - -[Discord](https://rivet.dev/discord) ā€” [Documentation](https://rivetkit.org) ā€” [Issues](https://github.com/rivet-dev/rivetkit/issues) - -## License - -Apache 2.0 \ No newline at end of file diff --git a/rivetkit-typescript/packages/db/src/config.ts b/rivetkit-typescript/packages/db/src/config.ts deleted file mode 100644 index 03dac5f13e..0000000000 --- a/rivetkit-typescript/packages/db/src/config.ts +++ /dev/null @@ -1,30 +0,0 @@ -export type AnyDatabaseProvider = DatabaseProvider | undefined; - -export type DatabaseProvider = { - /** - * Creates a new database client for the actor. - * The result is passed to the actor context as `c.db`. - * @experimental - */ - createClient: (ctx: { - getDatabase: () => Promise; - }) => Promise; - /** - * Runs before the actor has started. - * Use this to run migrations or other setup tasks. - * @experimental - */ - onMigrate: (client: DB) => void | Promise; -}; - -type ExecuteFunction = ( - query: string, - ...args: unknown[] -) => Promise; - -export type RawAccess = { - /** - * Executes a raw SQL query. - */ - execute: ExecuteFunction; -}; diff --git a/rivetkit-typescript/packages/db/src/drizzle/mod.ts b/rivetkit-typescript/packages/db/src/drizzle/mod.ts deleted file mode 100644 index 4a70e31a9f..0000000000 --- a/rivetkit-typescript/packages/db/src/drizzle/mod.ts +++ /dev/null @@ -1,100 +0,0 @@ -import Database from "better-sqlite3"; -import { - type BetterSQLite3Database, - drizzle as sqliteDrizzle, -} from "drizzle-orm/better-sqlite3"; -import { drizzle as durableDrizzle } from "drizzle-orm/durable-sqlite"; -import { migrate as durableMigrate } from "drizzle-orm/durable-sqlite/migrator"; -import type { DatabaseProvider, RawAccess } from "@/config"; - -export * from "drizzle-orm/sqlite-core"; - -import { type Config, defineConfig as originalDefineConfig } from "drizzle-kit"; -import type { SQLiteShim } from "@/utils"; - -export function defineConfig( - config: Partial, -): Config { - // This is a workaround to avoid the "drizzle-kit" import issue in the examples. - // It allows us to use the same defineConfig function in both the main package and the examples. - return originalDefineConfig({ - dialect: "sqlite", - driver: "durable-sqlite", - ...config, - }); -} - -interface DatabaseFactoryConfig< - TSchema extends Record = Record, -> { - /** - * The database schema. - */ - schema?: TSchema; - migrations?: any; -} - -export function db< - TSchema extends Record = Record, ->( - config?: DatabaseFactoryConfig, -): DatabaseProvider & RawAccess> { - return { - createClient: async (ctx) => { - // Create a database connection using the provided context - if (!ctx.getDatabase) { - throw new Error( - "createDatabase method is not available in context.", - ); - } - - const conn = await ctx.getDatabase(); - - if (!conn) { - throw new Error( - "Cannot create database connection, or database feature is not enabled.", - ); - } - - if (isSQLiteShim(conn)) { - // If the connection is already an object with exec method, return it - // i.e. in serverless environments (Cloudflare Workers) - const client = durableDrizzle( - conn, - config, - ); - return Object.assign(client, { - // client.$client.exec is the underlying SQLite client - execute: async (query, ...args) => - client.$client.exec(query, ...args), - } satisfies RawAccess); - } - - // Create a database client using the connection - const client = sqliteDrizzle({ - client: new Database(conn as string), - ...config, - }); - - return Object.assign(client, { - execute: async (query, ...args) => - client.$client.prepare(query).all(...args), - } satisfies RawAccess); - }, - onMigrate: async (client) => { - // Run migrations if provided in the config - if (config?.migrations) { - await durableMigrate(client, config?.migrations); - } - }, - }; -} - -function isSQLiteShim(conn: unknown): conn is SQLiteShim { - return ( - typeof conn === "object" && - conn !== null && - "exec" in conn && - typeof (conn as any).exec === "function" - ); -} diff --git a/rivetkit-typescript/packages/db/src/mod.ts b/rivetkit-typescript/packages/db/src/mod.ts deleted file mode 100644 index 1bcd0d4f2a..0000000000 --- a/rivetkit-typescript/packages/db/src/mod.ts +++ /dev/null @@ -1,53 +0,0 @@ -import SQLite from "better-sqlite3"; -import type { DatabaseProvider, RawAccess } from "./config"; -import { isSQLiteShim, type SQLiteShim } from "./utils"; - -interface DatabaseFactoryConfig { - onMigrate?: (db: RawAccess) => Promise | void; -} - -export function db({ - onMigrate, -}: DatabaseFactoryConfig = {}): DatabaseProvider { - return { - createClient: async (ctx) => { - // Create a database connection using the provided context - if (!ctx.getDatabase) { - throw new Error( - "createDatabase method is not available in context.", - ); - } - - const conn = await ctx.getDatabase(); - - if (!conn) { - throw new Error( - "Cannot create database connection, or database feature is not enabled.", - ); - } - - if (isSQLiteShim(conn)) { - // If the connection is already an object with exec method, return it - // i.e. in serverless environments (Cloudflare Workers) - return Object.assign({}, conn, { - execute: async (query, ...args) => { - return conn.exec(query, ...args); - }, - } satisfies RawAccess) as SQLiteShim & RawAccess; - } - - const client = new SQLite(conn as string); - return Object.assign({}, client, { - execute: async (query, ...args) => { - return client.prepare(query).all(...args); - }, - } satisfies RawAccess) as RawAccess; - }, - onMigrate: async (client) => { - // Run migrations if provided in the config - if (onMigrate) { - await onMigrate(client); - } - }, - }; -} diff --git a/rivetkit-typescript/packages/db/src/utils.ts b/rivetkit-typescript/packages/db/src/utils.ts deleted file mode 100644 index 82daaf7591..0000000000 --- a/rivetkit-typescript/packages/db/src/utils.ts +++ /dev/null @@ -1,17 +0,0 @@ -/** - * On serverless environments, we use a shim, as not all methods are available. - * This is a minimal shim that only includes the `exec` method, which is used for - * running raw SQL commands. - */ -export type SQLiteShim = { - exec: (query: string, ...args: unknown[]) => unknown[]; -}; - -export function isSQLiteShim(conn: unknown): conn is SQLiteShim & T { - return ( - typeof conn === "object" && - conn !== null && - "exec" in conn && - typeof (conn as SQLiteShim).exec === "function" - ); -} diff --git a/rivetkit-typescript/packages/db/tsup.config.ts b/rivetkit-typescript/packages/db/tsup.config.ts deleted file mode 100644 index b5ef019d94..0000000000 --- a/rivetkit-typescript/packages/db/tsup.config.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { defineConfig } from "tsup"; -import defaultConfig from "../../../tsup.base.ts"; - -export default defineConfig({ - ...defaultConfig, -}); diff --git a/rivetkit-typescript/packages/db/turbo.json b/rivetkit-typescript/packages/db/turbo.json deleted file mode 100644 index 29d4cb2625..0000000000 --- a/rivetkit-typescript/packages/db/turbo.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "$schema": "https://turbo.build/schema.json", - "extends": ["//"] -} diff --git a/rivetkit-typescript/packages/rivetkit/fixtures/driver-test-suite/actor-db-raw.ts b/rivetkit-typescript/packages/rivetkit/fixtures/driver-test-suite/actor-db-raw.ts new file mode 100644 index 0000000000..eef687cccd --- /dev/null +++ b/rivetkit-typescript/packages/rivetkit/fixtures/driver-test-suite/actor-db-raw.ts @@ -0,0 +1,82 @@ +import { actor } from "rivetkit"; +import { db } from "rivetkit/db"; + +export const dbActorRaw = actor({ + db: db({ + onMigrate: async (db) => { + await db.execute(` + CREATE TABLE IF NOT EXISTS test_data ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + value TEXT NOT NULL, + created_at INTEGER NOT NULL + ) + `); + }, + }), + actions: { + insertValue: async (c, value: string) => { + await c.db.execute( + `INSERT INTO test_data (value, created_at) VALUES ('${value}', ${Date.now()})`, + ); + return { success: true }; + }, + getValues: async (c) => { + const results = (await c.db.execute( + `SELECT * FROM test_data ORDER BY id`, + )) as Array<{ + id: number; + value: string; + created_at: number; + }>; + return results; + }, + getCount: async (c) => { + const results = (await c.db.execute( + `SELECT COUNT(*) as count FROM test_data`, + )) as Array<{ count: number }>; + return results[0].count; + }, + clearData: async (c) => { + await c.db.execute(`DELETE FROM test_data`); + }, + // Bulk operations for benchmarking (loop inside actor) + bulkInsert: async (c, count: number) => { + const start = performance.now(); + await c.db.execute("BEGIN TRANSACTION"); + for (let i = 0; i < count; i++) { + await c.db.execute( + `INSERT INTO test_data (value, created_at) VALUES ('User ${i}', ${Date.now()})`, + ); + } + await c.db.execute("COMMIT"); + const elapsed = performance.now() - start; + return { count, elapsed }; + }, + bulkGet: async (c, count: number) => { + const start = performance.now(); + for (let i = 0; i < count; i++) { + await c.db.execute(`SELECT COUNT(*) as count FROM test_data`); + } + const elapsed = performance.now() - start; + return { count, elapsed }; + }, + updateValue: async (c, id: number, value: string) => { + await c.db.execute( + `UPDATE test_data SET value = '${value}' WHERE id = ${id}`, + ); + return { success: true }; + }, + bulkUpdate: async (c, count: number) => { + const start = performance.now(); + await c.db.execute("BEGIN TRANSACTION"); + for (let i = 1; i <= count; i++) { + await c.db.execute( + `UPDATE test_data SET value = 'Updated ${i}' WHERE id = ${i}`, + ); + } + await c.db.execute("COMMIT"); + const elapsed = performance.now() - start; + return { count, elapsed }; + }, + }, +}); diff --git a/rivetkit-typescript/packages/rivetkit/fixtures/driver-test-suite/registry.ts b/rivetkit-typescript/packages/rivetkit/fixtures/driver-test-suite/registry.ts index d7136b39f2..3973d5d7f6 100644 --- a/rivetkit-typescript/packages/rivetkit/fixtures/driver-test-suite/registry.ts +++ b/rivetkit-typescript/packages/rivetkit/fixtures/driver-test-suite/registry.ts @@ -12,6 +12,7 @@ import { promiseActor, syncActionActor, } from "./action-types"; +import { dbActorRaw } from "./actor-db-raw"; import { onStateChangeActor } from "./actor-onstatechange"; import { counterWithParams } from "./conn-params"; import { connStateActor } from "./conn-state"; @@ -42,6 +43,7 @@ import { sleepWithRawHttp, sleepWithRawWebSocket, } from "./sleep"; +import { statelessActor } from "./stateless"; import { driverCtxActor, dynamicVarActor, @@ -117,5 +119,9 @@ export const registry = setup({ // From large-payloads.ts largePayloadActor, largePayloadConnActor, + // From actor-db-raw.ts + dbActorRaw, + // From stateless.ts + statelessActor, }, }); diff --git a/rivetkit-typescript/packages/rivetkit/fixtures/driver-test-suite/stateless.ts b/rivetkit-typescript/packages/rivetkit/fixtures/driver-test-suite/stateless.ts new file mode 100644 index 0000000000..0f2b8bbd35 --- /dev/null +++ b/rivetkit-typescript/packages/rivetkit/fixtures/driver-test-suite/stateless.ts @@ -0,0 +1,30 @@ +import { actor } from "rivetkit"; + +// Actor without state - only has actions +export const statelessActor = actor({ + actions: { + ping: () => "pong", + echo: (c, message: string) => message, + getActorId: (c) => c.actorId, + // Try to access state - should throw StateNotEnabled + tryGetState: (c) => { + try { + // State is typed as undefined, but we want to test runtime behavior + const state = c.state; + return { success: true, state }; + } catch (error) { + return { success: false, error: (error as Error).message }; + } + }, + // Try to access db - should throw DatabaseNotEnabled + tryGetDb: (c) => { + try { + // DB is typed as undefined, but we want to test runtime behavior + const db = c.db; + return { success: true, db }; + } catch (error) { + return { success: false, error: (error as Error).message }; + } + }, + }, +}); diff --git a/rivetkit-typescript/packages/rivetkit/package.json b/rivetkit-typescript/packages/rivetkit/package.json index da6100b80f..45682d7101 100644 --- a/rivetkit-typescript/packages/rivetkit/package.json +++ b/rivetkit-typescript/packages/rivetkit/package.json @@ -142,6 +142,26 @@ "types": "./dist/tsup/inspector/mod.d.cts", "default": "./dist/tsup/inspector/mod.cjs" } + }, + "./db": { + "import": { + "types": "./dist/tsup/db/mod.d.ts", + "default": "./dist/tsup/db/mod.js" + }, + "require": { + "types": "./dist/tsup/db/mod.d.cts", + "default": "./dist/tsup/db/mod.cjs" + } + }, + "./db/drizzle": { + "import": { + "types": "./dist/tsup/db/drizzle/mod.d.ts", + "default": "./dist/tsup/db/drizzle/mod.js" + }, + "require": { + "types": "./dist/tsup/db/drizzle/mod.d.cts", + "default": "./dist/tsup/db/drizzle/mod.cjs" + } } }, "engines": { @@ -152,7 +172,7 @@ "./dist/tsup/chunk-*.cjs" ], "scripts": { - "build": "tsup src/mod.ts src/client/mod.ts src/common/log.ts src/common/websocket.ts src/actor/errors.ts src/topologies/coordinate/mod.ts src/topologies/partition/mod.ts src/utils.ts src/driver-helpers/mod.ts src/driver-test-suite/mod.ts src/test/mod.ts src/inspector/mod.ts", + "build": "tsup src/mod.ts src/client/mod.ts src/common/log.ts src/common/websocket.ts src/actor/errors.ts src/topologies/coordinate/mod.ts src/topologies/partition/mod.ts src/utils.ts src/driver-helpers/mod.ts src/driver-test-suite/mod.ts src/test/mod.ts src/inspector/mod.ts src/db/mod.ts src/db/drizzle/mod.ts", "build:schema": "./scripts/compile-bare.ts compile schemas/client-protocol/v1.bare -o dist/schemas/client-protocol/v1.ts && ./scripts/compile-bare.ts compile schemas/client-protocol/v2.bare -o dist/schemas/client-protocol/v2.ts && ./scripts/compile-bare.ts compile schemas/file-system-driver/v1.bare -o dist/schemas/file-system-driver/v1.ts && ./scripts/compile-bare.ts compile schemas/file-system-driver/v2.bare -o dist/schemas/file-system-driver/v2.ts && ./scripts/compile-bare.ts compile schemas/file-system-driver/v3.bare -o dist/schemas/file-system-driver/v3.ts && ./scripts/compile-bare.ts compile schemas/actor-persist/v1.bare -o dist/schemas/actor-persist/v1.ts && ./scripts/compile-bare.ts compile schemas/actor-persist/v2.bare -o dist/schemas/actor-persist/v2.ts && ./scripts/compile-bare.ts compile schemas/actor-persist/v3.bare -o dist/schemas/actor-persist/v3.ts", "check-types": "tsc --noEmit", "lint": "biome check .", @@ -171,27 +191,30 @@ "@hono/zod-openapi": "^1.1.5", "@rivetkit/engine-runner": "workspace:*", "@rivetkit/fast-json-patch": "^3.1.2", + "@rivetkit/on-change": "^6.0.2-rc.1", "cbor-x": "^1.6.0", "get-port": "^7.1.0", "hono": "^4.7.0", "invariant": "^2.2.4", "nanoevents": "^9.1.0", - "@rivetkit/on-change": "^6.0.2-rc.1", "p-retry": "^6.2.1", "pino": "^9.5.0", "uuid": "^12.0.0", - "zod": "^4.1.0", - "vbare": "^0.0.4" + "vbare": "^0.0.4", + "wa-sqlite": "^1.0.0", + "zod": "^4.1.0" }, "devDependencies": { "@bare-ts/tools": "^0.13.0", "@biomejs/biome": "^2.2.3", "@hono/node-server": "^1.18.2", "@hono/node-ws": "^1.1.1", + "@types/better-sqlite3": "^7.6.13", "@types/invariant": "^2", "@types/node": "^22.13.1", "@types/ws": "^8", "@vitest/ui": "3.1.1", + "cli-table3": "^0.6.5", "commander": "^12.1.0", "eventsource": "^4.0.0", "tsup": "^8.4.0", @@ -205,6 +228,9 @@ "peerDependencies": { "@hono/node-server": "^1.14.0", "@hono/node-ws": "^1.1.1", + "better-sqlite3": "^11.0.0", + "drizzle-kit": "^0.31.2", + "drizzle-orm": "^0.44.2", "eventsource": "^4.0.0", "ws": "^8.0.0" }, @@ -215,6 +241,15 @@ "@hono/node-ws": { "optional": true }, + "better-sqlite3": { + "optional": true + }, + "drizzle-kit": { + "optional": true + }, + "drizzle-orm": { + "optional": true + }, "eventsource": { "optional": true }, diff --git a/rivetkit-typescript/packages/rivetkit/scripts/bench-sqlite.ts b/rivetkit-typescript/packages/rivetkit/scripts/bench-sqlite.ts new file mode 100755 index 0000000000..c70c75c6f9 --- /dev/null +++ b/rivetkit-typescript/packages/rivetkit/scripts/bench-sqlite.ts @@ -0,0 +1,231 @@ +#!/usr/bin/env -S tsx + +/** + * SQLite Benchmark Script + * + * Compares batch vs non-batch performance: + * 1. Filesystem + Native SQLite (better-sqlite3) + * 2. Filesystem + KV Filesystem (wa-sqlite with file-backed KV) + */ + +import Table from "cli-table3"; +import { createFileSystemDriver } from "@/drivers/file-system/mod"; +import { registry } from "../fixtures/driver-test-suite/registry"; + +interface TimingResult { + batch: number; + nonBatch: number; +} + +interface BenchmarkResult { + name: string; + insert: TimingResult; + select: TimingResult; + update: TimingResult; +} + +const ROW_COUNT = 100; +const QUERY_COUNT = 100; + +type Client = Awaited>["client"]; + +async function runBenchmark(client: Client, name: string): Promise { + const results: BenchmarkResult = { + name, + insert: { batch: 0, nonBatch: 0 }, + select: { batch: 0, nonBatch: 0 }, + update: { batch: 0, nonBatch: 0 }, + }; + + // --- INSERT --- + // Non-batch + { + const handle = await client.dbActorRaw.getOrCreate([`bench-nonbatch-${Date.now()}`]); + const start = performance.now(); + for (let i = 0; i < ROW_COUNT; i++) { + await handle.insertValue(`User ${i}`); + } + results.insert.nonBatch = performance.now() - start; + } + + // Batch + { + const handle = await client.dbActorRaw.getOrCreate([`bench-batch-${Date.now()}`]); + const start = performance.now(); + await handle.bulkInsert(ROW_COUNT); + results.insert.batch = performance.now() - start; + } + + // --- SELECT --- + const handle = await client.dbActorRaw.getOrCreate([`bench-select-${Date.now()}`]); + await handle.bulkInsert(ROW_COUNT); + + // Batch (single query) + { + const start = performance.now(); + await handle.getValues(); + results.select.batch = performance.now() - start; + } + + // Non-batch (100 queries) + { + const start = performance.now(); + for (let i = 0; i < QUERY_COUNT; i++) { + await handle.getCount(); + } + results.select.nonBatch = performance.now() - start; + } + + // --- UPDATE --- + // Non-batch + { + const start = performance.now(); + for (let i = 1; i <= QUERY_COUNT; i++) { + await handle.updateValue(i, `Updated ${i}`); + } + results.update.nonBatch = performance.now() - start; + } + + // Batch + { + const start = performance.now(); + await handle.bulkUpdate(QUERY_COUNT); + results.update.batch = performance.now() - start; + } + + return results; +} + +function ms(n: number): string { + return n === 0 ? "-" : `${n.toFixed(2)}ms`; +} + +function perOp(total: number, count: number): string { + return total === 0 ? "-" : `${(total / count).toFixed(3)}ms`; +} + +function speedup(nonBatch: number, batch: number): string { + if (nonBatch === 0 || batch === 0) return "-"; + return `${(nonBatch / batch).toFixed(1)}x`; +} + +function printResults(results: BenchmarkResult[]): void { + console.log(`\nBenchmark: ${ROW_COUNT} rows, ${QUERY_COUNT} queries\n`); + + // INSERT table + const insertTable = new Table({ + head: ["Driver", "Batch", "Per-Op", "Non-Batch", "Per-Op", "Speedup"], + }); + for (const r of results) { + insertTable.push([ + r.name, + ms(r.insert.batch), + perOp(r.insert.batch, ROW_COUNT), + ms(r.insert.nonBatch), + perOp(r.insert.nonBatch, ROW_COUNT), + speedup(r.insert.nonBatch, r.insert.batch), + ]); + } + console.log("INSERT"); + console.log(insertTable.toString()); + + // SELECT table + const selectTable = new Table({ + head: ["Driver", "Batch", "Non-Batch", "Per-Query", "Speedup"], + }); + for (const r of results) { + selectTable.push([ + r.name, + ms(r.select.batch), + ms(r.select.nonBatch), + perOp(r.select.nonBatch, QUERY_COUNT), + speedup(r.select.nonBatch, r.select.batch), + ]); + } + console.log("\nSELECT"); + console.log(selectTable.toString()); + + // UPDATE table + const updateTable = new Table({ + head: ["Driver", "Batch", "Per-Op", "Non-Batch", "Per-Op", "Speedup"], + }); + for (const r of results) { + updateTable.push([ + r.name, + ms(r.update.batch), + perOp(r.update.batch, QUERY_COUNT), + ms(r.update.nonBatch), + perOp(r.update.nonBatch, QUERY_COUNT), + speedup(r.update.nonBatch, r.update.batch), + ]); + } + console.log("\nUPDATE"); + console.log(updateTable.toString()); + + // Cross-driver comparison + const baseline = results[0]; + if (baseline && results.length > 1) { + const compTable = new Table({ + head: ["Driver", "Insert (batch)", "vs Baseline", "Select (batch)", "vs Baseline"], + }); + for (const r of results) { + compTable.push([ + r.name, + ms(r.insert.batch), + `${(r.insert.batch / baseline.insert.batch).toFixed(1)}x`, + ms(r.select.batch), + `${(r.select.batch / baseline.select.batch).toFixed(1)}x`, + ]); + } + console.log("\nCROSS-DRIVER (batch mode)"); + console.log(compTable.toString()); + } +} + +async function main(): Promise { + console.log("SQLite Benchmark\n"); + + const results: BenchmarkResult[] = []; + + // 1. Native SQLite + console.log("1. Native SQLite..."); + try { + const { client } = await registry.start({ + driver: createFileSystemDriver({ useNativeSqlite: true }), + defaultServerPort: 6430, + }); + results.push(await runBenchmark(client, "Native SQLite")); + console.log(" Done"); + } catch (err) { + console.log(` Skipped: ${err}`); + } + + // 2. KV Filesystem + console.log("2. KV Filesystem..."); + try { + const { client } = await registry.start({ + driver: createFileSystemDriver({ useNativeSqlite: false }), + defaultServerPort: 6431, + }); + results.push(await runBenchmark(client, "KV Filesystem")); + console.log(" Done"); + } catch (err) { + console.log(` Skipped: ${err}`); + } + + // 3. Engine (connects to running engine on 6420) + console.log("3. Engine (localhost:6420)..."); + try { + const { client } = await registry.start({ + endpoint: "http://localhost:6420", + }); + results.push(await runBenchmark(client, "Engine")); + console.log(" Done"); + } catch (err) { + console.log(` Skipped: ${err}`); + } + + printResults(results); +} + +main().catch(console.error); diff --git a/rivetkit-typescript/packages/rivetkit/src/actor/database.ts b/rivetkit-typescript/packages/rivetkit/src/actor/database.ts index a563475f0a..4280e70cf9 100644 --- a/rivetkit-typescript/packages/rivetkit/src/actor/database.ts +++ b/rivetkit-typescript/packages/rivetkit/src/actor/database.ts @@ -1,23 +1,13 @@ +import type { + AnyDatabaseProvider, + DatabaseProvider, + RawDatabaseClient, + DrizzleDatabaseClient, +} from "@/db/config"; + export type InferDatabaseClient = DBProvider extends DatabaseProvider ? Awaited> : never; -export type AnyDatabaseProvider = DatabaseProvider | undefined; - -export type DatabaseProvider any }> = { - /** - * Creates a new database client for the actor. - * The result is passed to the actor context as `c.db`. - * @experimental - */ - createClient: (ctx: { - getDatabase: () => Promise; - }) => Promise; - /** - * Runs before the actor has started. - * Use this to run migrations or other setup tasks. - * @experimental - */ - onMigrate: (client: DB) => void | Promise; -}; +export type { AnyDatabaseProvider, DatabaseProvider, RawDatabaseClient, DrizzleDatabaseClient }; diff --git a/rivetkit-typescript/packages/rivetkit/src/actor/driver.ts b/rivetkit-typescript/packages/rivetkit/src/actor/driver.ts index a36fe5f6b0..838ad2d4d6 100644 --- a/rivetkit-typescript/packages/rivetkit/src/actor/driver.ts +++ b/rivetkit-typescript/packages/rivetkit/src/actor/driver.ts @@ -3,7 +3,12 @@ import type { AnyClient } from "@/client/client"; import type { ManagerDriver } from "@/manager/driver"; import { type AnyConn } from "./conn/mod"; import type { AnyActorInstance } from "./instance/mod"; -import { RegistryConfig } from "@/registry/config"; +import type { RegistryConfig } from "@/registry/config"; +import type { + RawDatabaseClient, +} from "@/db/config"; +import type { SqliteVfs } from "@/db/vfs/mod"; +import { BaseSQLiteDatabase } from "drizzle-orm/sqlite-core"; export type ActorDriverBuilder = ( config: RegistryConfig, @@ -46,10 +51,30 @@ export interface ActorDriver { // Database /** + * Override the default raw database client for the actor. + * If not provided, rivetkit will construct a KV-backed SQLite client. * @experimental - * This is an experimental API that may change in the future. */ - getDatabase(actorId: string): Promise; + overrideRawDatabaseClient?( + actorId: string, + ): Promise; + + /** + * Override the default Drizzle database client for the actor. + * If not provided, rivetkit will construct a KV-backed Drizzle client. + * @experimental + */ + overrideDrizzleDatabaseClient?( + actorId: string, + ): Promise | undefined>; + + /** + * SQLite VFS instance for creating KV-backed databases. + * Each driver should create its own instance to avoid concurrency issues. + * If not provided, the db() provider will throw an error. + * @experimental + */ + sqliteVfs?: SqliteVfs; /** * Requests the actor to go to sleep. diff --git a/rivetkit-typescript/packages/rivetkit/src/actor/instance/kv.ts b/rivetkit-typescript/packages/rivetkit/src/actor/instance/kv.ts index d046f6bbab..cad0ca4dbf 100644 --- a/rivetkit-typescript/packages/rivetkit/src/actor/instance/kv.ts +++ b/rivetkit-typescript/packages/rivetkit/src/actor/instance/kv.ts @@ -2,6 +2,7 @@ export const KEYS = { PERSIST_DATA: Uint8Array.from([1]), CONN_PREFIX: Uint8Array.from([2]), // Prefix for connection keys INSPECTOR_TOKEN: Uint8Array.from([3]), // Inspector token key + SQLITE_PREFIX: Uint8Array.from([4]), // Prefix for SQLite VFS data (see @rivetkit/sqlite-vfs) }; // Helper to create a connection key diff --git a/rivetkit-typescript/packages/rivetkit/src/actor/instance/mod.ts b/rivetkit-typescript/packages/rivetkit/src/actor/instance/mod.ts index c40b7ffaa4..8f0331f2a3 100644 --- a/rivetkit-typescript/packages/rivetkit/src/actor/instance/mod.ts +++ b/rivetkit-typescript/packages/rivetkit/src/actor/instance/mod.ts @@ -946,6 +946,21 @@ export class ActorInstance { } async #callOnDestroy() { + // Clean up database first + if ("db" in this.#config && this.#config.db && this.#db) { + try { + this.#rLog.debug({ msg: "cleaning up database" }); + await this.#config.db.onDestroy?.(this.#db); + this.#rLog.debug({ msg: "database cleanup completed" }); + } catch (error) { + this.#rLog.error({ + msg: "error cleaning up database", + error: stringifyError(error), + }); + } + } + + // Then call user's onDestroy if (this.#config.onDestroy) { try { this.#rLog.debug({ msg: "calling onDestroy" }); @@ -972,13 +987,48 @@ export class ActorInstance { async #setupDatabase() { if ("db" in this.#config && this.#config.db) { - const client = await this.#config.db.createClient({ - getDatabase: () => this.driver.getDatabase(this.#actorId), - }); - this.#rLog.info({ msg: "database migration starting" }); - await this.#config.db.onMigrate?.(client); - this.#rLog.info({ msg: "database migration complete" }); - this.#db = client; + try { + const client = await this.#config.db.createClient({ + actorId: this.#actorId, + overrideRawDatabaseClient: this.driver.overrideRawDatabaseClient + ? () => this.driver.overrideRawDatabaseClient!(this.#actorId) + : undefined, + overrideDrizzleDatabaseClient: this.driver + .overrideDrizzleDatabaseClient + ? () => this.driver.overrideDrizzleDatabaseClient!(this.#actorId) + : undefined, + kv: { + batchPut: (entries) => + this.driver.kvBatchPut(this.#actorId, entries), + batchGet: (keys) => this.driver.kvBatchGet(this.#actorId, keys), + batchDelete: (keys) => + this.driver.kvBatchDelete(this.#actorId, keys), + }, + sqliteVfs: this.driver.sqliteVfs, + }); + this.#rLog.info({ msg: "database migration starting" }); + await this.#config.db.onMigrate?.(client); + this.#rLog.info({ msg: "database migration complete" }); + this.#db = client; + } catch (error) { + // Ensure error is properly formatted + if (error instanceof Error) { + this.#rLog.error({ + msg: "database setup failed", + error: stringifyError(error), + }); + throw error; + } + const wrappedError = new Error( + `Database setup failed: ${String(error)}`, + ); + this.#rLog.error({ + msg: "database setup failed with non-Error object", + error: String(error), + errorType: typeof error, + }); + throw wrappedError; + } } } diff --git a/rivetkit-typescript/packages/rivetkit/src/db/config.ts b/rivetkit-typescript/packages/rivetkit/src/db/config.ts new file mode 100644 index 0000000000..20fd37988f --- /dev/null +++ b/rivetkit-typescript/packages/rivetkit/src/db/config.ts @@ -0,0 +1,94 @@ +import type { SqliteVfs } from "./vfs/mod"; + +export type AnyDatabaseProvider = DatabaseProvider | undefined; + +/** + * Context provided to database providers for creating database clients + */ +export interface DatabaseProviderContext { + /** + * Actor ID + */ + actorId: string; + + /** + * Override the default raw database client (optional). + * If not provided, a KV-backed client will be constructed. + */ + overrideRawDatabaseClient?: () => Promise; + + /** + * Override the default Drizzle database client (optional). + * If not provided, a KV-backed client will be constructed. + */ + overrideDrizzleDatabaseClient?: () => Promise< + DrizzleDatabaseClient | undefined + >; + + /** + * KV operations for constructing KV-backed database clients + */ + kv: { + batchPut: (entries: [Uint8Array, Uint8Array][]) => Promise; + batchGet: (keys: Uint8Array[]) => Promise<(Uint8Array | null)[]>; + batchDelete: (keys: Uint8Array[]) => Promise; + }; + + /** + * SQLite VFS instance for creating KV-backed databases. + * Each driver creates its own instance to avoid concurrency issues. + */ + sqliteVfs?: SqliteVfs; +} + +export type DatabaseProvider = { + /** + * Creates a new database client for the actor. + * The result is passed to the actor context as `c.db`. + * @experimental + */ + createClient: (ctx: DatabaseProviderContext) => Promise; + /** + * Runs before the actor has started. + * Use this to run migrations or other setup tasks. + * @experimental + */ + onMigrate: (client: DB) => void | Promise; + /** + * Runs when the actor is being destroyed. + * Use this to clean up database connections and release resources. + * @experimental + */ + onDestroy?: (client: DB) => void | Promise; +}; + +/** + * Raw database client with basic exec method + */ +export interface RawDatabaseClient { + exec: (query: string, ...args: unknown[]) => Promise | unknown[]; +} + +/** + * Drizzle database client interface (will be extended by drizzle-orm types) + */ +export interface DrizzleDatabaseClient { + // This will be extended by BaseSQLiteDatabase from drizzle-orm + // For now, just a marker interface +} + +type ExecuteFunction = ( + query: string, + ...args: unknown[] +) => Promise; + +export type RawAccess = { + /** + * Executes a raw SQL query. + */ + execute: ExecuteFunction; + /** + * Closes the database connection and releases resources. + */ + close: () => Promise; +}; diff --git a/rivetkit-typescript/packages/rivetkit/src/db/drizzle/mod.ts b/rivetkit-typescript/packages/rivetkit/src/db/drizzle/mod.ts new file mode 100644 index 0000000000..bfce27de4d --- /dev/null +++ b/rivetkit-typescript/packages/rivetkit/src/db/drizzle/mod.ts @@ -0,0 +1,121 @@ +import { + type BetterSQLite3Database, + drizzle as sqliteDrizzle, +} from "drizzle-orm/better-sqlite3"; +import { drizzle as durableDrizzle } from "drizzle-orm/durable-sqlite"; +import { migrate as durableMigrate } from "drizzle-orm/durable-sqlite/migrator"; +import type { KvVfsOptions } from "../vfs/mod"; +import type { DatabaseProvider, RawAccess } from "../config"; + +export * from "drizzle-orm/sqlite-core"; + +import { type Config, defineConfig as originalDefineConfig } from "drizzle-kit"; + +export function defineConfig( + config: Partial, +): Config { + return originalDefineConfig({ + dialect: "sqlite", + driver: "durable-sqlite", + ...config, + }); +} + +interface DatabaseFactoryConfig< + TSchema extends Record = Record, +> { + schema?: TSchema; + migrations?: any; +} + +/** + * Create a KV store wrapper that uses the actor driver's KV operations + */ +function createActorKvStore(kv: { + batchPut: (entries: [Uint8Array, Uint8Array][]) => Promise; + batchGet: (keys: Uint8Array[]) => Promise<(Uint8Array | null)[]>; + batchDelete: (keys: Uint8Array[]) => Promise; +}): KvVfsOptions { + return { + get: async (key: Uint8Array) => { + const results = await kv.batchGet([key]); + return results[0]; + }, + getBatch: async (keys: Uint8Array[]) => { + return await kv.batchGet(keys); + }, + put: async (key: Uint8Array, value: Uint8Array) => { + await kv.batchPut([[key, value]]); + }, + putBatch: async (entries: [Uint8Array, Uint8Array][]) => { + await kv.batchPut(entries); + }, + deleteBatch: async (keys: Uint8Array[]) => { + await kv.batchDelete(keys); + }, + }; +} + +export function db< + TSchema extends Record = Record, +>( + config?: DatabaseFactoryConfig, +): DatabaseProvider & RawAccess> { + return { + createClient: async (ctx) => { + // Check if override is provided + const override = ctx.overrideDrizzleDatabaseClient + ? await ctx.overrideDrizzleDatabaseClient() + : undefined; + + if (override) { + // Use the override (wrap with Drizzle) + const client = durableDrizzle(override, config); + + return Object.assign(client, { + execute: async (query, ...args) => { + return client.$client.exec(query, ...args); + }, + close: async () => { + // Override clients don't need cleanup + }, + } satisfies RawAccess); + } + + // Construct KV-backed client using actor driver's KV operations + if (!ctx.sqliteVfs) { + throw new Error("SqliteVfs instance not provided in context. The driver must provide a sqliteVfs instance."); + } + + const kvStore = createActorKvStore(ctx.kv); + const db = await ctx.sqliteVfs.open(ctx.actorId, kvStore); + + // Wrap the KV-backed client with Drizzle + const rawClient = { + exec: async (query: string, ...args: unknown[]) => { + await db.exec(query); + return []; + }, + }; + + const client = durableDrizzle(rawClient, config); + + return Object.assign(client, { + execute: async (query, ...args) => { + return client.$client.exec(query, ...args); + }, + close: async () => { + await db.close(); + }, + } satisfies RawAccess); + }, + onMigrate: async (client) => { + if (config?.migrations) { + await durableMigrate(client, config?.migrations); + } + }, + onDestroy: async (client) => { + await client.close(); + }, + }; +} diff --git a/rivetkit-typescript/packages/rivetkit/src/db/mod.ts b/rivetkit-typescript/packages/rivetkit/src/db/mod.ts new file mode 100644 index 0000000000..775992e6df --- /dev/null +++ b/rivetkit-typescript/packages/rivetkit/src/db/mod.ts @@ -0,0 +1,98 @@ +import type { KvVfsOptions } from "./vfs/mod"; +import type { DatabaseProvider, RawAccess } from "./config"; + +interface DatabaseFactoryConfig { + onMigrate?: (db: RawAccess) => Promise | void; +} + +/** + * Create a KV store wrapper that uses the actor driver's KV operations + */ +function createActorKvStore(kv: { + batchPut: (entries: [Uint8Array, Uint8Array][]) => Promise; + batchGet: (keys: Uint8Array[]) => Promise<(Uint8Array | null)[]>; + batchDelete: (keys: Uint8Array[]) => Promise; +}): KvVfsOptions { + return { + get: async (key: Uint8Array) => { + const results = await kv.batchGet([key]); + return results[0]; + }, + getBatch: async (keys: Uint8Array[]) => { + return await kv.batchGet(keys); + }, + put: async (key: Uint8Array, value: Uint8Array) => { + await kv.batchPut([[key, value]]); + }, + putBatch: async (entries: [Uint8Array, Uint8Array][]) => { + await kv.batchPut(entries); + }, + deleteBatch: async (keys: Uint8Array[]) => { + await kv.batchDelete(keys); + }, + }; +} + +export function db({ + onMigrate, +}: DatabaseFactoryConfig = {}): DatabaseProvider { + return { + createClient: async (ctx) => { + // Check if override is provided + const override = ctx.overrideRawDatabaseClient + ? await ctx.overrideRawDatabaseClient() + : undefined; + + if (override) { + // Use the override + return { + execute: async (query, ...args) => { + return override.exec(query, ...args); + }, + close: async () => { + // Override clients don't need cleanup + }, + } satisfies RawAccess; + } + + // Construct KV-backed client using actor driver's KV operations + if (!ctx.sqliteVfs) { + throw new Error("SqliteVfs instance not provided in context. The driver must provide a sqliteVfs instance."); + } + + const kvStore = createActorKvStore(ctx.kv); + const db = await ctx.sqliteVfs.open(ctx.actorId, kvStore); + + return { + execute: async (query, ...args) => { + const results: Record[] = []; + let columnNames: string[] | null = null; + await db.exec(query, (row: unknown[], columns: string[]) => { + // Capture column names on first row + if (!columnNames) { + columnNames = columns; + } + // Convert array row to object + const rowObj: Record = {}; + for (let i = 0; i < row.length; i++) { + rowObj[columnNames[i]] = row[i]; + } + results.push(rowObj); + }); + return results; + }, + close: async () => { + await db.close(); + }, + } satisfies RawAccess; + }, + onMigrate: async (client) => { + if (onMigrate) { + await onMigrate(client); + } + }, + onDestroy: async (client) => { + await client.close(); + }, + }; +} diff --git a/rivetkit-typescript/packages/rivetkit/src/db/vfs/kv.ts b/rivetkit-typescript/packages/rivetkit/src/db/vfs/kv.ts new file mode 100644 index 0000000000..6b854c6381 --- /dev/null +++ b/rivetkit-typescript/packages/rivetkit/src/db/vfs/kv.ts @@ -0,0 +1,50 @@ +/** + * Key management for SQLite VFS storage + * + * This module contains constants and utilities for building keys used in the + * key-value store for SQLite file storage. + */ + +/** Size of each chunk stored in KV (4KB) */ +export const CHUNK_SIZE = 4096; + +/** Top-level SQLite prefix (must match SQLITE_PREFIX in actor KV system) */ +export const SQLITE_PREFIX = 4; + +/** Key prefix byte for file metadata (after SQLITE_PREFIX) */ +export const META_PREFIX = 0; + +/** Key prefix byte for file chunks (after SQLITE_PREFIX) */ +export const CHUNK_PREFIX = 1; + +/** + * Gets the key for file metadata + * Format: [SQLITE_PREFIX (1 byte), META_PREFIX (1 byte), filename (UTF-8 encoded)] + */ +export function getMetaKey(fileName: string): Uint8Array { + const encoder = new TextEncoder(); + const fileNameBytes = encoder.encode(fileName); + const key = new Uint8Array(2 + fileNameBytes.length); + key[0] = SQLITE_PREFIX; + key[1] = META_PREFIX; + key.set(fileNameBytes, 2); + return key; +} + +/** + * Gets the key for a file chunk + * Format: [SQLITE_PREFIX (1 byte), CHUNK_PREFIX (1 byte), filename (UTF-8), null separator (1 byte), chunk index (4 bytes, big-endian)] + */ +export function getChunkKey(fileName: string, chunkIndex: number): Uint8Array { + const encoder = new TextEncoder(); + const fileNameBytes = encoder.encode(fileName); + const key = new Uint8Array(2 + fileNameBytes.length + 1 + 4); + key[0] = SQLITE_PREFIX; + key[1] = CHUNK_PREFIX; + key.set(fileNameBytes, 2); + key[2 + fileNameBytes.length] = 0; // null separator + // Encode chunk index as 32-bit unsigned integer (big-endian for proper ordering) + const view = new DataView(key.buffer); + view.setUint32(2 + fileNameBytes.length + 1, chunkIndex, false); + return key; +} diff --git a/rivetkit-typescript/packages/rivetkit/src/db/vfs/mod.ts b/rivetkit-typescript/packages/rivetkit/src/db/vfs/mod.ts new file mode 100644 index 0000000000..b68ecdbfcf --- /dev/null +++ b/rivetkit-typescript/packages/rivetkit/src/db/vfs/mod.ts @@ -0,0 +1,711 @@ +/** + * SQLite raw database with KV storage backend + * + * This module provides a SQLite API that uses a KV-backed VFS + * for storage. Each SqliteVfs instance is independent and can be + * used concurrently with other instances. + */ + +// Note: wa-sqlite VFS.Base type definitions have incorrect types for xRead/xWrite +// The actual runtime uses Uint8Array, not the {size, value} object shown in types +import * as VFS from "wa-sqlite/src/VFS.js"; + +// VFS debug logging - set VFS_DEBUG=1 to enable +const VFS_DEBUG = process.env.VFS_DEBUG === "1"; +function vfsLog(op: string, details: Record) { + if (VFS_DEBUG) { + console.log(`[VFS] ${op}`, JSON.stringify(details)); + } +} +import SQLiteESMFactory from "wa-sqlite/dist/wa-sqlite-async.mjs"; +import { Factory } from "wa-sqlite"; +import { readFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { CHUNK_SIZE, getMetaKey, getChunkKey } from "./kv"; +import { + FILE_META_VERSIONED, + CURRENT_VERSION, +} from "./schemas/file-meta/versioned"; +import type { FileMeta } from "./schemas/file-meta/mod"; + +/** + * Options for creating the KV VFS + * Operations are scoped to a specific actor's KV store + */ +export interface KvVfsOptions { + /** Get a single value by key. Returns null if missing. */ + get: (key: Uint8Array) => Promise; + /** Get multiple values by keys. Returns null for missing keys. */ + getBatch: (keys: Uint8Array[]) => Promise<(Uint8Array | null)[]>; + /** Put a single key-value pair */ + put: (key: Uint8Array, value: Uint8Array) => Promise; + /** Put multiple key-value pairs */ + putBatch: (entries: [Uint8Array, Uint8Array][]) => Promise; + /** Delete multiple keys */ + deleteBatch: (keys: Uint8Array[]) => Promise; +} + +/** + * Represents an open file + */ +interface OpenFile { + /** File path */ + path: string; + /** File size in bytes */ + size: number; + /** Open flags */ + flags: number; + /** KV options for this file */ + options: KvVfsOptions; +} + +/** + * Encodes file metadata to a Uint8Array using BARE schema + */ +function encodeFileMeta(size: number): Uint8Array { + const meta: FileMeta = { size: BigInt(size) }; + return FILE_META_VERSIONED.serializeWithEmbeddedVersion( + meta, + CURRENT_VERSION, + ); +} + +/** + * Decodes file metadata from a Uint8Array using BARE schema + */ +function decodeFileMeta(data: Uint8Array): number { + const meta = FILE_META_VERSIONED.deserializeWithEmbeddedVersion(data); + return Number(meta.size); +} + +/** + * SQLite API interface (subset needed for VFS registration) + * This is part of wa-sqlite but not exported in TypeScript types + */ +interface SQLite3Api { + vfs_register: (vfs: unknown, makeDefault?: boolean) => number; + open_v2: ( + filename: string, + flags: number, + vfsName?: string, + ) => Promise; + close: (db: number) => Promise; + exec: ( + db: number, + sql: string, + callback?: (row: unknown[], columns: string[]) => void, + ) => Promise; + SQLITE_OPEN_READWRITE: number; + SQLITE_OPEN_CREATE: number; +} + +/** + * Simple async mutex for serializing database operations + * wa-sqlite is not safe for concurrent open_v2 calls + */ +class AsyncMutex { + #locked = false; + #waiting: (() => void)[] = []; + + async acquire(): Promise { + while (this.#locked) { + await new Promise((resolve) => this.#waiting.push(resolve)); + } + this.#locked = true; + } + + release(): void { + this.#locked = false; + const next = this.#waiting.shift(); + if (next) { + next(); + } + } +} + +/** + * Database wrapper that provides a simplified SQLite API + */ +export class Database { + readonly #sqlite3: SQLite3Api; + readonly #handle: number; + readonly #fileName: string; + readonly #onClose: () => void; + + constructor(sqlite3: SQLite3Api, handle: number, fileName: string, onClose: () => void) { + this.#sqlite3 = sqlite3; + this.#handle = handle; + this.#fileName = fileName; + this.#onClose = onClose; + } + + /** + * Execute SQL with optional row callback + * @param sql - SQL statement to execute + * @param callback - Called for each result row with (row, columns) where row is an array of values and columns is an array of column names + */ + async exec(sql: string, callback?: (row: unknown[], columns: string[]) => void): Promise { + return this.#sqlite3.exec(this.#handle, sql, callback); + } + + /** + * Close the database + */ + async close(): Promise { + await this.#sqlite3.close(this.#handle); + this.#onClose(); + } + + /** + * Get the raw wa-sqlite API (for advanced usage) + */ + get sqlite3(): SQLite3Api { + return this.#sqlite3; + } + + /** + * Get the raw database handle (for advanced usage) + */ + get handle(): number { + return this.#handle; + } +} + +/** + * SQLite VFS backed by KV storage. + * + * Each instance is independent and has its own wa-sqlite WASM module. + * This allows multiple instances to operate concurrently without interference. + */ +export class SqliteVfs { + #sqlite3: SQLite3Api | null = null; + #sqliteSystem: SqliteSystem | null = null; + #initPromise: Promise | null = null; + #openMutex = new AsyncMutex(); + #instanceId: string; + + constructor() { + // Generate unique instance ID for VFS name + this.#instanceId = crypto.randomUUID().replace(/-/g, '').slice(0, 8); + } + + /** + * Initialize wa-sqlite and VFS (called once per instance) + */ + async #ensureInitialized(): Promise { + // Fast path: already initialized + if (this.#sqlite3 && this.#sqliteSystem) { + return; + } + + // Synchronously create the promise if not started + if (!this.#initPromise) { + this.#initPromise = (async () => { + // Load WASM binary (Node.js environment) + const require = createRequire(import.meta.url); + const wasmPath = require.resolve("wa-sqlite/dist/wa-sqlite-async.wasm"); + const wasmBinary = readFileSync(wasmPath); + + // Initialize wa-sqlite module - each instance gets its own module + const module = await SQLiteESMFactory({ wasmBinary }); + this.#sqlite3 = Factory(module) as SQLite3Api; + + // Create and register VFS with unique name + this.#sqliteSystem = new SqliteSystem(this.#sqlite3, `kv-vfs-${this.#instanceId}`); + this.#sqliteSystem.register(); + })(); + } + + // Wait for initialization + await this.#initPromise; + } + + /** + * Open a SQLite database using KV storage backend + * + * @param fileName - The database file name (typically the actor ID) + * @param options - KV storage operations for this database + * @returns A Database instance + */ + async open( + fileName: string, + options: KvVfsOptions, + ): Promise { + // Serialize all open operations within this instance + await this.#openMutex.acquire(); + try { + // Initialize wa-sqlite and SqliteSystem on first call + await this.#ensureInitialized(); + + if (!this.#sqlite3 || !this.#sqliteSystem) { + throw new Error("Failed to initialize SQLite"); + } + + // Register this filename with its KV options + this.#sqliteSystem.registerFile(fileName, options); + + // Open database + const db = await this.#sqlite3.open_v2( + fileName, + this.#sqlite3.SQLITE_OPEN_READWRITE | this.#sqlite3.SQLITE_OPEN_CREATE, + this.#sqliteSystem.name, + ); + + // Create cleanup callback + const sqliteSystem = this.#sqliteSystem; + const onClose = () => { + sqliteSystem.unregisterFile(fileName); + }; + + return new Database(this.#sqlite3, db, fileName, onClose); + } finally { + this.#openMutex.release(); + } + } +} + +/** + * Internal VFS implementation + */ +class SqliteSystem extends VFS.Base { + readonly name: string; + readonly #fileOptions: Map = new Map(); + readonly #openFiles: Map = new Map(); + readonly #sqlite3: SQLite3Api; + + constructor(sqlite3: SQLite3Api, name: string) { + super(); + this.#sqlite3 = sqlite3; + this.name = name; + } + + /** + * Registers the VFS with SQLite + */ + register(): void { + this.#sqlite3.vfs_register(this, false); + } + + /** + * Registers a file with its KV options (before opening) + */ + registerFile(fileName: string, options: KvVfsOptions): void { + this.#fileOptions.set(fileName, options); + } + + /** + * Unregisters a file's KV options (after closing) + */ + unregisterFile(fileName: string): void { + this.#fileOptions.delete(fileName); + } + + /** + * Gets KV options for a file, handling journal/wal files by using the main database's options + */ + #getOptionsForPath(path: string): KvVfsOptions | undefined { + let options = this.#fileOptions.get(path); + if (!options) { + // Try to find the main database file by removing common SQLite suffixes + const mainDbPath = path + .replace(/-journal$/, "") + .replace(/-wal$/, "") + .replace(/-shm$/, ""); + + if (mainDbPath !== path) { + options = this.#fileOptions.get(mainDbPath); + } + } + return options; + } + + /** + * Opens a file + */ + xOpen( + path: string | null, + fileId: number, + flags: number, + pOutFlags: DataView, + ): number { + return this.handleAsync(async () => { + if (!path) { + return VFS.SQLITE_CANTOPEN; + } + + // Get the registered KV options for this file + // For journal/wal files, use the main database's options + const options = this.#getOptionsForPath(path); + if (!options) { + throw new Error(`No KV options registered for file: ${path}`); + } + + // Get existing file size if the file exists + const metaKey = getMetaKey(path); + const sizeData = await options.get(metaKey); + + let size: number; + + if (sizeData) { + // File exists, use existing size + size = decodeFileMeta(sizeData); + } else if (flags & VFS.SQLITE_OPEN_CREATE) { + // File doesn't exist, create it + size = 0; + await options.put(metaKey, encodeFileMeta(size)); + } else { + // File doesn't exist and we're not creating it + return VFS.SQLITE_CANTOPEN; + } + + // Store open file info with options + this.#openFiles.set(fileId, { + path, + size, + flags, + options, + }); + + // Set output flags + pOutFlags.setInt32(0, flags & VFS.SQLITE_OPEN_READONLY ? 1 : 0, true); + + return VFS.SQLITE_OK; + }); + } + + /** + * Closes a file + */ + xClose(fileId: number): number { + return this.handleAsync(async () => { + const file = this.#openFiles.get(fileId); + if (!file) { + return VFS.SQLITE_OK; + } + + // Delete file if SQLITE_OPEN_DELETEONCLOSE flag was set + if (file.flags & VFS.SQLITE_OPEN_DELETEONCLOSE) { + await this.#delete(file.path); + } + + this.#openFiles.delete(fileId); + return VFS.SQLITE_OK; + }); + } + + /** + * Reads data from a file + */ + // @ts-expect-error - VFS.Base types are incorrect, runtime uses Uint8Array + xRead(fileId: number, pData: Uint8Array, iOffset: number): number { + return this.handleAsync(async () => { + const file = this.#openFiles.get(fileId); + if (!file) { + return VFS.SQLITE_IOERR_READ; + } + + const options = file.options; + const requestedLength = pData.length; + const fileSize = file.size; + + // If offset is beyond file size, return short read with zeroed buffer + if (iOffset >= fileSize) { + pData.fill(0); + return VFS.SQLITE_IOERR_SHORT_READ; + } + + // Calculate which chunks we need to read + const startChunk = Math.floor(iOffset / CHUNK_SIZE); + const endChunk = Math.floor((iOffset + requestedLength - 1) / CHUNK_SIZE); + + // Fetch all needed chunks + const chunkKeys: Uint8Array[] = []; + for (let i = startChunk; i <= endChunk; i++) { + chunkKeys.push(getChunkKey(file.path, i)); + } + + const readStart = performance.now(); + const chunks = await options.getBatch(chunkKeys); + vfsLog("xRead", { file: file.path, offset: iOffset, len: requestedLength, chunks: chunkKeys.length, ms: (performance.now() - readStart).toFixed(2) }); + + // Copy data from chunks to output buffer + for (let i = startChunk; i <= endChunk; i++) { + const chunkData = chunks[i - startChunk]; + const chunkOffset = i * CHUNK_SIZE; + + // Calculate the range within this chunk + const readStart = Math.max(0, iOffset - chunkOffset); + const readEnd = Math.min( + CHUNK_SIZE, + iOffset + requestedLength - chunkOffset, + ); + + if (chunkData) { + // Copy available data + const sourceStart = readStart; + const sourceEnd = Math.min(readEnd, chunkData.length); + const destStart = chunkOffset + readStart - iOffset; + + if (sourceEnd > sourceStart) { + pData.set( + chunkData.slice(sourceStart, sourceEnd), + destStart, + ); + } + + // Zero-fill if chunk is smaller than expected + if (sourceEnd < readEnd) { + const zeroStart = destStart + (sourceEnd - sourceStart); + const zeroEnd = destStart + (readEnd - readStart); + pData.fill(0, zeroStart, zeroEnd); + } + } else { + // Chunk doesn't exist, zero-fill + const destStart = chunkOffset + readStart - iOffset; + const destEnd = destStart + (readEnd - readStart); + pData.fill(0, destStart, destEnd); + } + } + + // If we read less than requested (past EOF), return short read + const actualBytes = Math.min(requestedLength, fileSize - iOffset); + if (actualBytes < requestedLength) { + pData.fill(0, actualBytes); + return VFS.SQLITE_IOERR_SHORT_READ; + } + + return VFS.SQLITE_OK; + }); + } + + /** + * Writes data to a file + */ + // @ts-expect-error - VFS.Base types are incorrect, runtime uses Uint8Array + xWrite(fileId: number, pData: Uint8Array, iOffset: number): number { + return this.handleAsync(async () => { + const file = this.#openFiles.get(fileId); + if (!file) { + return VFS.SQLITE_IOERR_WRITE; + } + + const options = file.options; + const writeLength = pData.length; + + // Calculate which chunks we need to modify + const startChunk = Math.floor(iOffset / CHUNK_SIZE); + const endChunk = Math.floor((iOffset + writeLength - 1) / CHUNK_SIZE); + + // Fetch existing chunks that we'll need to modify + const chunkKeys: Uint8Array[] = []; + for (let i = startChunk; i <= endChunk; i++) { + chunkKeys.push(getChunkKey(file.path, i)); + } + + const getBatchStart = performance.now(); + const existingChunks = await options.getBatch(chunkKeys); + const getBatchMs = performance.now() - getBatchStart; + + // Prepare new chunk data + const entriesToWrite: [Uint8Array, Uint8Array][] = []; + + for (let i = startChunk; i <= endChunk; i++) { + const chunkOffset = i * CHUNK_SIZE; + const existingChunk = existingChunks[i - startChunk]; + + // Calculate the range within this chunk that we're writing + const writeStart = Math.max(0, iOffset - chunkOffset); + const writeEnd = Math.min( + CHUNK_SIZE, + iOffset + writeLength - chunkOffset, + ); + + // Calculate the size this chunk needs to be + const requiredSize = writeEnd; + + // Create new chunk data + let newChunk: Uint8Array; + if (existingChunk && existingChunk.length >= requiredSize) { + // Use existing chunk (copy it so we can modify) + newChunk = new Uint8Array(Math.max(existingChunk.length, requiredSize)); + newChunk.set(existingChunk); + } else if (existingChunk) { + // Need to expand existing chunk + newChunk = new Uint8Array(requiredSize); + newChunk.set(existingChunk); + } else { + // Create new chunk + newChunk = new Uint8Array(requiredSize); + } + + // Copy data from input buffer to chunk + const sourceStart = chunkOffset + writeStart - iOffset; + const sourceEnd = sourceStart + (writeEnd - writeStart); + newChunk.set(pData.slice(sourceStart, sourceEnd), writeStart); + + entriesToWrite.push([getChunkKey(file.path, i), newChunk]); + } + + // Update file size if we wrote past the end + const newSize = Math.max(file.size, iOffset + writeLength); + if (newSize !== file.size) { + file.size = newSize; + entriesToWrite.push([getMetaKey(file.path), encodeFileMeta(file.size)]); + } + + // Write all chunks and metadata + const putBatchStart = performance.now(); + await options.putBatch(entriesToWrite); + vfsLog("xWrite", { file: file.path, offset: iOffset, len: writeLength, readChunks: chunkKeys.length, writeEntries: entriesToWrite.length, getBatchMs: getBatchMs.toFixed(2), putBatchMs: (performance.now() - putBatchStart).toFixed(2) }); + + return VFS.SQLITE_OK; + }); + } + + /** + * Truncates a file + */ + xTruncate(fileId: number, size: number): number { + return this.handleAsync(async () => { + const file = this.#openFiles.get(fileId); + if (!file) { + return VFS.SQLITE_IOERR_TRUNCATE; + } + + const options = file.options; + + // If truncating to larger size, just update metadata + if (size >= file.size) { + return VFS.SQLITE_OK; + } + + // Calculate which chunks to delete + // Note: When size=0, lastChunkToKeep = floor(-1/4096) = -1, which means + // all chunks (starting from index 0) will be deleted in the loop below. + const lastChunkToKeep = Math.floor((size - 1) / CHUNK_SIZE); + const lastExistingChunk = Math.floor((file.size - 1) / CHUNK_SIZE); + + // Delete chunks beyond the new size + const keysToDelete: Uint8Array[] = []; + for (let i = lastChunkToKeep + 1; i <= lastExistingChunk; i++) { + keysToDelete.push(getChunkKey(file.path, i)); + } + + if (keysToDelete.length > 0) { + await options.deleteBatch(keysToDelete); + } + + // Truncate the last kept chunk if needed + if (size > 0 && size % CHUNK_SIZE !== 0) { + const lastChunkKey = getChunkKey(file.path, lastChunkToKeep); + const lastChunkData = await options.get(lastChunkKey); + + if (lastChunkData && lastChunkData.length > size % CHUNK_SIZE) { + const truncatedChunk = lastChunkData.slice(0, size % CHUNK_SIZE); + await options.put(lastChunkKey, truncatedChunk); + } + } + + // Update file size + file.size = size; + await options.put(getMetaKey(file.path), encodeFileMeta(file.size)); + + return VFS.SQLITE_OK; + }); + } + + /** + * Syncs file data to storage + */ + xSync(fileId: number, _flags: number): number { + return this.handleAsync(async () => { + // KV storage is immediately durable, so sync is a no-op + // But we should ensure size is persisted + const file = this.#openFiles.get(fileId); + if (!file) { + return VFS.SQLITE_OK; + } + + const options = file.options; + await options.put(getMetaKey(file.path), encodeFileMeta(file.size)); + return VFS.SQLITE_OK; + }); + } + + /** + * Gets the file size + */ + xFileSize(fileId: number, pSize: DataView): number { + return this.handleAsync(async () => { + const file = this.#openFiles.get(fileId); + if (!file) { + return VFS.SQLITE_IOERR_FSTAT; + } + + // Set size as 64-bit integer (low and high parts) + pSize.setBigInt64(0, BigInt(file.size), true); + return VFS.SQLITE_OK; + }); + } + + /** + * Deletes a file + */ + xDelete(path: string, _syncDir: number): number { + return this.handleAsync(async () => { + await this.#delete(path); + return VFS.SQLITE_OK; + }); + } + + /** + * Internal delete implementation + */ + async #delete(path: string): Promise { + const options = this.#getOptionsForPath(path); + if (!options) { + throw new Error(`No KV options registered for file: ${path}`); + } + + // Get file size to find out how many chunks to delete + const metaKey = getMetaKey(path); + const sizeData = await options.get(metaKey); + + if (!sizeData) { + // File doesn't exist, that's OK + return; + } + + const size = decodeFileMeta(sizeData); + + // Delete all chunks + const keysToDelete: Uint8Array[] = [metaKey]; + const numChunks = Math.ceil(size / CHUNK_SIZE); + for (let i = 0; i < numChunks; i++) { + keysToDelete.push(getChunkKey(path, i)); + } + + await options.deleteBatch(keysToDelete); + } + + /** + * Checks file accessibility + */ + xAccess(path: string, _flags: number, pResOut: DataView): number { + return this.handleAsync(async () => { + const options = this.#getOptionsForPath(path); + if (!options) { + // File not registered, doesn't exist + pResOut.setInt32(0, 0, true); + return VFS.SQLITE_OK; + } + + const metaKey = getMetaKey(path); + const metaData = await options.get(metaKey); + + // Set result: 1 if file exists, 0 otherwise + pResOut.setInt32(0, metaData ? 1 : 0, true); + return VFS.SQLITE_OK; + }); + } +} diff --git a/rivetkit-typescript/packages/rivetkit/src/db/vfs/schemas/file-meta/mod.ts b/rivetkit-typescript/packages/rivetkit/src/db/vfs/schemas/file-meta/mod.ts new file mode 100644 index 0000000000..c16a04f105 --- /dev/null +++ b/rivetkit-typescript/packages/rivetkit/src/db/vfs/schemas/file-meta/mod.ts @@ -0,0 +1 @@ +export * from "./v1"; diff --git a/rivetkit-typescript/packages/rivetkit/src/db/vfs/schemas/file-meta/v1.ts b/rivetkit-typescript/packages/rivetkit/src/db/vfs/schemas/file-meta/v1.ts new file mode 100644 index 0000000000..e091877457 --- /dev/null +++ b/rivetkit-typescript/packages/rivetkit/src/db/vfs/schemas/file-meta/v1.ts @@ -0,0 +1,37 @@ +import * as bare from "@bare-ts/lib" + +const config = /* @__PURE__ */ bare.Config({}) + +export type u64 = bigint + +export type FileMeta = { + readonly size: u64, +} + +export function readFileMeta(bc: bare.ByteCursor): FileMeta { + return { + size: bare.readU64(bc), + } +} + +export function writeFileMeta(bc: bare.ByteCursor, x: FileMeta): void { + bare.writeU64(bc, x.size) +} + +export function encodeFileMeta(x: FileMeta): Uint8Array { + const bc = new bare.ByteCursor( + new Uint8Array(config.initialBufferLength), + config + ) + writeFileMeta(bc, x) + return new Uint8Array(bc.view.buffer, bc.view.byteOffset, bc.offset) +} + +export function decodeFileMeta(bytes: Uint8Array): FileMeta { + const bc = new bare.ByteCursor(bytes, config) + const result = readFileMeta(bc) + if (bc.offset < bc.view.byteLength) { + throw new bare.BareError(bc.offset, "remaining bytes") + } + return result +} diff --git a/rivetkit-typescript/packages/rivetkit/src/db/vfs/schemas/file-meta/versioned.ts b/rivetkit-typescript/packages/rivetkit/src/db/vfs/schemas/file-meta/versioned.ts new file mode 100644 index 0000000000..870d999e8e --- /dev/null +++ b/rivetkit-typescript/packages/rivetkit/src/db/vfs/schemas/file-meta/versioned.ts @@ -0,0 +1,25 @@ +import { createVersionedDataHandler } from "vbare"; +import * as v1 from "./v1"; + +export const CURRENT_VERSION = 1; + +export const FILE_META_VERSIONED = createVersionedDataHandler({ + deserializeVersion: (bytes, version) => { + switch (version) { + case 1: + return v1.decodeFileMeta(bytes); + default: + throw new Error(`Unknown version ${version}`); + } + }, + serializeVersion: (data, version) => { + switch (version) { + case 1: + return v1.encodeFileMeta(data as v1.FileMeta); + default: + throw new Error(`Unknown version ${version}`); + } + }, + deserializeConverters: () => [], + serializeConverters: () => [], +}); diff --git a/rivetkit-typescript/packages/rivetkit/src/db/vfs/wa-sqlite.d.ts b/rivetkit-typescript/packages/rivetkit/src/db/vfs/wa-sqlite.d.ts new file mode 100644 index 0000000000..4b8db17c9e --- /dev/null +++ b/rivetkit-typescript/packages/rivetkit/src/db/vfs/wa-sqlite.d.ts @@ -0,0 +1,84 @@ +declare module "wa-sqlite/src/VFS.js" { + export const SQLITE_OK: number; + export const SQLITE_IOERR: number; + export const SQLITE_IOERR_READ: number; + export const SQLITE_IOERR_SHORT_READ: number; + export const SQLITE_IOERR_WRITE: number; + export const SQLITE_IOERR_TRUNCATE: number; + export const SQLITE_IOERR_FSTAT: number; + export const SQLITE_CANTOPEN: number; + export const SQLITE_OPEN_CREATE: number; + export const SQLITE_OPEN_READONLY: number; + export const SQLITE_OPEN_DELETEONCLOSE: number; + export const SQLITE_NOTFOUND: number; + + /** + * Base class for SQLite VFS implementations. + * Extend this class and override methods to implement custom file systems. + */ + export class Base { + mxPathName: number; + + /** Close a file */ + xClose(fileId: number): number; + + /** Read data from a file */ + xRead(fileId: number, pData: Uint8Array, iOffset: number): number; + + /** Write data to a file */ + xWrite(fileId: number, pData: Uint8Array, iOffset: number): number; + + /** Truncate a file */ + xTruncate(fileId: number, iSize: number): number; + + /** Sync file data to storage */ + xSync(fileId: number, flags: number): number; + + /** Get file size */ + xFileSize(fileId: number, pSize64: DataView): number; + + /** Lock a file */ + xLock(fileId: number, flags: number): number; + + /** Unlock a file */ + xUnlock(fileId: number, flags: number): number; + + /** Check for reserved lock */ + xCheckReservedLock(fileId: number, pResOut: DataView): number; + + /** File control operations */ + xFileControl(fileId: number, op: number, pArg: DataView): number; + + /** Get sector size */ + xSectorSize(fileId: number): number; + + /** Get device characteristics */ + xDeviceCharacteristics(fileId: number): number; + + /** Open a file */ + xOpen( + name: string | null, + fileId: number, + flags: number, + pOutFlags: DataView, + ): number; + + /** Delete a file */ + xDelete(name: string, syncDir: number): number; + + /** Check file accessibility */ + xAccess(name: string, flags: number, pResOut: DataView): number; + + /** Handle asynchronous operations */ + handleAsync(fn: () => Promise): T; + } +} + +declare module "wa-sqlite/dist/wa-sqlite-async.mjs" { + const factory: (options?: { wasmBinary?: ArrayBuffer | Uint8Array }) => Promise; + export default factory; +} + +declare module "wa-sqlite" { + export function Factory(module: any): any; +} diff --git a/rivetkit-typescript/packages/rivetkit/src/driver-helpers/mod.ts b/rivetkit-typescript/packages/rivetkit/src/driver-helpers/mod.ts index 86cd233704..d5f9f1c479 100644 --- a/rivetkit-typescript/packages/rivetkit/src/driver-helpers/mod.ts +++ b/rivetkit-typescript/packages/rivetkit/src/driver-helpers/mod.ts @@ -1,6 +1,7 @@ export type { ActorDriver } from "@/actor/driver"; export type { ActorInstance, AnyActorInstance } from "@/actor/instance/mod"; export { generateRandomString } from "@/actor/utils"; +export { KEYS, makeConnKey } from "@/actor/instance/kv"; export { ALLOWED_PUBLIC_HEADERS, HEADER_ACTOR_ID, diff --git a/rivetkit-typescript/packages/rivetkit/src/driver-test-suite/mod.ts b/rivetkit-typescript/packages/rivetkit/src/driver-test-suite/mod.ts index 2c0f7bb550..902ae3a2d9 100644 --- a/rivetkit-typescript/packages/rivetkit/src/driver-test-suite/mod.ts +++ b/rivetkit-typescript/packages/rivetkit/src/driver-test-suite/mod.ts @@ -12,6 +12,7 @@ import { runActionFeaturesTests } from "./tests/action-features"; import { runActorConnTests } from "./tests/actor-conn"; import { runActorConnHibernationTests } from "./tests/actor-conn-hibernation"; import { runActorConnStateTests } from "./tests/actor-conn-state"; +import { runActorDbRawTests } from "./tests/actor-db-raw"; import { runActorDestroyTests } from "./tests/actor-destroy"; import { runActorDriverTests } from "./tests/actor-driver"; import { runActorErrorHandlingTests } from "./tests/actor-error-handling"; @@ -20,6 +21,7 @@ import { runActorInlineClientTests } from "./tests/actor-inline-client"; import { runActorInspectorTests } from "./tests/actor-inspector"; import { runActorMetadataTests } from "./tests/actor-metadata"; import { runActorOnStateChangeTests } from "./tests/actor-onstatechange"; +import { runActorStatelessTests } from "./tests/actor-stateless"; import { runActorVarsTests } from "./tests/actor-vars"; import { runManagerDriverTests } from "./tests/manager-driver"; import { runRawHttpTests } from "./tests/raw-http"; @@ -79,65 +81,71 @@ export interface DriverDeployOutput { export function runDriverTests( driverTestConfigPartial: Omit, ) { - const clientTypes: ClientType[] = driverTestConfigPartial.skip?.inline - ? ["http"] - : ["http", "inline"]; - for (const clientType of clientTypes) { - describe(`client type (${clientType})`, () => { - const encodings: Encoding[] = ["bare", "cbor", "json"]; + describe("Driver Tests", () => { + const clientTypes: ClientType[] = driverTestConfigPartial.skip?.inline + ? ["http"] + : ["http", "inline"]; + for (const clientType of clientTypes) { + describe(`client type (${clientType})`, () => { + const encodings: Encoding[] = ["bare", "cbor", "json"]; - for (const encoding of encodings) { - describe(`encoding (${encoding})`, () => { - const driverTestConfig: DriverTestConfig = { - ...driverTestConfigPartial, - clientType, - encoding, - }; + for (const encoding of encodings) { + describe(`encoding (${encoding})`, () => { + const driverTestConfig: DriverTestConfig = { + ...driverTestConfigPartial, + clientType, + encoding, + }; - runActorDriverTests(driverTestConfig); - runManagerDriverTests(driverTestConfig); + runActorDriverTests(driverTestConfig); + runManagerDriverTests(driverTestConfig); - runActorConnTests(driverTestConfig); + runActorConnTests(driverTestConfig); - runActorConnStateTests(driverTestConfig); + runActorConnStateTests(driverTestConfig); - runActorConnHibernationTests(driverTestConfig); + runActorConnHibernationTests(driverTestConfig); - runActorDestroyTests(driverTestConfig); + runActorDbRawTests(driverTestConfig); - runRequestAccessTests(driverTestConfig); + runActorDestroyTests(driverTestConfig); - runActorHandleTests(driverTestConfig); + runRequestAccessTests(driverTestConfig); - runActionFeaturesTests(driverTestConfig); + runActorHandleTests(driverTestConfig); - runActorVarsTests(driverTestConfig); + runActionFeaturesTests(driverTestConfig); - runActorMetadataTests(driverTestConfig); + runActorVarsTests(driverTestConfig); - runActorOnStateChangeTests(driverTestConfig); + runActorMetadataTests(driverTestConfig); - runActorErrorHandlingTests(driverTestConfig); + runActorOnStateChangeTests(driverTestConfig); - runActorInlineClientTests(driverTestConfig); + runActorErrorHandlingTests(driverTestConfig); - runRawHttpTests(driverTestConfig); + runActorInlineClientTests(driverTestConfig); - runRawHttpRequestPropertiesTests(driverTestConfig); + runActorStatelessTests(driverTestConfig); - runRawWebSocketTests(driverTestConfig); + runRawHttpTests(driverTestConfig); - // TODO: re-expose this once we can have actor queries on the gateway - // runRawHttpDirectRegistryTests(driverTestConfig); + runRawHttpRequestPropertiesTests(driverTestConfig); - // TODO: re-expose this once we can have actor queries on the gateway - // runRawWebSocketDirectRegistryTests(driverTestConfig); + runRawWebSocketTests(driverTestConfig); - runActorInspectorTests(driverTestConfig); - }); - } - }); - } + // TODO: re-expose this once we can have actor queries on the gateway + // runRawHttpDirectRegistryTests(driverTestConfig); + + // TODO: re-expose this once we can have actor queries on the gateway + // runRawWebSocketDirectRegistryTests(driverTestConfig); + + runActorInspectorTests(driverTestConfig); + }); + } + }); + } + }); } /** diff --git a/rivetkit-typescript/packages/rivetkit/src/driver-test-suite/tests/actor-db-raw.ts b/rivetkit-typescript/packages/rivetkit/src/driver-test-suite/tests/actor-db-raw.ts new file mode 100644 index 0000000000..fd1b65f228 --- /dev/null +++ b/rivetkit-typescript/packages/rivetkit/src/driver-test-suite/tests/actor-db-raw.ts @@ -0,0 +1,73 @@ +import { describe, expect, test } from "vitest"; +import type { DriverTestConfig } from "../mod"; +import { setupDriverTest } from "../utils"; + +export function runActorDbRawTests(driverTestConfig: DriverTestConfig) { + describe("Actor Database (Raw) Tests", () => { + describe("Database Basic Operations", () => { + test("creates and queries database tables", async (c) => { + const { client } = await setupDriverTest(c, driverTestConfig); + + const instance = client.dbActorRaw.getOrCreate(); + + // Add values + await instance.insertValue("Alice"); + await instance.insertValue("Bob"); + + // Query values + const values = await instance.getValues(); + expect(values).toHaveLength(2); + expect(values[0].value).toBe("Alice"); + expect(values[1].value).toBe("Bob"); + }); + + test("persists data across actor instances", async (c) => { + const { client } = await setupDriverTest(c, driverTestConfig); + + // First instance adds items + const instance1 = client.dbActorRaw.getOrCreate(["test-persistence"]); + await instance1.insertValue("Item 1"); + await instance1.insertValue("Item 2"); + + // Second instance (same actor) should see persisted data + const instance2 = client.dbActorRaw.getOrCreate(["test-persistence"]); + const count = await instance2.getCount(); + expect(count).toBe(2); + }); + + test("maintains separate databases for different actors", async (c) => { + const { client } = await setupDriverTest(c, driverTestConfig); + + // First actor + const actor1 = client.dbActorRaw.getOrCreate(["actor-1"]); + await actor1.insertValue("A"); + await actor1.insertValue("B"); + + // Second actor + const actor2 = client.dbActorRaw.getOrCreate(["actor-2"]); + await actor2.insertValue("X"); + + // Verify separate data + const count1 = await actor1.getCount(); + const count2 = await actor2.getCount(); + expect(count1).toBe(2); + expect(count2).toBe(1); + }); + }); + + describe("Database Migrations", () => { + test("runs migrations on actor startup", async (c) => { + const { client } = await setupDriverTest(c, driverTestConfig); + + const instance = client.dbActorRaw.getOrCreate(); + + // Try to insert into the table to verify it exists + await instance.insertValue("test"); + const values = await instance.getValues(); + + expect(values).toHaveLength(1); + expect(values[0].value).toBe("test"); + }); + }); + }); +} diff --git a/rivetkit-typescript/packages/rivetkit/src/driver-test-suite/tests/actor-stateless.ts b/rivetkit-typescript/packages/rivetkit/src/driver-test-suite/tests/actor-stateless.ts new file mode 100644 index 0000000000..063e526759 --- /dev/null +++ b/rivetkit-typescript/packages/rivetkit/src/driver-test-suite/tests/actor-stateless.ts @@ -0,0 +1,70 @@ +import { describe, expect, test } from "vitest"; +import type { DriverTestConfig } from "../mod"; +import { setupDriverTest } from "../utils"; + +export function runActorStatelessTests(driverTestConfig: DriverTestConfig) { + describe("Actor Stateless Tests", () => { + describe("Stateless Actor Operations", () => { + test("can call actions on stateless actor", async (c) => { + const { client } = await setupDriverTest(c, driverTestConfig); + + const instance = client.statelessActor.getOrCreate(); + + const result = await instance.ping(); + expect(result).toBe("pong"); + }); + + test("can echo messages", async (c) => { + const { client } = await setupDriverTest(c, driverTestConfig); + + const instance = client.statelessActor.getOrCreate(); + + const message = "Hello, World!"; + const result = await instance.echo(message); + expect(result).toBe(message); + }); + + test("can access actorId", async (c) => { + const { client } = await setupDriverTest(c, driverTestConfig); + + const instance = client.statelessActor.getOrCreate(["test-id"]); + + const actorId = await instance.getActorId(); + expect(actorId).toBeDefined(); + expect(typeof actorId).toBe("string"); + }); + + test("accessing state throws StateNotEnabled", async (c) => { + const { client } = await setupDriverTest(c, driverTestConfig); + + const instance = client.statelessActor.getOrCreate(); + + const result = await instance.tryGetState(); + expect(result.success).toBe(false); + expect(result.error).toContain("state"); + }); + + test("accessing db throws DatabaseNotEnabled", async (c) => { + const { client } = await setupDriverTest(c, driverTestConfig); + + const instance = client.statelessActor.getOrCreate(); + + const result = await instance.tryGetDb(); + expect(result.success).toBe(false); + expect(result.error).toContain("database"); + }); + + test("multiple stateless actors can exist independently", async (c) => { + const { client } = await setupDriverTest(c, driverTestConfig); + + const actor1 = client.statelessActor.getOrCreate(["actor-1"]); + const actor2 = client.statelessActor.getOrCreate(["actor-2"]); + + const id1 = await actor1.getActorId(); + const id2 = await actor2.getActorId(); + + expect(id1).not.toBe(id2); + }); + }); + }); +} diff --git a/rivetkit-typescript/packages/rivetkit/src/drivers/engine/actor-driver.ts b/rivetkit-typescript/packages/rivetkit/src/drivers/engine/actor-driver.ts index 80a28eff0d..8b88dc7b37 100644 --- a/rivetkit-typescript/packages/rivetkit/src/drivers/engine/actor-driver.ts +++ b/rivetkit-typescript/packages/rivetkit/src/drivers/engine/actor-driver.ts @@ -33,6 +33,7 @@ import type { RivetMessageEvent, UniversalWebSocket, } from "@/common/websocket-interface"; +import { SqliteVfs } from "@/db/vfs/mod"; import { type ActorDriver, type AnyActorInstance, @@ -82,6 +83,9 @@ export class EngineActorDriver implements ActorDriver { #version: number = 1; // Version for the runner protocol #alarmTimeout?: LongTimeoutHandle; + /** SQLite VFS instance for creating KV-backed databases */ + readonly sqliteVfs = new SqliteVfs(); + #runnerStarted: PromiseWithResolvers = promiseWithResolvers(); #runnerStopped: PromiseWithResolvers = promiseWithResolvers(); #isRunnerStopped: boolean = false; @@ -223,9 +227,7 @@ export class EngineActorDriver implements ActorDriver { this.#runner.setAlarm(actor.id, timestamp); } - async getDatabase(_actorId: string): Promise { - return undefined; - } + // No database overrides - will use KV-backed implementation from rivetkit/db // MARK: - Batch KV operations async kvBatchPut( diff --git a/rivetkit-typescript/packages/rivetkit/src/drivers/file-system/actor.ts b/rivetkit-typescript/packages/rivetkit/src/drivers/file-system/actor.ts index 12335e48d0..bbfe8a5ccf 100644 --- a/rivetkit-typescript/packages/rivetkit/src/drivers/file-system/actor.ts +++ b/rivetkit-typescript/packages/rivetkit/src/drivers/file-system/actor.ts @@ -1,4 +1,6 @@ import type { AnyClient } from "@/client/client"; +import type { RawDatabaseClient } from "@/db/config"; +import type { SqliteVfs } from "@/db/vfs/mod"; import type { ActorDriver, AnyActorInstance, @@ -9,6 +11,21 @@ import { RegistryConfig } from "@/registry/config"; export type ActorDriverContext = Record; +/** + * Type alias for better-sqlite3 Database. + * We define this inline to avoid importing from better-sqlite3 directly, + * since it's an optional peer dependency. + */ +type BetterSQLite3Database = { + prepare(sql: string): { + run(...args: unknown[]): { changes: number; lastInsertRowid: number | bigint }; + all(...args: unknown[]): unknown[]; + get(...args: unknown[]): unknown; + }; + exec(sql: string): void; + close(): void; +}; + /** * File System implementation of the Actor Driver */ @@ -17,6 +34,8 @@ export class FileSystemActorDriver implements ActorDriver { #managerDriver: ManagerDriver; #inlineClient: AnyClient; #state: FileSystemGlobalState; + #nativeDatabases: Map = new Map(); + #drizzleDatabases: Map = new Map(); constructor( config: RegistryConfig, @@ -79,8 +98,106 @@ export class FileSystemActorDriver implements ActorDriver { await this.#state.setActorAlarm(actor.id, timestamp); } - getDatabase(actorId: string): Promise { - return this.#state.createDatabase(actorId); + async overrideRawDatabaseClient( + actorId: string, + ): Promise { + if (!this.#state.useNativeSqlite) { + return undefined; + } + + // Check if we already have a cached database for this actor + const existingDb = this.#nativeDatabases.get(actorId); + if (existingDb) { + return { + exec: (query: string) => { + const trimmed = query.trim().toUpperCase(); + if (trimmed.startsWith("SELECT") || trimmed.startsWith("PRAGMA")) { + // SELECT/PRAGMA queries return data + return existingDb.prepare(query).all(); + } + // Non-SELECT queries (INSERT, UPDATE, DELETE, CREATE, etc.) + // Use run() which doesn't throw for non-returning queries + existingDb.prepare(query).run(); + return []; + }, + }; + } + + // Dynamically import better-sqlite3 + try { + const Database = (await import("better-sqlite3")).default; + + const dbPath = this.#state.getActorDbPath(actorId); + const db = new Database(dbPath) as BetterSQLite3Database; + + this.#nativeDatabases.set(actorId, db); + + return { + exec: (query: string) => { + // HACK: sqlite3 throws error if not using a SELECT statement + const trimmed = query.trim().toUpperCase(); + if (trimmed.startsWith("SELECT") || trimmed.startsWith("PRAGMA")) { + // SELECT/PRAGMA queries return data + return db.prepare(query).all(); + } else { + // Non-SELECT queries (INSERT, UPDATE, DELETE, CREATE, etc.) + // Use run() which doesn't throw for non-returning queries + db.prepare(query).run(); + return []; + } + }, + }; + } catch (error) { + throw new Error( + `Failed to load better-sqlite3. Make sure it's installed: ${error}`, + ); + } + } + + async overrideDrizzleDatabaseClient( + actorId: string, + ): Promise { + if (!this.#state.useNativeSqlite) { + return undefined; + } + + // Check if we already have a cached drizzle database for this actor + const existingDrizzleDb = this.#drizzleDatabases.get(actorId); + if (existingDrizzleDb) { + return existingDrizzleDb; + } + + // Get or create the raw better-sqlite3 database + let rawDb = this.#nativeDatabases.get(actorId); + if (!rawDb) { + // Create it via overrideRawDatabaseClient + await this.overrideRawDatabaseClient(actorId); + rawDb = this.#nativeDatabases.get(actorId); + if (!rawDb) { + throw new Error( + "Failed to initialize native database for actor", + ); + } + } + + // Dynamically import drizzle and wrap the raw database + try { + const { drizzle } = await import("drizzle-orm/better-sqlite3"); + const drizzleDb = drizzle(rawDb as any); + + this.#drizzleDatabases.set(actorId, drizzleDb); + + return drizzleDb; + } catch (error) { + throw new Error( + `Failed to load drizzle-orm. Make sure it's installed: ${error}`, + ); + } + } + + /** SQLite VFS instance for creating KV-backed databases */ + get sqliteVfs(): SqliteVfs { + return this.#state.sqliteVfs; } startSleep(actorId: string): void { diff --git a/rivetkit-typescript/packages/rivetkit/src/drivers/file-system/global-state.ts b/rivetkit-typescript/packages/rivetkit/src/drivers/file-system/global-state.ts index aa68872822..c1486e6ff9 100644 --- a/rivetkit-typescript/packages/rivetkit/src/drivers/file-system/global-state.ts +++ b/rivetkit-typescript/packages/rivetkit/src/drivers/file-system/global-state.ts @@ -6,6 +6,9 @@ import type { ActorKey } from "@/actor/mod"; import { generateRandomString } from "@/actor/utils"; import type { AnyClient } from "@/client/client"; import { type ActorDriver, getInitialActorKvState } from "@/driver-helpers/mod"; +import { SqliteVfs } from "@/db/vfs/mod"; +import type { RegistryConfig } from "@/registry/config"; +import type { RunnerConfig } from "@/registry/run-config"; import type * as schema from "@/schemas/file-system-driver/mod"; import { ACTOR_ALARM_VERSIONED, @@ -71,6 +74,15 @@ interface ActorEntry { generation: string; } +export interface FileSystemDriverOptions { + /** Whether to persist data to disk */ + persist?: boolean; + /** Custom path for storage */ + customPath?: string; + /** Use native SQLite instead of KV-backed SQLite */ + useNativeSqlite?: boolean; +} + /** * Global state for the file system driver */ @@ -81,6 +93,10 @@ export class FileSystemGlobalState { #alarmsDir: string; #persist: boolean; + #useNativeSqlite: boolean; + + /** SQLite VFS instance for this driver. */ + readonly sqliteVfs = new SqliteVfs(); // IMPORTANT: Never delete from this map. Doing so will result in race // conditions since the actor generation will cease to be tracked @@ -107,8 +123,14 @@ export class FileSystemGlobalState { return this.#actorCountOnStartup; } - constructor(persist: boolean = true, customPath?: string) { + get useNativeSqlite(): boolean { + return this.#useNativeSqlite; + } + + constructor(options: FileSystemDriverOptions = {}) { + const { persist = true, customPath, useNativeSqlite = false } = options; this.#persist = persist; + this.#useNativeSqlite = useNativeSqlite; this.#storagePath = persist ? (customPath ?? getStoragePath()) : "/tmp"; const path = getNodePath(); this.#stateDir = path.join(this.#storagePath, "state"); diff --git a/rivetkit-typescript/packages/rivetkit/src/drivers/file-system/mod.ts b/rivetkit-typescript/packages/rivetkit/src/drivers/file-system/mod.ts index 7ec2027fe5..639fcb6bfd 100644 --- a/rivetkit-typescript/packages/rivetkit/src/drivers/file-system/mod.ts +++ b/rivetkit-typescript/packages/rivetkit/src/drivers/file-system/mod.ts @@ -1,6 +1,11 @@ +import { z } from "zod"; +import type { DriverConfig } from "@/registry/run-config"; import { importNodeDependencies } from "@/utils/node"; import { FileSystemActorDriver } from "./actor"; -import { FileSystemGlobalState } from "./global-state"; +import { + FileSystemGlobalState, + type FileSystemDriverOptions, +} from "./global-state"; import { FileSystemManagerDriver } from "./manager"; import { DriverConfig } from "@/registry/config"; @@ -9,13 +14,33 @@ export { FileSystemGlobalState } from "./global-state"; export { FileSystemManagerDriver } from "./manager"; export { getStoragePath } from "./utils"; +const CreateFileSystemDriverOptionsSchema = z.object({ + /** Custom path for storage. */ + path: z.string().optional(), + /** + * Use native SQLite (better-sqlite3) instead of KV-backed SQLite. + * Requires better-sqlite3 to be installed. + * @default false + */ + useNativeSqlite: z.boolean().optional().default(false), +}); + +type CreateFileSystemDriverOptionsInput = z.input< + typeof CreateFileSystemDriverOptionsSchema +>; + export function createFileSystemOrMemoryDriver( persist: boolean = true, - customPath?: string, + options?: CreateFileSystemDriverOptionsInput, ): DriverConfig { importNodeDependencies(); - const state = new FileSystemGlobalState(persist, customPath); + const stateOptions: FileSystemDriverOptions = { + persist, + customPath: options?.path, + useNativeSqlite: options?.useNativeSqlite ?? false, + }; + const state = new FileSystemGlobalState(stateOptions); const driverConfig: DriverConfig = { name: persist ? "file-system" : "memory", displayName: persist ? "File System" : "Memory", @@ -46,8 +71,13 @@ export function createFileSystemOrMemoryDriver( return driverConfig; } -export function createFileSystemDriver(opts?: { path?: string }): DriverConfig { - return createFileSystemOrMemoryDriver(true, opts?.path); +export function createFileSystemDriver( + opts?: CreateFileSystemDriverOptionsInput, +): DriverConfig { + const validatedOpts = opts + ? CreateFileSystemDriverOptionsSchema.parse(opts) + : undefined; + return createFileSystemOrMemoryDriver(true, validatedOpts); } export function createMemoryDriver(): DriverConfig { diff --git a/rivetkit-typescript/packages/rivetkit/src/test/mod.ts b/rivetkit-typescript/packages/rivetkit/src/test/mod.ts index a2acc272dd..a833e951af 100644 --- a/rivetkit-typescript/packages/rivetkit/src/test/mod.ts +++ b/rivetkit-typescript/packages/rivetkit/src/test/mod.ts @@ -25,9 +25,9 @@ export async function setupTest>( registry.config.test.enabled = true; // Create driver - const driver = await createFileSystemOrMemoryDriver( + const driver = createFileSystemOrMemoryDriver( true, - `/tmp/rivetkit-test-${crypto.randomUUID()}`, + { path: `/tmp/rivetkit-test-${crypto.randomUUID()}` }, ); // Build driver config diff --git a/rivetkit-typescript/packages/rivetkit/tests/actor-types.test.ts b/rivetkit-typescript/packages/rivetkit/tests/actor-types.test.ts index e6f16d7944..35ffbed4cd 100644 --- a/rivetkit-typescript/packages/rivetkit/tests/actor-types.test.ts +++ b/rivetkit-typescript/packages/rivetkit/tests/actor-types.test.ts @@ -1,6 +1,7 @@ import { describe, expectTypeOf, it } from "vitest"; import type { ActorContext } from "@/actor/contexts/actor"; import type { ActorContextOf, ActorDefinition } from "@/actor/definition"; +import type { DatabaseProviderContext } from "@/db/config"; describe("ActorDefinition", () => { describe("ActorContextOf type utility", () => { @@ -27,9 +28,7 @@ describe("ActorDefinition", () => { } interface TestDatabase { - createClient: (ctx: { - getDatabase: () => Promise; - }) => Promise<{ execute: (query: string) => any }>; + createClient: (ctx: DatabaseProviderContext) => Promise<{ execute: (query: string) => any }>; onMigrate: () => void; } diff --git a/rivetkit-typescript/packages/rivetkit/tests/driver-file-system-native-sqlite.test.ts b/rivetkit-typescript/packages/rivetkit/tests/driver-file-system-native-sqlite.test.ts new file mode 100644 index 0000000000..ff665f20d2 --- /dev/null +++ b/rivetkit-typescript/packages/rivetkit/tests/driver-file-system-native-sqlite.test.ts @@ -0,0 +1,28 @@ +import { join } from "node:path"; +import { createTestRuntime, runDriverTests } from "@/driver-test-suite/mod"; +import { createFileSystemOrMemoryDriver } from "@/drivers/file-system/mod"; + +runDriverTests({ + skip: { + // Does not support WS hibernation + hibernation: true, + }, + // TODO: Remove this once timer issues are fixed in actor-sleep.ts + useRealTimers: true, + async start() { + return await createTestRuntime( + join(__dirname, "../fixtures/driver-test-suite/registry.ts"), + async () => { + return { + driver: createFileSystemOrMemoryDriver( + true, + { + path: `/tmp/test-${crypto.randomUUID()}`, + useNativeSqlite: true, + }, + ), + }; + }, + ); + }, +}); diff --git a/rivetkit-typescript/packages/rivetkit/tests/driver-file-system.test.ts b/rivetkit-typescript/packages/rivetkit/tests/driver-file-system.test.ts index 7ab266f555..12af1ecd68 100644 --- a/rivetkit-typescript/packages/rivetkit/tests/driver-file-system.test.ts +++ b/rivetkit-typescript/packages/rivetkit/tests/driver-file-system.test.ts @@ -14,9 +14,9 @@ runDriverTests({ join(__dirname, "../fixtures/driver-test-suite/registry.ts"), async () => { return { - driver: await createFileSystemOrMemoryDriver( + driver: createFileSystemOrMemoryDriver( true, - `/tmp/test-${crypto.randomUUID()}`, + { path: `/tmp/test-${crypto.randomUUID()}` }, ), }; }, diff --git a/rivetkit-typescript/packages/rivetkit/tsconfig.json b/rivetkit-typescript/packages/rivetkit/tsconfig.json index abc57f2f60..d9361a34e7 100644 --- a/rivetkit-typescript/packages/rivetkit/tsconfig.json +++ b/rivetkit-typescript/packages/rivetkit/tsconfig.json @@ -7,7 +7,9 @@ "@/*": ["./src/*"], // Used for test fixtures "rivetkit": ["./src/mod.ts"], - "rivetkit/utils": ["./src/utils.ts"] + "rivetkit/utils": ["./src/utils.ts"], + "rivetkit/db": ["./src/db/mod.ts"], + "rivetkit/db/drizzle": ["./src/db/drizzle/mod.ts"] } }, "include": [