D1 beta support (#329)

* Adding first version of D1 support for beta

* Bunch of tweaks based on feedback

* Added D1 API directly, rather than through .fetch()

* rebased, bumped d1 version to 2.6, npm installed

* Fixing types and tests

* npm install

* lazy-installing better-sqlite3 using npx-import

* fixed tests and removed type alias

* fixing npx-import problems with local installs of sqlite3

* Invoking better-sqlite3 through @miniflare/d1 instead

* bumping npx-import to version compatible with npx 7

* bumping to npx-import v1.1.0 which should work on windows

* npx import v1.1.2

* Refactored Storage to now provide getSqliteDatabase()

* Added Jest integration test for D1 with push/pop & caching DB in storage

* Added vitest integration test for D1 as well

Co-authored-by: Glen Maddern <[email protected]>

Author: geelen

package-lock.json

@@ -0,0 +1,44 @@
+# `@miniflare/d1`
+
+Workers D1 module for [Miniflare](https://github.com/cloudflare/miniflare): a
+fun, full-featured, fully-local simulator for Cloudflare Workers. See
+[📦 D1](https://miniflare.dev/storage/d1) for more details.
+
+## Example
+
+```js
+import { BetaDatabase } from "@miniflare/d1";
+import { MemoryStorage } from "@miniflare/storage-memory";
+const db = new BetaDatabase(new MemoryStorage());
+
+// BetaDatabase only supports .fetch(), once D1 is out of beta the full API will be available here:
+await db.fetch("/execute", {
+  method: "POST",
+  body: JSON.stringify({
+    sql: `CREATE TABLE my_table (cid INTEGER PRIMARY KEY, name TEXT NOT NULL);`,
+  }),
+});
+const response = await db.fetch("/query", {
+  method: "POST",
+  body: JSON.stringify({
+    sql: `SELECT * FROM sqlite_schema`,
+  }),
+});
+console.log(await response.json());
+/*
+{
+  "success": true,
+  "result": [
+    [
+      {
+        "type": "table",
+        "name": "my_table",
+        "tbl_name": "my_table",
+        "rootpage": 2,
+        "sql": "CREATE TABLE my_table (cid INTEGER PRIMARY KEY, name TEXT NOT NULL)"
+      }
+    ]
+  ]
+}
+*/
+```

packages/d1/README.md

@@ -0,0 +1,45 @@
+{
+  "name": "@miniflare/d1",
+  "version": "2.8.2",
+  "description": "Workers D1 module for Miniflare: a fun, full-featured, fully-local simulator for Cloudflare Workers",
+  "keywords": [
+    "cloudflare",
+    "workers",
+    "worker",
+    "local",
+    "d1",
+    "sqlite"
+  ],
+  "author": "Glen Maddern <[email protected]>",
+  "license": "MIT",
+  "main": "./dist/src/index.js",
+  "types": "./dist/src/index.d.ts",
+  "files": [
+    "dist/src"
+  ],
+  "engines": {
+    "node": ">=16.7"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/cloudflare/miniflare.git",
+    "directory": "packages/d1"
+  },
+  "bugs": {
+    "url": "https://github.com/cloudflare/miniflare/issues"
+  },
+  "homepage": "https://github.com/cloudflare/miniflare/tree/master/packages/d1#readme",
+  "volta": {
+    "extends": "../../package.json"
+  },
+  "dependencies": {
+    "@miniflare/core": "2.8.2",
+    "@miniflare/shared": "2.8.2"
+  },
+  "devDependencies": {
+    "@miniflare/shared-test": "2.8.2"
+  }
+}

packages/d1/package.json

@@ -0,0 +1,38 @@
+import { performance } from "node:perf_hooks";
+import type { SqliteDB } from "@miniflare/shared";
+import { Statement } from "./statement";
+
+export class BetaDatabase {
+  #db: SqliteDB;
+
+  constructor(db: SqliteDB) {
+    this.#db = db;
+  }
+
+  prepare(source: string) {
+    return new Statement(this.#db, source);
+  }
+
+  async batch(statements: Statement[]) {
+    return await Promise.all(statements.map((s) => s.all()));
+  }
+
+  async exec(multiLineStatements: string) {
+    const statements = multiLineStatements
+      .split("\n")
+      .map((line) => line.trim())
+      .filter((line) => line.length > 0);
+    const start = performance.now();
+    for (const statement of statements) {
+      await new Statement(this.#db, statement).all();
+    }
+    return {
+      count: statements.length,
+      duration: performance.now() - start,
+    };
+  }
+
+  async dump() {
+    throw new Error("DB.dump() not implemented locally!");
+  }
+}

packages/d1/src/database.ts

@@ -0,0 +1,3 @@
+export * from "./database";
+export * from "./plugin";
+export * from "./statement";

packages/d1/src/index.ts

@@ -0,0 +1,70 @@
+import {
+  Context,
+  Option,
+  OptionType,
+  Plugin,
+  PluginContext,
+  SetupResult,
+  StorageFactory,
+  resolveStoragePersist,
+} from "@miniflare/shared";
+import { BetaDatabase } from "./database";
+
+export interface D1Options {
+  d1Databases?: string[];
+  d1Persist?: boolean | string;
+}
+const D1_BETA_PREFIX = `__D1_BETA__`;
+
+export class D1Plugin extends Plugin<D1Options> implements D1Options {
+  @Option({
+    type: OptionType.ARRAY,
+    name: "d1",
+    description: "D1 namespace to bind",
+    logName: "D1 Namespaces",
+    fromWrangler: ({ d1_databases }) =>
+      d1_databases?.map(({ binding }) => binding),
+  })
+  d1Databases?: string[];
+
+  @Option({
+    type: OptionType.BOOLEAN_STRING,
+    description: "Persist D1 data (to optional path)",
+    logName: "D1 Persistence",
+    fromWrangler: ({ miniflare }) => miniflare?.d1_persist,
+  })
+  d1Persist?: boolean | string;
+  readonly #persist?: boolean | string;
+
+  constructor(ctx: PluginContext, options?: D1Options) {
+    super(ctx);
+    this.assignOptions(options);
+    this.#persist = resolveStoragePersist(ctx.rootPath, this.d1Persist);
+  }
+
+  async getBetaDatabase(
+    storageFactory: StorageFactory,
+    dbName: string
+  ): Promise<BetaDatabase> {
+    const storage = await storageFactory.storage(dbName, this.#persist);
+    return new BetaDatabase(await storage.getSqliteDatabase());
+  }
+
+  async setup(storageFactory: StorageFactory): Promise<SetupResult> {
+    const bindings: Context = {};
+    for (const dbName of this.d1Databases ?? []) {
+      if (dbName.startsWith(D1_BETA_PREFIX)) {
+        bindings[dbName] = await this.getBetaDatabase(
+          storageFactory,
+          // Store it locally without the prefix
+          dbName.slice(D1_BETA_PREFIX.length)
+        );
+      } else {
+        console.warn(
+          `Not injecting D1 Database for '${dbName}' as this version of Miniflare only supports D1 beta bindings. Upgrade Wrangler and/or Miniflare and try again.`
+        );
+      }
+    }
+    return { bindings };
+  }
+}

packages/d1/src/plugin.ts

@@ -0,0 +1,137 @@
+import { performance } from "node:perf_hooks";
+import type {
+  Database as SqliteDB,
+  Statement as SqliteStatement,
+} from "better-sqlite3";
+
+export type BindParams = any[] | [Record<string, any>];
+
+function errorWithCause(message: string, e: unknown) {
+  // @ts-ignore Errors have causes now, why don't you know this Typescript?
+  return new Error(message, { cause: e });
+}
+
+export class Statement {
+  readonly #db: SqliteDB;
+  readonly #query: string;
+  readonly #bindings: BindParams | undefined;
+
+  constructor(db: SqliteDB, query: string, bindings?: BindParams) {
+    this.#db = db;
+    this.#query = query;
+    this.#bindings = bindings;
+  }
+
+  // Lazily accumulate binding instructions, because ".bind" in better-sqlite3
+  // is a real action that means the query must be valid when it's written,
+  // not when it's about to be executed (i.e. in a batch).
+  bind(...params: BindParams) {
+    // Adopting better-sqlite3 behaviour—once bound, a statement cannot be bound again
+    if (this.#bindings !== undefined) {
+      throw new TypeError(
+        "The bind() method can only be invoked once per statement object"
+      );
+    }
+    return new Statement(this.#db, this.#query, params);
+  }
+
+  private prepareAndBind() {
+    const prepared = this.#db.prepare(this.#query);
+    if (this.#bindings === undefined) return prepared;
+    try {
+      return prepared.bind(this.#bindings);
+    } catch (e) {
+      // For statements using ?1 ?2, etc, we want to pass them as varargs but
+      // "better" sqlite3 wants them as an object of {1: params[0], 2: params[1], ...}
+      if (this.#bindings.length > 0 && typeof this.#bindings[0] !== "object") {
+        return prepared.bind(
+          Object.fromEntries(this.#bindings.map((v, i) => [i + 1, v]))
+        );
+      } else {
+        throw e;
+      }
+    }
+  }
+
+  async all() {
+    const start = performance.now();
+    const statementWithBindings = this.prepareAndBind();
+    try {
+      const results = Statement.#all(statementWithBindings);
+      return {
+        results,
+        duration: performance.now() - start,
+        lastRowId: null,
+        changes: null,
+        success: true,
+        served_by: "x-miniflare.db3",
+      };
+    } catch (e) {
+      throw errorWithCause("D1_ALL_ERROR", e);
+    }
+  }
+
+  static #all(statementWithBindings: SqliteStatement) {
+    try {
+      return statementWithBindings.all();
+    } catch (e: unknown) {
+      // This is the quickest/simplest way I could find to return results by
+      // default, falling back to .run()
+      if (
+        /This statement does not return data\. Use run\(\) instead/.exec(
+          (e as Error).message
+        )
+      ) {
+        return Statement.#run(statementWithBindings);
+      }
+      throw e;
+    }
+  }
+
+  async first(col?: string) {
+    const statementWithBindings = this.prepareAndBind();
+    try {
+      const data = Statement.#first(statementWithBindings);
+      return typeof col === "string" ? data[col] : data;
+    } catch (e) {
+      throw errorWithCause("D1_FIRST_ERROR", e);
+    }
+  }
+
+  static #first(statementWithBindings: SqliteStatement) {
+    return statementWithBindings.get();
+  }
+
+  async run() {
+    const start = performance.now();
+    const statementWithBindings = this.prepareAndBind();
+    try {
+      const { changes, lastInsertRowid } = Statement.#run(
+        statementWithBindings
+      );
+      return {
+        results: null,
+        duration: performance.now() - start,
+        lastRowId: lastInsertRowid,
+        changes,
+        success: true,
+        served_by: "x-miniflare.db3",
+      };
+    } catch (e) {
+      throw errorWithCause("D1_RUN_ERROR", e);
+    }
+  }
+
+  static #run(statementWithBindings: SqliteStatement) {
+    return statementWithBindings.run();
+  }
+
+  async raw() {
+    const statementWithBindings = this.prepareAndBind();
+    return Statement.#raw(statementWithBindings);
+  }
+
+  static #raw(statementWithBindings: SqliteStatement) {
+    return statementWithBindings.raw() as any;
+  }
+}

packages/d1/src/statement.ts

@@ -0,0 +1,46 @@
+import { BetaDatabase } from "@miniflare/d1";
+import { Storage } from "@miniflare/shared";
+import { testClock } from "@miniflare/shared-test";
+import { MemoryStorage } from "@miniflare/storage-memory";
+import anyTest, { TestInterface } from "ava";
+
+interface Context {
+  storage: Storage;
+  db: BetaDatabase;
+}
+
+const test = anyTest as TestInterface<Context>;
+
+test.beforeEach(async (t) => {
+  const storage = new MemoryStorage(undefined, testClock);
+  const db = new BetaDatabase(await storage.getSqliteDatabase());
+  t.context = { storage, db };
+});
+
+test("batch, prepare & all", async (t) => {
+  const { db } = t.context;
+
+  await db.batch([
+    db.prepare(
+      `CREATE TABLE my_table (cid INTEGER PRIMARY KEY, name TEXT NOT NULL);`
+    ),
+  ]);
+  const response = await db.prepare(`SELECT * FROM sqlite_schema`).all();
+  t.deepEqual(Object.keys(response), [
+    "results",
+    "duration",
+    "lastRowId",
+    "changes",
+    "success",
+    "served_by",
+  ]);
+  t.deepEqual(response.results, [
+    {
+      type: "table",
+      name: "my_table",
+      tbl_name: "my_table",
+      rootpage: 2,
+      sql: "CREATE TABLE my_table (cid INTEGER PRIMARY KEY, name TEXT NOT NULL)",
+    },
+  ]);
+});