In-Memory Durable Objects (#391)

* Add support for in-memory Durable Objects

* Rename Durable Object `klass` to `designator`

Author: mrbbot

packages/tre/src/index.ts

@@ -26,8 +26,9 @@ import {
   Plugins,
   SERVICE_ENTRY,
   SOCKET_ENTRY,
+  normaliseDurableObject,
 } from "./plugins";
-import { HEADER_CUSTOM_SERVICE } from "./plugins/core";
+import { HEADER_CUSTOM_SERVICE, getUserServiceName } from "./plugins/core";
 import {
   Config,
   Runtime,
@@ -86,6 +87,33 @@ function validateOptions(
   return [pluginSharedOpts, pluginWorkerOpts];
 }
 
+// When creating user worker services, we need to know which Durable Objects
+// they export. Rather than parsing JavaScript to search for class exports
+// (which would have to be recursive because of `export * from ...`), we collect
+// all Durable Object bindings, noting that bindings may be defined for objects
+// in other services.
+function getDurableObjectClassNames(
+  allWorkerOpts: PluginWorkerOptions[]
+): Map<string, string[]> {
+  const serviceClassNames = new Map<string, string[]>();
+  for (const workerOpts of allWorkerOpts) {
+    const workerServiceName = getUserServiceName(workerOpts.core.name);
+    for (const designator of Object.values(
+      workerOpts.do.durableObjects ?? {}
+    )) {
+      // Fallback to current worker service if name not defined
+      const [className, serviceName = workerServiceName] =
+        normaliseDurableObject(designator);
+      let classNames = serviceClassNames.get(serviceName);
+      if (classNames === undefined) {
+        serviceClassNames.set(serviceName, (classNames = []));
+      }
+      classNames.push(className);
+    }
+  }
+  return serviceClassNames;
+}
+
 // ===== `Miniflare` Internal Storage & Routing =====
 type OptionalGatewayFactoryType<
   Gateway extends GatewayConstructor<any> | undefined
@@ -343,6 +371,8 @@ export class Miniflare {
       },
     ];
 
+    const durableObjectClassNames = getDurableObjectClassNames(allWorkerOpts);
+
     // Dedupe services by name
     const serviceNames = new Set<string>();
 
@@ -365,6 +395,7 @@ export class Miniflare {
           sharedOptions: sharedOpts[key],
           workerBindings,
           workerIndex: i,
+          durableObjectClassNames,
         });
         if (pluginServices !== undefined) {
           for (const service of pluginServices) {

packages/tre/src/plugins/core/index.ts

@@ -3,7 +3,7 @@ import fs from "fs/promises";
 import { Request, Response } from "undici";
 import { z } from "zod";
 import { Awaitable, JsonSchema } from "../../helpers";
-import { Service, Worker_Binding, Worker_Module } from "../../runtime";
+import { Service, Worker_Binding, Worker_Module, kVoid } from "../../runtime";
 import { BINDING_SERVICE_LOOPBACK, Plugin } from "../shared";
 import {
   ModuleDefinitionSchema,
@@ -107,6 +107,10 @@ const SERVICE_USER_PREFIX = `${CORE_PLUGIN_NAME}:user`;
 // Service prefix for custom fetch functions defined in `serviceBindings` option
 const SERVICE_CUSTOM_PREFIX = `${CORE_PLUGIN_NAME}:custom`;
 
+export function getUserServiceName(name = "") {
+  return `${SERVICE_USER_PREFIX}:${name}`;
+}
+
 export const HEADER_PROBE = "MF-Probe";
 export const HEADER_CUSTOM_SERVICE = "MF-Custom-Service";
 
@@ -141,12 +145,7 @@ export const SCRIPT_CUSTOM_SERVICE = `addEventListener("fetch", (event) => {
   event.respondWith(${BINDING_SERVICE_LOOPBACK}.fetch(request));
 })`;
 
-const now = new Date();
-const fallbackCompatibilityDate = [
-  now.getFullYear(),
-  (now.getMonth() + 1).toString().padStart(2, "0"),
-  now.getDate().toString().padStart(2, "0"),
-].join("-");
+const FALLBACK_COMPATIBILITY_DATE = "2000-01-01";
 
 export const CORE_PLUGIN: Plugin<
   typeof CoreOptionsSchema,
@@ -208,6 +207,7 @@ export const CORE_PLUGIN: Plugin<
     workerBindings,
     workerIndex,
     sharedOptions,
+    durableObjectClassNames,
   }) {
     // Define core/shared services.
     // Services get de-duped by name, so only the first worker's
@@ -224,23 +224,29 @@ export const CORE_PLUGIN: Plugin<
           serviceWorkerScript: SCRIPT_ENTRY,
           compatibilityDate: "2022-09-01",
           bindings: serviceEntryBindings,
-          compatibilityDate: "2022-09-01",
         },
       },
     ];
 
     // Define regular user worker if script is set
     const workerScript = getWorkerScript(options);
     if (workerScript !== undefined) {
-      const name = `${SERVICE_USER_PREFIX}:${options.name ?? ""}`;
+      const name = getUserServiceName(options.name);
+      const classNames = durableObjectClassNames.get(name) ?? [];
+
       services.push({
         name,
         worker: {
           ...workerScript,
           compatibilityDate:
-            options.compatibilityDate ?? fallbackCompatibilityDate,
+            options.compatibilityDate ?? FALLBACK_COMPATIBILITY_DATE,
           compatibilityFlags: options.compatibilityFlags,
           bindings: workerBindings,
+          durableObjectNamespaces: classNames.map((className) => ({
+            className,
+            uniqueKey: className,
+          })),
+          durableObjectStorage: { inMemory: kVoid },
         },
       });
       serviceEntryBindings.push({

packages/tre/src/plugins/do/index.ts

@@ -1,8 +1,14 @@
 import { z } from "zod";
+import { MiniflareError } from "../../helpers";
+import { Worker_Binding } from "../../runtime";
+import { getUserServiceName } from "../core";
 import { PersistenceSchema, Plugin } from "../shared";
 import { DurableObjectsStorageGateway } from "./gateway";
 import { DurableObjectsStorageRouter } from "./router";
 
+export type DurableObjectsErrorCode = "ERR_PERSIST_UNSUPPORTED"; // Durable Object persistence is not yet supported
+export class DurableObjectsError extends MiniflareError<DurableObjectsErrorCode> {}
+
 export const DurableObjectsOptionsSchema = z.object({
   durableObjects: z
     .record(
@@ -20,6 +26,20 @@ export const DurableObjectsSharedOptionsSchema = z.object({
   durableObjectsPersist: PersistenceSchema,
 });
 
+export function normaliseDurableObject(
+  designator: NonNullable<
+    z.infer<typeof DurableObjectsOptionsSchema>["durableObjects"]
+  >[string]
+): [className: string, serviceName: string | undefined] {
+  const isObject = typeof designator === "object";
+  const className = isObject ? designator.className : designator;
+  const serviceName =
+    isObject && designator.scriptName !== undefined
+      ? getUserServiceName(designator.scriptName)
+      : undefined;
+  return [className, serviceName];
+}
+
 export const DURABLE_OBJECTS_PLUGIN_NAME = "do";
 export const DURABLE_OBJECTS_PLUGIN: Plugin<
   typeof DurableObjectsOptionsSchema,
@@ -31,10 +51,29 @@ export const DURABLE_OBJECTS_PLUGIN: Plugin<
   options: DurableObjectsOptionsSchema,
   sharedOptions: DurableObjectsSharedOptionsSchema,
   getBindings(options) {
-    return undefined;
+    return Object.entries(options.durableObjects ?? {}).map<Worker_Binding>(
+      ([name, klass]) => {
+        const [className, serviceName] = normaliseDurableObject(klass);
+        return {
+          name,
+          durableObjectNamespace: { className, serviceName },
+        };
+      }
+    );
   },
-  getServices(options) {
-    return undefined;
+  getServices({ options, sharedOptions }) {
+    if (
+      // If we have Durable Object bindings...
+      Object.keys(options.durableObjects ?? {}).length > 0 &&
+      // ...and persistence is enabled...
+      sharedOptions.durableObjectsPersist
+    ) {
+      // ...throw, as Durable-Durable Objects are not yet supported
+      throw new DurableObjectsError(
+        "ERR_PERSIST_UNSUPPORTED",
+        "Persisted Durable Objects are not yet supported"
+      );
+    }
   },
 };
 

packages/tre/src/plugins/kv/index.ts

@@ -46,7 +46,7 @@ export const KV_PLUGIN: Plugin<
   sharedOptions: KVSharedOptionsSchema,
   async getBindings(options) {
     const bindings = Object.entries(
-      options.kvNamespaces ?? []
+      options.kvNamespaces ?? {}
     ).map<Worker_Binding>(([name, id]) => ({
       name,
       kvNamespace: { name: `${SERVICE_NAMESPACE_PREFIX}:${id}` },
@@ -75,7 +75,6 @@ export const KV_PLUGIN: Plugin<
               service: { name: SERVICE_LOOPBACK },
             },
           ],
-          compatibilityDate: "2022-09-01",
         },
       })
     );

packages/tre/src/plugins/shared/index.ts

@@ -4,6 +4,8 @@ import { Service, Worker_Binding } from "../../runtime";
 import { GatewayConstructor } from "./gateway";
 import { RouterConstructor } from "./router";
 
+export type DurableObjectClassNames = Map<string, string[]>;
+
 export interface PluginServicesOptions<
   Options extends z.ZodType,
   SharedOptions extends z.ZodType | undefined
@@ -13,19 +15,18 @@ export interface PluginServicesOptions<
   sharedOptions: OptionalZodTypeOf<SharedOptions>;
   workerBindings: Worker_Binding[];
   workerIndex: number;
+  durableObjectClassNames: DurableObjectClassNames;
 }
 
 export interface PluginBase<
   Options extends z.ZodType,
   SharedOptions extends z.ZodType | undefined
 > {
   options: Options;
-  getBindings(
-    options: z.infer<Options>
-  ): Awaitable<Worker_Binding[] | undefined>;
+  getBindings(options: z.infer<Options>): Awaitable<Worker_Binding[] | void>;
   getServices(
     options: PluginServicesOptions<Options, SharedOptions>
-  ): Awaitable<Service[] | undefined>;
+  ): Awaitable<Service[] | void>;
 }
 
 export type Plugin<

packages/tre/src/runtime/config/.gitignore

@@ -1,2 +1,3 @@
 # TODO: remove prior to release
 sserve-conf*
+workerd*