Skip to content

Commit 336a75d

Browse files
dxh9845Daniel Herzigpenalosa
authored
DEVX-2073: Support unsafe external plugins to Wrangler / Miniflare (#10119)
* DEVX-2073: Add stronger typing for unsafe bindings * DEVX-2073: Support 'unsafe' plugins in Miniflare as specified in unsafe.bindings Wrangler config * DEVX-2073: Support shared miniflare package for reusing zod and esbuild plugins between plugin implementations * DEVX-2073: Add fixtures showing an example of an unsafe external plugin * DEVX-2073: Add tests for unsafe external plugins * DEVX-2073: Update miniflare to better type unsafe bindings * Update ESBuild for Miniflare to build a single fixtures folder * Update tsconfigs to allow single Miniflare test fixture folder to be considered via TS * DEVX-2073: Add comments to PluginBase for more clarity on purpose * NOJIRA: Fixup lint / CI failures * chore: Fix failing formats * DEVX-2073: Add changelog for external Miniflare plugins * Fix unallowed any * chore: Address comments, remove superfluous options checks * NOJIRA: Fixup lockfile after rebase * Improve config & plugin loading * fix tests * fix lockfile * fix lockfile * fix lint * rename plugins export in comments * Remove miniflare-shared * DEVX-2073: Add tests to worker-with-unsafe-external-plugin fixture * fix vitest --------- Co-authored-by: Daniel Herzig <[email protected]> Co-authored-by: Samuel Macleod <[email protected]>
1 parent 51553ef commit 336a75d

File tree

39 files changed

+1212
-29
lines changed

39 files changed

+1212
-29
lines changed

‎.changeset/rich-cups-refuse.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
"miniflare": minor
3+
"wrangler": minor
4+
---
5+
6+
Add support for dynamically loading 'external' Miniflare plugins for unsafe Worker bindings (developed outside of the workers-sdk repo)

‎fixtures/unsafe-external-plugin/README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# Unsafe External Plugin
2+
3+
This folder contains an example of an Unsafe External Miniflare Plugin that extends Miniflare's suite of plugins with local development simulators. An example of usage can be found in the [worker-with-unsafe-exteral-plugin fixture](../worker-with-unsafe-external-plugin/).

‎fixtures/unsafe-external-plugin/package.json‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
{
2+
"name": "@fixture/unsafe-external-plugin",
3+
"private": true,
4+
"description": "An example of an unsafe external plugin for Miniflare",
5+
"license": "ISC",
6+
"author": "",
7+
"main": "dist/index.js",
8+
"scripts": {
9+
"build": "tsx tools/build.ts"
10+
},
11+
"devDependencies": {
12+
"@cloudflare/workers-tsconfig": "workspace:*",
13+
"@cloudflare/workers-types": "catalog:default",
14+
"esbuild": "catalog:default",
15+
"miniflare": "workspace:*",
16+
"tsx": "^3.12.8",
17+
"zod": "3.22.3"
18+
}
19+
}

‎fixtures/unsafe-external-plugin/src/index.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
import {
2+
UNSAFE_PLUGIN_NAME,
3+
UNSAFE_SERVICE_PLUGIN,
4+
} from "./plugins/unsafe-service";
5+
6+
export const plugins = {
7+
[UNSAFE_PLUGIN_NAME]: UNSAFE_SERVICE_PLUGIN,
8+
};

‎fixtures/unsafe-external-plugin/src/plugins/unsafe-service.ts‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
import fs from "node:fs/promises";
2+
import {
3+
getMiniflareObjectBindings,
4+
getPersistPath,
5+
Plugin,
6+
ProxyNodeBinding,
7+
SERVICE_LOOPBACK,
8+
SharedBindings,
9+
} from "miniflare";
10+
// The below imports (prefixed with `worker:`)
11+
// will be converted by our ESBuild plugin
12+
// into functions that load the transpiled Workers as JS
13+
import BINDING_WORKER from "worker:binding.worker";
14+
import OBJECT_WORKER from "worker:object.worker";
15+
import { z } from "zod";
16+
import type { Service, Worker_Binding } from "miniflare";
17+
18+
export const UNSAFE_PLUGIN_NAME = "unsafe-plugin";
19+
20+
export const UnsafeServiceBindingOptionSchema = z
21+
.array(
22+
z.object({
23+
name: z.string(),
24+
type: z.string(),
25+
plugin: z.object({
26+
package: z.string(),
27+
name: z.string(),
28+
}),
29+
options: z.object({ emitLogs: z.boolean() }),
30+
})
31+
)
32+
.or(z.undefined());
33+
34+
export const UNSAFE_SERVICE_PLUGIN: Plugin<
35+
typeof UnsafeServiceBindingOptionSchema
36+
> = {
37+
options: UnsafeServiceBindingOptionSchema,
38+
/**
39+
* getBindings will add bindings to the user's Workers. Specifically, we add a binding to a service
40+
* that will expose an `UnsafeBindingServiceEntrypoint`
41+
* @param options - A map of bindings names to options provided for that binding.
42+
* @returns
43+
*/
44+
async getBindings(options) {
45+
return options?.map<Worker_Binding>((binding) => {
46+
return {
47+
name: binding.name,
48+
service: {
49+
name: `${UNSAFE_PLUGIN_NAME}:${binding.name}`,
50+
entrypoint: "UnsafeBindingServiceEntrypoint",
51+
},
52+
};
53+
});
54+
},
55+
getNodeBindings(options) {
56+
return Object.fromEntries(
57+
options?.map((binding) => [binding.name, new ProxyNodeBinding()]) ?? []
58+
);
59+
},
60+
async getServices({
61+
options,
62+
tmpPath,
63+
defaultPersistRoot,
64+
unsafeStickyBlobs,
65+
}) {
66+
if (!options || options.length === 0) {
67+
return [];
68+
}
69+
70+
const persistPath = getPersistPath(
71+
UNSAFE_PLUGIN_NAME,
72+
tmpPath,
73+
defaultPersistRoot,
74+
undefined
75+
);
76+
77+
await fs.mkdir(persistPath, { recursive: true });
78+
79+
// Create a service that will persist any data
80+
const storageService = {
81+
name: `${UNSAFE_PLUGIN_NAME}:storage`,
82+
disk: { path: persistPath, writable: true },
83+
} satisfies Service;
84+
85+
const objectService = {
86+
name: `${UNSAFE_PLUGIN_NAME}:object`,
87+
worker: {
88+
compatibilityDate: "2025-01-01",
89+
modules: [
90+
{
91+
name: "object.worker.js",
92+
esModule: OBJECT_WORKER(),
93+
},
94+
],
95+
durableObjectNamespaces: [
96+
{
97+
className: "UnsafeBindingObject",
98+
uniqueKey: `miniflare-unsafe-binding-UnsafeBindingObject`,
99+
},
100+
],
101+
// Store Durable Object SQL databases in persist path
102+
durableObjectStorage: { localDisk: storageService.name },
103+
// Bind blob disk directory service to object
104+
bindings: [
105+
{
106+
name: SharedBindings.MAYBE_SERVICE_BLOBS,
107+
service: { name: storageService.name },
108+
},
109+
{
110+
name: SharedBindings.MAYBE_SERVICE_LOOPBACK,
111+
service: { name: SERVICE_LOOPBACK },
112+
},
113+
...getMiniflareObjectBindings(unsafeStickyBlobs),
114+
],
115+
},
116+
} satisfies Service;
117+
118+
const bindingWorker = options.map<Service>(
119+
(binding) =>
120+
({
121+
name: `${UNSAFE_PLUGIN_NAME}:${binding.name}`,
122+
worker: {
123+
compatibilityDate: "2025-01-01",
124+
modules: [
125+
{
126+
name: "binding.worker.js",
127+
esModule: BINDING_WORKER(),
128+
},
129+
],
130+
bindings: [
131+
{
132+
name: "config",
133+
json: JSON.stringify(binding.options),
134+
},
135+
{
136+
name: "store",
137+
durableObjectNamespace: {
138+
className: "UnsafeBindingObject",
139+
serviceName: objectService.name,
140+
},
141+
},
142+
],
143+
},
144+
}) satisfies Service
145+
);
146+
147+
return [...bindingWorker, storageService, objectService];
148+
},
149+
};

‎fixtures/unsafe-external-plugin/src/plugins/worker-shim.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
/**
2+
* ESBuild will build the Workers from '../workers' and provide the built script
3+
* files as variables on the global scope.
4+
*/
5+
declare module "worker:*" {
6+
export default function (): string;
7+
}

‎fixtures/unsafe-external-plugin/src/workers/binding.worker.ts‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
import { WorkerEntrypoint } from "cloudflare:workers";
2+
import type { UnsafeBindingObject } from "./object.worker";
3+
4+
// ENV configuration
5+
interface Env {
6+
config: { emitLogs: boolean };
7+
store: DurableObjectNamespace<UnsafeBindingObject>;
8+
}
9+
10+
/**
11+
* UnsafeBinding offers two RPCs, `performUnsafeWrite` and `performUnsafeRead`.
12+
*/
13+
export class UnsafeBindingServiceEntrypoint extends WorkerEntrypoint<Env> {
14+
override async fetch(_request: Request): Promise<Response> {
15+
return new Response("This is a development stub for an unsafe worker.", {
16+
status: 200,
17+
statusText: "OK",
18+
headers: {
19+
"content-type": "text/plain",
20+
},
21+
});
22+
}
23+
24+
async performUnsafeWrite(key: string, value: number) {
25+
if (this.env.config.emitLogs) {
26+
console.log("Emitting a log for write operation");
27+
}
28+
const objectNamespace = this.env.store;
29+
const namespaceId = JSON.stringify(this.env.config);
30+
const id = objectNamespace.idFromName(namespaceId);
31+
const stub = objectNamespace.get(id);
32+
await stub.set(key, value);
33+
34+
return {
35+
ok: true,
36+
result: `Set key ${key} to ${value}`,
37+
meta: {
38+
workersVersion: "my-version-from-dev",
39+
},
40+
};
41+
}
42+
43+
async performUnsafeRead(key: string) {
44+
if (this.env.config.emitLogs) {
45+
console.log("Emitting a log for read operation");
46+
}
47+
const objectNamespace = this.env.store;
48+
const namespaceId = JSON.stringify(this.env.config);
49+
const id = objectNamespace.idFromName(namespaceId);
50+
const stub = objectNamespace.get(id);
51+
const value = await stub.get(key);
52+
53+
return {
54+
ok: true,
55+
result: value,
56+
meta: {
57+
workersVersion: "my-version-from-dev",
58+
},
59+
};
60+
}
61+
}

‎fixtures/unsafe-external-plugin/src/workers/object.worker.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
import { DurableObject } from "cloudflare:workers";
2+
3+
export class UnsafeBindingObject extends DurableObject {
4+
async get(tag: string) {
5+
return await this.ctx.storage.get<number>(tag);
6+
}
7+
8+
async set(key: string, value: number) {
9+
await this.ctx.storage.put<number>(key, value);
10+
}
11+
}

‎fixtures/unsafe-external-plugin/tools/build.ts‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
import { join, resolve } from "node:path";
2+
import {
3+
BuildContext,
4+
BuildOptions,
5+
context,
6+
Plugin,
7+
build as runBuild,
8+
} from "esbuild";
9+
10+
type EmbedWorkersOptions = {
11+
/**
12+
* workersRootDir is a path to a directory containing external Workers that
13+
* will be bundled
14+
*/
15+
workersRootDir: string;
16+
workerOutputDir: string;
17+
};
18+
19+
/**
20+
* embedWorkerPlugin is an ESBuild plugin that will look for imports to Workers specified by `worker:`
21+
* and bundle them into the final output.
22+
*/
23+
export const embedWorkersPlugin: (options: EmbedWorkersOptions) => Plugin = ({
24+
workerOutputDir,
25+
workersRootDir,
26+
}) => {
27+
return {
28+
name: "embed-workers",
29+
setup(build) {
30+
const namespace = "embed-worker";
31+
// For imports prefixed with `worker:`, attempt to resolve them from a directory containing
32+
// your Workers
33+
build.onResolve({ filter: /^worker:/ }, async (args) => {
34+
let name = args.path.substring("worker:".length);
35+
// Allow `.worker` to be omitted
36+
if (!name.endsWith(".worker")) name += ".worker";
37+
// Use `build.resolve()` API so Workers can be written as `m?[jt]s` files
38+
const result = await build.resolve("./" + name, {
39+
kind: "import-statement",
40+
// Resolve relative to the directory containing the Workers
41+
resolveDir: workersRootDir,
42+
});
43+
if (result.errors.length > 0) return { errors: result.errors };
44+
return { path: result.path, namespace };
45+
});
46+
build.onLoad({ filter: /.*/, namespace }, async (args) => {
47+
await runBuild({
48+
platform: "node", // Marks `node:*` imports as external
49+
format: "esm",
50+
target: "esnext",
51+
bundle: true,
52+
sourcemap: true,
53+
sourcesContent: false,
54+
external: ["cloudflare:workers"],
55+
entryPoints: [args.path],
56+
minifySyntax: true,
57+
outdir: workerOutputDir,
58+
plugins: [],
59+
});
60+
61+
let outPath = args.path.substring(workersRootDir.length + 1);
62+
outPath = outPath.substring(0, outPath.lastIndexOf(".")) + ".js";
63+
outPath = JSON.stringify(outPath);
64+
const contents = `
65+
import fs from "fs";
66+
import path from "path";
67+
import url from "url";
68+
let contents;
69+
export default function() {
70+
if (contents !== undefined) return contents;
71+
const filePath = path.join(__dirname, "workers", ${outPath});
72+
contents = fs.readFileSync(filePath, "utf8") + "//# sourceURL=" + url.pathToFileURL(filePath);
73+
return contents;
74+
}
75+
`;
76+
return { contents, loader: "js" };
77+
});
78+
},
79+
};
80+
};
81+
82+
const distDir = resolve(__dirname, "../dist");
83+
// When outputting the Worker, map to the structure of 'src'.
84+
// This means the plugin will outout the build Workers to a `workers` dist in `dir`
85+
const workerOutputDir = resolve(distDir, "workers");
86+
const workersDir = resolve(__dirname, "../src/workers");
87+
88+
export async function buildPackage() {
89+
console.log("Building the module");
90+
await runBuild({
91+
platform: "node",
92+
target: "esnext",
93+
format: "cjs",
94+
bundle: true,
95+
sourcemap: true,
96+
entryPoints: ["src/index.ts"],
97+
plugins: [
98+
embedWorkersPlugin({
99+
workersRootDir: workersDir,
100+
workerOutputDir,
101+
}),
102+
],
103+
external: ["@cloudflare/workers-types", "miniflare"],
104+
outdir: distDir,
105+
});
106+
}
107+
108+
buildPackage().catch((exc) => {
109+
console.error("Failed to build external package", `${exc}`);
110+
});

‎fixtures/unsafe-external-plugin/tsconfig.json‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{
2+
"compilerOptions": {
3+
"target": "es2021",
4+
"lib": ["es2021"],
5+
"module": "es2022",
6+
"types": ["@cloudflare/workers-types/experimental"],
7+
"noEmit": true,
8+
"moduleResolution": "bundler",
9+
"isolatedModules": true,
10+
"forceConsistentCasingInFileNames": true,
11+
"strict": true,
12+
"skipLibCheck": true
13+
}
14+
}

0 commit comments

Comments
 (0)