Skip to content
This repository was archived by the owner on Mar 13, 2025. It is now read-only.

Commit ff3c347

Browse files
mrbbotmrbbot
committed
Add KV/Workers Sites plugin
1 parent 3ee4d22 commit ff3c347

File tree

4 files changed

+486
-0
lines changed

4 files changed

+486
-0
lines changed

‎packages/tre/src/plugins/kv/constants.ts‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
export const MIN_CACHE_TTL = 60; /* 60s */
2+
export const MAX_LIST_KEYS = 1000;
3+
export const MAX_KEY_SIZE = 512; /* 512B */
4+
export const MAX_VALUE_SIZE = 25 * 1024 * 1024; /* 25MiB */
5+
export const MAX_METADATA_SIZE = 1024; /* 1KiB */
6+
7+
export const PARAM_URL_ENCODED = "urlencoded";
8+
export const PARAM_CACHE_TTL = "cache_ttl";
9+
export const PARAM_EXPIRATION = "expiration";
10+
export const PARAM_EXPIRATION_TTL = "expiration_ttl";
11+
export const PARAM_LIST_LIMIT = "key_count_limit";
12+
export const PARAM_LIST_PREFIX = "prefix";
13+
export const PARAM_LIST_CURSOR = "cursor";
14+
15+
export const HEADER_EXPIRATION = "CF-Expiration";
16+
export const HEADER_METADATA = "CF-KV-Metadata";
17+
export const HEADER_SITES = "MF-Sites";

‎packages/tre/src/plugins/kv/gateway.ts‎

Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
import {
2+
Clock,
3+
Storage,
4+
StoredKeyMeta,
5+
StoredValueMeta,
6+
millisToSeconds,
7+
} from "@miniflare/shared";
8+
import { HttpError } from "../../helpers";
9+
import {
10+
MAX_KEY_SIZE,
11+
MAX_LIST_KEYS,
12+
MAX_METADATA_SIZE,
13+
MAX_VALUE_SIZE,
14+
MIN_CACHE_TTL,
15+
PARAM_CACHE_TTL,
16+
PARAM_EXPIRATION,
17+
PARAM_EXPIRATION_TTL,
18+
PARAM_LIST_LIMIT,
19+
} from "./constants";
20+
21+
export class KVError extends HttpError {}
22+
23+
function validateKey(key: string): void {
24+
if (key === "") {
25+
throw new KVError(400, "Key names must not be empty");
26+
}
27+
if (key === "." || key === "..") {
28+
throw new KVError(
29+
400,
30+
`Illegal key name "${key}". Please use a different name.`
31+
);
32+
}
33+
validateKeyLength(key);
34+
}
35+
36+
function validateKeyLength(key: string): void {
37+
const keyLength = Buffer.byteLength(key);
38+
if (keyLength > MAX_KEY_SIZE) {
39+
throw new KVError(
40+
414,
41+
`UTF-8 encoded length of ${keyLength} exceeds key length limit of ${MAX_KEY_SIZE}.`
42+
);
43+
}
44+
}
45+
46+
// Returns value as an integer or undefined if it isn't one
47+
function normaliseInt(value: string | number | undefined): number | undefined {
48+
switch (typeof value) {
49+
case "string":
50+
return parseInt(value);
51+
case "number":
52+
return Math.round(value);
53+
}
54+
}
55+
56+
export interface KVGatewayGetOptions {
57+
cacheTtl?: number;
58+
}
59+
60+
export interface KVGatewayPutOptions<Meta = unknown> {
61+
expiration?: string | number;
62+
expirationTtl?: string | number;
63+
metadata?: Meta;
64+
}
65+
66+
export interface KVGatewayListOptions {
67+
limit?: number;
68+
prefix?: string;
69+
cursor?: string;
70+
}
71+
export interface KVGatewayListResult<Meta = unknown> {
72+
keys: StoredKeyMeta<Meta>[];
73+
cursor: string;
74+
list_complete: boolean;
75+
}
76+
77+
export class KVGateway {
78+
constructor(
79+
private readonly storage: Storage,
80+
private readonly clock: Clock
81+
) {}
82+
83+
async get(
84+
key: string,
85+
options?: KVGatewayGetOptions
86+
): Promise<StoredValueMeta | undefined> {
87+
validateKey(key);
88+
// Validate cacheTtl, but ignore it as there's only one "edge location":
89+
// the user's computer
90+
if (options?.cacheTtl !== undefined) {
91+
throw new KVError(
92+
400,
93+
`Invalid ${PARAM_CACHE_TTL} of ${options.cacheTtl}. Cache TTL must be at least ${MIN_CACHE_TTL}.`
94+
);
95+
}
96+
return this.storage.get(key);
97+
}
98+
99+
async put(
100+
key: string,
101+
value: Uint8Array,
102+
options: KVGatewayPutOptions = {}
103+
): Promise<void> {
104+
validateKey(key);
105+
106+
// Normalise and validate expiration
107+
const now = millisToSeconds(this.clock());
108+
let expiration = normaliseInt(options.expiration);
109+
const expirationTtl = normaliseInt(options.expirationTtl);
110+
if (expirationTtl !== undefined) {
111+
if (isNaN(expirationTtl) || expirationTtl <= 0) {
112+
throw new KVError(
113+
400,
114+
`Invalid ${PARAM_EXPIRATION_TTL} of ${options.expirationTtl}. Please specify integer greater than 0.`
115+
);
116+
}
117+
if (expirationTtl < MIN_CACHE_TTL) {
118+
throw new KVError(
119+
400,
120+
`Invalid ${PARAM_EXPIRATION_TTL} of ${options.expirationTtl}. Expiration TTL must be at least ${MIN_CACHE_TTL}.`
121+
);
122+
}
123+
expiration = now + expirationTtl;
124+
} else if (expiration !== undefined) {
125+
if (isNaN(expiration) || expiration <= now) {
126+
throw new KVError(
127+
400,
128+
`Invalid ${PARAM_EXPIRATION} of ${options.expiration}. Please specify integer greater than the current number of seconds since the UNIX epoch.`
129+
);
130+
}
131+
if (expiration < now + MIN_CACHE_TTL) {
132+
throw new KVError(
133+
400,
134+
`Invalid ${PARAM_EXPIRATION} of ${options.expiration}. Expiration times must be at least ${MIN_CACHE_TTL} seconds in the future.`
135+
);
136+
}
137+
}
138+
139+
// Validate value and metadata size
140+
if (value.byteLength > MAX_VALUE_SIZE) {
141+
throw new KVError(
142+
413,
143+
`Value length of ${value.byteLength} exceeds limit of ${MAX_VALUE_SIZE}.`
144+
);
145+
}
146+
if (options.metadata !== undefined) {
147+
const metadataJSON = JSON.stringify(options.metadata);
148+
const metadataLength = Buffer.byteLength(metadataJSON);
149+
if (metadataLength > MAX_METADATA_SIZE) {
150+
throw new KVError(
151+
413,
152+
`Metadata length of ${metadataLength} exceeds limit of ${MAX_METADATA_SIZE}.`
153+
);
154+
}
155+
}
156+
157+
return this.storage.put(key, {
158+
value,
159+
expiration,
160+
metadata: options.metadata,
161+
});
162+
}
163+
164+
async delete(key: string): Promise<void> {
165+
validateKey(key);
166+
await this.storage.delete(key);
167+
}
168+
169+
async list(options: KVGatewayListOptions = {}): Promise<KVGatewayListResult> {
170+
// Validate key limit
171+
const limit = options.limit ?? MAX_LIST_KEYS;
172+
if (isNaN(limit) || limit < 1) {
173+
throw new KVError(
174+
400,
175+
`Invalid ${PARAM_LIST_LIMIT} of ${limit}. Please specify an integer greater than 0.`
176+
);
177+
}
178+
if (limit > MAX_LIST_KEYS) {
179+
throw new KVError(
180+
400,
181+
`Invalid ${PARAM_LIST_LIMIT} of ${limit}. Please specify an integer less than ${MAX_LIST_KEYS}.`
182+
);
183+
}
184+
185+
// Validate key prefix
186+
const prefix = options.prefix;
187+
if (prefix !== undefined) validateKeyLength(prefix);
188+
189+
const cursor = options.cursor;
190+
const res = await this.storage.list({ limit, prefix, cursor });
191+
return {
192+
keys: res.keys,
193+
cursor: res.cursor,
194+
list_complete: res.cursor === "",
195+
};
196+
}
197+
}

‎packages/tre/src/plugins/kv/index.ts‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
import { z } from "zod";
2+
import { Service, Worker_Binding } from "../../runtime";
3+
import { SERVICE_LOOPBACK } from "../core";
4+
import {
5+
BINDING_SERVICE_LOOPBACK,
6+
BINDING_TEXT_NAMESPACE,
7+
BINDING_TEXT_PERSIST,
8+
BINDING_TEXT_PLUGIN,
9+
HEADER_PERSIST,
10+
PersistenceSchema,
11+
Plugin,
12+
SCRIPT_PLUGIN_NAMESPACE_PERSIST,
13+
encodePersist,
14+
} from "../shared";
15+
import { HEADER_SITES } from "./constants";
16+
import { KVGateway } from "./gateway";
17+
import { KVRouter } from "./router";
18+
19+
export const KVOptionsSchema = z.object({
20+
kvNamespaces: z.record(z.string()).optional(),
21+
22+
// Workers Sites
23+
sitePath: z.string().optional(),
24+
siteInclude: z.string().array().optional(),
25+
siteExclude: z.string().array().optional(),
26+
});
27+
export const KVSharedOptionsSchema = z.object({
28+
kvPersist: PersistenceSchema,
29+
});
30+
31+
export const KV_PLUGIN_NAME = "kv";
32+
const SERVICE_NAMESPACE_PREFIX = `${KV_PLUGIN_NAME}:ns`;
33+
const SERVICE_NAMESPACE_SITE = `${KV_PLUGIN_NAME}:site`;
34+
35+
// Workers Sites
36+
const BINDING_KV_NAMESPACE_SITE = "__STATIC_CONTENT";
37+
const BINDING_JSON_SITE_MANIFEST = "__STATIC_CONTENT_MANIFEST";
38+
// TODO: add header(s) for key filter, then filter keys in KVRouter
39+
const SCRIPT_SITE = `addEventListener("fetch", (event) => {
40+
let request = event.request;
41+
42+
if (request.method === "PUT" || request.method === "DELETE") {
43+
const message = \`Cannot \${request.method.toLowerCase()}() with read-only Workers Sites namespace\`;
44+
return event.respondWith(new Response(message, {
45+
status: 400,
46+
statusText: message,
47+
}));
48+
}
49+
50+
const url = new URL(event.request.url);
51+
url.pathname = \`/${KV_PLUGIN_NAME}/${BINDING_KV_NAMESPACE_SITE}\${url.pathname}\`;
52+
53+
request = new Request(url, event.request);
54+
request.headers.set("${HEADER_PERSIST}", ${BINDING_TEXT_PERSIST});
55+
request.headers.set("${HEADER_SITES}", "true");
56+
57+
event.respondWith(${BINDING_SERVICE_LOOPBACK}.fetch(request));
58+
})`;
59+
60+
export const KV_PLUGIN: Plugin<
61+
typeof KVOptionsSchema,
62+
typeof KVSharedOptionsSchema,
63+
KVGateway
64+
> = {
65+
gateway: KVGateway,
66+
router: KVRouter,
67+
options: KVOptionsSchema,
68+
sharedOptions: KVSharedOptionsSchema,
69+
getBindings(options) {
70+
const bindings = Object.entries(
71+
options.kvNamespaces ?? []
72+
).map<Worker_Binding>(([name, id]) => ({
73+
name,
74+
kvNamespace: { name: `${SERVICE_NAMESPACE_PREFIX}:${id}` },
75+
}));
76+
77+
if (options.sitePath !== undefined) {
78+
bindings.push(
79+
{
80+
name: BINDING_KV_NAMESPACE_SITE,
81+
kvNamespace: { name: SERVICE_NAMESPACE_SITE },
82+
},
83+
// TODO: actually populate manifest here, respecting key filters:
84+
// - https://github.com/cloudflare/miniflare/issues/233
85+
// - https://github.com/cloudflare/miniflare/issues/326
86+
{ name: BINDING_JSON_SITE_MANIFEST, json: "{}" }
87+
);
88+
}
89+
90+
return bindings;
91+
},
92+
getServices({ options, sharedOptions }) {
93+
const persistBinding = encodePersist(sharedOptions.kvPersist);
94+
const loopbackBinding: Worker_Binding = {
95+
name: BINDING_SERVICE_LOOPBACK,
96+
service: { name: SERVICE_LOOPBACK },
97+
};
98+
const services = Object.entries(options.kvNamespaces ?? []).map<Service>(
99+
([_, id]) => ({
100+
name: `${SERVICE_NAMESPACE_PREFIX}:${id}`,
101+
worker: {
102+
serviceWorkerScript: SCRIPT_PLUGIN_NAMESPACE_PERSIST,
103+
bindings: [
104+
...persistBinding,
105+
{ name: BINDING_TEXT_PLUGIN, text: KV_PLUGIN_NAME },
106+
{ name: BINDING_TEXT_NAMESPACE, text: id },
107+
loopbackBinding,
108+
],
109+
},
110+
})
111+
);
112+
113+
if (options.sitePath !== undefined) {
114+
if (
115+
options.siteInclude !== undefined ||
116+
options.siteExclude !== undefined
117+
) {
118+
throw new Error("Workers Sites include/exclude not yet implemented!");
119+
}
120+
121+
services.push({
122+
name: SERVICE_NAMESPACE_SITE,
123+
worker: {
124+
serviceWorkerScript: SCRIPT_SITE,
125+
bindings: [
126+
{
127+
name: BINDING_TEXT_PERSIST,
128+
text: JSON.stringify(options.sitePath),
129+
},
130+
loopbackBinding,
131+
],
132+
},
133+
});
134+
}
135+
136+
return services;
137+
},
138+
};
139+
140+
export * from "./gateway";

0 commit comments

Comments
 (0)