Populate __STATIC_CONTENT_MANIFEST, closes #233

`__STATIC_CONTENT_MANIFEST` was previously empty to disable
edge-caching. The manifest is now populated, with values having a
magic prefix. URLs starting with this magic prefix are never cached.

Closes #326.
Closes cloudflare/workers-sdk#1632.

Author: mrbbot

packages/cache/src/cache.ts

@@ -9,6 +9,7 @@ import {
 import {
   Awaitable,
   Clock,
+  SITES_NO_CACHE_PREFIX,
   Storage,
   assertInRequest,
   defaultClock,
@@ -184,6 +185,11 @@ export class Cache implements CacheInterface {
       throw new TypeError("Cannot cache response with 'Vary: *' header.");
     }
 
+    // Disable caching of Workers Sites files, so we always serve the latest
+    // version from disk
+    const url = new URL(req.url);
+    if (url.pathname.startsWith("/" + SITES_NO_CACHE_PREFIX)) return;
+
     // Check if response cacheable and get expiration TTL if any
     const expirationTtl = getExpirationTtl(this.#clock, req, res);
     if (expirationTtl === undefined) return;

packages/shared/src/data.ts

@@ -38,6 +38,8 @@ export function randomHex(digits = 8): string {
     .join("");
 }
 
+export const SITES_NO_CACHE_PREFIX = "$__MINIFLARE_SITES__$/";
+
 // Arbitrary string matcher, note RegExp adheres to this interface
 export interface Matcher {
   test(string: string): boolean;

packages/sites/src/filtered.ts

@@ -12,8 +12,14 @@ import {
 } from "@miniflare/kv";
 import { Matcher, Storage } from "@miniflare/shared";
 
+export interface KeyMapper {
+  lookup(key: string): string;
+  reverseLookup(key: string): string;
+}
+
 export interface FilteredKVStorageNamespaceOptions {
   readOnly?: boolean;
+  map?: KeyMapper;
   include?: Matcher;
   exclude?: Matcher;
 }
@@ -41,6 +47,7 @@ export class FilteredKVNamespace extends KVNamespace {
     key: string,
     options?: KVGetValueType | Partial<KVGetOptions>
   ): KVValue<any> {
+    key = this.#options.map?.lookup(key) ?? key;
     if (!this.#included(key)) return Promise.resolve(null);
     return super.get(key, options as any);
   }
@@ -49,6 +56,7 @@ export class FilteredKVNamespace extends KVNamespace {
     key: string,
     options?: KVGetValueType | Partial<KVGetOptions>
   ): KVValueMeta<any, Meta> {
+    key = this.#options.map?.lookup(key) ?? key;
     if (!this.#included(key)) {
       return Promise.resolve({ value: null, metadata: null });
     }
@@ -63,13 +71,15 @@ export class FilteredKVNamespace extends KVNamespace {
     if (this.#options.readOnly) {
       throw new TypeError("Unable to put into read-only namespace");
     }
+    key = this.#options.map?.lookup(key) ?? key;
     return super.put(key, value, options);
   }
 
   async delete(key: string): Promise<void> {
     if (this.#options.readOnly) {
       throw new TypeError("Unable to delete from read-only namespace");
     }
+    key = this.#options.map?.lookup(key) ?? key;
     return super.delete(key);
   }
 
@@ -78,7 +88,13 @@ export class FilteredKVNamespace extends KVNamespace {
   ): Promise<KVListResult<Meta>> {
     const { keys, list_complete, cursor } = await super.list<Meta>(options);
     return {
-      keys: keys.filter((key) => this.#included(key.name)),
+      keys: keys.filter((key) => {
+        if (!this.#included(key.name)) return false;
+        if (this.#options.map !== undefined) {
+          key.name = this.#options.map.reverseLookup(key.name);
+        }
+        return true;
+      }),
       list_complete,
       cursor,
     };

packages/sites/src/plugin.ts

@@ -1,20 +1,37 @@
+import assert from "assert";
 import path from "path";
 import {
+  Matcher,
   Option,
   OptionType,
   Plugin,
   PluginContext,
+  SITES_NO_CACHE_PREFIX,
   SetupResult,
   globsToMatcher,
 } from "@miniflare/shared";
-import { FilteredKVNamespace } from "./filtered";
+import type { FileStorage } from "@miniflare/storage-file";
+import { FilteredKVNamespace, KeyMapper } from "./filtered";
 
 export interface SitesOptions {
   sitePath?: string;
   siteInclude?: string[];
   siteExclude?: string[];
 }
 
+const SITES_KEY_MAPPER: KeyMapper = {
+  lookup(key: string): string {
+    return key.startsWith(SITES_NO_CACHE_PREFIX)
+      ? decodeURIComponent(key.substring(SITES_NO_CACHE_PREFIX.length))
+      : key;
+  },
+  reverseLookup(key: string): string {
+    // `encodeURIComponent()` ensures `E-Tag`s used by `@cloudflare/kv-asset-handler`
+    // are always byte strings, as required by `undici`
+    return SITES_NO_CACHE_PREFIX + encodeURIComponent(key);
+  },
+};
+
 export class SitesPlugin extends Plugin<SitesOptions> implements SitesOptions {
   @Option({
     type: OptionType.STRING,
@@ -42,48 +59,72 @@ export class SitesPlugin extends Plugin<SitesOptions> implements SitesOptions {
   })
   siteExclude?: string[];
 
-  readonly #setupResult: Promise<SetupResult>;
+  readonly #include?: Matcher;
+  readonly #exclude?: Matcher;
+
+  readonly #resolvedSitePath?: string;
+  readonly #storage?: FileStorage;
+  readonly #__STATIC_CONTENT?: FilteredKVNamespace;
 
   constructor(ctx: PluginContext, options?: SitesOptions) {
     super(ctx);
     this.assignOptions(options);
+    if (!this.sitePath) return;
 
-    // setup() will be called each time a site file changes, but there's no need
-    // to recreate the namespace each time, so create it once and then return it
-    this.#setupResult = this.#setup();
-  }
+    // Lots of sites stuff is constant between reloads, so initialise it once
 
-  async #setup(): Promise<SetupResult> {
-    if (!this.sitePath) return {};
+    // Create include/exclude filters
+    this.#include = this.siteInclude && globsToMatcher(this.siteInclude);
+    this.#exclude = this.siteExclude && globsToMatcher(this.siteExclude);
 
     // Create file KV storage with sanitisation DISABLED so paths containing
     // /'s resolve correctly
     const {
       FileStorage,
     }: typeof import("@miniflare/storage-file") = require("@miniflare/storage-file");
-    const sitePath = path.resolve(this.ctx.rootPath, this.sitePath);
-    const storage = new FileStorage(sitePath, false);
+    this.#resolvedSitePath = path.resolve(this.ctx.rootPath, this.sitePath);
+    this.#storage = new FileStorage(this.#resolvedSitePath, false);
+
+    // Build KV namespace that strips prefix, and only returns matched keys
+    this.#__STATIC_CONTENT = new FilteredKVNamespace(this.#storage, {
+      readOnly: true,
+      map: SITES_KEY_MAPPER,
+      include: this.#include,
+      exclude: this.#exclude,
+    });
+  }
+
+  async setup(): Promise<SetupResult> {
+    if (!this.sitePath) return {};
+    assert(
+      this.#resolvedSitePath !== undefined &&
+        this.#storage !== undefined &&
+        this.#__STATIC_CONTENT !== undefined
+    );
+
+    // Build manifest, including prefix to disable caching of sites files
+    const staticContentManifest: Record<string, string> = {};
+    const result = await this.#storage.list();
+    assert.strictEqual(result.cursor, "");
+    for (const { name } of result.keys) {
+      if (this.#include !== undefined && !this.#include.test(name)) continue;
+      if (this.#exclude !== undefined && this.#exclude.test(name)) continue;
+      staticContentManifest[name] = SITES_KEY_MAPPER.reverseLookup(name);
+    }
+    const __STATIC_CONTENT_MANIFEST = JSON.stringify(staticContentManifest);
+
     const bindings = {
-      __STATIC_CONTENT: new FilteredKVNamespace(storage, {
-        readOnly: true,
-        include: this.siteInclude && globsToMatcher(this.siteInclude),
-        exclude: this.siteExclude && globsToMatcher(this.siteExclude),
-      }),
-      // Empty manifest means @cloudflare/kv-asset-handler will use the request
-      // path as the file path and won't edge cache files
-      __STATIC_CONTENT_MANIFEST: {},
+      __STATIC_CONTENT: this.#__STATIC_CONTENT,
+      __STATIC_CONTENT_MANIFEST,
     };
     // Allow `import manifest from "__STATIC_CONTENT_MANIFEST"`
     const additionalModules = {
-      __STATIC_CONTENT_MANIFEST: { default: "{}" },
+      __STATIC_CONTENT_MANIFEST: { default: __STATIC_CONTENT_MANIFEST },
     };
 
     // Whilst FileStorage will always serve the latest files, we want to
-    // force a reload when these files change for live reload.
-    return { bindings, watch: [sitePath], additionalModules };
-  }
-
-  async setup(): Promise<SetupResult> {
-    return this.#setupResult;
+    // force a reload when these files change for live reload and to rebuild
+    // the manifest.
+    return { bindings, watch: [this.#resolvedSitePath], additionalModules };
   }
 }