Support siteInclude/siteExclude options and populate __STATIC_CONTENT_MANIFEST (#383)

* Use `glob-to-regexp` for glob parsing and matching

For Workers Sites, we'll need to match globs both outside and inside
workers. Splitting glob parsing into `RegExp`s and matching allows us
to serialise parsed `RegExp`s into worker bindings.

`glob-to-regexp` is also the package wrangler uses for globs.

* Support `file://` URLs in persistence options

File-system persistence can still be enabled using a regular non-URL
string root. Using a `file://` URL allows us to use query parameters
for controlling path sanitisation. Workers Sites requires unsanitised
storage as keys must exactly match paths (e.g. we want to lookup
`myfile.test.txt` not `myfile_test_txt`).

* Support `site{Include/Exclude}`, populate `__STATIC_CONTENT_MANIFEST`

This brings Miniflare 3's Workers Sites implementation up to parity
with Miniflare 2. Note as the Cache API is not yet supported,
`{ cacheControl: { bypassCache: true } }` must be passed as options
to `@cloudflare/kv-asset-handler`.

Closes #372

* Populate `compatibilityDate` for internal services

Recent open-source runtime binaries require this to be set

* PR suggestions

- Include full URL when storage protocol invalid
- Always try parse storage persistence URLs
- Status 405 for Method Not Allowed for Sites KV Namespace

Author: mrbbot

package-lock.json

@@ -41,13 +41,16 @@
     "capnp-ts": "^0.7.0",
     "exit-hook": "^2.2.1",
     "get-port": "^5.1.1",
+    "glob-to-regexp": "^0.4.1",
     "kleur": "^4.1.5",
     "stoppable": "^1.1.0",
+    "undici": "5.8.2",
     "zod": "^3.18.0"
   },
   "devDependencies": {
     "@types/debug": "^4.1.7",
     "@types/estree": "^1.0.0",
+    "@types/glob-to-regexp": "^0.4.1",
     "@types/stoppable": "^1.1.1"
   }
 }

packages/tre/package.json

@@ -1,3 +1,4 @@
+import globToRegexp from "glob-to-regexp";
 import { z } from "zod";
 
 export type Awaitable<T> = T | Promise<T>;
@@ -46,7 +47,8 @@ export type MiniflareCoreErrorCode =
   | "ERR_MODULE_PARSE" // SyntaxError when attempting to parse/locate modules
   | "ERR_MODULE_STRING_SCRIPT" // Attempt to resolve module within string script
   | "ERR_MODULE_DYNAMIC_SPEC" // Attempted to import/require a module without a literal spec
-  | "ERR_MODULE_RULE"; // No matching module rule for file
+  | "ERR_MODULE_RULE" // No matching module rule for file
+  | "ERR_PERSIST_UNSUPPORTED"; // Unsupported storage persistence protocol
 export class MiniflareCoreError extends MiniflareError<MiniflareCoreErrorCode> {}
 
 export class HttpError extends MiniflareError<number> {
@@ -74,3 +76,61 @@ export class DeferredPromise<T> extends Promise<T> {
     this.reject = promiseReject!;
   }
 }
+
+// Split conversion to RegExps and testing to allow RegExps to be serialised
+// into Workers Sites KV namespace script. This will apply filtering, before
+// passing back to Miniflare's loopback server for storage access.
+export interface MatcherRegExps {
+  include: RegExp[];
+  exclude: RegExp[];
+}
+export interface SerialisableMatcherRegExps {
+  include: string[];
+  exclude: string[];
+}
+export function globsToRegExps(globs: string[] = []): MatcherRegExps {
+  const include: RegExp[] = [];
+  const exclude: RegExp[] = [];
+  // Setting `flags: "g"` removes "^" and "$" from the generated regexp,
+  // allowing matches anywhere in the path...
+  // (https://github.com/fitzgen/glob-to-regexp/blob/2abf65a834259c6504ed3b80e85f893f8cd99127/index.js#L123-L127)
+  const opts: globToRegexp.Options = { globstar: true, flags: "g" };
+  for (const glob of globs) {
+    // ...however, we don't actually want to include the "g" flag, since it will
+    // change `lastIndex` as paths are matched, and we want to reuse `RegExp`s.
+    // So, reconstruct each `RegExp` without any flags.
+    if (glob.startsWith("!")) {
+      exclude.push(new RegExp(globToRegexp(glob.slice(1), opts), ""));
+    } else {
+      include.push(new RegExp(globToRegexp(glob, opts), ""));
+    }
+  }
+  return { include, exclude };
+}
+// NOTE: this function will be `toString()`ed and must not have dependencies
+export function testRegExps(matcher: MatcherRegExps, value: string): boolean {
+  for (const exclude of matcher.exclude) if (exclude.test(value)) return false;
+  for (const include of matcher.include) if (include.test(value)) return true;
+  return false;
+}
+function serialiseRegExp(regExp: RegExp): string {
+  const str = regExp.toString();
+  return str.substring(str.indexOf("/") + 1, str.lastIndexOf("/"));
+}
+export function serialiseRegExps(
+  matcher: MatcherRegExps
+): SerialisableMatcherRegExps {
+  return {
+    include: matcher.include.map(serialiseRegExp),
+    exclude: matcher.exclude.map(serialiseRegExp),
+  };
+}
+// NOTE: this function will be `toString()`ed and must not have dependencies
+export function deserialiseRegExps(
+  matcher: SerialisableMatcherRegExps
+): MatcherRegExps {
+  return {
+    include: matcher.include.map((regExp) => new RegExp(regExp)),
+    exclude: matcher.exclude.map((regExp) => new RegExp(regExp)),
+  };
+}

packages/tre/src/helpers.ts

@@ -89,6 +89,13 @@ export const SCRIPT_CUSTOM_SERVICE = `addEventListener("fetch", (event) => {
   event.respondWith(${BINDING_SERVICE_LOOPBACK}.fetch(request));
 })`;
 
+const now = new Date();
+const fallbackCompatibilityDate = [
+  now.getFullYear(),
+  (now.getMonth() + 1).toString().padStart(2, "0"),
+  now.getDate().toString().padStart(2, "0"),
+].join("-");
+
 export const CORE_PLUGIN: Plugin<
   typeof CoreOptionsSchema,
   typeof CoreSharedOptionsSchema
@@ -156,6 +163,7 @@ export const CORE_PLUGIN: Plugin<
         name: SERVICE_ENTRY,
         worker: {
           serviceWorkerScript: SCRIPT_ENTRY,
+          compatibilityDate: "2022-09-01",
           bindings: serviceEntryBindings,
           compatibilityDate: "2022-09-01",
         },
@@ -170,7 +178,8 @@ export const CORE_PLUGIN: Plugin<
         name,
         worker: {
           ...workerScript,
-          compatibilityDate: options.compatibilityDate,
+          compatibilityDate:
+            options.compatibilityDate ?? fallbackCompatibilityDate,
           compatibilityFlags: options.compatibilityFlags,
           bindings: workerBindings,
         },
@@ -189,6 +198,7 @@ export const CORE_PLUGIN: Plugin<
             name: `${SERVICE_CUSTOM_PREFIX}:${name}`,
             worker: {
               serviceWorkerScript: SCRIPT_CUSTOM_SERVICE,
+              compatibilityDate: "2022-09-01",
               bindings: [
                 {
                   name: BINDING_TEXT_CUSTOM_SERVICE,

packages/tre/src/plugins/core/index.ts

@@ -3,13 +3,17 @@ import { readFileSync } from "fs";
 import { builtinModules } from "module";
 import path from "path";
 import { TextDecoder, TextEncoder } from "util";
-import { Matcher, globsToMatcher } from "@miniflare/shared";
 import acorn from "acorn";
 import walk from "acorn-walk";
 import type estree from "estree";
 import { dim } from "kleur/colors";
 import { z } from "zod";
-import { MiniflareCoreError } from "../../helpers";
+import {
+  MatcherRegExps,
+  MiniflareCoreError,
+  globsToRegExps,
+  testRegExps,
+} from "../../helpers";
 import { Worker_Module } from "../../runtime";
 
 const SUGGEST_BUNDLE =
@@ -55,7 +59,7 @@ const DEFAULT_MODULE_RULES: ModuleRule[] = [
 
 interface CompiledModuleRule {
   type: ModuleRuleType;
-  include: Matcher;
+  include: MatcherRegExps;
 }
 
 function compileModuleRules(rules?: ModuleRule[]) {
@@ -66,7 +70,7 @@ function compileModuleRules(rules?: ModuleRule[]) {
     if (finalisedTypes.has(rule.type)) continue;
     compiledRules.push({
       type: rule.type,
-      include: globsToMatcher(rule.include),
+      include: globsToRegExps(rule.include),
     });
     if (!rule.fallthrough) finalisedTypes.add(rule.type);
   }
@@ -210,7 +214,7 @@ ${dim(modulesConfig)}`;
 
     // Find first matching module rule
     const rule = this.#compiledRules.find((rule) =>
-      rule.include.test(identifier)
+      testRegExps(rule.include, identifier)
     );
     if (rule === undefined) {
       const prefix = getResolveErrorPrefix(referencingPath);

packages/tre/src/plugins/core/modules.ts

@@ -1,3 +1,5 @@
+export const KV_PLUGIN_NAME = "kv";
+
 export const MIN_CACHE_TTL = 60; /* 60s */
 export const MAX_LIST_KEYS = 1000;
 export const MAX_KEY_SIZE = 512; /* 512B */