Merge branch 'master' into bump/selfsigned

Author: mrbbot

docs/docs-config.js

@@ -23,10 +23,12 @@ module.exports = {
     },
   ],
   search: {
-    // apiKey: "f0ffcd9dba78827de321d7fce21a8181",
-    // indexName: "miniflare",
-    apiKey: "",
-    indexName: "",
+    // TODO: migrate to https://docsearch.algolia.com/docs/migrating-from-legacy/
+    //  appId: AYF9TLWKTY
+    //  apiKey: fd375d87d44c5b94352fa1ddd580ebdc
+    //  indexName: miniflare
+    apiKey: "f0ffcd9dba78827de321d7fce21a8181",
+    indexName: "miniflare",
     algoliaOptions: {},
   },
   siteMetadata: {

package-lock.json

@@ -40,6 +40,7 @@
   "dependencies": {
     "@iarna/toml": "^2.2.5",
     "@miniflare/shared": "2.0.0",
+    "@miniflare/watcher": "2.0.0",
     "busboy": "^0.3.1",
     "dotenv": "^10.0.0",
     "kleur": "^4.1.4",
@@ -54,13 +55,5 @@
     "@types/busboy": "^0.3.1",
     "@types/set-cookie-parser": "^2.4.1",
     "dequal": "^2.0.2"
-  },
-  "peerDependencies": {
-    "@miniflare/watcher": "2.0.0"
-  },
-  "peerDependenciesMeta": {
-    "@miniflare/watcher": {
-      "optional": true
-    }
   }
 }

packages/core/package.json

@@ -862,7 +862,7 @@ export class MiniflareCore<
       }
       if (watchedPaths.size > 0) {
         this.#ctx.log.debug(`Watching ${pathsToString(newWatchPaths)}...`);
-        await watcher.watch(watchedPaths);
+        watcher.watch(watchedPaths);
       }
       this.#previousWatchPaths = newWatchPaths;
     }
@@ -1012,15 +1012,16 @@ export class MiniflareCore<
     }
 
     // If upstream set, and the request URL doesn't begin with it, rewrite it
+    // so fetching the incoming request gets a response from the upstream
     const { upstreamURL } = this.#instances!.CorePlugin;
     if (upstreamURL && !url.toString().startsWith(upstreamURL.toString())) {
       let path = url.pathname + url.search;
       // Remove leading slash so we resolve relative to upstream's path
       if (path.startsWith("/")) path = path.substring(1);
       const newURL = new URL(path, upstreamURL);
       request = new Request(newURL, request);
-      // Make sure Host header is correct
-      request.headers.set("host", upstreamURL.host);
+      // We don't set the Host header here, fetch will automatically set it
+      // based on the request url
     }
 
     // Each fetch gets its own context (e.g. 50 subrequests).

packages/core/src/index.ts

@@ -39,8 +39,12 @@ export class BuildPlugin extends Plugin<BuildOptions> implements BuildOptions {
   @Option({
     type: OptionType.ARRAY,
     description: "Directory to watch for rebuilding on changes",
-    fromWrangler: ({ build }) => {
-      if (build?.watch_dir) return [build.watch_dir];
+    fromWrangler: ({ build, miniflare }) => {
+      const watchPaths = miniflare?.build_watch_dirs ?? [];
+      if (build?.watch_dir) watchPaths.push(build.watch_dir);
+      if (watchPaths.length) return watchPaths;
+
+      // If build command set and no paths set, fallback to watching "src"
       if (build?.command) return ["src"];
     },
   })

packages/core/src/plugins/build.ts

@@ -120,6 +120,31 @@ Headers.prototype.getAll = function (key: string): string[] {
   return value ? splitCookiesString(value) : [];
 };
 
+/** @internal */
+export function _headersFromIncomingRequest(
+  req: http.IncomingMessage
+): Headers {
+  const headers = new Headers();
+  for (const [name, values] of Object.entries(req.headers)) {
+    // These headers are unsupported in undici fetch requests, they're added
+    // automatically
+    if (
+      name === "transfer-encoding" ||
+      name === "connection" ||
+      name === "keep-alive" ||
+      name === "expect"
+    ) {
+      continue;
+    }
+    if (Array.isArray(values)) {
+      for (const value of values) headers.append(name, value);
+    } else if (values !== undefined) {
+      headers.append(name, values);
+    }
+  }
+  return headers;
+}
+
 // Instead of subclassing our customised Request and Response classes from
 // BaseRequest and BaseResponse, we instead compose them and implement the same
 // interface.
@@ -630,6 +655,19 @@ export function _getURLList(res: BaseResponse): URL[] | undefined {
   return res[fetchSymbols.kState]?.urlList;
 }
 
+/** @internal */
+export function _getBodyLength(
+  res: Response | BaseResponse
+): number | undefined {
+  // Extract the actual body length of the Response body. Cloudflare will return
+  // this for the Content-Length header instead of the user specified value
+  // if its set. When the body is a stream, it's the user's responsibility to
+  // set the Content-Length header if they want to.
+  if (res instanceof Response) res = res[_kInner];
+  // @ts-expect-error symbol properties are not included in type definitions
+  return res[fetchSymbols.kState]?.body?.length ?? undefined; // (normalise nullish to undefined)
+}
+
 /** @internal */
 export const _kLoopHeader = "MF-Loop";
 

packages/core/src/standards/http.ts

@@ -4,6 +4,7 @@ export * from "./domexception";
 export * from "./encoding";
 export * from "./event";
 export {
+  _headersFromIncomingRequest,
   _kInner,
   _isByteStream,
   Body,
@@ -14,6 +15,7 @@ export {
   Response,
   withWaitUntil,
   _getURLList,
+  _getBodyLength,
   _kLoopHeader,
   fetch,
   _urlFromRequestInput,

packages/core/src/standards/index.ts

@@ -467,7 +467,7 @@ test("MiniflareCore: uses original protocol and host when matching mount routes"
       async (res) => resolve(await text(res))
     );
   });
-  t.is(body, "https://example.com/a:example.com");
+  t.is(body, "https://example.com/a:custom.mf");
 });
 
 // Shared storage persistence tests

packages/core/test/index.mounts.spec.ts

@@ -41,6 +41,7 @@ import {
   triggerPromise,
   useMiniflare,
   useMiniflareWithHandler,
+  useServer,
   useTmp,
   utf8Decode,
   utf8Encode,
@@ -1087,17 +1088,29 @@ test("MiniflareCore: dispatchFetch: rewrites url to match upstream if different"
       new globals.Response(`${req.url}:${req.headers.get("host")}`)
   );
   // Check url and host header are correct
-  let res = await mf.dispatchFetch("http://localhost/a");
-  t.is(await res.text(), "https://miniflare.dev/a:miniflare.dev");
+  const init: RequestInit = { headers: { host: "localhost" } };
+  let res = await mf.dispatchFetch("http://localhost/a", init);
+  t.is(await res.text(), "https://miniflare.dev/a:localhost");
 
   // Check includes query string
-  res = await mf.dispatchFetch("http://localhost/a?b=c");
-  t.is(await res.text(), "https://miniflare.dev/a?b=c:miniflare.dev");
+  res = await mf.dispatchFetch("http://localhost/a?b=c", init);
+  t.is(await res.text(), "https://miniflare.dev/a?b=c:localhost");
 
   // Check includes subpath
   await mf.setOptions({ upstream: "https://miniflare.dev/subpath/" });
-  res = await mf.dispatchFetch("http://localhost/a");
-  t.is(await res.text(), "https://miniflare.dev/subpath/a:miniflare.dev");
+  res = await mf.dispatchFetch("http://localhost/a", init);
+  t.is(await res.text(), "https://miniflare.dev/subpath/a:localhost");
+});
+test("MiniflareCore: dispatchFetch: fetching incoming request responds with upstream", async (t) => {
+  const upstream = (await useServer(t, (req, res) => res.end("upstream"))).http;
+  const mf = useMiniflareWithHandler(
+    {},
+    { upstream: upstream.toString() },
+    (globals, req) => globals.fetch(req)
+  );
+  // Host should be rewritten to match upstream
+  const res = await mf.dispatchFetch("https://random.mf/");
+  t.is(await res.text(), "upstream");
 });
 test("MiniflareCore: dispatchFetch: request gets immutable headers", async (t) => {
   const mf = useMiniflareWithHandler({}, {}, (globals, req) => {

packages/core/test/index.spec.ts

@@ -62,11 +62,14 @@ test("BuildPlugin: parses options from wrangler config", (t) => {
       cwd: "cwd",
       watch_dir: "source",
     },
+    miniflare: {
+      build_watch_dirs: ["source1", "source2"],
+    },
   });
   t.deepEqual(options, {
     buildCommand: "npm run build",
     buildBasePath: "cwd",
-    buildWatchPaths: ["source"],
+    buildWatchPaths: ["source1", "source2", "source"],
   });
   // Check buildWatchPaths defaults to "src" if any command specified
   options = parsePluginWranglerConfig(BuildPlugin, {