Remove node-fetch dependency (#19)

Author: fwang

README.md

@@ -139,6 +139,8 @@ Create a CloudFront distribution, and dispatch requests to their cooresponding h
 
 Create a Lambda function with the code from `.open-next/middleware-function`, and attach it to the `/_next/data/*` and `/*` behaviors as `viewer request` edge function. This allows the function to run your [Middleware](https://nextjs.org/docs/advanced-features/middleware) code before the request hits your server function, and also before cached content.
 
+The middleware function uses the Node.js 18 [global fetch API](https://nodejs.org/de/blog/announcements/v18-release-announce/#new-globally-available-browser-compatible-apis). It requires to run on Node.js 18 runtime. [See why Node.js 18 runtime is required.](#workaround-add-headersgetall-extension-to-the-middleware-function)
+
 Note that if middleware is not used in the Next.js app, the `middleware-function` will not be generated. In this case, you don't have to create the Lambda@Edge function, and configure it in the CloudFront distribution.
 
 ## Limitations and workarounds
@@ -177,6 +179,15 @@ To workaround the issue, the middleware function JSON encodes all request header
 
 Note that the `x-op-middleware-request-headers` and `x-op-middleware-response-headers` headers need to be added to CloudFront distribution's cache policy allowed list.
 
+#### WORKAROUND: Add `Headers.getAll()` extension to the middleware function
+
+Vercel uses the `Headers.getAll()` function in the middleware code. This function is not part of the Node.js 18 [global fetch API](https://nodejs.org/de/blog/announcements/v18-release-announce/#new-globally-available-browser-compatible-apis). We have two options:
+
+1. Inject the `getAll()` function to the global fetch API.
+2. Use the [`node-fetch`](https://github.com/node-fetch/node-fetch) package to polyfill the fetch API.
+
+We decided to go with option 1. It does not require addition dependency. And it is likely that Vercel removes the usage of the `getAll()` function down the road.
+
 ## Example
 
 In the `example` folder, you can find a Next.js feature test app. Here's a link deployed using SST's [`NextjsSite`](https://docs.sst.dev/constructs/NextjsSite) construct. It contains a handful of pages. Each page aims to test a single Next.js feature.

package.json

@@ -25,7 +25,6 @@
     "@vercel/build-utils": "^5.7.0",
     "@vercel/next": "^3.3.2",
     "esbuild": "^0.15.18",
-    "node-fetch": "^3.3.0",
     "serverless-http": "^3.1.0",
     "yargs": "^17.6.2"
   },
@@ -39,7 +38,6 @@
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/serverless-stack/open-next.git",
-    "directory": "cli"
+    "url": "git+https://github.com/serverless-stack/open-next.git"
   }
 }

src/adapters/middleware-adapter.ts

@@ -3,14 +3,7 @@ import type {
   CloudFrontRequestResult,
   CloudFrontHeaders
 } from "aws-lambda"
-import { default as fetch, Headers, Request, Response } from "node-fetch";
-Object.assign(globalThis, {
-  Request,
-  Response,
-  fetch,
-  Headers,
-  self: {}
-});
+
 // @ts-ignore
 const index = await (() => import("./middleware.js"))();
 
@@ -22,6 +15,7 @@ export async function handler(event: CloudFrontRequestEvent): Promise<CloudFront
   console.log(request.headers);
 
   // Convert CloudFront request to Node request
+  // @ts-ignore Types has not been added for Node 18 native fetch API
   const requestHeaders = new Headers();
   for (const [key, values] of Object.entries(headers)) {
     for (const { value } of values) {
@@ -33,6 +27,7 @@ export async function handler(event: CloudFrontRequestEvent): Promise<CloudFront
   const host = headers["host"][0].value;
   const qs = querystring.length > 0 ? `?${querystring}` : "";
   const url = new URL(`${uri}${qs}`, `https://${host}`);
+  // @ts-ignore Types has not been added for Node 18 native fetch API
   const nodeRequest = new Request(url.toString(), {
     method,
     headers: requestHeaders,
@@ -49,7 +44,7 @@ export async function handler(event: CloudFrontRequestEvent): Promise<CloudFront
   });
   console.log("middleware response header", response.headers);
 
-  // WORKAROUND (AWS): pass middleware headers to server
+  // WORKAROUND: Pass headers from middleware function to server function (AWS specific) — https://github.com/serverless-stack/open-next#workaround-pass-headers-from-middleware-function-to-server-function-aws-specific
   if (response.headers.get("x-middleware-next") === "1") {
     headers["x-op-middleware-request-headers"] = [{
       key: "x-op-middleware-request-headers",

src/adapters/server-adapter.ts

@@ -61,7 +61,7 @@ const server = slsHttp(
 export async function handler(event: APIGatewayProxyEventV2, context: Context): Promise<APIGatewayProxyResultV2> {
   console.log(event)
 
-  // WORKAROUND (AWS): pass middleware headers to server
+  // WORKAROUND: Pass headers from middleware function to server function (AWS specific) — https://github.com/serverless-stack/open-next#workaround-pass-headers-from-middleware-function-to-server-function-aws-specific
   const middlewareRequestHeaders = JSON.parse(
     event.headers["x-op-middleware-request-headers"] || "{}"
   );
@@ -70,12 +70,12 @@ export async function handler(event: APIGatewayProxyEventV2, context: Context):
   // Invoke NextServer
   const response: APIGatewayProxyResultV2 = await server(event, context);
 
-  // WORKAROUND: `NextServer` does not set cache response headers for HTML pages
+  // WORKAROUND: `NextServer` does not set cache response headers for HTML pages — https://github.com/serverless-stack/open-next#workaround-nextserver-does-not-set-cache-response-headers-for-html-pages
   if (htmlPages.includes(event.rawPath) && !response.headers?.["cache-control"]) {
     response.headers!["cache-control"] = "public, max-age=0, s-maxage=31536000, must-revalidate";
   }
 
-  // WORKAROUND (AWS): pass middleware headers to server
+  // WORKAROUND: Pass headers from middleware function to server function (AWS specific) — https://github.com/serverless-stack/open-next#workaround-pass-headers-from-middleware-function-to-server-function-aws-specific
   const middlewareResponseHeaders = JSON.parse(
     event.headers["x-op-middleware-response-headers"] || "{}"
   );

src/build.ts

@@ -20,6 +20,7 @@ export async function build() {
 
   // Generate deployable bundle
   printHeader("Generating OpenNext bundle");
+  printVersion();
   cleanupOutputDir();
   copyAdapterFiles();
   createServerBundle();
@@ -58,6 +59,12 @@ function printHeader(header: string) {
   ].join("\n"));
 }
 
+function printVersion() {
+  const pathToPackageJson = path.join(__dirname, "../package.json");
+  const pkg = JSON.parse(fs.readFileSync(pathToPackageJson, "utf-8"));
+  console.log(`Using v${pkg.version}`);
+}
+
 function cleanupOutputDir() {
   fs.rmSync(outputDir, { recursive: true, force: true });
 }
@@ -198,6 +205,30 @@ function createMiddlewareBundle(buildOutput: any) {
     write: true,
     allowOverwrite: true,
     outfile: path.join(outputPath, "index.mjs"),
+    banner: {
+      js: [
+        // WORKAROUND: Add `Headers.getAll()` extension to the middleware function — https://github.com/serverless-stack/open-next#workaround-add-headersgetall-extension-to-the-middleware-function
+        "class Response extends globalThis.Response {",
+        "  constructor(body, init) {",
+        "    super(body, init);",
+        "    this.headers.getAll = (name) => {",
+        "      name = name.toLowerCase();",
+        "      if (name !== 'set-cookie') {",
+        "        throw new Error('Headers.getAll is only supported for Set-Cookie');",
+        "      }",
+        "      return [...this.headers.entries()]",
+        "        .filter(([key]) => key === name)",
+        "        .map(([, value]) => value);",
+        "    };",
+        "  }",
+        "}",
+        // Polyfill Response and self
+        "Object.assign(globalThis, {",
+        "  Response,",
+        "  self: {},",
+        "});",
+      ].join(""),
+    },
   });
   if (result.errors.length > 0) {
     result.errors.forEach((error) => console.error(error));

tsconfig.json

@@ -9,22 +9,4 @@
   },
   "include": ["src"],
   "exclude": []
-}
-//{
-//  "compilerOptions": {
-//    "strict": true,
-//    "noEmitOnError": true,
-//    "noFallthroughCasesInSwitch": true,
-//    "moduleResolution": "node",
-//    "module": "commonjs",
-//    "target": "ES2020",
-//    "esModuleInterop": true,
-//    "allowJs": true,
-//    "lib": ["ES2020"],
-//    "resolveJsonModule": true,
-//    "sourceMap": true,
-//    "outDir": "./dist",
-//  },
-//  "include": ["src"],
-//  "exclude": []
-//}
+}

yarn.lock

@@ -1525,11 +1525,6 @@ csv@^5.5.0:
     csv-stringify "^5.6.5"
     stream-transform "^2.1.3"
 
-data-uri-to-buffer@^4.0.0:
-  version "4.0.0"
-  resolved "https://registry.yarnpkg.com/data-uri-to-buffer/-/data-uri-to-buffer-4.0.0.tgz#b5db46aea50f6176428ac05b73be39a57701a64b"
-  integrity sha512-Vr3mLBA8qWmcuschSLAOogKgQ/Jwxulv3RNE4FXnYWRGujzrRWQI4m12fQqRkwX06C0KanhLr4hK+GydchZsaA==
-
 dataloader@^1.4.0:
   version "1.4.0"
   resolved "https://registry.yarnpkg.com/dataloader/-/dataloader-1.4.0.tgz#bca11d867f5d3f1b9ed9f737bd15970c65dff5c8"
@@ -1828,14 +1823,6 @@ fastq@^1.6.0:
   dependencies:
     reusify "^1.0.4"
 
-fetch-blob@^3.1.2, fetch-blob@^3.1.4:
-  version "3.2.0"
-  resolved "https://registry.yarnpkg.com/fetch-blob/-/fetch-blob-3.2.0.tgz#f09b8d4bbd45adc6f0c20b7e787e793e309dcce9"
-  integrity sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==
-  dependencies:
-    node-domexception "^1.0.0"
-    web-streams-polyfill "^3.0.3"
-
 fill-range@^7.0.1:
   version "7.0.1"
   resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.0.1.tgz#1919a6a7c75fe38b2c7c77e5198535da9acdda40"
@@ -1867,13 +1854,6 @@ [email protected]:
     micromatch "^4.0.2"
     pkg-dir "^4.2.0"
 
-formdata-polyfill@^4.0.10:
-  version "4.0.10"
-  resolved "https://registry.yarnpkg.com/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz#24807c31c9d402e002ab3d8c720144ceb8848423"
-  integrity sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==
-  dependencies:
-    fetch-blob "^3.1.2"
-
 fs-extra@^7.0.1:
   version "7.0.1"
   resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-7.0.1.tgz#4f189c44aa123b895f722804f55ea23eadc348e9"
@@ -2326,27 +2306,13 @@ mixme@^0.5.1:
   resolved "https://registry.yarnpkg.com/mixme/-/mixme-0.5.4.tgz#8cb3bd0cd32a513c161bf1ca99d143f0bcf2eff3"
   integrity sha512-3KYa4m4Vlqx98GPdOHghxSdNtTvcP8E0kkaJ5Dlh+h2DRzF7zpuVVcA8B0QpKd11YJeP9QQ7ASkKzOeu195Wzw==
 
-node-domexception@^1.0.0:
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5"
-  integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==
-
 node-fetch@^2.5.0:
   version "2.6.7"
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad"
   integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==
   dependencies:
     whatwg-url "^5.0.0"
 
-node-fetch@^3.3.0:
-  version "3.3.0"
-  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-3.3.0.tgz#37e71db4ecc257057af828d523a7243d651d91e4"
-  integrity sha512-BKwRP/O0UvoMKp7GNdwPlObhYGB5DQqwhEDQlNKuoqwVYSxkSZCSbHjnFFmUEtwSKRPU4kNK8PbDYYitwaE3QA==
-  dependencies:
-    data-uri-to-buffer "^4.0.0"
-    fetch-blob "^3.1.4"
-    formdata-polyfill "^4.0.10"
-
 normalize-package-data@^2.5.0:
   version "2.5.0"
   resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.5.0.tgz#e66db1838b200c1dfc233225d12cb36520e234a8"
@@ -2886,11 +2852,6 @@ wcwidth@^1.0.1:
   dependencies:
     defaults "^1.0.3"
 
-web-streams-polyfill@^3.0.3:
-  version "3.2.1"
-  resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz#71c2718c52b45fd49dbeee88634b3a60ceab42a6"
-  integrity sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==
-
 webidl-conversions@^3.0.0:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871"