fix: vendor deno dependencies (#2302)

Skn0tt · pieh · web-flow · commit a8d06abe28cf · 2023-09-29T16:42:54.000+02:00
* fix: vendor deno dependencies

* chore: install deno along node

* fix: install deno in cypress tests

* fix: try scoping down the import map

* fix: import straight from vendor directory

* fix: run vendoring before testing deno

* fix: npm install needs deno

* fix: run build:vendor in right dir

* refactor: move deno tests into normal test file

* fix: include vendor files in package

* fix: patch html rewriter to fetch wasm from web, not from disk

* chore: use precise path

* Update packages/runtime/src/templates/vendor.ts

Co-authored-by: Michal Piechowiak &lt;misiek.piechowiak@gmail.com&gt;

---------

Co-authored-by: Michal Piechowiak &lt;misiek.piechowiak@gmail.com&gt;
diff --git a/.eslintignore b/.eslintignore
@@ -4,6 +4,8 @@ lib
 demos
 packages/runtime/src/templates/edge
 packages/runtime/src/templates/edge-shared
+packages/runtime/src/templates/vendor
+packages/runtime/src/templates/vendor.ts
 packages/runtime/lib
 packages/runtime/dist-types
 jestSetup.js
diff --git a/.github/workflows/cypress-canary.yml b/.github/workflows/cypress-canary.yml
@@ -47,6 +47,10 @@ jobs:
         with:
           node-version: '16'
 
+      - uses: denoland/setup-deno@v1
+        with:
+          deno-version: v1.x
+
       - run: npm install
 
       - name: Cypress run
diff --git a/.github/workflows/cypress-demo-nx.yml b/.github/workflows/cypress-demo-nx.yml
@@ -47,6 +47,10 @@ jobs:
         with:
           node-version: '16'
 
+      - uses: denoland/setup-deno@v1
+        with:
+          deno-version: v1.x
+
       - run: npm install
 
       - name: Cypress run
diff --git a/.github/workflows/cypress-demo-static.yml b/.github/workflows/cypress-demo-static.yml
@@ -51,6 +51,10 @@ jobs:
         with:
           node-version: '16'
 
+      - uses: denoland/setup-deno@v1
+        with:
+          deno-version: v1.x
+
       - run: npm install
 
       - name: Cypress run
diff --git a/.github/workflows/cypress-demo.yml b/.github/workflows/cypress-demo.yml
@@ -51,6 +51,10 @@ jobs:
         with:
           node-version: '16'
 
+      - uses: denoland/setup-deno@v1
+        with:
+          deno-version: v1.x
+
       - run: npm install
 
       - name: Cypress run
diff --git a/.github/workflows/cypress-middleware.yml b/.github/workflows/cypress-middleware.yml
@@ -51,6 +51,10 @@ jobs:
         with:
           node-version: '16'
 
+      - uses: denoland/setup-deno@v1
+        with:
+          deno-version: v1.x
+
       - run: npm install
 
       - name: Cypress run
diff --git a/.github/workflows/e2e-appdir.yml b/.github/workflows/e2e-appdir.yml
@@ -16,6 +16,9 @@ jobs:
       test-files: ${{ steps['set-test-files'].outputs['test-files'] }}
     steps:
       - uses: actions/checkout@v3
+      - uses: denoland/setup-deno@v1
+        with:
+          deno-version: v1.x
       - run: npm install
       - id: set-test-files
         name: Get test files
@@ -42,6 +45,9 @@ jobs:
         with:
           node-version: '16'
           cache: 'npm'
+      - uses: denoland/setup-deno@v1
+        with:
+          deno-version: v1.x
       - run: npm install
       - name: Install Netlify CLI
         run: npm install -g netlify-cli
diff --git a/.github/workflows/e2e-next.yml b/.github/workflows/e2e-next.yml
@@ -44,6 +44,9 @@ jobs:
         with:
           node-version: '16'
           cache: 'npm'
+      - uses: denoland/setup-deno@v1
+        with:
+          deno-version: v1.x
       - run: npm install
       - name: Install Netlify CLI
         run: npm install -g netlify-cli
diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml
@@ -15,6 +15,9 @@ jobs:
           cache: 'npm'
           check-latest: true
           registry-url: 'https://registry.npmjs.org'
+      - uses: denoland/setup-deno@v1
+        with:
+          deno-version: v1.x
       - name: Install core dependencies
         run: npm ci --no-audit
       - name: Extract tag and version
diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml
@@ -26,6 +26,10 @@ jobs:
           check-latest: true
           registry-url: 'https://registry.npmjs.org'
         if: ${{ steps.release.outputs.releases_created }}
+      - uses: denoland/setup-deno@v1
+        with:
+          deno-version: v1.x
+        if: ${{ steps.release.outputs.releases_created }}
       - name: Install dependencies
         run: CI=1 npm ci
         if: ${{ steps.release.outputs.releases_created }}
diff --git a/.github/workflows/test-deno.yml b/.github/workflows/test-deno.yml
diff --git a/.github/workflows/test-integration.yml b/.github/workflows/test-integration.yml
@@ -24,6 +24,9 @@ jobs:
         with:
           node-version: 18
           check-latest: true
+      - uses: denoland/setup-deno@v1
+        with:
+          deno-version: v1.x
       - name: Install netlify-cli and npm
         run: npm install -g netlify-cli npm
       - name: NPM Install
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -31,12 +31,17 @@ jobs:
         with:
           node-version: 16
           check-latest: true
+      - uses: denoland/setup-deno@v1
+        with:
+          deno-version: v1.x
       - name: NPM Install
         run: npm install
       - name: Linting
         run: npm run format:ci
       - name: Run tests against next@latest
         run: npm test
+      - name: Run Deno tests
+        run: npm run test:deno
   canary:
     name: Unit tests (Canary)
     runs-on: ${{ matrix.os }}
@@ -54,6 +59,9 @@ jobs:
         with:
           node-version: 16
           check-latest: true
+      - uses: denoland/setup-deno@v1
+        with:
+          deno-version: v1.x
       - name: NPM Install
         run: npm install
       - name: Install Next.js Canary
diff --git a/.prettierignore b/.prettierignore
@@ -26,4 +26,5 @@ test/e2e
 
 **/CHANGELOG.md
 packages/runtime/lib
-packages/runtime/dist-types
+packages/runtime/dist-types
+packages/runtime/src/templates/vendor
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -2,6 +2,8 @@
   "deno.enablePaths": [
     "packages/runtime/src/templates/edge",
     "packages/runtime/src/templates/edge-shared",
+    "packages/runtime/src/templates/vendor", 
+    "packages/runtime/src/templates/vendor.ts", 
     "demos/middleware/.netlify/edge-functions",
     "demos/server-components/.netlify/edge-functions",
   ],
diff --git a/packages/runtime/.gitignore b/packages/runtime/.gitignore
@@ -0,0 +1 @@
+src/templates/vendor
diff --git a/packages/runtime/html_rewriter.patch b/packages/runtime/html_rewriter.patch
@@ -0,0 +1,4 @@
+56c56
+<       fetch(new URL("./vendor/html_rewriter_bg.wasm", import.meta.url).href)
+---
+>       fetch("https://deno.land/x/html_rewriter@v0.1.0-pre.17/vendor/html_rewriter_bg.wasm")
diff --git a/packages/runtime/package.json b/packages/runtime/package.json
@@ -7,6 +7,7 @@
     "lib/**/*",
     "src/templates/edge/*",
     "src/templates/edge-shared/*",
+    "src/templates/vendor/*",
     "index.js",
     "manifest.yml"
   ],
@@ -52,7 +53,9 @@
     "publish:install": "npm ci",
     "publish:test": "cd .. && npm ci && npm test",
     "clean": "rimraf lib dist-types",
-    "build": "tsc",
+    "build": "run-s build:*",
+    "build:vendor": "rimraf src/templates/vendor && deno vendor src/templates/vendor.ts --output=src/templates/vendor && patch src/templates/vendor/deno.land/x/html_rewriter@v0.1.0-pre.17/index.ts html_rewriter.patch",
+    "build:tsc": "tsc",
     "watch": "tsc --watch",
     "prepare": "npm run build"
   },
diff --git a/packages/runtime/src/helpers/edge.ts b/packages/runtime/src/helpers/edge.ts
@@ -378,6 +378,7 @@ export const writeEdgeFunctions = async ({
   const nextConfig = nextConfigFile.config
   const usesAppDir = nextConfig.experimental?.appDir
 
+  await copy(getEdgeTemplatePath('../vendor'), join(edgeFunctionRoot, 'vendor'))
   await copy(getEdgeTemplatePath('../edge-shared'), join(edgeFunctionRoot, 'edge-shared'))
   await writeJSON(join(edgeFunctionRoot, 'edge-shared', 'nextConfig.json'), nextConfig)
   await copy(join(publish, 'prerender-manifest.json'), join(edgeFunctionRoot, 'edge-shared', 'prerender-manifest.json'))
diff --git a/packages/runtime/src/templates/edge-shared/next-utils.ts b/packages/runtime/src/templates/edge-shared/next-utils.ts
@@ -7,10 +7,10 @@
  */
 
 // Deno imports
-import type { Key } from 'https://deno.land/x/path_to_regexp@v6.2.1/index.ts'
+import type { Key } from '../vendor/deno.land/x/path_to_regexp@v6.2.1/index.ts'
 
-import { compile, pathToRegexp } from 'https://deno.land/x/path_to_regexp@v6.2.1/index.ts'
-import { getCookies } from 'https://deno.land/std@0.148.0/http/cookie.ts'
+import { compile, pathToRegexp } from '../vendor/deno.land/x/path_to_regexp@v6.2.1/index.ts'
+import { getCookies } from '../vendor/deno.land/std@0.175.0/http/cookie.ts'
 
 // Inlined/re-implemented types
 
diff --git a/packages/runtime/src/templates/edge-shared/utils.ts b/packages/runtime/src/templates/edge-shared/utils.ts
@@ -1,5 +1,5 @@
 import type { Context } from 'https://edge.netlify.com'
-import { ElementHandlers, HTMLRewriter } from 'https://deno.land/x/html_rewriter@v0.1.0-pre.17/index.ts'
+import { ElementHandlers, HTMLRewriter } from '../vendor/deno.land/x/html_rewriter@v0.1.0-pre.17/index.ts'
 
 export interface FetchEventResult {
   response: Response
diff --git a/packages/runtime/src/templates/edge/next-dev.js b/packages/runtime/src/templates/edge/next-dev.js
@@ -1,6 +1,6 @@
-import { NextRequest } from 'https://esm.sh/v91/next@12.2.5/deno/dist/server/web/spec-extension/request.js'
-import { NextResponse } from 'https://esm.sh/v91/next@12.2.5/deno/dist/server/web/spec-extension/response.js'
-import { fromFileUrl } from 'https://deno.land/std@0.151.0/path/mod.ts'
+import { NextRequest } from '../vendor/esm.sh/v91/next@12.2.5/deno/dist/server/web/spec-extension/request.js'
+import { NextResponse } from '../vendor/esm.sh/v91/next@12.2.5/deno/dist/server/web/spec-extension/response.js'
+import { fromFileUrl } from '../vendor/deno.land/std@0.175.0/path/mod.ts'
 import { buildResponse, isFunction } from '../edge-shared/utils.ts'
 
 globalThis.NFRequestContextMap ||= new Map()
diff --git a/packages/runtime/src/templates/edge/shims.js b/packages/runtime/src/templates/edge/shims.js
@@ -1,11 +1,11 @@
 // @ts-check
 // deno-lint-ignore-file prefer-const no-unused-vars
-import { decode as _base64Decode } from 'https://deno.land/std@0.175.0/encoding/base64.ts'
-import BufferCompat from 'https://deno.land/std@0.175.0/node/buffer.ts'
-import EventsCompat from 'https://deno.land/std@0.175.0/node/events.ts'
-import AsyncHooksCompat from 'https://deno.land/std@0.175.0/node/async_hooks.ts'
-import AssertCompat from 'https://deno.land/std@0.175.0/node/assert.ts'
-import UtilCompat from 'https://deno.land/std@0.175.0/node/util.ts'
+import { decode as _base64Decode } from '../vendor/deno.land/std@0.175.0/encoding/base64.ts'
+import BufferCompat from '../vendor/deno.land/std@0.175.0/node/buffer.ts'
+import EventsCompat from '../vendor/deno.land/std@0.175.0/node/events.ts'
+import AsyncHooksCompat from '../vendor/deno.land/std@0.175.0/node/async_hooks.ts'
+import AssertCompat from '../vendor/deno.land/std@0.175.0/node/assert.ts'
+import UtilCompat from '../vendor/deno.land/std@0.175.0/node/util.ts'
 
 /**
  * These are the shims, polyfills and other kludges to make Next.js work in standards-compliant runtime.
diff --git a/packages/runtime/src/templates/vendor.ts b/packages/runtime/src/templates/vendor.ts
@@ -0,0 +1,21 @@
+// this file isn't meant to be imported.
+// it's a list of all external modules that we use,
+// and we vendor those into the `vendor/` directory
+// for consumption in other files.
+// Vendoring happens automatically as part of runtime `build` script.
+// You can trigger just vendoring without full package build by running `build:vendor` script.
+
+import 'https://deno.land/std@0.175.0/encoding/base64.ts'
+import 'https://deno.land/std@0.175.0/http/cookie.ts'
+import 'https://deno.land/std@0.175.0/node/buffer.ts'
+import 'https://deno.land/std@0.175.0/node/events.ts'
+import 'https://deno.land/std@0.175.0/node/async_hooks.ts'
+import 'https://deno.land/std@0.175.0/node/assert.ts'
+import 'https://deno.land/std@0.175.0/node/util.ts'
+import 'https://deno.land/std@0.175.0/path/mod.ts'
+
+import 'https://deno.land/x/path_to_regexp@v6.2.1/index.ts'
+import 'https://deno.land/x/html_rewriter@v0.1.0-pre.17/index.ts'
+
+import 'https://esm.sh/v91/next@12.2.5/deno/dist/server/web/spec-extension/request.js'
+import 'https://esm.sh/v91/next@12.2.5/deno/dist/server/web/spec-extension/response.js'
diff --git a/packages/runtime/tsconfig.json b/packages/runtime/tsconfig.json
@@ -10,6 +10,7 @@
     ],
     "exclude": [
         "src/templates/edge/*",
-        "src/templates/edge-shared/*"
+        "src/templates/edge-shared/*",
+        "src/templates/vendor/*"
     ]
 }
