Add falling back to wasm swc build on native load failure (#36612)

Follow-up to #36527 this adds falling back to the wasm swc build when loading the native bindings fails so that we don't block the build on the native dependency being available.  

This continues off of #33496 but does not add a postinstall script yet and only downloads the fallback when the native dependency fails to load.

Author: ijjk

packages/next/build/swc/index.js

@@ -1,46 +1,81 @@
+import path from 'path'
+import { pathToFileURL } from 'url'
 import { platform, arch } from 'os'
 import { platformArchTriples } from 'next/dist/compiled/@napi-rs/triples'
-import { version as nextVersion, optionalDependencies } from 'next/package.json'
 import * as Log from '../output/log'
 import { getParserOptions } from './options'
 import { eventSwcLoadFailure } from '../../telemetry/events/swc-load-failure'
 import { patchIncorrectLockfile } from '../../lib/patch-incorrect-lockfile'
+import { downloadWasmSwc } from '../../lib/download-wasm-swc'
+import { version as nextVersion } from 'next/package.json'
 
 const ArchName = arch()
 const PlatformName = platform()
 const triples = platformArchTriples[PlatformName][ArchName] || []
 
 let nativeBindings
 let wasmBindings
+let downloadWasmPromise
+let pendingBindings
 export const lockfilePatchPromise = {}
 
 async function loadBindings() {
-  if (!lockfilePatchPromise.cur) {
-    // always run lockfile check once so that it gets patched
-    // even if it doesn't fail to load locally
-    lockfilePatchPromise.cur = patchIncorrectLockfile(process.cwd()).catch(
-      console.error
-    )
+  if (pendingBindings) {
+    return pendingBindings
   }
+  pendingBindings = new Promise(async (resolve, reject) => {
+    if (!lockfilePatchPromise.cur) {
+      // always run lockfile check once so that it gets patched
+      // even if it doesn't fail to load locally
+      lockfilePatchPromise.cur = patchIncorrectLockfile(process.cwd()).catch(
+        console.error
+      )
+    }
 
-  let attempts = []
-  try {
-    return loadNative()
-  } catch (a) {
-    attempts = attempts.concat(a)
-  }
+    let attempts = []
+    try {
+      return resolve(loadNative())
+    } catch (a) {
+      attempts = attempts.concat(a)
+    }
 
-  // TODO: fetch wasm and fallback when loading native fails
-  // so that users aren't blocked on this, we still want to
-  // report the native load failure so we can patch though
-  try {
-    let bindings = await loadWasm()
-    return bindings
-  } catch (a) {
-    attempts = attempts.concat(a)
-  }
+    try {
+      let bindings = await loadWasm()
+      eventSwcLoadFailure({ wasm: 'enabled' })
+      return resolve(bindings)
+    } catch (a) {
+      attempts = attempts.concat(a)
+    }
 
-  logLoadFailure(attempts)
+    try {
+      // if not installed already download wasm package on-demand
+      // we download to a custom directory instead of to node_modules
+      // as node_module import attempts are cached and can't be re-attempted
+      // x-ref: https://github.com/nodejs/modules/issues/307
+      const wasmDirectory = path.join(
+        path.dirname(require.resolve('next/package.json')),
+        'wasm'
+      )
+      if (!downloadWasmPromise) {
+        downloadWasmPromise = downloadWasmSwc(nextVersion, wasmDirectory)
+      }
+      await downloadWasmPromise
+      let bindings = await loadWasm(pathToFileURL(wasmDirectory).href)
+      eventSwcLoadFailure({ wasm: 'fallback' })
+
+      // still log native load attempts so user is
+      // aware it failed and should be fixed
+      for (const attempt of attempts) {
+        Log.warn(attempt)
+      }
+      return resolve(bindings)
+    } catch (a) {
+      attempts = attempts.concat(a)
+    }
+
+    logLoadFailure(attempts, true)
+  })
+  return pendingBindings
 }
 
 function loadBindingsSync() {
@@ -56,47 +91,16 @@ function loadBindingsSync() {
 
 let loggingLoadFailure = false
 
-function logLoadFailure(attempts) {
+function logLoadFailure(attempts, triedWasm = false) {
   // make sure we only emit the event and log the failure once
   if (loggingLoadFailure) return
   loggingLoadFailure = true
 
   for (let attempt of attempts) {
     Log.warn(attempt)
   }
-  let glibcVersion
-  let installedSwcPackages
 
-  try {
-    glibcVersion = process.report?.getReport().header.glibcVersionRuntime
-  } catch (_) {}
-
-  try {
-    const pkgNames = Object.keys(optionalDependencies || {}).filter((pkg) =>
-      pkg.startsWith('@next/swc')
-    )
-    const installedPkgs = []
-
-    for (const pkg of pkgNames) {
-      try {
-        const { version } = require(`${pkg}/package.json`)
-        installedPkgs.push(`${pkg}@${version}`)
-      } catch (_) {}
-    }
-
-    if (installedPkgs.length > 0) {
-      installedSwcPackages = installedPkgs.sort().join(',')
-    }
-  } catch (_) {}
-
-  eventSwcLoadFailure({
-    nextVersion,
-    glibcVersion,
-    installedSwcPackages,
-    arch: process.arch,
-    platform: process.platform,
-    nodeVersion: process.versions.node,
-  })
+  eventSwcLoadFailure({ wasm: triedWasm ? 'failed' : undefined })
     .then(() => lockfilePatchPromise.cur || Promise.resolve())
     .finally(() => {
       Log.error(
@@ -106,15 +110,21 @@ function logLoadFailure(attempts) {
     })
 }
 
-async function loadWasm() {
+async function loadWasm(importPath = '') {
   if (wasmBindings) {
     return wasmBindings
   }
 
   let attempts = []
   for (let pkg of ['@next/swc-wasm-nodejs', '@next/swc-wasm-web']) {
     try {
-      let bindings = await import(pkg)
+      let pkgPath = pkg
+
+      if (importPath) {
+        // the import path must be exact when not in node_modules
+        pkgPath = path.join(importPath, pkg, 'wasm.js')
+      }
+      let bindings = await import(pkgPath)
       if (pkg === '@next/swc-wasm-web') {
         bindings = await bindings.default()
       }
@@ -139,14 +149,16 @@ async function loadWasm() {
       }
       return wasmBindings
     } catch (e) {
-      // Do not report attempts to load wasm when it is still experimental
-      // if (e?.code === 'ERR_MODULE_NOT_FOUND') {
-      //   attempts.push(`Attempted to load ${pkg}, but it was not installed`)
-      // } else {
-      //   attempts.push(
-      //     `Attempted to load ${pkg}, but an error occurred: ${e.message ?? e}`
-      //   )
-      // }
+      // Only log attempts for loading wasm when loading as fallback
+      if (importPath) {
+        if (e?.code === 'ERR_MODULE_NOT_FOUND') {
+          attempts.push(`Attempted to load ${pkg}, but it was not installed`)
+        } else {
+          attempts.push(
+            `Attempted to load ${pkg}, but an error occurred: ${e.message ?? e}`
+          )
+        }
+      }
     }
   }
 

packages/next/compiled/tar/LICENSE

@@ -0,0 +1,15 @@
+The ISC License
+
+Copyright (c) Isaac Z. Schlueter and Contributors
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR
+IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

packages/next/compiled/tar/index.js

@@ -0,0 +1 @@
+{"name":"tar","main":"index.js","author":"Isaac Z. Schlueter <[email protected]> (http://blog.izs.me/)","license":"ISC"}

packages/next/compiled/tar/package.json

@@ -0,0 +1,118 @@
+import os from 'os'
+import fs from 'fs'
+import path from 'path'
+import * as Log from '../build/output/log'
+import { execSync } from 'child_process'
+import tar from 'next/dist/compiled/tar'
+import fetch from 'next/dist/compiled/node-fetch'
+import { fileExists } from './file-exists'
+
+const MAX_VERSIONS_TO_CACHE = 5
+
+export async function downloadWasmSwc(
+  version: string,
+  wasmDirectory: string,
+  variant: 'nodejs' | 'web' = 'nodejs'
+) {
+  const pkgName = `@next/swc-wasm-${variant}`
+  const tarFileName = `${pkgName.substring(6)}-${version}.tgz`
+  const outputDirectory = path.join(wasmDirectory, pkgName)
+
+  if (await fileExists(outputDirectory)) {
+    // if the package is already downloaded a different
+    // failure occurred than not being present
+    return
+  }
+
+  // get platform specific cache directory adapted from playwright's handling
+  // https://github.com/microsoft/playwright/blob/7d924470d397975a74a19184c136b3573a974e13/packages/playwright-core/src/utils/registry.ts#L141
+  const cacheDirectory = (() => {
+    let result
+    const envDefined = process.env['NEXT_SWC_PATH']
+
+    if (envDefined) {
+      result = envDefined
+    } else {
+      let systemCacheDirectory
+      if (process.platform === 'linux') {
+        systemCacheDirectory =
+          process.env.XDG_CACHE_HOME || path.join(os.homedir(), '.cache')
+      } else if (process.platform === 'darwin') {
+        systemCacheDirectory = path.join(os.homedir(), 'Library', 'Caches')
+      } else if (process.platform === 'win32') {
+        systemCacheDirectory =
+          process.env.LOCALAPPDATA ||
+          path.join(os.homedir(), 'AppData', 'Local')
+      } else {
+        console.error(new Error('Unsupported platform: ' + process.platform))
+        process.exit(0)
+      }
+      result = path.join(systemCacheDirectory, 'next-swc')
+    }
+
+    if (!path.isAbsolute(result)) {
+      // It is important to resolve to the absolute path:
+      //   - for unzipping to work correctly;
+      //   - so that registry directory matches between installation and execution.
+      // INIT_CWD points to the root of `npm/yarn install` and is probably what
+      // the user meant when typing the relative path.
+      result = path.resolve(process.env['INIT_CWD'] || process.cwd(), result)
+    }
+    return result
+  })()
+
+  await fs.promises.mkdir(outputDirectory, { recursive: true })
+
+  const extractFromTar = async () => {
+    await tar.x({
+      file: path.join(cacheDirectory, tarFileName),
+      cwd: outputDirectory,
+      strip: 1,
+    })
+  }
+
+  if (!(await fileExists(path.join(cacheDirectory, tarFileName)))) {
+    Log.info('Downloading WASM swc package...')
+    await fs.promises.mkdir(cacheDirectory, { recursive: true })
+    const tempFile = path.join(
+      cacheDirectory,
+      `${tarFileName}.temp-${Date.now()}`
+    )
+    let registry = `https://registry.npmjs.org/`
+
+    try {
+      const output = execSync('npm config get registry').toString().trim()
+      if (output.startsWith('http')) {
+        registry = output
+      }
+    } catch (_) {}
+
+    await fetch(`${registry}${pkgName}/-/${tarFileName}`).then((res) => {
+      if (!res.ok) {
+        throw new Error(`request failed with status ${res.status}`)
+      }
+      const cacheWriteStream = fs.createWriteStream(tempFile)
+
+      return new Promise<void>((resolve, reject) => {
+        res.body
+          .pipe(cacheWriteStream)
+          .on('error', (err) => reject(err))
+          .on('finish', () => resolve())
+      }).finally(() => cacheWriteStream.close())
+    })
+    await fs.promises.rename(tempFile, path.join(cacheDirectory, tarFileName))
+  }
+  await extractFromTar()
+
+  const cacheFiles = await fs.promises.readdir(cacheDirectory)
+
+  if (cacheFiles.length > MAX_VERSIONS_TO_CACHE) {
+    cacheFiles.sort()
+
+    for (let i = MAX_VERSIONS_TO_CACHE - 1; i++; i < cacheFiles.length) {
+      await fs.promises
+        .unlink(path.join(cacheDirectory, cacheFiles[i]))
+        .catch(() => {})
+    }
+  }
+}

packages/next/lib/download-wasm-swc.ts

@@ -252,6 +252,7 @@
     "string-hash": "1.1.3",
     "string_decoder": "1.3.0",
     "strip-ansi": "6.0.0",
+    "tar": "6.1.11",
     "taskr": "1.1.0",
     "terser": "5.10.0",
     "text-table": "0.2.0",

packages/next/package.json

@@ -1410,6 +1410,16 @@ export async function ncc_nft(task, opts) {
     .ncc({ packageName: '@vercel/nft', externals })
     .target('compiled/@vercel/nft')
 }
+
+// eslint-disable-next-line camelcase
+externals['tar'] = 'next/dist/compiled/tar'
+export async function ncc_tar(task, opts) {
+  await task
+    .source(opts.src || relative(__dirname, require.resolve('tar')))
+    .ncc({ packageName: 'tar', externals })
+    .target('compiled/tar')
+}
+
 // eslint-disable-next-line camelcase
 externals['terser'] = 'next/dist/compiled/terser'
 export async function ncc_terser(task, opts) {
@@ -1729,6 +1739,7 @@ export async function ncc(task, opts) {
         'ncc_string_hash',
         'ncc_strip_ansi',
         'ncc_nft',
+        'ncc_tar',
         'ncc_terser',
         'ncc_text_table',
         'ncc_unistore',