feat(turbopack): Experimental wasm build (#57906)

### What?

continuation of #57851, since it is from a remote branch that I don't have access to write.

Co-authored-by: Maia Teegarden <[email protected]>
Co-authored-by: Tim Neutkens <[email protected]>

Author: 3 people

.github/workflows/build_and_deploy.yml

@@ -340,12 +340,65 @@ jobs:
           name: turbo run summary
           path: .turbo/runs
 
+  build-wasm:
+    strategy:
+      matrix:
+        target: [web, nodejs]
+
+    runs-on:
+      - 'self-hosted'
+      - 'linux'
+      - 'x64'
+      - 'metal'
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup node
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ env.NODE_LTS_VERSION }}
+          check-latest: true
+      - run: corepack enable
+
+      - name: Install Rust
+        uses: ./.github/actions/setup-rust
+        with:
+          targets: wasm32-unknown-unknown
+
+      - run: npm i -g turbo@${{ env.TURBO_VERSION }}
+
+      - name: Install wasm-pack
+        run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
+
+      - name: normalize versions
+        run: node scripts/normalize-version-bump.js
+
+      - name: Build
+        run: turbo run build-wasm -vvv --remote-cache-timeout 90 --summarize -- --target ${{ matrix.target }} --features tracing/release_max_level_info
+
+      - name: Add target to folder name
+        run: '[[ -d "packages/next-swc/crates/wasm/pkg" ]] && mv packages/next-swc/crates/wasm/pkg packages/next-swc/crates/wasm/pkg-${{ matrix.target }} || ls packages/next-swc/crates/wasm'
+
+      - name: Upload turbo summary artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: turbo run summary
+          path: .turbo/runs
+
+      - name: Upload swc artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: wasm-binaries
+          path: packages/next-swc/crates/wasm/pkg-*
+
   publishRelease:
     if: ${{ needs.build.outputs.isRelease == 'true' }}
     name: Potentially publish release
     runs-on: ubuntu-latest
     needs:
       - build
+      - build-wasm
       - build-native
     permissions:
       contents: write
@@ -376,6 +429,11 @@ jobs:
           name: next-swc-binaries
           path: packages/next-swc/native
 
+      - uses: actions/download-artifact@v3
+        with:
+          name: wasm-binaries
+          path: packages/next-swc/crates/wasm
+
       - run: npm i -g [email protected] # need latest version for provenance (pinning to avoid bugs)
       - run: echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" >> ~/.npmrc
       - run: ./scripts/publish-native.js

.github/workflows/build_and_test.yml

@@ -152,14 +152,14 @@ jobs:
       afterBuild: RUST_BACKTRACE=0 NEXT_EXTERNAL_TESTS_FILTERS="$(pwd)/test/turbopack-tests-manifest.json" TURBOPACK=1 node run-tests.js --timings -g ${{ matrix.group }}/5 -c ${TEST_CONCURRENCY} --type integration
     secrets: inherit
 
-  # test-next-swc-wasm:
-  #   name: test next-swc wasm
-  #   needs: ['build-native', 'build-next']
-  #   uses: ./.github/workflows/build_reusable.yml
-  #   with:
-  #     skipForDocsOnly: 'yes'
-  #     afterBuild: rustup target add wasm32-unknown-unknown && curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh && node ./scripts/normalize-version-bump.js && turbo run build-wasm -- --target nodejs --features tracing/release_max_level_info && git checkout . && mv packages/next-swc/crates/wasm/pkg packages/next-swc/crates/wasm/pkg-nodejs && node ./scripts/setup-wasm.mjs && NEXT_TEST_MODE=start TEST_WASM=true node run-tests.js test/production/pages-dir/production/test/index.test.ts test/e2e/streaming-ssr/index.test.ts
-  #   secrets: inherit
+  test-next-swc-wasm:
+    name: test next-swc wasm
+    needs: ['build-native', 'build-next']
+    uses: ./.github/workflows/build_reusable.yml
+    with:
+      skipForDocsOnly: 'yes'
+      afterBuild: rustup target add wasm32-unknown-unknown && curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh && node ./scripts/normalize-version-bump.js && turbo run build-wasm -- --target nodejs --features tracing/release_max_level_info && git checkout . && mv packages/next-swc/crates/wasm/pkg packages/next-swc/crates/wasm/pkg-nodejs && node ./scripts/setup-wasm.mjs && NEXT_TEST_MODE=start TEST_WASM=true node run-tests.js test/production/pages-dir/production/test/index.test.ts test/e2e/streaming-ssr/index.test.ts
+    secrets: inherit
 
   test-unit:
     name: test unit
@@ -262,6 +262,7 @@ jobs:
         'test-cargo-integration',
         'test-cargo-bench',
         'rust-check',
+        'test-next-swc-wasm',
         'test-turbopack-dev',
         'test-turbopack-integration',
       ]

contributing/core/building.md

@@ -8,4 +8,6 @@ pnpm build
 
 By default, the latest canary of the `next-swc` binaries will be installed and used. If you are actively working on Rust code or you need to test out the most recent Rust code that hasn't been published as a canary yet, you can [install Rust](https://www.rust-lang.org/tools/install) and run `pnpm --filter=@next/swc build-native`.
 
+If you want to test out the wasm build locally, you will need to [install wasm-pack](https://rustwasm.github.io/wasm-pack/installer/). Run `pnpm --filter=@next/swc build-wasm --target <wasm_target>` to build and `node ./scripts/setup-wasm.mjs` to copy it into your `node_modules`. Run next with `NODE_OPTIONS='--no-addons'` to force it to use the wasm binary.
+
 If you need to clean the project for any reason, use `pnpm clean`.

docs/04-architecture/nextjs-compiler.mdx

@@ -17,6 +17,7 @@ We chose to build on SWC for a few reasons:
 
 - **Extensibility:** SWC can be used as a Crate inside Next.js, without having to fork the library or workaround design constraints.
 - **Performance:** We were able to achieve ~3x faster Fast Refresh and ~5x faster builds in Next.js by switching to SWC, with more room for optimization still in progress.
+- **WebAssembly:** Rust's support for WASM is essential for supporting all possible platforms and taking Next.js development everywhere.
 - **Community:** The Rust community and ecosystem are amazing and still growing.
 
 ## Supported Features

packages/next/src/build/index.ts

@@ -1030,7 +1030,7 @@ export default async function build(
         const turboJson = findUp.sync('turbo.json', { cwd: dir })
         // eslint-disable-next-line no-shadow
         const packagePath = findUp.sync('package.json', { cwd: dir })
-        let binding = await loadBindings()
+        let binding = await loadBindings(config?.experimental?.useWasmBinary)
 
         let root =
           turboNextBuildRoot ??

packages/next/src/build/jest/jest.ts

@@ -89,7 +89,7 @@ export default function nextJest(options: { dir?: string } = {}) {
           : customJestConfig) ?? {}
 
       // eagerly load swc bindings instead of waiting for transform calls
-      await loadBindings()
+      await loadBindings(nextConfig?.experimental?.useWasmBinary)
 
       if (lockfilePatchPromise.cur) {
         await lockfilePatchPromise.cur

packages/next/src/build/swc/index.ts

@@ -31,7 +31,7 @@ const infoLog = (...args: any[]) => {
  * Based on napi-rs's target triples, returns triples that have corresponding next-swc binaries.
  */
 export const getSupportedArchTriples: () => Record<string, any> = () => {
-  const { darwin, win32, linux } = platformArchTriples
+  const { darwin, win32, linux, freebsd, android } = platformArchTriples
 
   return {
     darwin,
@@ -48,6 +48,16 @@ export const getSupportedArchTriples: () => Record<string, any> = () => {
         (triple: { abi: string }) => triple.abi !== 'gnux32'
       ),
       arm64: linux.arm64,
+      // This target is being deprecated, however we keep it in `knownDefaultWasmFallbackTriples` for now
+      arm: linux.arm,
+    },
+    // Below targets are being deprecated, however we keep it in `knownDefaultWasmFallbackTriples` for now
+    freebsd: {
+      x64: freebsd.x64,
+    },
+    android: {
+      arm64: android.arm64,
+      arm: android.arm,
     },
   }
 }
@@ -67,7 +77,7 @@ const triples = (() => {
 
   if (rawTargetTriple) {
     Log.warn(
-      `Trying to load next-swc for target triple ${rawTargetTriple}, but this target is not supported`
+      `Trying to load next-swc for target triple ${rawTargetTriple}, but there next-swc does not have native bindings support`
     )
   } else {
     Log.warn(
@@ -97,6 +107,21 @@ function checkVersionMismatch(pkgData: any) {
   }
 }
 
+// These are the platforms we'll try to load wasm bindings first,
+// only try to load native bindings if loading wasm binding somehow fails.
+// Fallback to native binding is for migration period only,
+// once we can verify loading-wasm-first won't cause visible regressions,
+// we'll not include native bindings for these platform at all.
+const knownDefaultWasmFallbackTriples = [
+  'x86_64-unknown-freebsd',
+  'aarch64-linux-android',
+  'arm-linux-androideabi',
+  'armv7-unknown-linux-gnueabihf',
+  'i686-pc-windows-msvc',
+  // WOA targets are TBD, while current userbase is small we may support it in the future
+  //'aarch64-pc-windows-msvc',
+]
+
 // The last attempt's error code returned when cjs require to native bindings fails.
 // If node.js throws an error without error code, this should be `unknown` instead of undefined.
 // For the wasm-first targets (`knownDefaultWasmFallbackTriples`) this will be `unsupported_target`.
@@ -149,7 +174,9 @@ export interface Binding {
   teardownCrashReporter?: any
 }
 
-export async function loadBindings(): Promise<Binding> {
+export async function loadBindings(
+  useWasmBinary: boolean = false
+): Promise<Binding> {
   if (pendingBindings) {
     return pendingBindings
   }
@@ -176,6 +203,27 @@ export async function loadBindings(): Promise<Binding> {
     }
 
     let attempts: any[] = []
+    const disableWasmFallback = process.env.NEXT_DISABLE_SWC_WASM
+    const unsupportedPlatform = triples.some(
+      (triple: any) =>
+        !!triple?.raw && knownDefaultWasmFallbackTriples.includes(triple.raw)
+    )
+    const shouldLoadWasmFallbackFirst =
+      !disableWasmFallback && unsupportedPlatform && useWasmBinary
+
+    if (!unsupportedPlatform && useWasmBinary) {
+      Log.warn(
+        `experimental.useWasmBinary is not an option for supported platform ${PlatformName}/${ArchName} and will be ignored.`
+      )
+    }
+
+    if (shouldLoadWasmFallbackFirst) {
+      lastNativeBindingsLoadErrorCode = 'unsupported_target'
+      const fallbackBindings = await tryLoadWasmWithFallback(attempts)
+      if (fallbackBindings) {
+        return resolve(fallbackBindings)
+      }
+    }
 
     // Trickle down loading `fallback` bindings:
     //
@@ -184,6 +232,7 @@ export async function loadBindings(): Promise<Binding> {
     // that host system where generated package lock is not matching to the guest system running on, try to manually
     // download corresponding target triple and load it. This won't be triggered if native bindings are failed to load
     // with other reasons than `ERR_MODULE_NOT_FOUND`.
+    // - Lastly, falls back to wasm binding where possible.
     try {
       return resolve(loadNative())
     } catch (a) {
@@ -201,7 +250,7 @@ export async function loadBindings(): Promise<Binding> {
       attempts = attempts.concat(a)
     }
 
-    logLoadFailure(attempts)
+    logLoadFailure(attempts, true)
   })
   return pendingBindings
 }
@@ -230,8 +279,6 @@ async function tryLoadNativeWithFallback(attempts: Array<string>) {
   return undefined
 }
 
-// We may end up using this function in the future if/when we support wasm builds again
-// eslint-disable-next-line @typescript-eslint/no-unused-vars
 async function tryLoadWasmWithFallback(attempts: any) {
   try {
     let bindings = await loadWasm('')
@@ -284,12 +331,18 @@ function loadBindingsSync() {
     attempts = attempts.concat(a)
   }
 
+  // we can leverage the wasm bindings if they are already
+  // loaded
+  if (wasmBindings) {
+    return wasmBindings
+  }
+
   logLoadFailure(attempts)
 }
 
 let loggingLoadFailure = false
 
-function logLoadFailure(attempts: any) {
+function logLoadFailure(attempts: any, triedWasm = false) {
   // make sure we only emit the event and log the failure once
   if (loggingLoadFailure) return
   loggingLoadFailure = true
@@ -300,6 +353,7 @@ function logLoadFailure(attempts: any) {
 
   // @ts-expect-error TODO: this event has a wrong type.
   eventSwcLoadFailure({
+    wasm: triedWasm ? 'failed' : undefined,
     nativeBindingsErrorCode: lastNativeBindingsLoadErrorCode,
   })
     .then(() => lockfilePatchPromise.cur || Promise.resolve())

packages/next/src/build/webpack-config.ts

@@ -370,7 +370,7 @@ export default async function getBaseWebpackConfig(
 
   // eagerly load swc bindings instead of waiting for transform calls
   if (!babelConfigFile && isClient) {
-    await loadBindings()
+    await loadBindings(config.experimental.useWasmBinary)
   }
 
   if (!loggedIgnoredCompilerOptions && !useSWCLoader && config.compiler) {

packages/next/src/cli/next-info.ts

@@ -59,6 +59,9 @@ async function getNextConfig() {
 
   return {
     output: config.output ?? 'N/A',
+    experimental: {
+      useWasmBinary: config.experimental?.useWasmBinary,
+    },
   }
 }
 
@@ -350,8 +353,11 @@ async function printVerbose() {
 
           // First, try to load next-swc via loadBindings.
           try {
+            let nextConfig = await getNextConfig()
             const { loadBindings } = require('../build/swc')
-            const bindings = await loadBindings()
+            const bindings = await loadBindings(
+              nextConfig.experimental?.useWasmBinary
+            )
             // Run arbitary function to verify the bindings are loaded correctly.
             const target = bindings.getTargetTriple()
 

packages/next/src/server/config-schema.ts

@@ -367,6 +367,7 @@ export const configSchema: zod.ZodType<NextConfig> = z.lazy(() =>
         serverSourceMaps: z.boolean().optional(),
         bundlePagesExternals: z.boolean().optional(),
         staticWorkerRequestDeduping: z.boolean().optional(),
+        useWasmBinary: z.boolean().optional(),
       })
       .optional(),
     exportPathMap: z