move static worker IPC server behind experimental flag (#57943)

A shared incremental cache IPC server was introduced for build-time static workers as an optimization to dedupe fetch requests, however this can cause fetch-related flakiness to builds under certain conditions (e.g., large payloads).

This moves the optimization behind an experimental flag to unblock those running into IPC-related build time errors while we work on an alternative solution for the underlying issue

Fixes #53695

Author: ztanner

packages/next/src/build/index.ts

@@ -1218,8 +1218,8 @@ export default async function build(
         : config.experimental.cpus || 4
 
       function createStaticWorker(
-        incrementalCacheIpcPort: number,
-        incrementalCacheIpcValidationKey: string
+        incrementalCacheIpcPort?: number,
+        incrementalCacheIpcValidationKey?: string
       ) {
         let infoPrinted = false
 
@@ -1259,7 +1259,9 @@ export default async function build(
           forkOptions: {
             env: {
               ...process.env,
-              __NEXT_INCREMENTAL_CACHE_IPC_PORT: incrementalCacheIpcPort + '',
+              __NEXT_INCREMENTAL_CACHE_IPC_PORT: incrementalCacheIpcPort
+                ? incrementalCacheIpcPort + ''
+                : undefined,
               __NEXT_INCREMENTAL_CACHE_IPC_KEY:
                 incrementalCacheIpcValidationKey,
             },
@@ -1281,42 +1283,45 @@ export default async function build(
           >
       }
 
-      let CacheHandler: any
+      let incrementalCacheIpcPort
+      let incrementalCacheIpcValidationKey
 
-      if (incrementalCacheHandlerPath) {
-        CacheHandler = require(path.isAbsolute(incrementalCacheHandlerPath)
-          ? incrementalCacheHandlerPath
-          : path.join(dir, incrementalCacheHandlerPath))
-        CacheHandler = CacheHandler.default || CacheHandler
-      }
+      if (config.experimental.staticWorkerRequestDeduping) {
+        let CacheHandler
+        if (incrementalCacheHandlerPath) {
+          CacheHandler = require(path.isAbsolute(incrementalCacheHandlerPath)
+            ? incrementalCacheHandlerPath
+            : path.join(dir, incrementalCacheHandlerPath))
+          CacheHandler = CacheHandler.default || CacheHandler
+        }
 
-      const {
-        ipcPort: incrementalCacheIpcPort,
-        ipcValidationKey: incrementalCacheIpcValidationKey,
-      } = await initializeIncrementalCache({
-        fs: nodeFs,
-        dev: false,
-        pagesDir: true,
-        appDir: true,
-        fetchCache: true,
-        flushToDisk: config.experimental.isrFlushToDisk,
-        serverDistDir: path.join(distDir, 'server'),
-        fetchCacheKeyPrefix: config.experimental.fetchCacheKeyPrefix,
-        maxMemoryCacheSize: config.experimental.isrMemoryCacheSize,
-        getPrerenderManifest: () => ({
-          version: -1 as any, // letting us know this doesn't conform to spec
-          routes: {},
-          dynamicRoutes: {},
-          notFoundRoutes: [],
-          preview: null as any, // `preview` is special case read in next-dev-server
-        }),
-        requestHeaders: {},
-        CurCacheHandler: CacheHandler,
-        minimalMode: ciEnvironment.hasNextSupport,
+        const cacheInitialization = await initializeIncrementalCache({
+          fs: nodeFs,
+          dev: false,
+          pagesDir: true,
+          appDir: true,
+          fetchCache: true,
+          flushToDisk: config.experimental.isrFlushToDisk,
+          serverDistDir: path.join(distDir, 'server'),
+          fetchCacheKeyPrefix: config.experimental.fetchCacheKeyPrefix,
+          maxMemoryCacheSize: config.experimental.isrMemoryCacheSize,
+          getPrerenderManifest: () => ({
+            version: -1 as any, // letting us know this doesn't conform to spec
+            routes: {},
+            dynamicRoutes: {},
+            notFoundRoutes: [],
+            preview: null as any, // `preview` is special case read in next-dev-server
+          }),
+          requestHeaders: {},
+          CurCacheHandler: CacheHandler,
+          minimalMode: ciEnvironment.hasNextSupport,
+          allowedRevalidateHeaderKeys:
+            config.experimental.allowedRevalidateHeaderKeys,
+        })
 
-        allowedRevalidateHeaderKeys:
-          config.experimental.allowedRevalidateHeaderKeys,
-      })
+        incrementalCacheIpcPort = cacheInitialization.ipcPort
+        incrementalCacheIpcValidationKey = cacheInitialization.ipcValidationKey
+      }
 
       const pagesStaticWorkers = createStaticWorker(
         incrementalCacheIpcPort,

packages/next/src/server/config-schema.ts

@@ -366,6 +366,7 @@ export const configSchema: zod.ZodType<NextConfig> = z.lazy(() =>
         serverMinification: z.boolean().optional(),
         serverSourceMaps: z.boolean().optional(),
         bundlePagesExternals: z.boolean().optional(),
+        staticWorkerRequestDeduping: z.boolean().optional(),
       })
       .optional(),
     exportPathMap: z

packages/next/src/server/config-shared.ts

@@ -334,6 +334,10 @@ export interface ExperimentalConfig {
    * Enables the bundling of node_modules packages (externals) for pages server-side bundles.
    */
   bundlePagesExternals?: boolean
+  /**
+   * Uses an IPC server to dedupe build-time requests to the cache handler
+   */
+  staticWorkerRequestDeduping?: boolean
 }
 
 export type ExportPathMap = {
@@ -783,6 +787,7 @@ export const defaultConfig: NextConfig = {
     typedRoutes: false,
     instrumentationHook: false,
     bundlePagesExternals: false,
+    staticWorkerRequestDeduping: false,
   },
 }
 

test/e2e/app-dir/app-fetch-deduping/app-fetch-deduping.test.ts

@@ -34,10 +34,15 @@ describe('app-fetch-deduping', () => {
 
       afterAll(() => externalServer.close())
 
-      it('dedupes requests amongst static workers', async () => {
+      it('dedupes requests amongst static workers when experimental.staticWorkerRequestDeduping is enabled', async () => {
         const next = await createNext({
           files: new FileRef(__dirname),
           env: { TEST_SERVER_PORT: `${externalServerPort}` },
+          nextConfig: {
+            experimental: {
+              staticWorkerRequestDeduping: true,
+            },
+          },
         })
 
         expect(requests.length).toBe(1)