Reduce FS access for incremental cache (#57902)

This passes down the route kind information to the incrememntal cache so it no longer needs to test some files existing in order to validate if the file exists or not for a route.

Author: wyattjoh

packages/next/src/build/index.ts

@@ -81,6 +81,7 @@ import type { BuildManifest } from '../server/get-page-files'
 import { normalizePagePath } from '../shared/lib/page-path/normalize-page-path'
 import { getPagePath } from '../server/require'
 import * as ciEnvironment from '../telemetry/ci-info'
+
 import {
   eventBuildOptimize,
   eventCliSession,
@@ -155,6 +156,7 @@ import { collectBuildTraces } from './collect-build-traces'
 import type { BuildTraceContext } from './webpack/plugins/next-trace-entrypoints-plugin'
 import { formatManifest } from './manifests/formatter/format-manifest'
 import { getStartServerInfo, logStartInfo } from '../server/lib/app-info-log'
+import type { NextEnabledDirectories } from '../server/base-server'
 
 interface ExperimentalBypassForInfo {
   experimentalBypassFor?: RouteHas[]
@@ -347,7 +349,6 @@ export default async function build(
   const isCompile = buildMode === 'experimental-compile'
   const isGenerate = buildMode === 'experimental-generate'
 
-  let hasAppDir = false
   try {
     const nextBuildSpan = trace('next-build', undefined, {
       buildMode: buildMode,
@@ -432,11 +433,14 @@ export default async function build(
       setGlobal('telemetry', telemetry)
 
       const publicDir = path.join(dir, 'public')
-      const isAppDirEnabled = true
       const { pagesDir, appDir } = findPagesDir(dir)
       NextBuildContext.pagesDir = pagesDir
       NextBuildContext.appDir = appDir
-      hasAppDir = Boolean(appDir)
+
+      const enabledDirectories: NextEnabledDirectories = {
+        app: typeof appDir === 'string',
+        pages: typeof pagesDir === 'string',
+      }
 
       const isSrcDir = path
         .relative(dir, pagesDir || appDir || '')
@@ -1292,8 +1296,9 @@ export default async function build(
       } = await initializeIncrementalCache({
         fs: nodeFs,
         dev: false,
-        appDir: isAppDirEnabled,
-        fetchCache: isAppDirEnabled,
+        pagesDir: true,
+        appDir: true,
+        fetchCache: true,
         flushToDisk: config.experimental.isrFlushToDisk,
         serverDistDir: path.join(distDir, 'server'),
         fetchCacheKeyPrefix: config.experimental.fetchCacheKeyPrefix,
@@ -1317,7 +1322,7 @@ export default async function build(
         incrementalCacheIpcPort,
         incrementalCacheIpcValidationKey
       )
-      const appStaticWorkers = isAppDirEnabled
+      const appStaticWorkers = appDir
         ? createStaticWorker(
             incrementalCacheIpcPort,
             incrementalCacheIpcValidationKey
@@ -2032,7 +2037,7 @@ export default async function build(
         (combinedPages.length > 0 ||
           useStaticPages404 ||
           useDefaultStatic500 ||
-          isAppDirEnabled)
+          appDir)
       ) {
         const staticGenerationSpan =
           nextBuildSpan.traceChild('static-generation')
@@ -2172,7 +2177,7 @@ export default async function build(
 
           const exportOptions: ExportAppOptions = {
             nextConfig: exportConfig,
-            hasAppDir,
+            enabledDirectories,
             silent: false,
             buildExport: true,
             debugOutput,
@@ -2870,7 +2875,7 @@ export default async function build(
         const options: ExportAppOptions = {
           buildExport: false,
           nextConfig: config,
-          hasAppDir,
+          enabledDirectories,
           silent: true,
           threads: config.experimental.cpus,
           outdir: path.join(dir, configOutDir),

packages/next/src/build/utils.ts

@@ -69,8 +69,8 @@ import { nodeFs } from '../server/lib/node-fs-methods'
 import * as ciEnvironment from '../telemetry/ci-info'
 import { normalizeAppPath } from '../shared/lib/router/utils/app-paths'
 import { denormalizeAppPagePath } from '../shared/lib/page-path/denormalize-app-path'
-import { AppRouteRouteModule } from '../server/future/route-modules/app-route/module.compiled'
 import { RouteKind } from '../server/future/route-kind'
+import { isAppRouteRouteModule } from '../server/future/route-modules/checks'
 
 export type ROUTER_TYPE = 'pages' | 'app'
 
@@ -1267,6 +1267,8 @@ export async function buildAppStaticPaths({
   const incrementalCache = new IncrementalCache({
     fs: nodeFs,
     dev: true,
+    // Enabled both for build as we're only writing this cache, not reading it.
+    pagesDir: true,
     appDir: true,
     flushToDisk: isrFlushToDisk,
     serverDistDir: path.join(distDir, 'server'),
@@ -1506,7 +1508,7 @@ export async function isPageStatic({
         const { tree, staticGenerationAsyncStorage, serverHooks } = ComponentMod
 
         const generateParams: GenerateParams =
-          routeModule && AppRouteRouteModule.is(routeModule)
+          routeModule && isAppRouteRouteModule(routeModule)
             ? [
                 {
                   config: {

packages/next/src/export/helpers/create-incremental-cache.ts

@@ -1,14 +1,17 @@
+import type { NextEnabledDirectories } from '../../server/base-server'
+
 import path from 'path'
-import fs from 'fs'
 import { IncrementalCache } from '../../server/lib/incremental-cache'
 import { hasNextSupport } from '../../telemetry/ci-info'
+import { nodeFs } from '../../server/lib/node-fs-methods'
 
 export function createIncrementalCache(
   incrementalCacheHandlerPath: string | undefined,
   isrMemoryCacheSize: number | undefined,
   fetchCacheKeyPrefix: string | undefined,
   distDir: string,
-  dir: string
+  dir: string,
+  enabledDirectories: NextEnabledDirectories
 ) {
   // Custom cache handler overrides.
   let CacheHandler: any
@@ -37,13 +40,9 @@ export function createIncrementalCache(
       },
       notFoundRoutes: [],
     }),
-    fs: {
-      readFile: fs.promises.readFile,
-      readFileSync: fs.readFileSync,
-      writeFile: (f, d) => fs.promises.writeFile(f, d),
-      mkdir: (d) => fs.promises.mkdir(d, { recursive: true }),
-      stat: (f) => fs.promises.stat(f),
-    },
+    fs: nodeFs,
+    pagesDir: enabledDirectories.pages,
+    appDir: enabledDirectories.app,
     serverDistDir: path.join(distDir, 'server'),
     CurCacheHandler: CacheHandler,
     minimalMode: hasNextSupport,

packages/next/src/export/index.ts

@@ -215,6 +215,8 @@ export async function exportAppImpl(
   // attempt to load global env values so they are available in next.config.js
   span.traceChild('load-dotenv').traceFn(() => loadEnvConfig(dir, false, Log))
 
+  const { enabledDirectories } = options
+
   const nextConfig =
     options.nextConfig ||
     (await span
@@ -444,7 +446,7 @@ export async function exportAppImpl(
   }
 
   let serverActionsManifest
-  if (options.hasAppDir) {
+  if (enabledDirectories.app) {
     serverActionsManifest = require(join(
       distDir,
       SERVER_DIRECTORY,
@@ -487,15 +489,15 @@ export async function exportAppImpl(
     nextScriptWorkers: nextConfig.experimental.nextScriptWorkers,
     optimizeFonts: nextConfig.optimizeFonts as FontConfig,
     largePageDataBytes: nextConfig.experimental.largePageDataBytes,
-    serverComponents: options.hasAppDir,
     serverActions: nextConfig.experimental.serverActions,
+    serverComponents: enabledDirectories.app,
     nextFontManifest: require(join(
       distDir,
       'server',
       `${NEXT_FONT_MANIFEST}.json`
     )),
     images: nextConfig.images,
-    ...(options.hasAppDir
+    ...(enabledDirectories.app
       ? {
           serverActionsManifest,
         }
@@ -698,6 +700,7 @@ export async function exportAppImpl(
           incrementalCacheHandlerPath:
             nextConfig.experimental.incrementalCacheHandlerPath,
           enableExperimentalReact: needsExperimentalReact(nextConfig),
+          enabledDirectories,
         })
       })
 

packages/next/src/export/types.ts

@@ -8,6 +8,7 @@ import type { FontConfig } from '../server/font-utils'
 import type { ExportPathMap, NextConfigComplete } from '../server/config-shared'
 import type { Span } from '../trace'
 import type { Revalidate } from '../server/lib/revalidate'
+import type { NextEnabledDirectories } from '../server/base-server'
 
 export interface AmpValidation {
   page: string
@@ -58,6 +59,7 @@ export interface ExportPageInput {
   fetchCacheKeyPrefix?: string
   nextConfigOutput?: NextConfigComplete['output']
   enableExperimentalReact?: boolean
+  enabledDirectories: NextEnabledDirectories
 }
 
 export type ExportedPageFile = {
@@ -98,7 +100,7 @@ export type ExportWorker = (
 
 export interface ExportAppOptions {
   outdir: string
-  hasAppDir: boolean
+  enabledDirectories: NextEnabledDirectories
   silent?: boolean
   threads?: number
   debugOutput?: boolean

packages/next/src/export/worker.ts

@@ -65,6 +65,7 @@ async function exportPageImpl(
     enableExperimentalReact,
     ampValidatorPath,
     trailingSlash,
+    enabledDirectories,
   } = input
 
   if (enableExperimentalReact) {
@@ -222,7 +223,8 @@ async function exportPageImpl(
             isrMemoryCacheSize,
             fetchCacheKeyPrefix,
             distDir,
-            dir
+            dir,
+            enabledDirectories
           )
         : undefined
 

packages/next/src/server/app-render/strip-flight-headers.ts

@@ -0,0 +1,14 @@
+import type { IncomingHttpHeaders } from 'node:http'
+
+import { FLIGHT_PARAMETERS } from '../../client/components/app-router-headers'
+
+/**
+ * Removes the flight headers from the request.
+ *
+ * @param req the request to strip the headers from
+ */
+export function stripFlightHeaders(headers: IncomingHttpHeaders) {
+  for (const [header] of FLIGHT_PARAMETERS) {
+    delete headers[header.toLowerCase()]
+  }
+}