initial isr

Author: pieh

playwright.config.ts

@@ -26,6 +26,7 @@ export default defineConfig({
       /* Add debug logging for netlify cache headers */
       'x-nf-debug-logging': '1',
       'x-next-debug-logging': '1',
+      'x-nf-enable-tracing': '1',
     },
   },
   timeout: 10 * 60 * 1000,

src/adapter/build/netlify-adapter-context.ts

@@ -1,12 +1,49 @@
 import type { FrameworksAPIConfig } from './types.js'
 
+type Revalidate = number | false
+
+export type ISRCacheEntry = {
+  /** body of initial response */
+  content: string
+  /**
+   * initialStatus is the status code that should be applied
+   * when serving the fallback
+   */
+  status?: number
+  /**
+   * initialHeaders are the headers that should be sent when
+   * serving the fallback
+   */
+  headers?: Record<string, string>
+  /**
+   * initial expiration is how long until the fallback entry
+   * is considered expired and no longer valid to serve
+   */
+  expiration?: number
+  /**
+   * initial revalidate is how long until the fallback is
+   * considered stale and should be revalidated
+   */
+  revalidate?: Revalidate
+}
+
+export type ISRDef = {
+  pathname: string
+  queryParams: string[]
+  fallback?: ISRCacheEntry
+}
+
 export function createNetlifyAdapterContext() {
   return {
     frameworksAPIConfig: undefined as FrameworksAPIConfig | undefined,
     preparedOutputs: {
       staticAssets: [] as string[],
       staticAssetsAliases: {} as Record<string, string>,
-      endpoints: [] as string[],
+      endpoints: {} as Record<
+        string,
+        { entry: string; id: string } & ({ type: 'function' } | { type: 'isr'; isrGroup: number })
+      >,
+      isrGroups: {} as Record<number, ISRDef[]>,
       middleware: false,
     },
   }

src/adapter/build/pages-and-app-handlers.ts

@@ -1,4 +1,4 @@
-import { cp, mkdir, writeFile } from 'node:fs/promises'
+import { cp, mkdir, readFile, writeFile } from 'node:fs/promises'
 import { join, relative } from 'node:path/posix'
 
 import type { InSourceConfig } from '@netlify/zip-it-and-ship-it/dist/runtimes/node/in_source_config/index.js'
@@ -41,7 +41,7 @@ export async function onBuildComplete(
   netlifyAdapterContext: NetlifyAdapterContext,
 ) {
   const requiredFiles = new Set<string>()
-  const pathnameToEntry: Record<string, string> = {}
+  const { isrGroups, endpoints } = netlifyAdapterContext.preparedOutputs
 
   for (const outputs of [
     nextAdapterContext.outputs.pages,
@@ -59,31 +59,59 @@ export async function onBuildComplete(
       }
 
       requiredFiles.add(output.filePath)
-      pathnameToEntry[normalizeIndex(output.pathname)] = relative(
-        nextAdapterContext.repoRoot,
-        output.filePath,
-      )
+      endpoints[normalizeIndex(output.pathname)] = {
+        entry: relative(nextAdapterContext.repoRoot, output.filePath),
+        id: normalizeIndex(output.pathname),
+        type: 'function',
+      }
     }
   }
 
   for (const prerender of nextAdapterContext.outputs.prerenders) {
     const normalizedPathname = normalizeIndex(prerender.pathname)
     const normalizedParentOutputId = normalizeIndex(prerender.parentOutputId)
 
-    if (normalizedPathname in pathnameToEntry) {
-      // console.log('Skipping prerender, already have route:', normalizedPathname)
-    } else if (normalizedParentOutputId in pathnameToEntry) {
-      // if we don't have routing for this route yet, add it
-      // console.log('prerender mapping', {
-      //   from: normalizedPathname,
-      //   to: normalizedParentOutputId,
-      // })
-      pathnameToEntry[normalizedPathname] = pathnameToEntry[normalizedParentOutputId]
+    const existingEntryForParent = endpoints[normalizedParentOutputId]
+
+    if (existingEntryForParent) {
+      endpoints[normalizedPathname] = {
+        ...existingEntryForParent,
+        id: normalizedPathname,
+        type: 'isr',
+        isrGroup: prerender.groupId,
+      }
+
+      if (!isrGroups[prerender.groupId]) {
+        isrGroups[prerender.groupId] = []
+      }
+      const isrGroup: (typeof isrGroups)[number][number] = {
+        pathname: normalizedPathname,
+        queryParams: prerender.config.allowQuery ?? [],
+      }
+
+      if (prerender.fallback) {
+        isrGroup.fallback = {
+          content: await readFile(prerender.fallback.filePath, 'utf-8'),
+          status: prerender.fallback.initialStatus,
+          headers: prerender.fallback.initialHeaders
+            ? Object.fromEntries(
+                Object.entries(prerender.fallback.initialHeaders).map(([key, value]) => [
+                  key,
+                  Array.isArray(value) ? value.join(',') : value,
+                ]),
+              )
+            : undefined,
+          expiration: prerender.fallback.initialExpiration,
+          revalidate: prerender.fallback.initialRevalidate,
+        }
+      }
+
+      isrGroups[prerender.groupId].push(isrGroup)
     } else {
-      // console.warn('Could not find parent output for prerender:', {
-      //   pathname: normalizedPathname,
-      //   parentOutputId: normalizedParentOutputId,
-      // })
+      console.warn('Could not find parent output for prerender:', {
+        pathname: normalizedPathname,
+        parentOutputId: normalizedParentOutputId,
+      })
     }
   }
 
@@ -99,12 +127,14 @@ export async function onBuildComplete(
     )
   }
 
-  // copy needed runtime files
-
   await copyRuntime(join(PAGES_AND_APP_FUNCTION_DIR, RUNTIME_DIR))
 
+  const normalizedPathsForFunctionConfig = Object.keys(endpoints).map((pathname) =>
+    pathname.toLowerCase(),
+  )
+
   const functionConfig = {
-    path: Object.keys(pathnameToEntry).map((pathname) => pathname.toLowerCase()),
+    path: normalizedPathsForFunctionConfig,
     nodeBundler: 'none',
     includedFiles: ['**'],
     generator: GENERATOR,
@@ -113,45 +143,18 @@ export async function onBuildComplete(
 
   // generate needed runtime files
   const entrypoint = /* javascript */ `
-    import { AsyncLocalStorage } from 'node:async_hooks'
     import { createRequire } from 'node:module'
-    import { runNextHandler } from './${RUNTIME_DIR}/dist/adapter/run/pages-and-app-handler.js'
 
-    globalThis.AsyncLocalStorage = AsyncLocalStorage
+    import { runHandler } from './${RUNTIME_DIR}/dist/adapter/run/pages-and-app-handler.js'
 
-    const RouterServerContextSymbol = Symbol.for(
-      '@next/router-server-methods'
-    );
+    const pickedOutputs = ${JSON.stringify({ isrGroups, endpoints }, null, 2)}
 
-    if (!globalThis[RouterServerContextSymbol]) {
-      globalThis[RouterServerContextSymbol] = {};
-    }
-
-    globalThis[RouterServerContextSymbol]['.'] = {
-      revalidate: (...args) => {
-        console.log('revalidate called with args:', ...args); 
-      }
-    }
-    
     const require = createRequire(import.meta.url)
 
-    const pathnameToEntry = ${JSON.stringify(pathnameToEntry, null, 2)}
-
     export default async function handler(request, context) {
-      const url = new URL(request.url)
-
-      const entry = pathnameToEntry[url.pathname]
-      if (!entry) {
-        return new Response('Not Found', { status: 404 })
-      }
-
-      const nextHandler = await require('./' + entry)
-
-      if (typeof nextHandler.handler !== 'function') {
-        console.log('.handler is not a function', { nextHandler })
-      }
-
-      return runNextHandler(request, context, nextHandler.handler)
+      const response = await runHandler(request, context, pickedOutputs, require)
+      console.log('Serving response with status:', response.status)
+      return response
     }
 
     export const config = ${JSON.stringify(functionConfig, null, 2)}
@@ -161,7 +164,7 @@ export async function onBuildComplete(
     entrypoint,
   )
 
-  netlifyAdapterContext.preparedOutputs.endpoints.push(...functionConfig.path)
+  // netlifyAdapterContext.preparedOutputs.endpoints.push(...functionConfig.path)
 }
 
 const copyRuntime = async (handlerDirectory: string): Promise<void> => {

src/adapter/build/routing.ts

@@ -647,8 +647,6 @@ export async function onBuildComplete(
 
   await copyRuntime(ROUTING_FUNCTION_DIR)
 
-  // TODO(pieh): middleware case would need to be split in 2 functions
-
   const entrypoint = /* javascript */ `
     import { runNextRouting } from "./dist/adapter/run/routing.js";
 

src/adapter/run/isr.ts

@@ -0,0 +1,136 @@
+import { getDeployStore } from '@netlify/blobs'
+
+import type { ISRCacheEntry } from '../build/netlify-adapter-context.js'
+import type { NetlifyAdapterContext } from '../build/types.js'
+
+function cacheEntryToResponse(entry: ISRCacheEntry, source: 'fallback' | 'blobs') {
+  const headers = new Headers(entry.headers ?? {})
+
+  headers.set('x-isr-revalidate', String(entry.revalidate ?? 'undefined'))
+  headers.set('x-isr-expiration', String(entry.expiration ?? 'undefined'))
+  headers.set('x-isr-source', source)
+
+  return new Response(entry.content, {
+    status: entry.status ?? 200,
+    headers,
+  })
+}
+
+export async function getIsrResponse(
+  request: Request,
+  outputs: NetlifyAdapterContext['preparedOutputs'],
+) {
+  const def = matchIsrDefinitionFromOutputs(request, outputs)
+  if (!def) {
+    return
+  }
+
+  const cacheKey = generateIsrCacheKey(request, def)
+  const store = getDeployStore({ consistency: 'strong', region: 'us-east-2' })
+
+  const cachedEntry = await store.get(cacheKey, { type: 'json' })
+  if (cachedEntry) {
+    return cacheEntryToResponse(cachedEntry as ISRCacheEntry, 'blobs')
+  }
+
+  if (!def.fallback) {
+    return
+  }
+
+  return cacheEntryToResponse(def.fallback, 'fallback')
+}
+
+export function matchIsrGroupFromOutputs(
+  request: Request,
+  outputs: Pick<NetlifyAdapterContext['preparedOutputs'], 'endpoints' | 'isrGroups'>,
+) {
+  const { pathname } = new URL(request.url)
+
+  const endpoint = outputs.endpoints[pathname.toLowerCase()]
+  if (!endpoint || endpoint.type !== 'isr') {
+    return
+  }
+
+  return outputs.isrGroups[endpoint.isrGroup]
+}
+
+export function matchIsrDefinitionFromIsrGroup(
+  request: Request,
+  isrGroup: NetlifyAdapterContext['preparedOutputs']['isrGroups'][number],
+) {
+  const { pathname } = new URL(request.url)
+
+  return isrGroup.find((def) => def.pathname === pathname)
+}
+
+export function matchIsrDefinitionFromOutputs(
+  request: Request,
+  outputs: Pick<NetlifyAdapterContext['preparedOutputs'], 'endpoints' | 'isrGroups'>,
+) {
+  const defs = matchIsrGroupFromOutputs(request, outputs)
+
+  if (!defs) {
+    return
+  }
+
+  return matchIsrDefinitionFromIsrGroup(request, defs)
+}
+
+export function requestToIsrRequest(
+  request: Request,
+  def: NetlifyAdapterContext['preparedOutputs']['isrGroups'][number][number],
+) {
+  const isrUrl = new URL(request.url)
+
+  // eslint-disable-next-line unicorn/no-useless-spread
+  for (const queryKey of [...isrUrl.searchParams.keys()]) {
+    if (!def.queryParams.includes(queryKey)) {
+      isrUrl.searchParams.delete(queryKey)
+    }
+  }
+
+  // we should strip headers as well - at very least conditional ones, but probably better to just use allowed headers like so
+  //       "allowHeader": [
+  //   "host",
+  //   "x-matched-path",
+  //   "x-prerender-revalidate",
+  //   "x-prerender-revalidate-if-generated",
+  //   "x-next-revalidated-tags",
+  //   "x-next-revalidate-tag-token"
+  // ],
+
+  return new Request(isrUrl, request)
+}
+
+export function generateIsrCacheKey(
+  request: Request,
+  def: NetlifyAdapterContext['preparedOutputs']['isrGroups'][number][number],
+) {
+  const parts = ['isr', def.pathname]
+
+  const url = new URL(request.url)
+
+  for (const queryParamName of def.queryParams) {
+    const value = url.searchParams.get(queryParamName) ?? ''
+    parts.push(`${queryParamName}=${value}`)
+  }
+
+  return parts.join(':')
+}
+
+export async function responseToCacheEntry(response: Response): Promise<ISRCacheEntry> {
+  const content = await response.text()
+  const headers: Record<string, string> = Object.fromEntries(response.headers.entries())
+
+  return { content, headers, status: response.status }
+}
+
+export async function storeIsrGroupUpdate(update: Record<string, ISRCacheEntry>) {
+  const store = getDeployStore({ consistency: 'strong', region: 'us-east-2' })
+
+  await Promise.all(
+    Object.entries(update).map(async ([key, entry]) => {
+      await store.setJSON(key, entry)
+    }),
+  )
+}