feat: cache 404s indefinitely for bot probes

Author: orinokai

src/run/handlers/server.ts

@@ -112,7 +112,7 @@ export default async (request: Request, context: FutureContext) => {
 
     await adjustDateHeader({ headers: response.headers, request, span, tracer, requestContext })
 
-    setCacheControlHeaders(response.headers, request, requestContext)
+    setCacheControlHeaders(response.headers, response.status, request, requestContext)
     setCacheTagsHeaders(response.headers, requestContext)
     setVaryHeaders(response.headers, request, nextConfig)
     setCacheStatusHeader(response.headers)

src/run/headers.ts

@@ -211,6 +211,7 @@ export const adjustDateHeader = async ({
  */
 export const setCacheControlHeaders = (
   headers: Headers,
+  status: number,
   request: Request,
   requestContext: RequestContext,
 ) => {
@@ -231,6 +232,12 @@ export const setCacheControlHeaders = (
     return
   }
 
+  if (status === 404 && request.url.endsWith('.php')) {
+    // temporary CDN Cache Control handling for bot probes
+    headers.set('cache-control', 'public, max-age=0, must-revalidate')
+    headers.set('netlify-cdn-cache-control', `max-age=31536000, durable`)
+  }
+
   const cacheControl = headers.get('cache-control')
   if (
     cacheControl !== null &&

tests/integration/simple-app.test.ts

@@ -158,6 +158,13 @@ test<FixtureTestContext>('stale-while-revalidate headers should be normalized to
   )
 })
 
+test<FixtureTestContext>('404 responses for PHP pages should be cached indefinitely', async (ctx) => {
+  await createFixture('simple', ctx)
+  await runPlugin(ctx)
+  const index = await invokeFunction(ctx, { url: '/admin.php' })
+  expect(index.headers?.['netlify-cdn-cache-control']).toContain('max-age=31536000, durable')
+})
+
 test<FixtureTestContext>('handlers receive correct site domain', async (ctx) => {
   await createFixture('simple', ctx)
   await runPlugin(ctx)