Improve catch-all rewriter caching strategy (#36)

#34 introduced a catch-all route handler that acts as a fallback
rewriter for our content pages. It had implementations of caching
mechanisms, but these did not seem to work as expected in production.

This pr introduces the use of `generateStaticParams` to cache route
invocations for the paths present in our `sitemap` on build time, which
results in instant server responses on our rewritten pages.

Author: ben-fornefeld

src/app/(rewrites)/[[...path]]/route.ts src/app/(rewrites)/[[...slug]]/route.tssrc/app/(rewrites)/[[...path]]/route.ts renamed to src/app/(rewrites)/[[...slug]]/route.ts

@@ -5,10 +5,12 @@ import {
   replaceUrls,
 } from '@/configs/domains'
 import { ERROR_CODES } from '@/configs/logs'
-import { logError } from '@/lib/clients/logger'
 import { NextRequest } from 'next/server'
+import sitemap from '@/app/sitemap'
+import { BASE_URL } from '@/configs/urls'
 
 export const revalidate = 900
+export const dynamic = 'force-static'
 
 const REVALIDATE_TIME = 900 // 15 minutes ttl
 
@@ -48,21 +50,24 @@ export async function GET(request: NextRequest): Promise<Response> {
     }
   }
 
-  if (url.hostname === requestHostname) {
-    url.pathname = '/not-found'
-  }
-
   try {
-    const headers = new Headers(request.headers)
-    headers.delete('host') // prevent host header conflicts
-
-    const res = await fetch(url.toString(), {
-      headers,
+    // if hostname did not change, we want to make sure it does not cache the route based on the build times hostname (127.0.0.1:3000)
+    const fetchUrl =
+      url.hostname === requestHostname
+        ? `${BASE_URL}/not-found`
+        : url.toString()
+
+    const res = await fetch(fetchUrl, {
+      headers: new Headers(request.headers),
       redirect: 'follow',
-      cache: 'force-cache',
-      next: {
-        revalidate: REVALIDATE_TIME,
-      },
+      // if the hostname is the same, we don't want to cache the response, since it will not be available in build time
+      ...(url.hostname === requestHostname
+        ? { cache: 'no-store' }
+        : {
+            next: {
+              revalidate: REVALIDATE_TIME,
+            },
+          }),
     })
 
     const contentType = res.headers.get('Content-Type')
@@ -75,14 +80,6 @@ export async function GET(request: NextRequest): Promise<Response> {
       const newHeaders = new Headers(res.headers)
       newHeaders.delete('content-encoding')
 
-      // add cache control headers if not already present
-      if (!newHeaders.has('cache-control')) {
-        newHeaders.set(
-          'cache-control',
-          `public, max-age=${REVALIDATE_TIME}, s-maxage=${REVALIDATE_TIME}, stale-while-revalidate=86400`
-        )
-      }
-
       return new Response(modifiedHtmlBody, {
         status: res.status,
         headers: newHeaders,
@@ -91,7 +88,7 @@ export async function GET(request: NextRequest): Promise<Response> {
 
     return res
   } catch (error) {
-    logError(ERROR_CODES.URL_REWRITE, error)
+    console.error(ERROR_CODES.URL_REWRITE, error)
 
     return new Response(
       `Proxy Error: ${error instanceof Error ? error.message : 'Unknown error'}`,
@@ -102,3 +99,17 @@ export async function GET(request: NextRequest): Promise<Response> {
     )
   }
 }
+
+export async function generateStaticParams() {
+  const sitemapEntries = await sitemap()
+
+  const slugs = sitemapEntries.map((entry) => {
+    const url = new URL(entry.url)
+    const pathname = url.pathname
+    const pathSegments = pathname.split('/').filter((segment) => segment !== '')
+
+    return { slug: pathSegments.length > 0 ? pathSegments : undefined }
+  })
+
+  return slugs
+}

src/app/(rewrites)/not-found/page.tsx

@@ -1,5 +1,7 @@
-import { notFound } from 'next/navigation'
+import NotFound from '@/ui/not-found'
 
-export default function NotFound() {
-  throw notFound()
+export const dynamic = 'force-static'
+
+export default function NotFoundShell() {
+  return <NotFound />
 }

src/configs/urls.ts

@@ -21,6 +21,8 @@ export const PROTECTED_URLS = {
   KEYS: (teamId: string) => `/dashboard/${teamId}/keys`,
 }
 
-export const BASE_URL = process.env.VERCEL_URL
-  ? `https://${process.env.VERCEL_URL}`
+export const BASE_URL = process.env.VERCEL_ENV
+  ? process.env.VERCEL_ENV === 'production'
+    ? `https://${process.env.VERCEL_PROJECT_PRODUCTION_URL}`
+    : `https://${process.env.VERCEL_BRANCH_URL}`
   : 'http://localhost:3000'

src/lib/env.ts

@@ -5,10 +5,13 @@ export const serverSchema = z.object({
   COOKIE_ENCRYPTION_KEY: z.string().min(32),
 
   BILLING_API_URL: z.string().url().optional(),
-  VERCEL_URL: z.string().optional(),
   DEVELOPMENT_INFRA_API_DOMAIN: z.string().optional(),
   SENTRY_AUTH_TOKEN: z.string().optional(),
   ZEROBOUNCE_API_KEY: z.string().optional(),
+  VERCEL_ENV: z.enum(['production', 'preview', 'development']).optional(),
+  VERCEL_URL: z.string().optional(),
+  VERCEL_PROJECT_PRODUCTION_URL: z.string().optional(),
+  VERCEL_BRANCH_URL: z.string().optional(),
 })
 
 export const clientSchema = z.object({

src/middleware.ts

@@ -78,8 +78,8 @@ export const config = {
      * - favicon.ico (favicon file)
      * - images - .svg, .png, .jpg, .jpeg, .gif, .webp
      * - api routes
-     * Feel free to modify this pattern to include more paths.
+     * - paths handled by the catchall route.ts (terms, privacy, pricing, cookbook, changelog, blog, ai-agents, docs)
      */
-    '/((?!_next/static|_next/image|favicon.ico|api/|.*\\.(?:svg|png|jpg|jpeg|gif|webp)$).*)',
+    '/((?!_next/static|_next/image|favicon.ico|api/|terms|privacy|pricing|cookbook|changelog|blog|ai-agents|docs|.*\\.(?:svg|png|jpg|jpeg|gif|webp)$).*)',
   ],
 }