Render raw markdown pages for llms to read, ref in llms.txt (#501)

* render raw markdown pages for llms to read, ref in llms.txt

* make async

Author: evantahler

app/api/markdown/[[...slug]]/route.ts

@@ -0,0 +1,49 @@
+import { access, readFile } from "node:fs/promises";
+import { join } from "node:path";
+import { type NextRequest, NextResponse } from "next/server";
+
+export const dynamic = "force-dynamic";
+
+// Regex pattern for removing .md extension
+const MD_EXTENSION_REGEX = /\.md$/;
+
+export async function GET(
+  request: NextRequest,
+  _context: { params: Promise<{ slug?: string[] }> }
+) {
+  try {
+    // Get the original pathname from the request
+    const url = new URL(request.url);
+    // Remove /api/markdown prefix to get the original path
+    const originalPath = url.pathname.replace("/api/markdown", "");
+
+    // Remove .md extension
+    const pathWithoutMd = originalPath.replace(MD_EXTENSION_REGEX, "");
+
+    // Map URL to file path
+    // e.g., /en/home/quickstart -> app/en/home/quickstart/page.mdx
+    const filePath = join(process.cwd(), "app", `${pathWithoutMd}/page.mdx`);
+
+    // Check if file exists
+    try {
+      await access(filePath);
+    } catch {
+      return new NextResponse("Markdown file not found", { status: 404 });
+    }
+
+    const content = await readFile(filePath, "utf-8");
+
+    // Return the raw markdown with proper headers
+    return new NextResponse(content, {
+      status: 200,
+      headers: {
+        "Content-Type": "text/plain; charset=utf-8",
+        "Content-Disposition": "inline",
+      },
+    });
+  } catch (error) {
+    return new NextResponse(`Internal server error: ${error}`, {
+      status: 500,
+    });
+  }
+}

middleware.ts

@@ -2,6 +2,9 @@ import { type NextRequest, NextResponse } from "next/server";
 
 const SUPPORTED_LOCALES = ["en", "es", "pt-BR"];
 
+// Regex pattern for removing .md extension
+const MD_EXTENSION_REGEX = /\.md$/;
+
 /**
  * Parse Accept-Language header and normalize locale codes
  */
@@ -60,6 +63,21 @@ function pathnameIsMissingLocale(pathname: string): boolean {
 export function middleware(request: NextRequest) {
   const pathname = request.nextUrl.pathname;
 
+  // Handle .md requests without locale - redirect to add locale first
+  if (pathname.endsWith(".md") && pathnameIsMissingLocale(pathname)) {
+    const locale = getPreferredLocale(request);
+    const pathWithoutMd = pathname.replace(MD_EXTENSION_REGEX, "");
+    const redirectPath = `/${locale}${pathWithoutMd}.md`;
+    return NextResponse.redirect(new URL(redirectPath, request.url));
+  }
+
+  // Rewrite .md requests (with locale) to the markdown API route
+  if (pathname.endsWith(".md") && !pathname.startsWith("/api/")) {
+    const url = request.nextUrl.clone();
+    url.pathname = `/api/markdown${pathname}`;
+    return NextResponse.rewrite(url);
+  }
+
   // Redirect if there is no locale
   if (pathnameIsMissingLocale(pathname)) {
     const locale = getPreferredLocale(request);