From 8ad4901fbf6bf2b8b07cf44398bfb68765e11729 Mon Sep 17 00:00:00 2001 From: Cody De Arkland Date: Sun, 28 Sep 2025 11:56:29 -0700 Subject: [PATCH 01/11] feat: add AI client user-agent detection for automatic markdown serving MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Detect AI/LLM tools (Claude, Cursor, Copilot, etc.) via user-agent - Automatically redirect to .md versions for better LLM consumption - Add manual override with ?format=md query parameter - Preserve existing redirect functionality 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- src/middleware.ts | 70 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/src/middleware.ts b/src/middleware.ts index d84e86501bf50c..3d1e5ba78275c1 100644 --- a/src/middleware.ts +++ b/src/middleware.ts @@ -19,12 +19,82 @@ export const config = { // This function can be marked `async` if using `await` inside export function middleware(request: NextRequest) { + // Check for AI/LLM clients and redirect to markdown if appropriate + const markdownRedirect = handleAIClientRedirect(request); + if (markdownRedirect) { + return markdownRedirect; + } + return handleRedirects(request); } // don't send Permanent Redirects (301) in dev mode - it gets cached for "localhost" by the browser const redirectStatusCode = process.env.NODE_ENV === 'development' ? 302 : 301; +/** + * Detects if the user agent belongs to an AI/LLM tool or development environment + * that would benefit from markdown format + */ +function isAIOrDevTool(userAgent: string): boolean { + const patterns = [ + /claude/i, // Claude Desktop/Code + /cursor/i, // Cursor IDE + /copilot/i, // GitHub Copilot + /chatgpt/i, // ChatGPT + /openai/i, // OpenAI tools + /anthropic/i, // Anthropic tools + /vscode/i, // VS Code extensions + /intellij/i, // IntelliJ plugins + /sublime/i, // Sublime Text plugins + // Add more patterns as needed + ]; + + return patterns.some(pattern => pattern.test(userAgent)); +} + +/** + * Handles redirection to markdown versions for AI/LLM clients + */ +const handleAIClientRedirect = (request: NextRequest) => { + const userAgent = request.headers.get('user-agent') || ''; + const url = request.nextUrl; + + // Skip if already requesting a markdown file + if (url.pathname.endsWith('.md')) { + return undefined; + } + + // Skip API routes and static assets (should already be filtered by matcher) + if (url.pathname.startsWith('/api/') || + url.pathname.startsWith('/_next/') || + url.pathname.includes('.')) { + return undefined; + } + + // Check for explicit format request via query parameter + const forceMarkdown = url.searchParams.get('format') === 'md'; + const isAIClient = isAIOrDevTool(userAgent); + + if (isAIClient || forceMarkdown) { + // Create new URL with .md extension + const newUrl = url.clone(); + // Handle root path and ensure proper .md extension + const pathname = url.pathname === '/' ? '/index' : url.pathname.replace(/\/$/, ''); + newUrl.pathname = pathname + '.md'; + + // Clean up the format query parameter if it was used + if (forceMarkdown) { + newUrl.searchParams.delete('format'); + } + + return NextResponse.redirect(newUrl, { + status: redirectStatusCode, + }); + } + + return undefined; +}; + const handleRedirects = (request: NextRequest) => { const urlPath = request.nextUrl.pathname; From 6df27415cdcc53de2dfc79d1006e41e19df6f2b7 Mon Sep 17 00:00:00 2001 From: Cody De Arkland Date: Sun, 28 Sep 2025 12:01:14 -0700 Subject: [PATCH 02/11] fix: handle trailing slashes correctly in markdown redirects --- src/middleware.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/middleware.ts b/src/middleware.ts index 3d1e5ba78275c1..da6e87df3e5b42 100644 --- a/src/middleware.ts +++ b/src/middleware.ts @@ -79,8 +79,10 @@ const handleAIClientRedirect = (request: NextRequest) => { // Create new URL with .md extension const newUrl = url.clone(); // Handle root path and ensure proper .md extension - const pathname = url.pathname === '/' ? '/index' : url.pathname.replace(/\/$/, ''); - newUrl.pathname = pathname + '.md'; + let pathname = url.pathname === '/' ? '/index' : url.pathname; + // Remove trailing slash if present, then add .md + pathname = pathname.replace(/\/$/, '') + '.md'; + newUrl.pathname = pathname; // Clean up the format query parameter if it was used if (forceMarkdown) { From 8e2e6cd2a3de8cc3057db08be17bcd195e9fa6d4 Mon Sep 17 00:00:00 2001 From: Cody De Arkland Date: Sun, 28 Sep 2025 12:14:25 -0700 Subject: [PATCH 03/11] add: user agent logging for debugging AI client detection - Log all user agents for non-static requests - Log when redirects to markdown occur - Helps identify what user agents AI tools are sending --- src/middleware.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/middleware.ts b/src/middleware.ts index da6e87df3e5b42..fb377cad7f7723 100644 --- a/src/middleware.ts +++ b/src/middleware.ts @@ -59,6 +59,13 @@ const handleAIClientRedirect = (request: NextRequest) => { const userAgent = request.headers.get('user-agent') || ''; const url = request.nextUrl; + // Log user agent for debugging (only for non-static assets) + if (!url.pathname.startsWith('/_next/') && + !url.pathname.includes('.') && + !url.pathname.startsWith('/api/')) { + console.log(`[Middleware] ${url.pathname} - User-Agent: ${userAgent}`); + } + // Skip if already requesting a markdown file if (url.pathname.endsWith('.md')) { return undefined; @@ -76,6 +83,9 @@ const handleAIClientRedirect = (request: NextRequest) => { const isAIClient = isAIOrDevTool(userAgent); if (isAIClient || forceMarkdown) { + // Log the redirect for debugging + console.log(`[Middleware] Redirecting to markdown: ${isAIClient ? 'AI client detected' : 'Manual format=md'}`); + // Create new URL with .md extension const newUrl = url.clone(); // Handle root path and ensure proper .md extension From 9e46f5ec2ae631030da3eff524321a68659a8afb Mon Sep 17 00:00:00 2001 From: Cody De Arkland Date: Sun, 28 Sep 2025 12:42:53 -0700 Subject: [PATCH 04/11] add: support for 'got' HTTP library user agent - Add /got/i pattern to detect sindresorhus/got library - Common HTTP client used by Node.js tools and automation - Ensures markdown serving for tools using got for requests --- src/middleware.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/middleware.ts b/src/middleware.ts index fb377cad7f7723..4fcdd728ece6d3 100644 --- a/src/middleware.ts +++ b/src/middleware.ts @@ -46,6 +46,7 @@ function isAIOrDevTool(userAgent: string): boolean { /vscode/i, // VS Code extensions /intellij/i, // IntelliJ plugins /sublime/i, // Sublime Text plugins + /got/i, // Got HTTP library (sindresorhus/got) // Add more patterns as needed ]; From 2404cf3def92ced01aa49f6ffe088a1d3ae7192a Mon Sep 17 00:00:00 2001 From: Cody De Arkland Date: Sun, 28 Sep 2025 13:13:06 -0700 Subject: [PATCH 05/11] enhance: add content type indicators to middleware logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Show 📄 MARKDOWN or 🌐 HTML indicators in logs - Determine content type before logging for better debugging - Makes it clear which requests get markdown vs HTML content --- src/middleware.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/middleware.ts b/src/middleware.ts index 4fcdd728ece6d3..4f0dbb8d1befb9 100644 --- a/src/middleware.ts +++ b/src/middleware.ts @@ -60,11 +60,17 @@ const handleAIClientRedirect = (request: NextRequest) => { const userAgent = request.headers.get('user-agent') || ''; const url = request.nextUrl; + // Determine if this will be served as markdown + const forceMarkdown = url.searchParams.get('format') === 'md'; + const isAIClient = isAIOrDevTool(userAgent); + const willServeMarkdown = (isAIClient || forceMarkdown) && !url.pathname.endsWith('.md'); + // Log user agent for debugging (only for non-static assets) if (!url.pathname.startsWith('/_next/') && !url.pathname.includes('.') && !url.pathname.startsWith('/api/')) { - console.log(`[Middleware] ${url.pathname} - User-Agent: ${userAgent}`); + const contentType = willServeMarkdown ? '📄 MARKDOWN' : '🌐 HTML'; + console.log(`[Middleware] ${url.pathname} - ${contentType} - User-Agent: ${userAgent}`); } // Skip if already requesting a markdown file @@ -79,9 +85,7 @@ const handleAIClientRedirect = (request: NextRequest) => { return undefined; } - // Check for explicit format request via query parameter - const forceMarkdown = url.searchParams.get('format') === 'md'; - const isAIClient = isAIOrDevTool(userAgent); + // Check for AI client detection if (isAIClient || forceMarkdown) { // Log the redirect for debugging From f625e51fd9a0bfdae5692dcde8f0e68e57772d39 Mon Sep 17 00:00:00 2001 From: Cody De Arkland Date: Sun, 28 Sep 2025 18:09:43 -0700 Subject: [PATCH 06/11] fix: replace overly broad static file detection with precise regex - Replace url.pathname.includes('.') with specific file extension regex - Fixes AI client detection for .NET, Node.js, Next.js documentation - Now only excludes actual static files (.js, .png, .css, etc.) - Resolves Sentry bug report about .NET docs not getting markdown redirects Fixes: Critical bug where paths like /platforms/dotnet/ were incorrectly excluded from AI client detection due to containing dots in the path. --- src/middleware.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/middleware.ts b/src/middleware.ts index 4f0dbb8d1befb9..f1bc0833378a30 100644 --- a/src/middleware.ts +++ b/src/middleware.ts @@ -81,7 +81,7 @@ const handleAIClientRedirect = (request: NextRequest) => { // Skip API routes and static assets (should already be filtered by matcher) if (url.pathname.startsWith('/api/') || url.pathname.startsWith('/_next/') || - url.pathname.includes('.')) { + /\.(js|json|png|jpg|jpeg|gif|ico|pdf|css|woff|woff2|ttf|map|xml|txt|zip|svg)$/i.test(url.pathname)) { return undefined; } From 94210d2830eb4b8f0d2fdfed491c3e326c85908d Mon Sep 17 00:00:00 2001 From: Cody De Arkland Date: Sun, 28 Sep 2025 18:12:53 -0700 Subject: [PATCH 07/11] fix: handle multiple trailing slashes in markdown redirect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replace /\/$/ with /\/+$/ to remove ALL trailing slashes - Prevents malformed URLs like /path//.md when multiple slashes present - Defensive programming for edge cases bypassing Next.js normalization - Reported by Cursor - simple one-character regex fix Examples: - /platforms/react/// -> /platforms/react.md ✅ - /platforms/react/ -> /platforms/react.md ✅ - /platforms/react -> /platforms/react.md ✅ --- src/middleware.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/middleware.ts b/src/middleware.ts index f1bc0833378a30..7b23f3381cd725 100644 --- a/src/middleware.ts +++ b/src/middleware.ts @@ -95,8 +95,8 @@ const handleAIClientRedirect = (request: NextRequest) => { const newUrl = url.clone(); // Handle root path and ensure proper .md extension let pathname = url.pathname === '/' ? '/index' : url.pathname; - // Remove trailing slash if present, then add .md - pathname = pathname.replace(/\/$/, '') + '.md'; + // Remove all trailing slashes if present, then add .md + pathname = pathname.replace(/\/+$/, '') + '.md'; newUrl.pathname = pathname; // Clean up the format query parameter if it was used From 1a242d3b88704586541939118765f35346bcf25c Mon Sep 17 00:00:00 2001 From: "getsantry[bot]" <66042841+getsantry[bot]@users.noreply.github.com> Date: Mon, 29 Sep 2025 01:15:35 +0000 Subject: [PATCH 08/11] [getsentry/action-github-commit] Auto commit --- src/middleware.ts | 49 +++++++++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/src/middleware.ts b/src/middleware.ts index 7b23f3381cd725..aaa9d087830197 100644 --- a/src/middleware.ts +++ b/src/middleware.ts @@ -37,16 +37,16 @@ const redirectStatusCode = process.env.NODE_ENV === 'development' ? 302 : 301; */ function isAIOrDevTool(userAgent: string): boolean { const patterns = [ - /claude/i, // Claude Desktop/Code - /cursor/i, // Cursor IDE - /copilot/i, // GitHub Copilot - /chatgpt/i, // ChatGPT - /openai/i, // OpenAI tools - /anthropic/i, // Anthropic tools - /vscode/i, // VS Code extensions - /intellij/i, // IntelliJ plugins - /sublime/i, // Sublime Text plugins - /got/i, // Got HTTP library (sindresorhus/got) + /claude/i, // Claude Desktop/Code + /cursor/i, // Cursor IDE + /copilot/i, // GitHub Copilot + /chatgpt/i, // ChatGPT + /openai/i, // OpenAI tools + /anthropic/i, // Anthropic tools + /vscode/i, // VS Code extensions + /intellij/i, // IntelliJ plugins + /sublime/i, // Sublime Text plugins + /got/i, // Got HTTP library (sindresorhus/got) // Add more patterns as needed ]; @@ -63,14 +63,19 @@ const handleAIClientRedirect = (request: NextRequest) => { // Determine if this will be served as markdown const forceMarkdown = url.searchParams.get('format') === 'md'; const isAIClient = isAIOrDevTool(userAgent); - const willServeMarkdown = (isAIClient || forceMarkdown) && !url.pathname.endsWith('.md'); + const willServeMarkdown = + (isAIClient || forceMarkdown) && !url.pathname.endsWith('.md'); // Log user agent for debugging (only for non-static assets) - if (!url.pathname.startsWith('/_next/') && - !url.pathname.includes('.') && - !url.pathname.startsWith('/api/')) { + if ( + !url.pathname.startsWith('/_next/') && + !url.pathname.includes('.') && + !url.pathname.startsWith('/api/') + ) { const contentType = willServeMarkdown ? '📄 MARKDOWN' : '🌐 HTML'; - console.log(`[Middleware] ${url.pathname} - ${contentType} - User-Agent: ${userAgent}`); + console.log( + `[Middleware] ${url.pathname} - ${contentType} - User-Agent: ${userAgent}` + ); } // Skip if already requesting a markdown file @@ -79,9 +84,13 @@ const handleAIClientRedirect = (request: NextRequest) => { } // Skip API routes and static assets (should already be filtered by matcher) - if (url.pathname.startsWith('/api/') || - url.pathname.startsWith('/_next/') || - /\.(js|json|png|jpg|jpeg|gif|ico|pdf|css|woff|woff2|ttf|map|xml|txt|zip|svg)$/i.test(url.pathname)) { + if ( + url.pathname.startsWith('/api/') || + url.pathname.startsWith('/_next/') || + /\.(js|json|png|jpg|jpeg|gif|ico|pdf|css|woff|woff2|ttf|map|xml|txt|zip|svg)$/i.test( + url.pathname + ) + ) { return undefined; } @@ -89,7 +98,9 @@ const handleAIClientRedirect = (request: NextRequest) => { if (isAIClient || forceMarkdown) { // Log the redirect for debugging - console.log(`[Middleware] Redirecting to markdown: ${isAIClient ? 'AI client detected' : 'Manual format=md'}`); + console.log( + `[Middleware] Redirecting to markdown: ${isAIClient ? 'AI client detected' : 'Manual format=md'}` + ); // Create new URL with .md extension const newUrl = url.clone(); From 471a1e52d455710802b1fdf72d817162b788feb9 Mon Sep 17 00:00:00 2001 From: Cody De Arkland Date: Sun, 28 Sep 2025 19:46:45 -0700 Subject: [PATCH 09/11] fix: prioritize canonical redirects over AI client detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Process canonical URL redirects first before AI client detection - Ensures deprecated URLs redirect to current paths before markdown conversion - Prevents AI clients from getting outdated content via old URLs - Fixes Cursor bug report about bypassing canonical URL redirects Example flow: 1. /old-path/ → /new-path/ (canonical redirect) 2. AI client requests /new-path/ → /new-path.md (markdown) Instead of: 1. AI client requests /old-path/ → /old-path.md (404/outdated) --- src/middleware.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/middleware.ts b/src/middleware.ts index aaa9d087830197..120c038ac025d7 100644 --- a/src/middleware.ts +++ b/src/middleware.ts @@ -19,13 +19,14 @@ export const config = { // This function can be marked `async` if using `await` inside export function middleware(request: NextRequest) { - // Check for AI/LLM clients and redirect to markdown if appropriate - const markdownRedirect = handleAIClientRedirect(request); - if (markdownRedirect) { - return markdownRedirect; + // First, handle canonical URL redirects for deprecated paths + const canonicalRedirect = handleRedirects(request); + if (canonicalRedirect) { + return canonicalRedirect; } - return handleRedirects(request); + // Then, check for AI/LLM clients and redirect to markdown if appropriate + return handleAIClientRedirect(request); } // don't send Permanent Redirects (301) in dev mode - it gets cached for "localhost" by the browser From 115f34baa94e2b13fcf9fe28d76ad64a30b2c9e1 Mon Sep 17 00:00:00 2001 From: Cody De Arkland Date: Sun, 28 Sep 2025 22:03:37 -0700 Subject: [PATCH 10/11] feat: add Accept header content negotiation for markdown requests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Implement hybrid strategy: Accept header + user-agent detection - Support standards-compliant text/markdown and text/x-markdown Accept headers - Maintain backward compatibility with existing user-agent detection - Enhanced logging shows detection method (Accept header, User-agent, Manual) - Inspired by OpenCode's content negotiation approach Examples: - Accept: text/markdown → markdown (standards-compliant) - User-Agent: Claude/1.0 → markdown (fallback for existing tools) - ?format=md → markdown (manual override) This makes docs accessible to more tools while following HTTP standards. --- src/middleware.ts | 43 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/src/middleware.ts b/src/middleware.ts index 120c038ac025d7..b5d0ff9416246d 100644 --- a/src/middleware.ts +++ b/src/middleware.ts @@ -54,18 +54,47 @@ function isAIOrDevTool(userAgent: string): boolean { return patterns.some(pattern => pattern.test(userAgent)); } +/** + * Detects if client wants markdown via Accept header (standards-compliant) + */ +function wantsMarkdownViaAccept(acceptHeader: string): boolean { + return acceptHeader.includes('text/markdown') || + acceptHeader.includes('text/x-markdown'); +} + +/** + * Detects if client wants markdown via Accept header or user-agent + */ +function wantsMarkdown(request: NextRequest): boolean { + const userAgent = request.headers.get('user-agent') || ''; + const acceptHeader = request.headers.get('accept') || ''; + + // Strategy 1: Accept header content negotiation (standards-compliant) + if (wantsMarkdownViaAccept(acceptHeader)) { + return true; + } + + // Strategy 2: User-agent detection (fallback for tools that don't set Accept) + return isAIOrDevTool(userAgent); +} + /** * Handles redirection to markdown versions for AI/LLM clients */ const handleAIClientRedirect = (request: NextRequest) => { const userAgent = request.headers.get('user-agent') || ''; + const acceptHeader = request.headers.get('accept') || ''; const url = request.nextUrl; // Determine if this will be served as markdown const forceMarkdown = url.searchParams.get('format') === 'md'; - const isAIClient = isAIOrDevTool(userAgent); + const clientWantsMarkdown = wantsMarkdown(request); const willServeMarkdown = - (isAIClient || forceMarkdown) && !url.pathname.endsWith('.md'); + (clientWantsMarkdown || forceMarkdown) && !url.pathname.endsWith('.md'); + + // Determine detection method for logging + const detectionMethod = wantsMarkdownViaAccept(acceptHeader) ? 'Accept header' : + isAIOrDevTool(userAgent) ? 'User-agent' : 'Manual'; // Log user agent for debugging (only for non-static assets) if ( @@ -74,8 +103,9 @@ const handleAIClientRedirect = (request: NextRequest) => { !url.pathname.startsWith('/api/') ) { const contentType = willServeMarkdown ? '📄 MARKDOWN' : '🌐 HTML'; + const methodInfo = willServeMarkdown ? ` (${detectionMethod})` : ''; console.log( - `[Middleware] ${url.pathname} - ${contentType} - User-Agent: ${userAgent}` + `[Middleware] ${url.pathname} - ${contentType}${methodInfo} - User-Agent: ${userAgent}` ); } @@ -95,12 +125,11 @@ const handleAIClientRedirect = (request: NextRequest) => { return undefined; } - // Check for AI client detection - - if (isAIClient || forceMarkdown) { + // Check for markdown request (Accept header, user-agent, or manual) + if (clientWantsMarkdown || forceMarkdown) { // Log the redirect for debugging console.log( - `[Middleware] Redirecting to markdown: ${isAIClient ? 'AI client detected' : 'Manual format=md'}` + `[Middleware] Redirecting to markdown: ${forceMarkdown ? 'Manual format=md' : detectionMethod}` ); // Create new URL with .md extension From 62ffeaeb9b3e21831cd17de358d2f98909f6d63e Mon Sep 17 00:00:00 2001 From: "getsantry[bot]" <66042841+getsantry[bot]@users.noreply.github.com> Date: Mon, 29 Sep 2025 05:04:35 +0000 Subject: [PATCH 11/11] [getsentry/action-github-commit] Auto commit --- src/middleware.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/middleware.ts b/src/middleware.ts index b5d0ff9416246d..fb21ff92c9e5ae 100644 --- a/src/middleware.ts +++ b/src/middleware.ts @@ -58,8 +58,9 @@ function isAIOrDevTool(userAgent: string): boolean { * Detects if client wants markdown via Accept header (standards-compliant) */ function wantsMarkdownViaAccept(acceptHeader: string): boolean { - return acceptHeader.includes('text/markdown') || - acceptHeader.includes('text/x-markdown'); + return ( + acceptHeader.includes('text/markdown') || acceptHeader.includes('text/x-markdown') + ); } /** @@ -93,8 +94,11 @@ const handleAIClientRedirect = (request: NextRequest) => { (clientWantsMarkdown || forceMarkdown) && !url.pathname.endsWith('.md'); // Determine detection method for logging - const detectionMethod = wantsMarkdownViaAccept(acceptHeader) ? 'Accept header' : - isAIOrDevTool(userAgent) ? 'User-agent' : 'Manual'; + const detectionMethod = wantsMarkdownViaAccept(acceptHeader) + ? 'Accept header' + : isAIOrDevTool(userAgent) + ? 'User-agent' + : 'Manual'; // Log user agent for debugging (only for non-static assets) if (