feat(agents): serve markdown for versioned api-reference URLs

app/api/api-reference-markdown/[version]/route.ts

@@ -0,0 +1,24 @@
+import { agentResponse, agentNotFoundResponse } from '@/utils/agentResponseHeaders'
+import { buildApiReferenceMarkdown } from '@/utils/openapiMarkdown'
+import { getOpenAPISpecForVersion } from '@/utils/openapiSpec'
+import { resolveLatestVersion } from '@/utils/apiReference'
+
+export const revalidate = 86400 // 24h — see API_SPEC_REVALIDATE_SECONDS
+
+/**
+ * Markdown twin of /api-reference/<version>. Agents request these directly
+ * (31 requests over 30 days across 7 release tags, all previously 404) and
+ * `.md` is the convention llms.txt advertises for every other page.
+ */
+export async function GET(_: Request, props: { params: Promise<{ version: string }> }) {
+  const params = await props.params
+  const raw = params.version
+  const version = raw === 'latest' ? await resolveLatestVersion() : raw
+
+  if (!version) return agentNotFoundResponse(`/api-reference/${raw}.md`)
+
+  const spec = await getOpenAPISpecForVersion(version)
+  if (!spec) return agentNotFoundResponse(`/api-reference/${raw}.md`)
+
+  return agentResponse(buildApiReferenceMarkdown(spec), { varyAccept: true })
+}

proxy.ts

@@ -15,9 +15,11 @@ import { QUERY_PARAMS } from '@/constants/queryParams'
 import {
   API_REFERENCE_MARKDOWN_PATH,
   buildApiReferenceOpenAPISpecRewritePath,
+  buildApiReferenceVersionMarkdownRewritePath,
   isApiReferenceIndexPath,
   shouldRewriteApiReferenceIndexToMarkdown,
   shouldRewriteApiReferenceToOpenAPISpec,
+  shouldRewriteApiReferenceVersionToMarkdown,
 } from '@/utils/apiReferenceMarkdownRouting'
 import {
   AGENT_MARKDOWN_SELF_FETCH_HEADER,
@@ -128,7 +130,11 @@ export function proxy(req: NextRequest) {
 
   const docsMarkdownRewrite =
     !isAgentMarkdownSelfFetch && shouldRewriteDocsToMarkdown(pathname, prefersMarkdown)
-  const apiRefYamlRewrite = shouldRewriteApiReferenceToOpenAPISpec(pathname, prefersMarkdown, isBot)
+  const apiRefYamlRewrite = shouldRewriteApiReferenceToOpenAPISpec(pathname, acceptHeader)
+  const apiRefVersionMarkdownRewrite = shouldRewriteApiReferenceVersionToMarkdown(
+    pathname,
+    prefersMarkdown
+  )
   const apiRefIndexMarkdownRewrite = shouldRewriteApiReferenceIndexToMarkdown(
     pathname,
     prefersMarkdown
@@ -146,13 +152,15 @@ export function proxy(req: NextRequest) {
     ? buildDocsMarkdownRewritePath(pathname)
     : apiRefYamlRewrite
       ? buildApiReferenceOpenAPISpecRewritePath(pathname)
-      : apiRefIndexMarkdownRewrite
-        ? API_REFERENCE_MARKDOWN_PATH
-        : contentMarkdownRewrite
-          ? buildContentMarkdownRewritePath(pathname)
-          : pageMarkdownRewrite
-            ? buildPageMarkdownRewritePath(pathname)
-            : null
+      : apiRefVersionMarkdownRewrite
+        ? buildApiReferenceVersionMarkdownRewritePath(pathname)
+        : apiRefIndexMarkdownRewrite
+          ? API_REFERENCE_MARKDOWN_PATH
+          : contentMarkdownRewrite
+            ? buildContentMarkdownRewritePath(pathname)
+            : pageMarkdownRewrite
+              ? buildPageMarkdownRewritePath(pathname)
+              : null
 
   if (markdownRewritePath) {
     const rewriteUrl = req.nextUrl.clone()

tests/api-reference-version-markdown.test.js

@@ -0,0 +1,89 @@
+const test = require('node:test')
+const assert = require('node:assert/strict')
+const { loadTsModule } = require('./helpers/loadTsModule')
+
+const {
+  parseApiReferenceVersionPath,
+  shouldRewriteApiReferenceVersionToMarkdown,
+  buildApiReferenceVersionMarkdownRewritePath,
+  shouldRewriteApiReferenceToOpenAPISpec,
+  shouldRewriteApiReferenceIndexToMarkdown,
+} = loadTsModule('utils/apiReferenceMarkdownRouting.ts')
+
+test('parseApiReferenceVersionPath accepts release tags, latest, and .md twins', () => {
+  assert.equal(parseApiReferenceVersionPath('/api-reference/v0.139.0'), 'v0.139.0')
+  assert.equal(parseApiReferenceVersionPath('/api-reference/v0.139.0/'), 'v0.139.0')
+  assert.equal(parseApiReferenceVersionPath('/api-reference/v0.139.0.md'), 'v0.139.0')
+  assert.equal(parseApiReferenceVersionPath('/api-reference/latest'), 'latest')
+  assert.equal(parseApiReferenceVersionPath('/api-reference/latest.md'), 'latest')
+})
+
+test('parseApiReferenceVersionPath rejects the index and non-version segments', () => {
+  assert.equal(parseApiReferenceVersionPath('/api-reference'), null)
+  assert.equal(parseApiReferenceVersionPath('/api-reference/'), null)
+  assert.equal(parseApiReferenceVersionPath('/api-reference/not-a-version'), null)
+  assert.equal(parseApiReferenceVersionPath('/api-reference/v0.139.0/query-range'), null)
+  assert.equal(parseApiReferenceVersionPath('/docs/introduction'), null)
+})
+
+// 31 requests over 30 days across 7 release tags, all previously 404.
+test('versioned .md URLs rewrite to the markdown twin', () => {
+  assert.equal(
+    shouldRewriteApiReferenceVersionToMarkdown('/api-reference/v0.139.0.md', false),
+    true
+  )
+  assert.equal(shouldRewriteApiReferenceVersionToMarkdown('/api-reference/latest.md', false), true)
+  assert.equal(
+    buildApiReferenceVersionMarkdownRewritePath('/api-reference/v0.139.0.md'),
+    '/api/api-reference-markdown/v0.139.0'
+  )
+  assert.equal(
+    buildApiReferenceVersionMarkdownRewritePath('/api-reference/latest'),
+    '/api/api-reference-markdown/latest'
+  )
+})
+
+test('Accept: text/markdown on a version now gets markdown, not YAML', () => {
+  assert.equal(shouldRewriteApiReferenceVersionToMarkdown('/api-reference/v0.139.0', true), true)
+  assert.equal(
+    shouldRewriteApiReferenceToOpenAPISpec('/api-reference/v0.139.0', 'text/markdown'),
+    false
+  )
+})
+
+test('YAML is still served when the client actually asks for YAML', () => {
+  ;['text/yaml', 'application/yaml', 'application/x-yaml', 'application/vnd.oai.openapi'].forEach(
+    (accept) => {
+      assert.equal(
+        shouldRewriteApiReferenceToOpenAPISpec('/api-reference/v0.139.0', accept),
+        true,
+        `expected YAML for Accept: ${accept}`
+      )
+    }
+  )
+  assert.equal(shouldRewriteApiReferenceToOpenAPISpec('/api-reference/latest', 'text/yaml'), true)
+})
+
+test('the YAML path no longer depends on user-agent sniffing', () => {
+  // Previously gated on isBot, so a browser UA sending the same header got HTML.
+  assert.equal(shouldRewriteApiReferenceToOpenAPISpec('/api-reference/v0.139.0', 'text/yaml'), true)
+  assert.equal(
+    shouldRewriteApiReferenceToOpenAPISpec('/api-reference/v0.139.0', 'text/html'),
+    false
+  )
+  assert.equal(shouldRewriteApiReferenceToOpenAPISpec('/api-reference/v0.139.0', ''), false)
+})
+
+test('the index keeps its own markdown twin and is never treated as a version', () => {
+  assert.equal(shouldRewriteApiReferenceIndexToMarkdown('/api-reference', true), true)
+  assert.equal(shouldRewriteApiReferenceVersionToMarkdown('/api-reference', true), false)
+  assert.equal(shouldRewriteApiReferenceToOpenAPISpec('/api-reference', 'text/yaml'), false)
+})
+
+test('plain HTML requests to a version are untouched', () => {
+  assert.equal(shouldRewriteApiReferenceVersionToMarkdown('/api-reference/v0.139.0', false), false)
+  assert.equal(
+    shouldRewriteApiReferenceToOpenAPISpec('/api-reference/v0.139.0', 'text/html'),
+    false
+  )
+})

tests/proxy.test.js

@@ -114,28 +114,50 @@ test('does not re-enter docs markdown routing for agent self-fetches', () => {
   assert.equal(isPassthrough(res), true)
 })
 
-test('rewrites versioned api-reference pages to the OpenAPI spec for markdown-preferring bots', () => {
+test('markdown requests for a versioned api-reference serve the markdown twin', () => {
+  // Previously these answered `Accept: text/markdown` with raw YAML — the wrong
+  // media type for the request. The spec stays reachable via a YAML Accept and
+  // at /api/api-reference-openapi/<tag>.
   assert.equal(
     rewriteTarget(
       run('/api-reference/latest', { headers: { 'user-agent': BOT_UA, accept: 'text/markdown' } })
     ),
+    '/api/api-reference-markdown/latest'
+  )
+  assert.equal(
+    rewriteTarget(run('/api-reference/v1.2.3', { headers: { accept: 'text/markdown' } })),
+    '/api/api-reference-markdown/v1.2.3'
+  )
+})
+
+test('versioned api-reference .md URLs serve the markdown twin without any Accept header', () => {
+  assert.equal(rewriteTarget(run('/api-reference/v1.2.3.md')), '/api/api-reference-markdown/v1.2.3')
+  assert.equal(rewriteTarget(run('/api-reference/latest.md')), '/api/api-reference-markdown/latest')
+})
+
+test('the OpenAPI spec is served when the client asks for YAML, regardless of user-agent', () => {
+  // The rewrite used to require a bot UA, so a browser sending the same header
+  // got HTML. It now depends on the Accept header alone.
+  assert.equal(
+    rewriteTarget(run('/api-reference/latest', { headers: { accept: 'application/yaml' } })),
     '/api/api-reference-openapi/latest'
   )
   assert.equal(
     rewriteTarget(
-      run('/api-reference/v1.2.3', { headers: { 'user-agent': BOT_UA, accept: 'text/markdown' } })
+      run('/api-reference/v1.2.3', { headers: { 'user-agent': BOT_UA, accept: 'text/yaml' } })
     ),
     '/api/api-reference-openapi/v1.2.3'
   )
 })
 
-test('api-reference OpenAPI rewrite requires both a bot UA and markdown Accept', () => {
+test('plain HTML requests to a versioned api-reference pass through untouched', () => {
+  assert.equal(isPassthrough(run('/api-reference/latest')), true)
   assert.equal(
-    isPassthrough(run('/api-reference/latest', { headers: { accept: 'text/markdown' } })),
+    isPassthrough(run('/api-reference/latest', { headers: { 'user-agent': BOT_UA } })),
     true
   )
   assert.equal(
-    isPassthrough(run('/api-reference/latest', { headers: { 'user-agent': BOT_UA } })),
+    isPassthrough(run('/api-reference/latest', { headers: { accept: 'text/html' } })),
     true
   )
 })

utils/apiReferenceMarkdownRouting.ts

@@ -1,26 +1,59 @@
 import { parseSemverTag } from '@/utils/semverTags'
 
-export function shouldRewriteApiReferenceToOpenAPISpec(
+/**
+ * The release tag in `/api-reference/<tag>`, or null when the path is not a
+ * single-segment versioned api-reference URL. A trailing `.md` is accepted so
+ * the markdown twin resolves to the same tag.
+ */
+export function parseApiReferenceVersionPath(pathname: string): string | null {
+  const normalized = pathname.replace(/\/+$/, '') || '/'
+  if (!normalized.startsWith('/api-reference/')) return null
+
+  const rest = normalized.slice('/api-reference/'.length).replace(/\.md$/, '')
+  if (!rest || rest.includes('/')) return null
+  if (rest === 'latest') return 'latest'
+
+  return parseSemverTag(rest) !== null ? rest : null
+}
+
+const isYamlAccept = (accept: string): boolean =>
+  /(?:application|text)\/(?:x-)?yaml/i.test(accept) ||
+  /application\/vnd\.oai\.openapi/i.test(accept)
+
+/**
+ * Versioned api-reference URLs keep serving the raw spec, but only when the
+ * client actually asked for YAML. Previously any `Accept: text/markdown` from
+ * a bot got YAML back — the wrong media type for the request, and dependent on
+ * user-agent sniffing. Markdown requests now go to the markdown twin instead;
+ * the spec stays at /api/api-reference-openapi/<tag>, /openapi.yaml and here.
+ */
+export function shouldRewriteApiReferenceToOpenAPISpec(pathname: string, accept: string): boolean {
+  if (!isYamlAccept(accept)) return false
+
+  return parseApiReferenceVersionPath(pathname) !== null
+}
+
+/** `/api-reference/<tag>.md`, or `/api-reference/<tag>` asking for markdown. */
+export function shouldRewriteApiReferenceVersionToMarkdown(
   pathname: string,
-  prefersMarkdown: boolean,
-  isBot: boolean
+  prefersMarkdown: boolean
 ): boolean {
-  if (!isBot || !prefersMarkdown) return false
-
   const normalized = pathname.replace(/\/+$/, '') || '/'
-  if (normalized === '/api-reference') return false
-  if (!normalized.startsWith('/api-reference/')) return false
+  const hasMarkdownExtension = normalized.endsWith('.md')
 
-  const rest = normalized.slice('/api-reference/'.length)
-  if (!rest || rest.includes('/')) return false
-  if (rest === 'latest') return true
-  return parseSemverTag(rest) !== null
+  if (!prefersMarkdown && !hasMarkdownExtension) return false
+
+  return parseApiReferenceVersionPath(pathname) !== null
+}
+
+export function buildApiReferenceVersionMarkdownRewritePath(pathname: string): string {
+  const version = parseApiReferenceVersionPath(pathname)
+  return `/api/api-reference-markdown/${encodeURIComponent(version || 'latest')}`
 }
 
 export function buildApiReferenceOpenAPISpecRewritePath(pathname: string): string {
-  const normalized = pathname.replace(/\/+$/, '') || '/'
-  const version = normalized.slice('/api-reference/'.length)
-  return `/api/api-reference-openapi/${encodeURIComponent(version)}`
+  const version = parseApiReferenceVersionPath(pathname)
+  return `/api/api-reference-openapi/${encodeURIComponent(version || 'latest')}`
 }
 
 export const API_REFERENCE_MARKDOWN_PATH = '/api-reference.md'

utils/openapiSpec.ts

@@ -28,11 +28,11 @@ export const stampSpecVersion = (document: OpenAPIDocument, version: string): Op
   return { ...document, info }
 }
 
-/** Fetch and parse the spec for the newest published SigNoz release. */
-export async function getLatestOpenAPISpec(): Promise<LatestOpenAPISpec | null> {
-  const version = await resolveLatestVersion()
-  if (!version) return null
-
+/**
+ * Fetch and parse the spec for one published release. `version` is a release
+ * tag; callers resolve `latest` themselves.
+ */
+export async function getOpenAPISpecForVersion(version: string): Promise<LatestOpenAPISpec | null> {
   const yaml = await fetchOpenAPISpec(version)
   if (!yaml) return null
 
@@ -52,3 +52,11 @@ export async function getLatestOpenAPISpec(): Promise<LatestOpenAPISpec | null>
     document: stampSpecVersion(parsed as OpenAPIDocument, version),
   }
 }
+
+/** Fetch and parse the spec for the newest published SigNoz release. */
+export async function getLatestOpenAPISpec(): Promise<LatestOpenAPISpec | null> {
+  const version = await resolveLatestVersion()
+  if (!version) return null
+
+  return getOpenAPISpecForVersion(version)
+}