Skip to content

fix(docs): correct llms.txt, llms-full.txt, and .md routing #4675

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: app
Choose a base branch
from
Open

fix(docs): correct llms.txt, llms-full.txt, and .md routing #4675

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions packages/fern-docs/bundle/src/app/[host]/[domain]/[lang]/fern-docs/llms-full.txt/route.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,13 @@ export async function GET(req: NextRequest, props: { params: Promise<BaseParams>
});
} catch (error) {
console.error(`[llmsFull:${domain}] Stream error:`, error);
track("static_content_failed", {
domain,
host,
path,
staticContentType: "llms-full.txt",
error: error instanceof Error ? error.message : String(error)
});
controller.error(error);
}
}
Expand Down
7 changes: 7 additions & 0 deletions packages/fern-docs/bundle/src/app/[host]/[domain]/[lang]/fern-docs/llms.txt/route.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,13 @@ export async function GET(req: NextRequest, props: { params: Promise<BaseParams>
});
} catch (error) {
console.error(`[llmsTxt:${domain}] Stream error:`, error);
track("static_content_failed", {
domain,
host,
path,
staticContentType: "llms.txt",
error: error instanceof Error ? error.message : String(error)
});
controller.error(error);
}
}
Expand Down
35 changes: 29 additions & 6 deletions packages/fern-docs/bundle/src/app/[host]/[domain]/[lang]/fern-docs/markdown/route.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,15 @@ export async function GET(req: NextRequest, props: { params: Promise<BaseParams>

const fernToken = (await cookies()).get(COOKIE_FERN_TOKEN)?.value;

const path = req.nextUrl.pathname;
const slug = path.replace(MARKDOWN_PATTERN, "");
const slugParam = req.nextUrl.searchParams.get("slug");
const slug = slugParam ?? req.nextUrl.pathname.replace(MARKDOWN_PATTERN, "");
const cleanSlug = removeLeadingSlash(slug);

const loader = await createCachedDocsLoader(host, domain, fernToken);
const node = getPageNodeForPath(await loader.getRoot(), cleanSlug);

if (node == null) {
console.error(`[${domain}] Node not found: ${path}`);
console.error(`[${domain}] Node not found: ${cleanSlug}`);
notFound();
}

Expand All @@ -43,9 +43,32 @@ export async function GET(req: NextRequest, props: { params: Promise<BaseParams>
return new NextResponse("User is not logged in", { status: 403 });
}

const markdown = await getMarkdownForPath(node, loader);
let markdown;
try {
markdown = await getMarkdownForPath(node, loader);
} catch (error) {
console.error(`[${domain}] Error getting markdown for path: ${cleanSlug}`, error);
track("static_content_failed", {
domain,
path: cleanSlug,
slug: cleanSlug,
host,
staticContentType: "markdown",
error: error instanceof Error ? error.message : String(error)
});
throw error;
}

if (markdown == null) {
console.error(`[${domain}] Markdown not found: ${path}`);
console.error(`[${domain}] Markdown not found: ${cleanSlug}`);
track("static_content_failed", {
domain,
path: cleanSlug,
slug: cleanSlug,
host,
staticContentType: "markdown",
error: "Markdown not found"
});
notFound();
}

Expand All @@ -57,7 +80,7 @@ export async function GET(req: NextRequest, props: { params: Promise<BaseParams>

track("static_content_served", {
domain,
path,
path: cleanSlug,
slug: cleanSlug,
host,
staticContentType: "markdown",
Expand Down
13 changes: 7 additions & 6 deletions packages/fern-docs/bundle/src/middleware.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,18 +158,19 @@ export const middleware: NextMiddleware = async (request) => {
}

/**
* Rewrite API routes to /api/fern-docs
* Rewrite API routes to /api/fern-docs (legacy compatibility - redirect to /fern-docs)
*/
if (pathname.includes("/api/fern-docs/")) {
return rewrite(withDomain(withoutBasepath("/api/fern-docs/")));
const rel = withoutBasepath("/api/fern-docs/").replace(/^\/api\/fern-docs/, "/fern-docs");
Copy link
Contributor

@vercel vercel bot Nov 3, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
const rel = withoutBasepath("/api/fern-docs/").replace(/^\/api\/fern-docs/, "/fern-docs");
const afterBasepath = withoutBasepath("/api/fern-docs/");
const cleanPath = afterBasepath.replace(/^\/api\/fern-docs/, "");
// Helper function to extract path before a given ending pattern
const extractSlugFrom = (path: string, pattern: RegExp): string => {
const match = path.search(pattern);
if (match < 0) {
return path;
}
return path.slice(0, match);
};
// Extract slug for specific endpoints that require it
if (cleanPath.endsWith("/llms.txt")) {
const slug = removeLeadingSlash(extractSlugFrom(cleanPath, /\/llms\.txt$/));
return rewrite(withDomain("/fern-docs/llms.txt"), { slug });
}
if (cleanPath.endsWith("/llms-full.txt")) {
const slug = removeLeadingSlash(extractSlugFrom(cleanPath, /\/llms-full\.txt$/));
return rewrite(withDomain("/fern-docs/llms-full.txt"), { slug });
}
if (cleanPath.match(MARKDOWN_PATTERN)) {
const slug = removeLeadingSlash(extractSlugFrom(cleanPath, MARKDOWN_PATTERN));
return rewrite(withDomain("/fern-docs/markdown"), { slug });
}
const rel = "/fern-docs" + cleanPath;

The legacy redirect block for /api/fern-docs/* requests doesn't extract or pass the slug as a search parameter, which will cause nested paths in llms.txt, llms-full.txt, and markdown content to be served incorrectly (always serving root content instead of the requested path).

View Details

Analysis

Legacy redirect doesn't extract slug for /api/fern-docs/* endpoints

What fails: Legacy redirect for /api/fern-docs/* paths (middleware.ts lines 163-165) doesn't extract slug parameter for llms.txt, llms-full.txt, and markdown endpoints, causing them to serve root content instead of requested path content.

How to reproduce:

# Request nested llms.txt content via legacy API path:
curl "https://docs.example.com/api/fern-docs/docs/v2/llms.txt"

Result: Returns root llms.txt content instead of /docs/v2/ content because req.nextUrl.searchParams.get("slug") returns null, defaulting to empty string in slugToHref().

Expected: Should return /docs/v2/ content, matching behavior of direct path https://docs.example.com/docs/v2/llms.txt which correctly extracts slug parameter.

Root cause: Unlike specific endpoint handlers (lines 171-173, 186-188, 194-196), legacy redirect doesn't extract slug from path before rewriting to /fern-docs/* endpoints that require it.

return rewrite(withDomain(rel));
}

/**
* Rewrite llms.txt
*/
if (pathname.endsWith("/llms.txt")) {
const slug = removeLeadingSlash(withoutEnding(/\/llms\.txt$/));
return rewrite(withDomain("/api/fern-docs/llms.txt"), { slug });
return rewrite(withDomain("/fern-docs/llms.txt"), { slug });
}

/**
Expand All @@ -184,15 +185,15 @@ export const middleware: NextMiddleware = async (request) => {
*/
if (pathname.endsWith("/llms-full.txt")) {
const slug = removeLeadingSlash(withoutEnding(/\/llms-full\.txt$/));
return rewrite(withDomain("/api/fern-docs/llms-full.txt"), { slug });
return rewrite(withDomain("/fern-docs/llms-full.txt"), { slug });
}

/**
* Rewrite markdown
*/
if (pathname.match(MARKDOWN_PATTERN)) {
const slug = removeLeadingSlash(withoutEnding(MARKDOWN_PATTERN));
return rewrite(withDomain("/api/fern-docs/markdown"), { slug });
return rewrite(withDomain("/fern-docs/markdown"), { slug });
}

/**
Expand Down Expand Up @@ -228,7 +229,7 @@ export const middleware: NextMiddleware = async (request) => {
const acceptHeader = request.headers.get("accept");
if (acceptHeader && (acceptHeader.includes("text/plain") || acceptHeader.includes("text/markdown"))) {
const slug = removeLeadingSlash(pathname);
return rewrite(withDomain("/api/fern-docs/llms.txt"), { slug });
return rewrite(withDomain("/fern-docs/llms.txt"), { slug });
}

/**
Expand Down
Loading