Serve a markdown version of the page when adding a .md extension (#3055)

SamyPesse · Copilot · web-flow · commit c85d3f97521f · 2025-03-27T12:35:18.000Z
Co-authored-by: Copilot &lt;175728472+Copilot@users.noreply.github.com&gt;
diff --git a/packages/gitbook-v2/src/app/sites/dynamic/[mode]/[siteURL]/[siteData]/llms.txt/route.ts b/packages/gitbook-v2/src/app/sites/dynamic/[mode]/[siteURL]/[siteData]/llms.txt/route.ts
diff --git a/packages/gitbook-v2/src/app/sites/dynamic/[mode]/[siteURL]/[siteData]/robots.txt/route.ts b/packages/gitbook-v2/src/app/sites/dynamic/[mode]/[siteURL]/[siteData]/robots.txt/route.ts
diff --git a/packages/gitbook-v2/src/app/sites/dynamic/[mode]/[siteURL]/[siteData]/sitemap-pages.xml/route.ts b/packages/gitbook-v2/src/app/sites/dynamic/[mode]/[siteURL]/[siteData]/sitemap-pages.xml/route.ts
diff --git a/packages/gitbook-v2/src/app/sites/dynamic/[mode]/[siteURL]/[siteData]/sitemap.xml/route.ts b/packages/gitbook-v2/src/app/sites/dynamic/[mode]/[siteURL]/[siteData]/sitemap.xml/route.ts
diff --git a/packages/gitbook-v2/src/app/sites/static/[mode]/[siteURL]/[siteData]/llms.txt/route.ts b/packages/gitbook-v2/src/app/sites/static/[mode]/[siteURL]/[siteData]/llms.txt/route.ts
@@ -10,5 +10,5 @@ export async function GET(
     { params }: { params: Promise<RouteLayoutParams> }
 ) {
     const { context } = await getStaticSiteContext(await params);
-    return serveLLMsTxt(context);
+    return serveLLMsTxt(context, { withMarkdownPages: true });
 }
diff --git a/packages/gitbook-v2/src/app/sites/static/[mode]/[siteURL]/[siteData]/~gitbook/markdown/[pagePath]/route.ts b/packages/gitbook-v2/src/app/sites/static/[mode]/[siteURL]/[siteData]/~gitbook/markdown/[pagePath]/route.ts
@@ -0,0 +1,11 @@
+import { servePageMarkdown } from '@/routes/markdownPage';
+import { type RouteParams, getPagePathFromParams, getStaticSiteContext } from '@v2/app/utils';
+import type { NextRequest } from 'next/server';
+
+export const dynamic = 'force-static';
+
+export async function GET(_request: NextRequest, { params }: { params: Promise<RouteParams> }) {
+    const { context } = await getStaticSiteContext(await params);
+    const pathname = getPagePathFromParams(await params);
+    return servePageMarkdown(context, pathname);
+}
diff --git a/packages/gitbook-v2/src/lib/data/api.ts b/packages/gitbook-v2/src/lib/data/api.ts
@@ -8,7 +8,7 @@ import {
 import { getCacheTag, getComputedContentSourceCacheTags } from '@gitbook/cache-tags';
 import { GITBOOK_API_TOKEN, GITBOOK_API_URL, GITBOOK_USER_AGENT } from '@v2/lib/env';
 import { unstable_cacheLife as cacheLife, unstable_cacheTag as cacheTag } from 'next/cache';
-import { wrapDataFetcherError } from './errors';
+import { DataFetcherError, wrapDataFetcherError } from './errors';
 import { memoize } from './memoize';
 import type { GitBookDataFetcher } from './types';
 
@@ -95,6 +95,15 @@ export function createDataFetcher(
                 })
             );
         },
+        getRevisionPageMarkdown(params) {
+            return trace('getRevisionPageMarkdown', () =>
+                getRevisionPageMarkdown(input, {
+                    spaceId: params.spaceId,
+                    revisionId: params.revisionId,
+                    pageId: params.pageId,
+                })
+            );
+        },
         getReusableContent(params) {
             return trace('getReusableContent', () =>
                 getReusableContent(input, {
@@ -324,6 +333,39 @@ const getRevisionFile = memoize(async function getRevisionFile(
     });
 });
 
+const getRevisionPageMarkdown = memoize(async function getRevisionPageMarkdown(
+    input: DataFetcherInput,
+    params: {
+        spaceId: string;
+        revisionId: string;
+        pageId: string;
+    }
+) {
+    'use cache';
+
+    return trace('getRevisionPageMarkdown.uncached', () => {
+        cacheLife('max');
+
+        return wrapDataFetcherError(async () => {
+            const api = await apiClient(input);
+            const res = await api.spaces.getPageInRevisionById(
+                params.spaceId,
+                params.revisionId,
+                params.pageId,
+                {
+                    format: 'markdown',
+                }
+            );
+
+            if (!('markdown' in res.data)) {
+                throw new DataFetcherError('Page is not a document', 404);
+            }
+
+            return res.data.markdown;
+        });
+    });
+});
+
 const getRevisionPageByPath = memoize(async function getRevisionPageByPath(
     input: DataFetcherInput,
     params: {
diff --git a/packages/gitbook-v2/src/lib/data/types.ts b/packages/gitbook-v2/src/lib/data/types.ts
@@ -97,6 +97,15 @@ export interface GitBookDataFetcher {
         path: string;
     }): Promise<DataFetcherResponse<api.RevisionPageDocument | api.RevisionPageGroup>>;
 
+    /**
+     * Get the markdown content of a page by its path.
+     */
+    getRevisionPageMarkdown(params: {
+        spaceId: string;
+        revisionId: string;
+        pageId: string;
+    }): Promise<DataFetcherResponse<string>>;
+
     /**
      * Get a document by its space ID and document ID.
      */
diff --git a/packages/gitbook-v2/src/middleware.ts b/packages/gitbook-v2/src/middleware.ts
@@ -389,13 +389,26 @@ function encodePathInSiteContent(rawPathname: string): {
         return { pathname };
     }
 
+    // If the pathname is a markdown file, we rewrite it to ~gitbook/markdown/:pathname
+    if (pathname.match(/\.md$/)) {
+        const pagePathWithoutMD = pathname.slice(0, -3);
+        return {
+            pathname: `~gitbook/markdown/${encodeURIComponent(pagePathWithoutMD)}`,
+            // The markdown content is always static and doesn't depend on the dynamic parameter (customization, theme, etc)
+            routeType: 'static',
+        };
+    }
+
     switch (pathname) {
         case '~gitbook/icon':
+            return { pathname };
         case 'llms.txt':
         case 'sitemap.xml':
         case 'sitemap-pages.xml':
         case 'robots.txt':
-            return { pathname };
+            // LLMs.txt, sitemap, sitemap-pages and robots.txt are always static
+            // as they only depend on the site structure / pages.
+            return { pathname, routeType: 'static' };
         case '~gitbook/pdf':
             // PDF routes are always dynamic as they depend on the search params.
             return { pathname, routeType: 'dynamic' };
diff --git a/packages/gitbook/e2e/internal.spec.ts b/packages/gitbook/e2e/internal.spec.ts
@@ -403,6 +403,22 @@ const testCases: TestsCase[] = [
             },
         ],
     },
+    {
+        name: 'Markdown page',
+        skip: process.env.ARGOS_BUILD_NAME !== 'v2-vercel',
+        contentBaseURL: 'https://gitbook.gitbook.io/test-gitbook-open/',
+        tests: [
+            {
+                name: 'Text page',
+                url: 'text-page.md',
+                screenshot: false,
+                run: async (_page, response) => {
+                    expect(response?.status()).toBe(200);
+                    expect(response?.headers()['content-type']).toContain('text/markdown');
+                },
+            },
+        ],
+    },
     {
         name: 'Content tests',
         contentBaseURL: 'https://gitbook.gitbook.io/test-gitbook-open/',
diff --git a/packages/gitbook/src/lib/pages.ts b/packages/gitbook/src/lib/pages.ts
@@ -104,7 +104,7 @@ export function resolvePrevNextPages(
 }
 
 /**
- * Resolve a page to its canonical path.
+ * Resolve a page to its canonical path. The first page of the site will return "".
  * The path will NOT start with "/".
  */
 export function getPagePath(
diff --git a/packages/gitbook/src/lib/v1.ts b/packages/gitbook/src/lib/v1.ts
@@ -153,6 +153,10 @@ async function getDataFetcherV1(): Promise<GitBookDataFetcher> {
             });
         },
 
+        getRevisionPageMarkdown() {
+            throw new Error('Not implemented in v1');
+        },
+
         getDocument(params) {
             return wrapDataFetcherError(async () => {
                 const document = await getDocument(params.spaceId, params.documentId);
diff --git a/packages/gitbook/src/routes/llms.ts b/packages/gitbook/src/routes/llms.ts
@@ -1,19 +1,27 @@
-import type { SiteSection, SiteSpace } from '@gitbook/api';
-import assertNever from 'assert-never';
-import type { ListItem, Paragraph, Root, RootContent } from 'mdast';
-import { toMarkdown } from 'mdast-util-to-markdown';
-
-import { getPagePath } from '@/lib/pages';
 import { joinPath } from '@/lib/paths';
 import { getIndexablePages } from '@/lib/sitemap';
 import { getSiteStructureSections } from '@/lib/sites';
+import type { SiteSection, SiteSpace } from '@gitbook/api';
 import { type GitBookSiteContext, checkIsRootSiteContext } from '@v2/lib/context';
 import { throwIfDataError } from '@v2/lib/data';
+import assertNever from 'assert-never';
+import type { ListItem, Paragraph, Root, RootContent } from 'mdast';
+import { toMarkdown } from 'mdast-util-to-markdown';
 
 /**
  * Generate a llms.txt file for the site.
  */
-export async function serveLLMsTxt(context: GitBookSiteContext) {
+export async function serveLLMsTxt(
+    context: GitBookSiteContext,
+    {
+        withMarkdownPages = false,
+    }: {
+        /**
+         * If true, a markdown extension will be added to the page path.
+         */
+        withMarkdownPages?: boolean;
+    } = {}
+) {
     const { site } = context;
 
     if (!checkIsRootSiteContext(context)) {
@@ -28,7 +36,7 @@ export async function serveLLMsTxt(context: GitBookSiteContext) {
                 depth: 1,
                 children: [{ type: 'text', value: site.title }],
             },
-            ...(await getNodesFromSiteStructure(context)),
+            ...(await getNodesFromSiteStructure(context, { withMarkdownPages })),
         ],
     };
 
@@ -47,15 +55,24 @@ export async function serveLLMsTxt(context: GitBookSiteContext) {
 /**
  * Get MDAST nodes from site structure.
  */
-async function getNodesFromSiteStructure(context: GitBookSiteContext): Promise<RootContent[]> {
+async function getNodesFromSiteStructure(
+    context: GitBookSiteContext,
+    options: {
+        withMarkdownPages: boolean;
+    }
+): Promise<RootContent[]> {
     switch (context.structure.type) {
         case 'sections':
             return getNodesFromSections(
                 context,
-                getSiteStructureSections(context.structure, { ignoreGroups: true })
+                getSiteStructureSections(context.structure, { ignoreGroups: true }),
+                { withMarkdownPages: options.withMarkdownPages }
             );
         case 'siteSpaces':
-            return getNodesFromSiteSpaces(context, context.structure.structure, { heading: true });
+            return getNodesFromSiteSpaces(context, context.structure.structure, {
+                heading: true,
+                withMarkdownPages: options.withMarkdownPages,
+            });
         default:
             assertNever(context.structure);
     }
@@ -66,12 +83,16 @@ async function getNodesFromSiteStructure(context: GitBookSiteContext): Promise<R
  */
 async function getNodesFromSections(
     context: GitBookSiteContext,
-    siteSections: SiteSection[]
+    siteSections: SiteSection[],
+    options: {
+        withMarkdownPages: boolean;
+    }
 ): Promise<RootContent[]> {
     const all = await Promise.all(
         siteSections.map(async (siteSection): Promise<RootContent[]> => {
             const siteSpaceNodes = await getNodesFromSiteSpaces(context, siteSection.siteSpaces, {
                 heading: false,
+                withMarkdownPages: options.withMarkdownPages,
             });
             return [
                 {
@@ -97,6 +118,11 @@ async function getNodesFromSiteSpaces(
          * Includes a heading for each site space.
          */
         heading?: boolean;
+
+        /**
+         * If true, a markdown extension will be added to the page path.
+         */
+        withMarkdownPages: boolean;
     }
 ): Promise<RootContent[]> {
     const { dataFetcher, linker } = context;
@@ -118,7 +144,10 @@ async function getNodesFromSiteSpaces(
             const listChildren = await Promise.all(
                 pages.map(async ({ page }): Promise<ListItem> => {
                     const pageURL = new URL(siteSpaceUrl);
-                    pageURL.pathname = joinPath(pageURL.pathname, getPagePath(rootPages, page));
+                    pageURL.pathname = joinPath(pageURL.pathname, page.path);
+                    if (options.withMarkdownPages) {
+                        pageURL.pathname = `${pageURL.pathname}.md`;
+                    }
 
                     const url = linker.toLinkForContent(pageURL.toString());
                     const children: Paragraph['children'] = [
diff --git a/packages/gitbook/src/routes/markdownPage.ts b/packages/gitbook/src/routes/markdownPage.ts
@@ -0,0 +1,34 @@
+import { resolvePagePath } from '@/lib/pages';
+import { RevisionPageType } from '@gitbook/api';
+import type { GitBookSiteContext } from '@v2/lib/context';
+import { throwIfDataError } from '@v2/lib/data';
+
+/**
+ * Generate a markdown version of a page.
+ */
+export async function servePageMarkdown(context: GitBookSiteContext, pagePath: string) {
+    const pageLookup = resolvePagePath(context.pages, pagePath);
+    if (!pageLookup) {
+        return new Response(`Page "${pagePath}" not found`, { status: 404 });
+    }
+
+    const { page } = pageLookup;
+
+    if (page.type !== RevisionPageType.Document) {
+        return new Response(`Page "${pagePath}" is not a document`, { status: 404 });
+    }
+
+    const markdown = await throwIfDataError(
+        context.dataFetcher.getRevisionPageMarkdown({
+            spaceId: context.space.id,
+            revisionId: context.revisionId,
+            pageId: page.id,
+        })
+    );
+
+    return new Response(markdown, {
+        headers: {
+            'Content-Type': 'text/markdown; charset=utf-8',
+        },
+    });
+}

Original file line number	Diff line number	Diff line change
`@@ -10,5 +10,5 @@ export async function GET(`
`10`	`10`	`{ params }: { params: Promise<RouteLayoutParams> }`
`11`	`11`	`) {`
`12`	`12`	`const { context } = await getStaticSiteContext(await params);`
`13`		`- return serveLLMsTxt(context);`
	`13`	`+ return serveLLMsTxt(context, { withMarkdownPages: true });`
`14`	`14`	`}`