Skip to content

Commit 6d025d1

Browse files
authored
Merge pull request #2171 from brefphp/agent-readability
Make the website easier to use for AI agents
2 parents 083cb38 + 63a96e3 commit 6d025d1

11 files changed

Lines changed: 375 additions & 49 deletions

File tree

website/app/[[...mdxPath]]/page.jsx

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@ export async function generateMetadata(props) {
2323
const mdxPath = params.mdxPath ?? []
2424
const { metadata } = await importPage(mdxPath)
2525
const isHome = mdxPath.length === 0
26-
const isDocsPage = mdxPath[0] === 'docs' && mdxPath.length > 1
26+
// Pages that have a Markdown version (see src/lib/markdown.js)
27+
const hasMarkdown = (mdxPath[0] === 'docs' && mdxPath.length > 1) || (mdxPath[0] === 'news' && mdxPath.length > 1)
2728
// `seoTitle` frontmatter sets the exact <title> (no "– Bref" suffix) without
2829
// affecting the sidebar label; `ogImage` overrides the default social card.
2930
const seoTitle = metadata.seoTitle
@@ -38,9 +39,9 @@ export async function generateMetadata(props) {
3839
images: [{ url: ogImage ?? 'https://bref.sh/social-card.png' }],
3940
...(metadata.openGraph || {}),
4041
},
41-
// Per-docs-page alternate markdown link (was theme.config.head conditional)
42-
...(isDocsPage
43-
? { alternates: { types: { 'text/markdown': `/docs/${mdxPath.slice(1).join('/')}.md` } } }
42+
// Alternate Markdown version of the page, for AI agents
43+
...(hasMarkdown
44+
? { alternates: { types: { 'text/markdown': `/${mdxPath.join('/')}.md` } } }
4445
: {}),
4546
}
4647
}
Lines changed: 16 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,46 +1,28 @@
1-
import fs from 'fs'
2-
import path from 'path'
1+
import { readMarkdown, routeToUrl } from '../../../../src/lib/markdown'
32

4-
// App Router Route Handler replacing pages/api/md/[...slug].js.
5-
// Reads raw MDX from content/docs and strips imports + YAML frontmatter
6-
// (NextSeo is gone in v4, so the old <NextSeo> stripper is replaced by a frontmatter strip).
3+
// Markdown version of docs and news pages, for AI agents. Reached via:
4+
// - /docs/<page>.md and /news/<page>.md (rewrites in next.config.mjs)
5+
// - `Accept: text/markdown` on the HTML URL (middleware.js)
76
export async function GET(req, { params }) {
87
const { slug } = await params
9-
const slugPath = Array.isArray(slug) ? slug.join('/') : (slug ?? '')
8+
const route = Array.isArray(slug) ? slug.join('/') : (slug ?? '')
109

11-
const docsDir = path.join(process.cwd(), 'content/docs')
12-
const candidates = slugPath
13-
? [
14-
path.join(docsDir, `${slugPath}.mdx`),
15-
path.join(docsDir, `${slugPath}.md`),
16-
path.join(docsDir, slugPath, 'index.mdx'),
17-
]
18-
: [path.join(docsDir, 'index.mdx')]
19-
20-
// Reject any path that escapes content/docs (e.g. via `..` segments).
21-
const resolvedDocsDir = path.resolve(docsDir) + path.sep
22-
let filePath = candidates.find(
23-
candidate => path.resolve(candidate).startsWith(resolvedDocsDir) && fs.existsSync(candidate)
24-
)
25-
if (!filePath) {
26-
return new Response(JSON.stringify({ error: 'Page not found' }), {
10+
const content = readMarkdown(route)
11+
if (content === null) {
12+
return new Response('Page not found', {
2713
status: 404,
28-
headers: { 'Content-Type': 'application/json' },
14+
headers: { 'Content-Type': 'text/plain; charset=utf-8' },
2915
})
3016
}
3117

32-
let content = fs.readFileSync(filePath, 'utf8')
33-
// Strip YAML frontmatter
34-
content = content.replace(/^---\n[\s\S]*?\n---\n/, '')
35-
// Strip import statements
36-
content = content.replace(/^import\s+.*?(?:from\s+['"].*?['"])?;?\s*$/gm, '')
37-
// Strip JSX comments (invisible when rendered, but not valid Markdown)
38-
content = content.replace(/\{\/\*[\s\S]*?\*\/\}\n?/g, '')
39-
// Clean up excessive blank lines at the start
40-
content = content.replace(/^\s*\n+/, '')
41-
4218
return new Response(content, {
4319
status: 200,
44-
headers: { 'Content-Type': 'text/markdown; charset=utf-8' },
20+
headers: {
21+
'Content-Type': 'text/markdown; charset=utf-8',
22+
// The HTML page is the canonical version of this content
23+
Link: `<${routeToUrl(route)}>; rel="canonical"`,
24+
// The same URL serves HTML or Markdown depending on the Accept header
25+
Vary: 'Accept',
26+
},
4527
})
4628
}

website/app/llms-full.txt/route.js

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
import { getDocsSections } from '../../src/lib/site-index'
2+
import { readMarkdown, SITE_URL } from '../../src/lib/markdown'
3+
4+
// The whole documentation as a single Markdown file, for AI agents.
5+
export const dynamic = 'force-static'
6+
7+
export async function GET() {
8+
const sections = await getDocsSections()
9+
10+
const parts = [
11+
'# Bref documentation',
12+
'',
13+
`> Bref is an open-source framework to run PHP applications on AWS Lambda (serverless). This file contains the whole documentation of ${SITE_URL}. Each page starts with its canonical URL.`,
14+
'',
15+
]
16+
for (const section of sections) {
17+
for (const page of section.pages) {
18+
const content = readMarkdown(page.route)
19+
if (content === null) continue
20+
parts.push('---', '', `Source: ${SITE_URL}${page.route}`, '', content.trim(), '')
21+
}
22+
}
23+
24+
return new Response(parts.join('\n'), {
25+
headers: { 'Content-Type': 'text/plain; charset=utf-8' },
26+
})
27+
}

website/app/llms.txt/route.js

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
import { getDocsSections, getNewsPages } from '../../src/lib/site-index'
2+
import { SITE_URL } from '../../src/lib/markdown'
3+
4+
// https://llmstxt.org/ - an index of the website for AI agents.
5+
export const dynamic = 'force-static'
6+
7+
const intro = `# Bref
8+
9+
> Bref is an open-source framework to run PHP applications on AWS Lambda (serverless): PHP runtimes for Lambda, deployment tooling, and integrations for Laravel and Symfony. Bref Cloud is the hosted service that deploys, monitors and operates Bref applications in your own AWS account.
10+
11+
Every documentation page is available as Markdown: append \`.md\` to its URL (for example ${SITE_URL}/docs/laravel/getting-started.md), or request the HTML URL with an \`Accept: text/markdown\` header. The whole documentation is also available as a single file: ${SITE_URL}/llms-full.txt
12+
13+
Bref is free and open-source (MIT license). Bref Cloud has a free plan for personal projects and paid plans with a free trial: ${SITE_URL}/cloud#pricing
14+
`
15+
16+
const line = page => `- [${page.title}](${SITE_URL}${page.route}${page.md ? '.md' : ''})${page.description ? `: ${page.description}` : ''}`
17+
18+
export async function GET() {
19+
const sections = await getDocsSections()
20+
const news = await getNewsPages()
21+
22+
const docs = sections.map(section => [
23+
`## ${section.title ? `Documentation: ${section.title}` : 'Documentation'}`,
24+
'',
25+
...section.pages.map(page => line({ ...page, md: true })),
26+
'',
27+
].join('\n'))
28+
29+
const site = [
30+
'## Bref Cloud',
31+
'',
32+
line({ title: 'Bref Cloud', route: '/cloud', description: 'Serverless PHP hosting on AWS Lambda: deploy, monitor and operate PHP applications in your own AWS account. Features, how it works, plans and pricing.' }),
33+
line({ title: 'Bref Cloud documentation', route: '/docs/cloud', md: true, description: 'What Bref Cloud does and how to get started.' }),
34+
'',
35+
'## Support and community',
36+
'',
37+
line({ title: 'Support plans', route: '/support', description: 'Consulting and support plans for serverless migrations to AWS with Bref and PHP.' }),
38+
line({ title: 'Community', route: '/docs/community', md: true, description: 'Slack, GitHub and other places to get help.' }),
39+
'- [GitHub repository](https://github.com/brefphp/bref): source code, issues and releases of the open-source project.',
40+
'',
41+
'## News',
42+
'',
43+
...news.map(page => line({ ...page, md: true })),
44+
'',
45+
'## Optional',
46+
'',
47+
`- [Full documentation in one file](${SITE_URL}/llms-full.txt)`,
48+
`- [Sitemap](${SITE_URL}/sitemap.xml)`,
49+
'',
50+
].join('\n')
51+
52+
return new Response([intro, ...docs, site].join('\n'), {
53+
headers: { 'Content-Type': 'text/plain; charset=utf-8' },
54+
})
55+
}

website/middleware.js

Lines changed: 26 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,36 @@
11
import { NextResponse } from 'next/server';
22

3+
const SITE_URL = 'https://bref.sh';
4+
5+
// Routes that have a Markdown version (see app/api/md and src/lib/markdown.js)
6+
const MARKDOWN_ROUTES = /^\/(docs|news)(\/|$)/;
7+
38
export function middleware(request) {
4-
const accept = request.headers.get('accept') || '';
9+
const { pathname } = request.nextUrl;
10+
// `.md` URLs are already served as Markdown (rewrites in next.config.mjs)
11+
if (pathname.endsWith('.md')) return NextResponse.next();
12+
const hasMarkdown = MARKDOWN_ROUTES.test(pathname);
513

6-
// Content negotiation: serve Markdown for AI crawlers requesting it
7-
if (accept.includes('text/markdown') && request.nextUrl.pathname.startsWith('/docs/')) {
8-
const mdPath = request.nextUrl.pathname.replace('/docs/', '/api/md/');
9-
return NextResponse.rewrite(new URL(mdPath, request.url));
14+
// Content negotiation: serve Markdown to AI agents that ask for it
15+
const accept = request.headers.get('accept') || '';
16+
if (hasMarkdown && accept.includes('text/markdown')) {
17+
return NextResponse.rewrite(new URL('/api/md' + pathname, request.url));
1018
}
1119

12-
return NextResponse.next();
20+
// Advertise the machine-readable resources in HTTP headers, so that agents
21+
// that do not parse the HTML can still discover them.
22+
const links = [`<${SITE_URL}/llms.txt>; rel="llms-txt"`];
23+
if (hasMarkdown) {
24+
links.push(`<${SITE_URL}${pathname.replace(/\/$/, '')}.md>; rel="alternate"; type="text/markdown"`);
25+
}
26+
const response = NextResponse.next();
27+
response.headers.set('Link', links.join(', '));
28+
return response;
1329
}
1430

1531
export const config = {
16-
matcher: '/docs/:path*',
32+
// All pages, excluding Next.js internals, API routes and static files.
33+
// (page routes can contain dots, e.g. /news/01-bref-1.0, so we cannot
34+
// simply exclude every path that contains a dot)
35+
matcher: ['/((?!_next/|api/|_pagefind/|js/|.*\\.(?:txt|xml|json|png|jpe?g|gif|svg|ico|webp|woff2?|css|js|map)$).*)'],
1736
};

website/next.config.mjs

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -63,16 +63,21 @@ export default withNextra(withPlausibleProxy()({
6363
...redirectList,
6464
]
6565
},
66-
// Serve Markdown versions of docs for AI crawlers
66+
// Serve Markdown versions of docs and news pages for AI agents
67+
// (see also middleware.js for the `Accept: text/markdown` content negotiation)
6768
async rewrites() {
6869
return [
6970
{
7071
source: '/docs/:path*.md',
71-
destination: '/api/md/:path*',
72+
destination: '/api/md/docs/:path*',
7273
},
7374
{
7475
source: '/docs.md',
75-
destination: '/api/md',
76+
destination: '/api/md/docs',
77+
},
78+
{
79+
source: '/news/:path*.md',
80+
destination: '/api/md/news/:path*',
7681
},
7782
]
7883
},

website/package.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,8 @@
33
"dev": "next dev -p 8000",
44
"build": "next build",
55
"postbuild": "next-sitemap && pagefind --site .next/server/app --glob \"{docs.html,docs/**/*.html}\" --output-path public/_pagefind",
6-
"start": "next start"
6+
"start": "next start",
7+
"test": "node --test \"src/**/*.test.js\""
78
},
89
"dependencies": {
910
"@aws-sdk/client-cloudwatch": "^3.1109.0",

website/redirects.js

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@ module.exports.redirects = {
33
'/#ecosystem': '/support',
44
'/#plans': '/support',
55
'/#enterprise': '/support',
6+
// URLs that people (and AI agents) guess
7+
'/pricing': '/cloud#pricing',
8+
'/cloud/pricing': '/cloud#pricing',
69
'/docs/news': '/news',
710
'/docs/news/01-bref-1.0': '/news/01-bref-1.0',
811
'/docs/news/02-bref-2.0': '/news/02-bref-2.0',

website/src/lib/markdown.js

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
import fs from 'fs'
2+
import path from 'path'
3+
4+
export const SITE_URL = 'https://bref.sh'
5+
6+
const contentDir = path.join(process.cwd(), 'content')
7+
8+
// Top-level content folders whose pages are served as Markdown (for AI agents).
9+
// The marketing pages (home, /cloud, /support...) are React components, not
10+
// prose, so they are not exposed here.
11+
const MARKDOWN_ROOTS = ['docs', 'news']
12+
13+
/**
14+
* Resolve a site route (e.g. "docs/laravel/getting-started", "docs", "news/04-august-2026")
15+
* to its MDX source file, or null if none exists.
16+
*/
17+
export function findSourceFile(route) {
18+
// Normalize first so that `..` segments cannot escape the allowed roots
19+
const clean = path.posix.normalize('/' + route).replace(/^\/+|\/+$/g, '')
20+
const root = clean.split('/')[0]
21+
if (!MARKDOWN_ROOTS.includes(root)) return null
22+
23+
const candidates = [
24+
path.join(contentDir, `${clean}.mdx`),
25+
path.join(contentDir, `${clean}.md`),
26+
path.join(contentDir, clean, 'index.mdx'),
27+
]
28+
// Reject any path that escapes content/ (e.g. via `..` segments).
29+
const resolvedContentDir = path.resolve(contentDir) + path.sep
30+
return (
31+
candidates.find(
32+
candidate => path.resolve(candidate).startsWith(resolvedContentDir) && fs.existsSync(candidate)
33+
) ?? null
34+
)
35+
}
36+
37+
/**
38+
* Turn the URL of a page (relative to the site root, without leading slash)
39+
* into the canonical HTML URL of that page.
40+
*/
41+
export function routeToUrl(route) {
42+
const clean = route.replace(/^\/+|\/+$/g, '')
43+
return clean ? `${SITE_URL}/${clean}` : SITE_URL
44+
}
45+
46+
/**
47+
* Rewrite a link target found in a MDX file into an absolute URL.
48+
* `fileRoute` is the route of the file that contains the link (e.g. "docs/laravel/index").
49+
*/
50+
function absoluteUrl(target, fileRoute) {
51+
if (/^(https?:|mailto:|#|data:)/.test(target)) return target
52+
53+
const [pathPart, hash] = target.split('#')
54+
// Relative links are resolved from the directory of the file
55+
let resolved = pathPart.startsWith('/')
56+
? pathPart
57+
: path.posix.join(path.posix.dirname('/' + fileRoute), pathPart)
58+
// `./getting-started.mdx` -> `/docs/laravel/getting-started`
59+
resolved = resolved.replace(/\.mdx?$/, '').replace(/\/index$/, '')
60+
return `${SITE_URL}${resolved}${hash ? '#' + hash : ''}`
61+
}
62+
63+
/**
64+
* Convert the raw MDX source of a page into Markdown suitable for AI agents:
65+
* strips the frontmatter, imports and JSX comments, and makes all links absolute.
66+
* The occasional JSX component (tabs, cards...) is left as is.
67+
*
68+
* `fileRoute` is the route of the source file relative to content/, without
69+
* extension (e.g. "docs/laravel/index").
70+
*/
71+
export function toMarkdown(source, fileRoute) {
72+
let content = source
73+
// Strip YAML frontmatter
74+
content = content.replace(/^---\n[\s\S]*?\n---\n/, '')
75+
// Strip JSX comments (invisible when rendered, but not valid Markdown)
76+
content = content.replace(/\{\/\*[\s\S]*?\*\/\}\n?/g, '')
77+
78+
// The rest only applies outside of code blocks and inline code, so that code
79+
// samples (e.g. JS `import` statements or HTML links) are preserved.
80+
content = content
81+
.split(/(```[\s\S]*?```|`[^`\n]*`)/)
82+
.map((part, index) => (index % 2 === 1 ? part : transformProse(part, fileRoute)))
83+
.join('')
84+
85+
// Clean up excessive blank lines at the start
86+
content = content.replace(/^\s*\n+/, '')
87+
return content
88+
}
89+
90+
function transformProse(content, fileRoute) {
91+
// Strip import statements
92+
content = content.replace(/^import\s+.*?(?:from\s+['"].*?['"])?;?\s*$/gm, '')
93+
// Absolute URLs in Markdown links: [text](./page.mdx#anchor) -> [text](https://bref.sh/docs/page#anchor)
94+
content = content.replace(/\]\(([^)\s]+)((?:\s[^)]*)?)\)/g, (match, target, title) => {
95+
return `](${absoluteUrl(target, fileRoute)}${title})`
96+
})
97+
// Absolute URLs in JSX/HTML links: href="./page.mdx" -> href="https://bref.sh/docs/page"
98+
content = content.replace(/href="([^"]+)"/g, (match, target) => `href="${absoluteUrl(target, fileRoute)}"`)
99+
return content
100+
}
101+
102+
/**
103+
* Read a page and return its Markdown, or null if the page has no source file.
104+
*/
105+
export function readMarkdown(route) {
106+
const filePath = findSourceFile(route)
107+
if (!filePath) return null
108+
const fileRoute = path.relative(contentDir, filePath).replace(/\.mdx?$/, '')
109+
return toMarkdown(fs.readFileSync(filePath, 'utf8'), fileRoute)
110+
}

0 commit comments

Comments
 (0)