Skip to content

Commit 7721f60

Browse files
HonzaTuronHonzaTuron
authored
feat: Enable llms for api & sdk docs (#1736)
Closes #1711 - Enable merging of llms across all docusaurus plugins - Add markdown link to head - Setup route categories
1 parent 04c7efa commit 7721f60

File tree

4 files changed

+105
-10
lines changed

4 files changed

+105
-10
lines changed

‎apify-docs-theme/src/theme/Layout/index.jsx

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import Head from '@docusaurus/Head';
12
import { useLocation } from '@docusaurus/router';
23
// cannot use any of the theme aliases here as it causes a circular dependency :( ideas welcome
34
import Layout from '@docusaurus/theme-classic/lib/theme/Layout/index';
@@ -11,13 +12,19 @@ export default function LayoutWrapper(props) {
1112
const currentPath = useLocation().pathname.replace(new RegExp(`^${baseUrl}`), '');
1213

1314
return (
14-
<div style={{
15-
'--ifm-navbar-height': subNavbar && !currentPath.startsWith('api/v2') ? '126px' : '68px',
16-
margin: 0,
17-
padding: 0,
18-
boxSizing: 'border-box',
19-
}}>
20-
<Layout {...props} />
21-
</div>
15+
<>
16+
<Head>
17+
<link rel="alternate" type="text/markdown" href={`${currentPath}.md`}/>
18+
</Head>
19+
<div
20+
style={{
21+
'--ifm-navbar-height': subNavbar && !currentPath.startsWith('api/v2') ? '126px' : '68px',
22+
margin: 0,
23+
padding: 0,
24+
boxSizing: 'border-box',
25+
}}>
26+
<Layout {...props} />
27+
</div>
28+
</>
2229
);
2330
}

‎docusaurus.config.js

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -265,13 +265,34 @@ module.exports = {
265265
[
266266
'@signalwire/docusaurus-plugin-llms-txt',
267267
{
268-
enableDescriptions: false,
269268
content: {
270269
includeVersionedDocs: false,
271270
enableLlmsFullTxt: true,
272271
includeBlog: true,
272+
includeGeneratedIndex: false,
273273
includePages: true,
274274
relativePaths: false,
275+
excludeRoutes: [
276+
'/',
277+
],
278+
routeRules: [
279+
{
280+
route: '/api/**',
281+
categoryName: 'Apify API',
282+
},
283+
{
284+
route: '/academy/**',
285+
categoryName: 'Apify academy',
286+
},
287+
{
288+
route: '/legal/**',
289+
categoryName: 'Legal documents',
290+
},
291+
{
292+
route: '/platform/**',
293+
categoryName: 'Platform documentation',
294+
},
295+
],
275296
},
276297
},
277298
],

‎package.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,8 @@
4040
"lint:md:fix": "markdownlint '**/*.md' --fix",
4141
"lint:code": "eslint .",
4242
"lint:code:fix": "eslint . --fix",
43-
"postinstall": "patch-package"
43+
"postinstall": "patch-package",
44+
"postbuild": "node ./scripts/joinLlmsFiles.mjs"
4445
},
4546
"devDependencies": {
4647
"@apify/eslint-config": "^1.0.0",

‎scripts/joinLlmsFiles.mjs

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
import fs from 'node:fs/promises';
2+
import path from 'node:path';
3+
4+
const BUILD_DIR = path.resolve('build');
5+
6+
const FILES_ROUTES = {
7+
'llms.txt': [
8+
'https://docs.apify.com/api/client/js/llms.txt',
9+
'https://docs.apify.com/api/client/python/llms.txt',
10+
'https://docs.apify.com/sdk/js/llms.txt',
11+
'https://docs.apify.com/sdk/python/llms.txt',
12+
'https://docs.apify.com/cli/llms.txt',
13+
],
14+
'llms-full.txt': [
15+
'https://docs.apify.com/api/client/js/llms-full.txt',
16+
'https://docs.apify.com/api/client/python/llms-full.txt',
17+
'https://docs.apify.com/sdk/js/llms-full.txt',
18+
'https://docs.apify.com/sdk/python/llms-full.txt',
19+
'https://docs.apify.com/cli/llms-full.txt',
20+
'https://raw.githubusercontent.com/apify/actor-whitepaper/refs/heads/master/README.md',
21+
'https://raw.githubusercontent.com/apify/actor-whitepaper/refs/heads/master/pages/ACTOR_FILE.md',
22+
'https://raw.githubusercontent.com/apify/actor-whitepaper/refs/heads/master/pages/DATASET_SCHEMA.md',
23+
'https://raw.githubusercontent.com/apify/actor-whitepaper/refs/heads/master/pages/IDEAS.md',
24+
'https://raw.githubusercontent.com/apify/actor-whitepaper/refs/heads/master/pages/INPUT_SCHEMA.md',
25+
'https://raw.githubusercontent.com/apify/actor-whitepaper/refs/heads/master/pages/KEY_VALUE_STORE_SCHEMA.md',
26+
'https://raw.githubusercontent.com/apify/actor-whitepaper/refs/heads/master/pages/OUTPUT_SCHEMA.md',
27+
'https://raw.githubusercontent.com/apify/actor-whitepaper/refs/heads/master/pages/REQUEST_QUEUE_SCHEMA.md',
28+
],
29+
};
30+
31+
async function fetchFile(route) {
32+
try {
33+
const res = await fetch(route);
34+
if (!res.ok) throw new Error(`Failed to fetch ${route}: ${res.status}`);
35+
return await res.text();
36+
} catch (err) {
37+
console.error(`Error fetching ${route}:`, err.message);
38+
return '';
39+
}
40+
}
41+
42+
async function joinFiles() {
43+
await fs.mkdir(BUILD_DIR, { recursive: true });
44+
for (const [llmsFile, files] of Object.entries(FILES_ROUTES)) {
45+
const contents = await Promise.all(
46+
files.map((route) => fetchFile(route)),
47+
);
48+
const joined = contents.filter(Boolean).join('\n\n');
49+
await fs.appendFile(path.join(BUILD_DIR, llmsFile), joined, 'utf8');
50+
console.log(`Wrote ${llmsFile} to build/`);
51+
}
52+
}
53+
54+
async function sanitizeFile(filePath) {
55+
const content = await fs.readFile(filePath, 'utf8');
56+
const sanitizedContent = content.replace(/<[^>]*>/g, ''); // Remove HTML tags
57+
await fs.writeFile(filePath, sanitizedContent, 'utf8');
58+
console.log(`Sanitized ${filePath}`);
59+
}
60+
61+
joinFiles().catch((err) => {
62+
console.error('Failed to join LLMs files:', err);
63+
process.exit(1);
64+
});
65+
66+
Object.keys(FILES_ROUTES).forEach((llmsFile) => sanitizeFile(path.join(BUILD_DIR, llmsFile)));

0 commit comments

Comments
 (0)