apify · HonzaTuron · Aug 7, 2025 · Jul 29, 2025 · Jul 30, 2025 · Jul 31, 2025
diff --git a/apify-docs-theme/src/theme/Layout/index.jsx b/apify-docs-theme/src/theme/Layout/index.jsx
@@ -1,3 +1,4 @@
+import Head from '@docusaurus/Head';
 import { useLocation } from '@docusaurus/router';
 // cannot use any of the theme aliases here as it causes a circular dependency :( ideas welcome
 import Layout from '@docusaurus/theme-classic/lib/theme/Layout/index';
@@ -11,13 +12,19 @@ export default function LayoutWrapper(props) {
     const currentPath = useLocation().pathname.replace(new RegExp(`^${baseUrl}`), '');
 
     return (
-        <div style={{
-            '--ifm-navbar-height': subNavbar && !currentPath.startsWith('api/v2') ? '126px' : '68px',
-            margin: 0,
-            padding: 0,
-            boxSizing: 'border-box',
-        }}>
-            <Layout {...props} />
-        </div>
+        <>
+            <Head>
+                <link rel="alternate" type="text/markdown" href={`${currentPath}.md`}/>
+            </Head>
+            <div
+                style={{
+                    '--ifm-navbar-height': subNavbar && !currentPath.startsWith('api/v2') ? '126px' : '68px',
+                    margin: 0,
+                    padding: 0,
+                    boxSizing: 'border-box',
+                }}>
+                <Layout {...props} />
+            </div>
+        </>
     );
 }
diff --git a/docusaurus.config.js b/docusaurus.config.js
@@ -265,13 +265,34 @@ module.exports = {
         [
             '@signalwire/docusaurus-plugin-llms-txt',
             {
-                enableDescriptions: false,
                 content: {
                     includeVersionedDocs: false,
                     enableLlmsFullTxt: true,
                     includeBlog: true,
+                    includeGeneratedIndex: false,
                     includePages: true,
                     relativePaths: false,
+                    excludeRoutes: [
+                        '/',
+                    ],
+                    routeRules: [
+                        {
+                            route: '/api/**',
+                            categoryName: 'Apify API',
+                        },
+                        {
+                            route: '/academy/**',
+                            categoryName: 'Apify academy',
+                        },
+                        {
+                            route: '/legal/**',
+                            categoryName: 'Legal documents',
+                        },
+                        {
+                            route: '/platform/**',
+                            categoryName: 'Platform documentation',
+                        },
+                    ],
                 },
             },
         ],

diff --git a/package.json b/package.json
@@ -40,7 +40,8 @@
         "lint:md:fix": "markdownlint '**/*.md' --fix",
         "lint:code": "eslint .",
         "lint:code:fix": "eslint . --fix",
-        "postinstall": "patch-package"
+        "postinstall": "patch-package",
+        "postbuild": "node ./scripts/joinLlmsFiles.mjs"
     },
     "devDependencies": {
         "@apify/eslint-config": "^1.0.0",

diff --git a/scripts/joinLlmsFiles.mjs b/scripts/joinLlmsFiles.mjs
@@ -0,0 +1,66 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+const BUILD_DIR = path.resolve('build');
+
+const FILES_ROUTES = {
+    'llms.txt': [
+        'https://docs.apify.com/api/client/js/llms.txt',
+        'https://docs.apify.com/api/client/python/llms.txt',
+        'https://docs.apify.com/sdk/js/llms.txt',
+        'https://docs.apify.com/sdk/python/llms.txt',
+        'https://docs.apify.com/cli/llms.txt',
+    ],
+    'llms-full.txt': [
+        'https://docs.apify.com/api/client/js/llms-full.txt',
+        'https://docs.apify.com/api/client/python/llms-full.txt',
+        'https://docs.apify.com/sdk/js/llms-full.txt',
+        'https://docs.apify.com/sdk/python/llms-full.txt',
+        'https://docs.apify.com/cli/llms-full.txt',
+        'https://raw.githubusercontent.com/apify/actor-whitepaper/refs/heads/master/README.md',
+        'https://raw.githubusercontent.com/apify/actor-whitepaper/refs/heads/master/pages/ACTOR_FILE.md',
+        'https://raw.githubusercontent.com/apify/actor-whitepaper/refs/heads/master/pages/DATASET_SCHEMA.md',
+        'https://raw.githubusercontent.com/apify/actor-whitepaper/refs/heads/master/pages/IDEAS.md',
+        'https://raw.githubusercontent.com/apify/actor-whitepaper/refs/heads/master/pages/INPUT_SCHEMA.md',
+        'https://raw.githubusercontent.com/apify/actor-whitepaper/refs/heads/master/pages/KEY_VALUE_STORE_SCHEMA.md',
+        'https://raw.githubusercontent.com/apify/actor-whitepaper/refs/heads/master/pages/OUTPUT_SCHEMA.md',
+        'https://raw.githubusercontent.com/apify/actor-whitepaper/refs/heads/master/pages/REQUEST_QUEUE_SCHEMA.md',
+    ],
+};
+
+async function fetchFile(route) {
+    try {
+        const res = await fetch(route);
+        if (!res.ok) throw new Error(`Failed to fetch ${route}: ${res.status}`);
+        return await res.text();
+    } catch (err) {
+        console.error(`Error fetching ${route}:`, err.message);
+        return '';
+    }
+}
+
+async function joinFiles() {
+    await fs.mkdir(BUILD_DIR, { recursive: true });
+    for (const [llmsFile, files] of Object.entries(FILES_ROUTES)) {
+        const contents = await Promise.all(
+            files.map((route) => fetchFile(route)),
+        );
+        const joined = contents.filter(Boolean).join('\n\n');
+        await fs.appendFile(path.join(BUILD_DIR, llmsFile), joined, 'utf8');
+        console.log(`Wrote ${llmsFile} to build/`);
+    }
+}
+
+async function sanitizeFile(filePath) {
+    const content = await fs.readFile(filePath, 'utf8');
+    const sanitizedContent = content.replace(/<[^>]*>/g, ''); // Remove HTML tags
+    await fs.writeFile(filePath, sanitizedContent, 'utf8');
+    console.log(`Sanitized ${filePath}`);
+}
+
+joinFiles().catch((err) => {
+    console.error('Failed to join LLMs files:', err);
+    process.exit(1);
+});
+
+Object.keys(FILES_ROUTES).forEach((llmsFile) => sanitizeFile(path.join(BUILD_DIR, llmsFile)));