diff --git a/apify-docs-theme/src/theme/Layout/index.jsx b/apify-docs-theme/src/theme/Layout/index.jsx
index 08793f0b28..2ca67c8a9d 100644
--- a/apify-docs-theme/src/theme/Layout/index.jsx
+++ b/apify-docs-theme/src/theme/Layout/index.jsx
@@ -1,3 +1,4 @@
+import Head from '@docusaurus/Head';
import { useLocation } from '@docusaurus/router';
// cannot use any of the theme aliases here as it causes a circular dependency :( ideas welcome
import Layout from '@docusaurus/theme-classic/lib/theme/Layout/index';
@@ -11,13 +12,19 @@ export default function LayoutWrapper(props) {
const currentPath = useLocation().pathname.replace(new RegExp(`^${baseUrl}`), '');
return (
-
-
+ <>
+
+
+
+
);
}
diff --git a/docusaurus.config.js b/docusaurus.config.js
index b41e6db326..b5d81a2b4b 100644
--- a/docusaurus.config.js
+++ b/docusaurus.config.js
@@ -265,13 +265,34 @@ module.exports = {
[
'@signalwire/docusaurus-plugin-llms-txt',
{
- enableDescriptions: false,
content: {
includeVersionedDocs: false,
enableLlmsFullTxt: true,
includeBlog: true,
+ includeGeneratedIndex: false,
includePages: true,
relativePaths: false,
+ excludeRoutes: [
+ '/',
+ ],
+ routeRules: [
+ {
+ route: '/api/**',
+ categoryName: 'Apify API',
+ },
+ {
+ route: '/academy/**',
+ categoryName: 'Apify academy',
+ },
+ {
+ route: '/legal/**',
+ categoryName: 'Legal documents',
+ },
+ {
+ route: '/platform/**',
+ categoryName: 'Platform documentation',
+ },
+ ],
},
},
],
diff --git a/package.json b/package.json
index 952572138b..a40d40aac3 100644
--- a/package.json
+++ b/package.json
@@ -40,7 +40,8 @@
"lint:md:fix": "markdownlint '**/*.md' --fix",
"lint:code": "eslint .",
"lint:code:fix": "eslint . --fix",
- "postinstall": "patch-package"
+ "postinstall": "patch-package",
+ "postbuild": "node ./scripts/joinLlmsFiles.mjs"
},
"devDependencies": {
"@apify/eslint-config": "^1.0.0",
diff --git a/scripts/joinLlmsFiles.mjs b/scripts/joinLlmsFiles.mjs
new file mode 100644
index 0000000000..2a398fb3d8
--- /dev/null
+++ b/scripts/joinLlmsFiles.mjs
@@ -0,0 +1,66 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+const BUILD_DIR = path.resolve('build');
+
+const FILES_ROUTES = {
+ 'llms.txt': [
+ 'https://docs.apify.com/api/client/js/llms.txt',
+ 'https://docs.apify.com/api/client/python/llms.txt',
+ 'https://docs.apify.com/sdk/js/llms.txt',
+ 'https://docs.apify.com/sdk/python/llms.txt',
+ 'https://docs.apify.com/cli/llms.txt',
+ ],
+ 'llms-full.txt': [
+ 'https://docs.apify.com/api/client/js/llms-full.txt',
+ 'https://docs.apify.com/api/client/python/llms-full.txt',
+ 'https://docs.apify.com/sdk/js/llms-full.txt',
+ 'https://docs.apify.com/sdk/python/llms-full.txt',
+ 'https://docs.apify.com/cli/llms-full.txt',
+ 'https://raw.githubusercontent.com/apify/actor-whitepaper/refs/heads/master/README.md',
+ 'https://raw.githubusercontent.com/apify/actor-whitepaper/refs/heads/master/pages/ACTOR_FILE.md',
+ 'https://raw.githubusercontent.com/apify/actor-whitepaper/refs/heads/master/pages/DATASET_SCHEMA.md',
+ 'https://raw.githubusercontent.com/apify/actor-whitepaper/refs/heads/master/pages/IDEAS.md',
+ 'https://raw.githubusercontent.com/apify/actor-whitepaper/refs/heads/master/pages/INPUT_SCHEMA.md',
+ 'https://raw.githubusercontent.com/apify/actor-whitepaper/refs/heads/master/pages/KEY_VALUE_STORE_SCHEMA.md',
+ 'https://raw.githubusercontent.com/apify/actor-whitepaper/refs/heads/master/pages/OUTPUT_SCHEMA.md',
+ 'https://raw.githubusercontent.com/apify/actor-whitepaper/refs/heads/master/pages/REQUEST_QUEUE_SCHEMA.md',
+ ],
+};
+
+async function fetchFile(route) {
+ try {
+ const res = await fetch(route);
+ if (!res.ok) throw new Error(`Failed to fetch ${route}: ${res.status}`);
+ return await res.text();
+ } catch (err) {
+ console.error(`Error fetching ${route}:`, err.message);
+ return '';
+ }
+}
+
+async function joinFiles() {
+ await fs.mkdir(BUILD_DIR, { recursive: true });
+ for (const [llmsFile, files] of Object.entries(FILES_ROUTES)) {
+ const contents = await Promise.all(
+ files.map((route) => fetchFile(route)),
+ );
+ const joined = contents.filter(Boolean).join('\n\n');
+ await fs.appendFile(path.join(BUILD_DIR, llmsFile), joined, 'utf8');
+ console.log(`Wrote ${llmsFile} to build/`);
+ }
+}
+
+async function sanitizeFile(filePath) {
+ const content = await fs.readFile(filePath, 'utf8');
+ const sanitizedContent = content.replace(/<[^>]*>/g, ''); // Remove HTML tags
+ await fs.writeFile(filePath, sanitizedContent, 'utf8');
+ console.log(`Sanitized ${filePath}`);
+}
+
+joinFiles().catch((err) => {
+ console.error('Failed to join LLMs files:', err);
+ process.exit(1);
+});
+
+Object.keys(FILES_ROUTES).forEach((llmsFile) => sanitizeFile(path.join(BUILD_DIR, llmsFile)));