Merge pull request #245 from underctrl-io/llms.txt

fix: llms.txt generator

Author: twlite

apps/website/package.json

@@ -23,6 +23,7 @@
     "@docusaurus/theme-search-algolia": "^3.7.0",
     "@mdx-js/react": "^3.0.0",
     "clsx": "^2.0.0",
+    "gray-matter": "^4.0.3",
     "prism-react-renderer": "^2.3.0",
     "react": "^19.0.0",
     "react-dom": "^19.0.0"

apps/website/src/plugins/llms-txt.js

@@ -1,81 +1,75 @@
 const path = require('path');
 const fs = require('fs');
+const matter = require('gray-matter');
+
+const DOCS_URL = 'https://commandkit.dev/docs/next/guide';
 
 module.exports = function (context) {
   return {
     name: 'llms-txt-plugin',
     loadContent: async () => {
       const { siteDir } = context;
       const docsDir = path.join(siteDir, 'docs');
+      const guideDir = path.join(docsDir, 'guide');
       const versionedDocsDir = path.join(siteDir, 'versioned_docs');
-      const allMdx = [];
+      const allDocs = [];
 
-      // Recursive function to get all mdx and md files (excluding API reference)
       const getMdxFiles = async (dir, versionPrefix = '') => {
         if (!fs.existsSync(dir)) return;
 
+        // skip versioned docs
+        if (dir.includes(versionedDocsDir)) return;
+
         const entries = await fs.promises.readdir(dir, { withFileTypes: true });
 
         for (const entry of entries) {
           const fullPath = path.join(dir, entry.name);
           if (entry.isDirectory()) {
-            // Skip api-reference directories
-            if (entry.name === 'api-reference') {
-              continue;
-            }
             await getMdxFiles(fullPath, versionPrefix);
-          } else if (
-            entry.name.endsWith('.mdx') ||
-            entry.name.endsWith('.md')
-          ) {
-            // Skip files in api-reference paths
-            if (fullPath.includes('api-reference')) {
-              continue;
-            }
+          } else if (entry.name.endsWith('.mdx') || entry.name.endsWith('.md')) {
             const content = await fs.promises.readFile(fullPath, 'utf8');
-            allMdx.push({
-              content,
-              path: fullPath,
+            const { data: frontmatter } = matter(content);
+
+            // Get relative path from guide directory
+            const relativePath = path.relative(guideDir, fullPath)
+              .replace(/\.mdx?$/, '')
+              .replace(/\\/g, '/');
+
+            allDocs.push({
+              title: frontmatter.title,
+              description: frontmatter.description,
+              path: relativePath,
               version: versionPrefix,
             });
           }
         }
       };
 
-      // Get current/latest docs
-      await getMdxFiles(docsDir, 'latest');
-
-      // Get versioned docs
-      if (fs.existsSync(versionedDocsDir)) {
-        const versionDirs = await fs.promises.readdir(versionedDocsDir, {
-          withFileTypes: true,
-        });
-        for (const versionDir of versionDirs) {
-          if (versionDir.isDirectory()) {
-            const versionName = versionDir.name.replace('version-', '');
-            await getMdxFiles(
-              path.join(versionedDocsDir, versionDir.name),
-              versionName,
-            );
-          }
-        }
+      // Only process guide directory
+      if (fs.existsSync(guideDir)) {
+        await getMdxFiles(guideDir, 'latest');
       }
 
-      return { allMdx };
+      return { allDocs };
     },
     postBuild: async ({ content, outDir }) => {
-      const { allMdx } = content;
+      const { allDocs } = content;
 
-      // Filter out any remaining API reference content and concatenate
-      const filteredContent = allMdx
-        .filter((item) => !item.path.includes('api-reference'))
-        .map((item) => item.content)
-        .join('\n\n---\n\n');
+      // Generate markdown content
+      const markdownContent = [
+        '# CommandKit Docs\n',
+        ...allDocs
+          .filter(doc => doc.title && doc.description)
+          .map(doc => {
+            const url = `${DOCS_URL}/${doc.path}`;
+            return `- [${doc.title}](${url}): ${doc.description || doc.title}`;
+          })
+      ].join('\n');
 
-      // Write concatenated content as llms.txt
+      // Write markdown content
       const llmsTxtPath = path.join(outDir, 'llms.txt');
       try {
-        await fs.promises.writeFile(llmsTxtPath, filteredContent);
+        await fs.promises.writeFile(llmsTxtPath, markdownContent);
         console.log('✅ llms.txt generated successfully');
       } catch (err) {
         console.error('❌ Error generating llms.txt:', err);