Add versioning to llms.txt (#3495)

* create versioned llms.txt files

* Refactor LLM file generation for versioned docs

Reworks the script to loop through all documentation versions and generate LLM files for each, removing redundant code and simplifying file output logic. The latest version now generates both llms.txt and llms-full.txt, while older versions generate llms-{version}.txt. Cleans up unused variables and functions, and improves maintainability.

* Update generate-llm-files.ts

* Update generate-llm-files.ts

* run prettier

* Refactor llms.txt file generation logic

Replaces the buildLlmsTxtContent function with a more concise implementation using template constants. Extracts file writing logic into dedicated writeLlmsTxtFile and writeLlmsFullTxtFile functions for better modularity and clarity.

Author: vincanger

web/.gitignore

@@ -21,5 +21,4 @@ yarn-error.log*
 .vercel
 
 # LLM files
-llms.txt
-llms-full.txt
+llms*.txt

web/scripts/generate-llm-files.ts

@@ -7,29 +7,22 @@ import fs from "fs/promises";
 import { globSync } from "glob";
 import path from "path";
 
-import docsSidebarConfig from "../sidebars.js";
+import versionsJson from "../versions.json";
 
 const SITE_ROOT = process.cwd();
 const STATIC_DIR = path.join(SITE_ROOT, "static/");
-const DOCS_DIR = path.join(SITE_ROOT, "docs/");
+const VERSIONED_DOCS_DIR = path.join(SITE_ROOT, "versioned_docs/");
+const VERSIONED_SIDEBARS_DIR = path.join(SITE_ROOT, "versioned_sidebars/");
 const BLOG_DIR = path.join(SITE_ROOT, "blog/");
 const GITHUB_RAW_BASE_URL =
   "https://raw.githubusercontent.com/wasp-lang/wasp/refs/heads/release/web/"; // Use the release branch
 const WASP_BASE_URL = "https://wasp.sh/";
-const LLM_FULL_FILENAME = "llms-full.txt";
-const LLM_OVERVIEW_FILENAME = "llms.txt";
-const LLMS_TXT_FILE_PATH = path.join(STATIC_DIR, LLM_OVERVIEW_FILENAME);
-const LLMS_FULL_TXT_FILE_PATH = path.join(STATIC_DIR, LLM_FULL_FILENAME);
 const CATEGORIES_TO_IGNORE = ["Miscellaneous"];
 
-const OVERVIEW_INTRO_CONTENT = `
-# Wasp
-Wasp is a full-stack framework with batteries included for React, Node.js, and Prisma.
+const LLMS_TXT_INTRO = `# Wasp
+Wasp is a full-stack framework with batteries included for React, Node.js, and Prisma.`;
 
-## Individual documentation sections and guides:
-`;
-const OVERVIEW_MISC_SECTION_CONTENT = `
-## Miscellaneous
+const LLMS_TXT_MISC = `## Miscellaneous
 - [Wasp Developer Discord](https://discord.com/invite/rzdnErX)
 - [Open SaaS -- Wasp's free, open-source SaaS boilerplate starter](https://opensaas.sh)
 `;
@@ -38,31 +31,68 @@ generateFiles();
 
 /**
  * Main function to generate the LLM-friendly doc files.
- * It orchestrates the process by fetching and processing documentation and blog files,
- * and then writing the final output to the static directory.
+ * Loops through all versioned docs and generates llms.txt files for each.
+ * The latest version also gets llms-full.txt generated.
  */
 async function generateFiles() {
   console.log("Starting LLM file generation...");
 
-  if (!Array.isArray(docsSidebarConfig.docs)) {
-    throw new Error(
-      "Sidebar configuration for docs is not an array. Please check the sidebars.ts file.",
-    );
+  const blogSectionContent = await processBlogFiles();
+  const latestVersion = versionsJson[0];
+
+  for (const version of versionsJson) {
+    const isLatest = version === latestVersion;
+    console.log(`Processing version ${version}...`);
+
+    const docsDir = path.join(VERSIONED_DOCS_DIR, `version-${version}`);
+    const sidebarItems = await loadVersionedSidebar(version);
+
+    const { overviewDocsSection, llmsFullTxtContent } =
+      await processDocumentationFiles(sidebarItems, docsDir, version, {
+        generateLlmsFullTxt: isLatest,
+      });
+
+    const filename = isLatest ? "llms.txt" : `llms-${version}.txt`;
+    await writeLlmsTxtFile(filename, overviewDocsSection, blogSectionContent);
+
+    if (isLatest) {
+      await writeLlmsFullTxtFile(llmsFullTxtContent);
+    }
   }
 
-  const { overviewDocsSection, fullConcatContent } =
-    await processDocumentationFiles(docsSidebarConfig.docs);
-  const blogSectionContent = await processBlogFiles();
+  console.log("🎉 LLM files generation completed successfully.");
+}
 
-  const llmsTxtContent =
-    OVERVIEW_INTRO_CONTENT +
-    overviewDocsSection +
-    blogSectionContent +
-    OVERVIEW_MISC_SECTION_CONTENT;
-  const llmsFullTxtContent = fullConcatContent;
+async function writeLlmsTxtFile(
+  filename: string,
+  overviewDocsSection: string,
+  blogSectionContent: string,
+): Promise<void> {
+  const content = buildLlmsTxtContent(overviewDocsSection, blogSectionContent);
+  const outputPath = path.join(STATIC_DIR, filename);
+  await fs.writeFile(outputPath, content, "utf8");
+  console.log(`  Generated: ${filename}`);
+}
+
+async function writeLlmsFullTxtFile(content: string): Promise<void> {
+  const outputPath = path.join(STATIC_DIR, "llms-full.txt");
+  await fs.writeFile(outputPath, content.trim(), "utf8");
+  console.log(`  Generated: llms-full.txt`);
+}
 
-  await writeOutputFiles(llmsTxtContent, llmsFullTxtContent);
-  console.log("🎉 LLM file generation complete.");
+/**
+ * Assembles the overview content for llms.txt from its component sections.
+ */
+function buildLlmsTxtContent(
+  overviewDocsSection: string,
+  blogSectionContent: string,
+): string {
+  return [
+    LLMS_TXT_INTRO,
+    overviewDocsSection,
+    blogSectionContent,
+    LLMS_TXT_MISC,
+  ].join("\n\n");
 }
 
 /**
@@ -72,9 +102,12 @@ async function generateFiles() {
  */
 async function processDocumentationFiles(
   docsSidebarItems: SidebarItemConfig[],
-): Promise<{ overviewDocsSection: string; fullConcatContent: string }> {
-  let overviewDocsSection = "";
-  let fullConcatContent = "";
+  docsDir: string,
+  version: string,
+  { generateLlmsFullTxt = false } = {},
+): Promise<{ overviewDocsSection: string; llmsFullTxtContent: string }> {
+  let overviewDocsSection = `## Documentation Raw Text URLs -- Version ${version}:\n`;
+  let llmsFullTxtContent = "";
 
   const orderedDocIds = flattenSidebarItemsToDocIds(docsSidebarItems);
   console.log(
@@ -84,9 +117,13 @@ async function processDocumentationFiles(
   const sidebarOverviewStructure =
     getDocsSidebarCategoryStructure(docsSidebarItems);
 
-  const docIdToPathMap = buildDocIdToPathMap(DOCS_DIR);
+  const docIdToPathMap = buildDocIdToPathMap(docsDir);
 
-  const docInfoMap = await populateDocInfoMap(orderedDocIds, docIdToPathMap);
+  const docInfoMap = await populateDocInfoMap(
+    orderedDocIds,
+    docIdToPathMap,
+    docsDir,
+  );
 
   for (const category of sidebarOverviewStructure) {
     overviewDocsSection += `${category.categoryLabel}\n`;
@@ -104,23 +141,24 @@ async function processDocumentationFiles(
     }
   }
 
-  // Build fullConcatContent using sidebar structure with proper heading hierarchy
-  for (const category of sidebarOverviewStructure) {
-    // Add category header as H1 and separator
-    fullConcatContent += `# ${category.categoryLabel}\n\n`;
-
-    for (const docId of category.docIds) {
-      if (docInfoMap.has(docId)) {
-        const info = docInfoMap.get(docId);
-        // Add document title as H2
-        fullConcatContent += `## ${info.title}\n\n${info.processedBody}\n\n`;
+  if (generateLlmsFullTxt) {
+    for (const category of sidebarOverviewStructure) {
+      // Add category header as H1 and separator
+      llmsFullTxtContent += `# ${category.categoryLabel}\n\n`;
+
+      for (const docId of category.docIds) {
+        if (docInfoMap.has(docId)) {
+          const info = docInfoMap.get(docId);
+          // Add document title as H2
+          llmsFullTxtContent += `## ${info.title}\n\n${info.processedBody}\n\n`;
+        }
       }
-    }
 
-    // Add category separator
-    fullConcatContent += `------\n\n`;
+      // Add category separator
+      llmsFullTxtContent += `------\n\n`;
+    }
   }
-  return { overviewDocsSection, fullConcatContent };
+  return { overviewDocsSection, llmsFullTxtContent };
 }
 
 /**
@@ -162,13 +200,14 @@ type DocDetails = {
 async function populateDocInfoMap(
   orderedDocIds: string[],
   docIdToPathMap: Map<string, string>,
+  docsDir: string,
 ): Promise<Map<string, DocDetails>> {
   const docInfoMap = new Map<string, DocDetails>();
   for (const docId of orderedDocIds) {
     const relativeDocPath = docIdToPathMap.get(docId);
 
     if (relativeDocPath) {
-      const absolutePath = path.join(DOCS_DIR, relativeDocPath);
+      const absolutePath = path.join(docsDir, relativeDocPath);
       try {
         const rawContent = await fs.readFile(absolutePath, "utf8");
         const { attributes, body } = fm(rawContent);
@@ -412,23 +451,26 @@ function constructBlogUrl(filename: string): string {
 }
 
 /**
- * Writes the LLM-friendly content to their respective output files.
+ * Loads and parses a versioned sidebar configuration file.
+ * Returns the docs array from the sidebar config.
  */
-async function writeOutputFiles(
-  llmsTxtContent: string,
-  llmsFullTxtContent: string,
-): Promise<void> {
-  console.log("Writing output files to static/ ...");
+async function loadVersionedSidebar(
+  version: string,
+): Promise<SidebarItemConfig[]> {
+  const sidebarPath = path.join(
+    VERSIONED_SIDEBARS_DIR,
+    `version-${version}-sidebars.json`,
+  );
+  const sidebarContent = await fs.readFile(sidebarPath, "utf8");
+  const sidebarConfig = JSON.parse(sidebarContent);
 
-  await fs.writeFile(LLMS_TXT_FILE_PATH, llmsTxtContent.trim(), "utf8");
-  console.log(`Generated overview file: ${LLMS_TXT_FILE_PATH}`);
+  if (!Array.isArray(sidebarConfig.docs)) {
+    throw new Error(
+      `Versioned sidebar configuration for ${version} does not have a docs array.`,
+    );
+  }
 
-  await fs.writeFile(
-    LLMS_FULL_TXT_FILE_PATH,
-    llmsFullTxtContent.trim(),
-    "utf8",
-  );
-  console.log(`Generated full concatenated file: ${LLMS_FULL_TXT_FILE_PATH}`);
+  return sidebarConfig.docs;
 }
 
 /**