llms.txt built in-project (#497)

* Generate LLMs.txt ourself.

* regenerate

* bring back github actions

Author: evantahler

.github/workflows/llmstxt.yml

@@ -2,7 +2,7 @@ name: Generate LLMs.txt
 
 on:
   schedule:
-    - cron: '0 0 * * 0'  # Run at 00:00 every Sunday
+    - cron: "0 0 * * 0" # Run at 00:00 every Sunday
   workflow_dispatch:
 
 permissions:
@@ -22,7 +22,7 @@ jobs:
       - name: Use Node.js
         uses: actions/setup-node@v4
         with:
-          node-version: '22.x'
+          node-version: "22.x"
 
       - name: Checkout code
         uses: actions/checkout@v4
@@ -38,7 +38,7 @@ jobs:
       - name: generate LLMs.txt
         run: pnpm llmstxt
         env:
-          FIRECRAWL_API_KEY: ${{ secrets.FIRECRAWL_API_KEY }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
 
       # commit the changes and make a PR (branch protection)
       - name: Create Pull Request
@@ -49,7 +49,7 @@ jobs:
           commit-message: Regenerate LLMs.txt and related files
           branch: auto-update-llms-txt
           delete-branch: true
-          title: '🤖 Regenerate LLMs.txt'
+          title: "🤖 Regenerate LLMs.txt"
           reviewers: >
             evantahler
             torresmateo

README.md

@@ -10,6 +10,26 @@ First, run `pnpm install` to install the dependencies.
 
 Then, run `pnpm dev` to start the development server and visit localhost:3000.
 
+### Environment Variables
+
+Copy `.env.example` to `.env.local` and fill in the required values:
+
+```bash
+cp .env.example .env.local
+```
+
+- `OPENAI_API_KEY` - Required for generating the llms.txt file with AI-powered summaries
+
+## llms.txt Generation
+
+The project includes a Next.js plugin that automatically generates an `llms.txt` file following the [llms.txt specification](https://llmstxt.org/). This file helps LLMs understand and navigate the documentation.
+
+**Automatic generation**: Runs during production builds (`pnpm build`)
+
+**Manual generation**: Run `pnpm llmstxt` to regenerate the file
+
+See [`scripts/README-llmstxt.md`](./scripts/README-llmstxt.md) for detailed documentation.
+
 ## Styling
 
 We use Nextra's built-in Callout component for callouts in MDX:

app/en/home/agentic-development/page.mdx

@@ -5,16 +5,13 @@ description: "Learn how to speed up your development with agents in your IDEs"
 
 # Agentic Development
 
-Give your AI IDE access to Arcade.dev's documentation using our llms.txt files (short [llms.txt](/llms.txt), [llms-full.txt](/llms-full.txt)), or use [context7](https://context7.com/arcadeai/docs). This allows Claude Code, Cursor, and other AI IDEs to access the documentation and help you write code.
+Give your AI IDE access to Arcade.dev's documentation using our [llms.txt](/llms.txt) file, or use [context7](https://context7.com/arcadeai/docs). This allows Claude Code, Cursor, and other AI IDEs to access the documentation and help you write code.
 
 ## LLMs.txt
 
 LLMs.txt is a file format that allows you to give your AI IDE access to Arcade.dev's documentation in a format that can be easily parsed by the LLM. All you need to do is paste in the conent of the file into your IDE's settings, or reference the docs, e.g. via [Cursor's `@docs` annotation](https://cursor.com/docs/context/symbols#docs).
 
-Our LLMs.txt files are available in two versions:
-
-- [`/llms.txt`](/llms.txt) - a short version of the documentation index
-- [`/llms-full.txt`](/llms-full.txt) - a full version of the documentation
+Our LLMs.txt files are available at [`/llms.txt`](/llms.txt).
 
 ![LLMs.txt example](/images/agentic-development/cursor-llms-txt.png)
 

lib/next-plugin-llmstxt.ts

@@ -0,0 +1,72 @@
+import type { NextConfig } from "next";
+
+type LlmsTxtPluginOptions = {
+  enabled?: boolean;
+  generateOnBuild?: boolean;
+};
+
+/**
+ * Next.js plugin that generates llms.txt at build time
+ *
+ * This plugin integrates with the Next.js build process to automatically
+ * generate an llms.txt file that follows the llms.txt specification.
+ *
+ * @param options - Plugin configuration options
+ * @returns Next.js config wrapper function
+ */
+export function withLlmsTxt(options: LlmsTxtPluginOptions = {}) {
+  const { enabled = true, generateOnBuild = true } = options;
+
+  return (nextConfig: NextConfig = {}): NextConfig => {
+    if (!enabled) {
+      return nextConfig;
+    }
+
+    return {
+      ...nextConfig,
+      webpack: (config, context) => {
+        // Only run during production build or when explicitly enabled
+        if (generateOnBuild && !context.isServer) {
+          // Add a custom plugin that runs after compilation
+          config.plugins = config.plugins || [];
+          config.plugins.push({
+            apply: (compiler: {
+              hooks: {
+                afterEmit: {
+                  tapPromise: (
+                    name: string,
+                    callback: () => Promise<void>
+                  ) => void;
+                };
+              };
+              options: { name: string };
+            }) => {
+              compiler.hooks.afterEmit.tapPromise("LlmsTxtPlugin", async () => {
+                // Only generate once (not on every rebuild)
+                if (compiler.options.name === "client") {
+                  try {
+                    // Dynamic import to avoid bundling issues
+                    const { generateLlmsTxt } = await import(
+                      "../scripts/generate-llmstxt.js"
+                    );
+
+                    await generateLlmsTxt();
+                  } catch (_error) {
+                    // Don't fail the build if llms.txt generation fails
+                  }
+                }
+              });
+            },
+          });
+        }
+
+        // Call the original webpack function if it exists
+        if (typeof nextConfig.webpack === "function") {
+          return nextConfig.webpack(config, context);
+        }
+
+        return config;
+      },
+    };
+  };
+}

next.config.ts

@@ -1,5 +1,6 @@
 import type { NextConfig } from "next";
 import nextra from "nextra";
+import { withLlmsTxt } from "./lib/next-plugin-llmstxt";
 import { remarkGlossary } from "./lib/remark-glossary";
 
 // Set up Nextra with its configuration
@@ -13,30 +14,34 @@ const withNextra = nextra({
   },
 });
 
-const nextConfig: NextConfig = withNextra({
-  headers: async () => [
-    {
-      source: "/(.*)",
-      headers: [
-        {
-          key: "X-Frame-Options",
-          value: "SAMEORIGIN",
-        },
-        {
-          key: "X-Content-Type-Options",
-          value: "nosniff",
-        },
-        {
-          key: "Permissions-Policy",
-          value: "camera=(), geolocation=(), microphone=()",
-        },
-        {
-          key: "Referrer-Policy",
-          value: "strict-origin-when-cross-origin",
-        },
-      ],
-    },
-  ],
-});
+const nextConfig: NextConfig = withLlmsTxt({
+  enabled: false, // disabled for now, we will recreate this every week
+})(
+  withNextra({
+    headers: async () => [
+      {
+        source: "/(.*)",
+        headers: [
+          {
+            key: "X-Frame-Options",
+            value: "SAMEORIGIN",
+          },
+          {
+            key: "X-Content-Type-Options",
+            value: "nosniff",
+          },
+          {
+            key: "Permissions-Policy",
+            value: "camera=(), geolocation=(), microphone=()",
+          },
+          {
+            key: "Referrer-Policy",
+            value: "strict-origin-when-cross-origin",
+          },
+        ],
+      },
+    ],
+  })
+);
 
 export default nextConfig;

package.json

@@ -13,7 +13,7 @@
     "postbuild": "pagefind --site .next/server/app --output-path public/_pagefind && next-sitemap",
     "translate": "pnpm dlx tsx scripts/i18n-sync/index.ts && pnpm format",
     "sync:metas": "pnpm dlx tsx scripts/sync-metas.ts app/en",
-    "llmstxt": "./scripts/generate-llmstxt.sh",
+    "llmstxt": "pnpm dlx tsx scripts/generate-llmstxt.ts",
     "test": "vitest --run",
     "test:watch": "vitest --watch"
   },