Docs LLM accessibility: add llms.txt, allow copying and viewing pages as markdown

docusaurus.config.ts

@@ -32,13 +32,13 @@ import { extendedPostcssConfigPlugin } from "./server/postcss";
 import { rehypeHLJS } from "./server/rehype-hljs";
 import { definer as hcl } from "highlightjs-terraform";
 import path from "path";
-import fs from "fs";
+import { llmsTxtPluginOptions } from "./server/llms";
 
 const latestVersion = getLatestVersion();
 
 function clayTrackingPlugin() {
   return {
-    name: 'clay-tracking',
+    name: "clay-tracking",
     injectHtmlTags() {
       return {
         postBodyTags: [
@@ -337,6 +337,7 @@ const config: Config = {
     extendedPostcssConfigPlugin,
     clayTrackingPlugin,
     process.env.NODE_ENV !== "production" && "@docusaurus/plugin-debug",
+    ["@signalwire/docusaurus-plugin-llms-txt", llmsTxtPluginOptions],
   ].filter(Boolean),
 };
 

package.json

@@ -59,6 +59,7 @@
     "@iconify-json/logos": "^1.2.10",
     "@iconify/tools": "^5.0.4",
     "@mdx-js/react": "^3.1.1",
+    "@signalwire/docusaurus-plugin-llms-txt": "2.0.0-alpha.7",
     "classnames": "^2.3.1",
     "clsx": "^2.1.1",
     "date-fns": "^4.1.0",
@@ -131,6 +132,10 @@
     "wait-on": "^9.0.4",
     "yaml": "^2.8.2"
   },
+  "resolutions": {
+    "vfile": "^6.0.0",
+    "vfile-message": "^4.0.0"
+  },
   "browserslist": {
     "production": [
       ">0.5%",

server/fixtures/llms/admonition.html

@@ -0,0 +1 @@
+<div class="theme-admonition"><div class="admonitionHeading_abc">note</div><div class="admonitionContent_abc"><p>This is a note.</p></div></div>

server/fixtures/llms/anchor-mixed-content-nested.html

@@ -0,0 +1 @@
+<a class="featureItem_abc" href="https://example.com"><div class="wrapper_xyz"><img src="https://example.com/image.png" alt="Feature image"></img><div class="inner_abc"><h4 class="featureTitle_xyz">Feature title</h4><p class="featureDescription_abc">Feature description</p></div></div></a>

server/fixtures/llms/anchor-mixed-content.html

@@ -0,0 +1 @@
+<a class="featureItem_abc" href="https://example.com"><svg class="iconSvg_abc"><path d="M1 1"></path></svg><h4 class="featureTitle_xyz">Feature title</h4><p class="featureDescription_abc">Feature description</p></a>

server/fixtures/llms/code-block-commands.html

@@ -0,0 +1 @@
+<div class="codeBlockContainer_abc"><div><pre class="code_xyz"><div class="scroll_abc"><div class="command_abc"><div class="line_pqr" data-content="$ ">tsh login --proxy=example.teleport.sh</div></div></div></pre></div></div>

server/fixtures/llms/code-block.html

@@ -0,0 +1 @@
+<div class="codeBlockContainer_abc"><div><pre class="code_xyz"><div class="scroll_abc"><div class="line_pqr">const x = 1;</div><div class="line_pqr">const y = 2;</div></div></pre></div></div>

server/fixtures/llms/empty-comments.html

@@ -0,0 +1 @@
+<p>Before comment.</p><!-- --><p>After comment.</p>

server/fixtures/llms/heading-anchor-links.html

@@ -0,0 +1 @@
+<h2 class="anchor anchorTargetStickyNavbar_abc123" id="how-it-works">How it works<a href="#how-it-works" class="hash-link">&ZeroWidthSpace;</a></h2><p>Some content</p>

server/fixtures/llms/heading-with-regular-link.html

@@ -0,0 +1 @@
+<h2>Check out <a href="https://example.com">this link</a></h2>

server/fixtures/llms/irrelevant-components.html

@@ -0,0 +1 @@
+<section class="docsHeader_abc"><div class="background_xyz"></div>Page header</section><p>Normal paragraph.</p><div class="thumbsFeedback_abc123">Did you find this page helpful?</div><div class="checkpoint_xyz">Step checkpoint</div><p>Another paragraph.</p>

server/fixtures/llms/reposition-h1-with-duplicate.html

@@ -0,0 +1 @@
+<header><h1 class="docItemTitle">Page Title</h1></header><p>Some content</p><h1>Duplicate Title</h1>

server/fixtures/llms/reposition-h1.html

@@ -0,0 +1 @@
+<header><h1 class="docItemTitle">Page title</h1><div class="pageActions_xyz"><button>Action 1</button><button>Action 2</button></div></header><p>Some content</p><p>More content</p>

server/fixtures/llms/result/admonition.html

@@ -0,0 +1 @@
+<hr><div class="theme-admonition"><div class="admonitionHeading_abc">NOTE</div><div class="admonitionContent_abc"><p>This is a note.</p></div></div><hr>

server/fixtures/llms/result/anchor-mixed-content-nested.html

@@ -0,0 +1 @@
+<a href="https://example.com"><h4 class="featureTitle_xyz">Feature title</h4></a><p class="featureDescription_abc">Feature description</p>

server/fixtures/llms/result/anchor-mixed-content.html

@@ -0,0 +1 @@
+<a href="https://example.com"><h4 class="featureTitle_xyz">Feature title</h4></a><p class="featureDescription_abc">Feature description</p>

server/fixtures/llms/result/code-block-commands.html

@@ -0,0 +1 @@
+<pre><code>$ tsh login --proxy=example.teleport.sh</code></pre>

server/fixtures/llms/result/code-block.html

@@ -0,0 +1,2 @@
+<pre><code>const x = 1;
+const y = 2;</code></pre>

server/fixtures/llms/result/empty-comments.html

@@ -0,0 +1 @@
+<p>Before comment.</p><p>After comment.</p>

server/fixtures/llms/result/heading-anchor-links.html

@@ -0,0 +1 @@
+<h2 class="anchor anchorTargetStickyNavbar_abc123" id="how-it-works">How it works</h2><p>Some content</p>

server/fixtures/llms/result/heading-with-regular-link.html

@@ -0,0 +1 @@
+<h2>Check out <a href="https://example.com">this link</a></h2>

server/fixtures/llms/result/irrelevant-components.html

@@ -0,0 +1 @@
+<p>Normal paragraph.</p><p>Another paragraph.</p>

server/fixtures/llms/result/reposition-h1-with-duplicate.html

@@ -0,0 +1 @@
+<h1 class="docItemTitle">Page Title</h1><p>Some content</p>

server/fixtures/llms/result/reposition-h1.html

@@ -0,0 +1 @@
+<h1 class="docItemTitle">Page title</h1><p>Some content</p><p>More content</p>

server/fixtures/llms/result/tabs.html

@@ -0,0 +1 @@
+<div class="theme-tabs-container"><strong>Tab 1</strong><div role="tabpanel" class="tabItem_xyz"><p>Tab 1 content</p></div><strong>Tab 2</strong><div role="tabpanel" class="tabItem_xyz"><p>Tab 2 content</p></div></div>

server/fixtures/llms/result/var-component.html

@@ -0,0 +1 @@
+<p>Connect to example.teleport.sh</p>

server/fixtures/llms/tabs.html

@@ -0,0 +1 @@
+<div class="theme-tabs-container"><ul role="tablist" class="tabs"><li role="tab" tabindex="0" class="tabs__item tabItem_abc tabs__item--active">Tab 1</li><li role="tab" tabindex="-1" class="tabs__item tabItem_abc">Tab 2</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_xyz"><p>Tab 1 content</p></div><div role="tabpanel" class="tabItem_xyz"><p>Tab 2 content</p></div></div></div>

server/fixtures/llms/var-component.html

@@ -0,0 +1 @@
+<p>Connect to <span class="wrapper-input wrapper_xyz"><input class="field_abc" type="text" size="19" placeholder="example.teleport.sh" name="example.teleport.sh" value=""><span class="fake-field_abc">example.teleport.sh</span></span></p>

server/llms/index.ts

@@ -0,0 +1,134 @@
+import type { Options as PluginOptions } from "@signalwire/docusaurus-plugin-llms-txt";
+import rehypeRemoveAnchorLinks from "./rehype-remove-heading-anchor-links";
+import rehypeRemoveIrrelevantComponents from "./rehype-remove-irrelevant-components";
+import rehypeRepositionH1 from "./rehype-reposition-h1";
+import rehypeProcessCustomComponentsForMarkdown from "./rehype-process-custom-components";
+
+// Define the sections for the llms.txt index file, including their routes and descriptions
+const sections = [
+  {
+    id: "get-started",
+    name: "Get Started",
+    description:
+      "Learn how to deploy a cluster, connect infrastructure, set up access controls, and review audit logs.",
+    routes: [{ route: "/docs/get-started/**" }],
+    position: 0,
+  },
+  {
+    id: "core-concepts",
+    name: "Core Concepts",
+    description: "Learn the key components that make up Teleport.",
+    routes: [{ route: "/docs/core-concepts/**" }],
+    position: 1,
+  },
+  {
+    id: "agentic-identity-framework",
+    name: "Agentic Identity Framework",
+    description:
+      "Design and reference implementation for the secure deployment of agents on infrastructure.",
+    routes: [{ route: "/docs/agentic-identity-framework/**" }],
+    position: 2,
+  },
+  {
+    id: "installation",
+    name: "Installation",
+    description:
+      "How to install Teleport and Teleport's client tools on your platform, including binaries and instructions for Docker and Helm.",
+    routes: [{ route: "/docs/installation/**" }],
+    position: 3,
+  },
+  {
+    id: "connect-your-client",
+    name: "Teleport User Guides",
+    description:
+      "Provides instructions to help users connect to infrastructure resources with Teleport.",
+    routes: [{ route: "/docs/connect-your-client/**" }],
+    position: 4,
+  },
+  {
+    id: "zero-trust-access",
+    name: "Teleport Zero Trust Access",
+    description:
+      "Easy access to all your infrastructure, on a foundation of cryptographic identity and zero trust.",
+    routes: [{ route: "/docs/zero-trust-access/**" }],
+    position: 5,
+  },
+  {
+    id: "machine-workload-identity",
+    name: "Machine & Workload Identity",
+    description:
+      "Use Teleport to replace long-lived secrets with identity-based authentication for your machines and workloads.",
+    routes: [{ route: "/docs/machine-workload-identity/**" }],
+    position: 6,
+  },
+  {
+    id: "identity-governance",
+    name: "Identity Governance",
+    description:
+      "Manage on-demand access, privileges, and compliance for all your infrastructure.",
+    routes: [{ route: "/docs/identity-governance/**" }],
+    position: 7,
+  },
+  {
+    id: "identity-security",
+    name: "Teleport Identity Security",
+    description:
+      "Teleport Identity Security centralizes access policy across your infrastructure, consolidates disparate identity audit logs, discovers shadow access, and alerts on access anomalies.",
+    routes: [{ route: "/docs/identity-security/**" }],
+    position: 8,
+  },
+  {
+    id: "enroll-resources",
+    name: "Enroll Resources",
+    description:
+      "Teleport protects infrastructure resources such as servers, databases, and Kubernetes clusters by enforcing strong access controls and auditability.",
+    routes: [{ route: "/docs/enroll-resources/**" }],
+    position: 9,
+  },
+  {
+    id: "reference-guides",
+    name: "Teleport Reference Guides",
+    description:
+      "Provides comprehensive information on configuration fields, Teleport commands, and other ways of interacting with Teleport.",
+    routes: [{ route: "/docs/reference/**" }],
+    position: 10,
+  },
+];
+
+export const llmsTxtPluginOptions: PluginOptions = {
+  // Top-level runtime options
+  logLevel: 1,
+  onRouteError: "throw",
+  onSectionError: "throw",
+  runOnPostBuild: true,
+
+  // Markdown file generation options
+  markdown: {
+    relativePaths: false, // Whether to use relative paths or absolute path URLs in the generated markdown files
+    beforeDefaultRehypePlugins: [
+      rehypeRemoveAnchorLinks,
+      rehypeRemoveIrrelevantComponents,
+      rehypeRepositionH1,
+      rehypeProcessCustomComponentsForMarkdown,
+    ], // Custom rehype plugins to clean up and reposition content for cleaner markdown output
+    remarkStringify: {
+      bullet: "-",
+      rule: "-",
+      ruleRepetition: 3,
+    },
+  },
+
+  // llms.txt index file options
+  llmsTxt: {
+    sections,
+    autoSectionPosition: 11,
+    includeDocs: true,
+    includeBlog: false,
+    includePages: false,
+    excludeRoutes: ["/docs/tags/**"],
+    enableDescriptions: true,
+    siteTitle: "Teleport documentation",
+    siteDescription:
+      "Teleport is an identity-based access platform that secures servers, Kubernetes clusters, databases, internal applications, and desktops using short-lived certificates, detailed audit logging, and fine-grained role-based access controls tied to your SSO provider (e.g., Okta, GitHub, Google Workspace).",
+  },
+};

server/llms/rehype-process-custom-components.test.ts

@@ -0,0 +1,122 @@
+import { describe, expect, test } from "@jest/globals";
+import { readFileSync } from "fs";
+import { resolve } from "path";
+import { unified } from "unified";
+import rehypeParse from "rehype-parse";
+import rehypeStringify from "rehype-stringify";
+import rehypeProcessCustomComponents from "./rehype-process-custom-components";
+
+describe("server/llms/rehype-process-custom-components", () => {
+  const transformer = (html: string) =>
+    unified()
+      .use(rehypeParse, { fragment: true })
+      .use(rehypeProcessCustomComponents)
+      .use(rehypeStringify)
+      .processSync(html);
+
+  test("Var: replaces wrapper-input span with the input placeholder text", () => {
+    const result = transformer(
+      readFileSync(
+        resolve("server/fixtures/llms/var-component.html"),
+        "utf-8",
+      ),
+    );
+
+    expect((result.value as string).trim()).toBe(
+      readFileSync(
+        resolve("server/fixtures/llms/result/var-component.html"),
+        "utf-8",
+      ).trim(),
+    );
+  });
+
+  test("Admonition: wraps block in hr separators and uppercases the type title", () => {
+    const result = transformer(
+      readFileSync(
+        resolve("server/fixtures/llms/admonition.html"),
+        "utf-8",
+      ),
+    );
+
+    expect((result.value as string).trim()).toBe(
+      readFileSync(
+        resolve("server/fixtures/llms/result/admonition.html"),
+        "utf-8",
+      ).trim(),
+    );
+  });
+
+  test("Tabs: restructures so each tab label is directly followed by its content", () => {
+    const result = transformer(
+      readFileSync(resolve("server/fixtures/llms/tabs.html"), "utf-8"),
+    );
+
+    expect((result.value as string).trim()).toBe(
+      readFileSync(
+        resolve("server/fixtures/llms/result/tabs.html"),
+        "utf-8",
+      ).trim(),
+    );
+  });
+
+  test("Code block: extracts line text into a clean pre/code structure", () => {
+    const result = transformer(
+      readFileSync(resolve("server/fixtures/llms/code-block.html"), "utf-8"),
+    );
+
+    expect((result.value as string).trim()).toBe(
+      readFileSync(
+        resolve("server/fixtures/llms/result/code-block.html"),
+        "utf-8",
+      ).trim(),
+    );
+  });
+
+  test("Code block: prepends data-content prefix to command lines", () => {
+    const result = transformer(
+      readFileSync(
+        resolve("server/fixtures/llms/code-block-commands.html"),
+        "utf-8",
+      ),
+    );
+
+    expect((result.value as string).trim()).toBe(
+      readFileSync(
+        resolve("server/fixtures/llms/result/code-block-commands.html"),
+        "utf-8",
+      ).trim(),
+    );
+  });
+
+  test("Link with inner content: only headings keep the link, other elements are unwrapped", () => {
+    const result = transformer(
+      readFileSync(
+        resolve("server/fixtures/llms/anchor-mixed-content.html"),
+        "utf-8",
+      ),
+    );
+
+    expect((result.value as string).trim()).toBe(
+      readFileSync(
+        resolve("server/fixtures/llms/result/anchor-mixed-content.html"),
+        "utf-8",
+      ).trim(),
+    );
+  });
+
+  test("Link with deeply nested content: traverses into wrapper divs, headings keep the link and other elements are unwrapped", () => {
+    const result = transformer(
+      readFileSync(
+        resolve("server/fixtures/llms/anchor-mixed-content-nested.html"),
+        "utf-8",
+      ),
+    );
+
+    expect((result.value as string).trim()).toBe(
+      readFileSync(
+        resolve("server/fixtures/llms/result/anchor-mixed-content-nested.html"),
+        "utf-8",
+      ).trim(),
+    );
+  });
+});