feat: add llms capture and export functionality for llmstxt standard

feat: enhance description handling and markdown export

fix: remove llms-capture plugin and update markdown processing

feat: llms capture and processing enhancements for md export

feat: enhance markdown processing

fix: docusaurus build

feat: type definitions for llms-export plugin

feat: normalized sidebar item types and markdown file generation

feat: implement URL conversion to .md format for llms links

refactor: streamline llms plugin code and enhance markdown processing

refactor: enhance type definitions and improve markdown processing in llms plugin

Author: uniyalabhishek

docusaurus.config.ts

@@ -5,8 +5,12 @@ import type * as Preset from '@docusaurus/preset-classic';
 import {editLinkUrl as editUrlFunction} from './src/editUrl';
 import crossRepoLinksPlugin from './src/remark/cross-repo-links';
 import {plugin as codeBlockSnippetsPlugin} from './src/remark/code-block-snippets';
+import llmsCapture from './src/remark/llms-capture';
+import llmsPlugin from './src/plugins/llms';
 import {redirectsOptions} from './redirects';
 
+const siteDir = __dirname;
+
 const config: Config = {
   title: 'Oasis Documentation',
   tagline: '',
@@ -46,6 +50,7 @@ const config: Config = {
                 converters:['pnpm','yarn'], // Package managers to use.
               },
             ],
+            [llmsCapture, {siteDir}],
           ],
           routeBasePath: '/',
           showLastUpdateTime: true,
@@ -89,6 +94,7 @@ const config: Config = {
         }
       };
     },
+    llmsPlugin,
     [
       '@docusaurus/plugin-client-redirects',
       redirectsOptions,

package.json

@@ -5,7 +5,7 @@
   "scripts": {
     "docusaurus": "docusaurus",
     "start": "docusaurus start --no-open",
-    "build": "docusaurus build",
+    "build": "docusaurus clear && docusaurus build",
     "mermaid:theme": "CSS_STRING=$(tr -s ' ' < mermaid-theme.css | tr -d '\n') && jq --arg themeCSS \"$CSS_STRING\" '.themeCSS = $themeCSS' mermaid.config.json > mermaid.config.json.temp && mv mermaid.config.json.temp mermaid.config.json",
     "diagrams": "yarn mermaid:theme && find -L docs -name '*.mmd' -exec mmdc -i {} -p puppeteer-config.json -c mermaid.config.json --backgroundColor transparent ';'",
     "swizzle": "docusaurus swizzle",

src/plugins/llms/capture.ts

@@ -0,0 +1,46 @@
+import * as crypto from 'crypto';
+import * as fs from 'fs';
+import * as path from 'path';
+
+import type {MdastNode} from './types';
+
+export const CAPTURE_TREE_VERSION = 1;
+
+function hash(value: string) {
+  return crypto.createHash('sha1').update(value).digest('hex');
+}
+
+export function captureDirForSite(siteDir: string) {
+  return path.join(siteDir, '.docusaurus', 'llms-export', 'trees');
+}
+
+export function capturedTreePathForSource(
+  sourceAbsPath: string,
+  siteDir: string,
+) {
+  let abs = sourceAbsPath;
+  try {
+    abs = fs.realpathSync(sourceAbsPath);
+  } catch {
+    abs = path.resolve(sourceAbsPath);
+  }
+  return path.join(captureDirForSite(siteDir), `${hash(abs)}.json`);
+}
+
+export function wrapCapturedTree(tree: unknown) {
+  return {version: CAPTURE_TREE_VERSION, tree};
+}
+
+export function unwrapCapturedTree(payload: unknown): MdastNode | null {
+  if (!payload || typeof payload !== 'object') return null;
+
+  const record = payload as Record<string, unknown>;
+  if ('version' in record && 'tree' in record) {
+    const version = Number(record.version);
+    if (version !== CAPTURE_TREE_VERSION) return null;
+    return (record.tree as MdastNode) ?? null;
+  }
+
+  // Legacy format support (version 1 only)
+  return CAPTURE_TREE_VERSION === 1 ? (payload as MdastNode) : null;
+}