feat(ai): Add .md extension to provide pages in markdown for LLMs

.gitignore

@@ -4,6 +4,9 @@ npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
 
+# Ignore generated export markdown files 
+/public/md-exports/
+
 # Runtime data
 pids
 *.pid

next.config.ts

@@ -1,7 +1,7 @@
 import {codecovNextJSWebpackPlugin} from '@codecov/nextjs-webpack-plugin';
 import {withSentryConfig} from '@sentry/nextjs';
 
-import {redirects} from './redirects';
+import {redirects} from './redirects.js';
 
 const outputFileTracingExcludes = process.env.NEXT_PUBLIC_DEVELOPER_DOCS
   ? {
@@ -24,6 +24,7 @@ const outputFileTracingExcludes = process.env.NEXT_PUBLIC_DEVELOPER_DOCS
     };
 
 if (
+  !process.env.NODE_ENV &&
   process.env.NODE_ENV !== 'development' &&
   (!process.env.NEXT_PUBLIC_SENTRY_DSN || !process.env.SENTRY_DSN)
 ) {
@@ -55,6 +56,12 @@ const nextConfig = {
     DEVELOPER_DOCS_: process.env.NEXT_PUBLIC_DEVELOPER_DOCS,
   },
   redirects,
+  rewrites: async () => [
+    {
+      source: '/:path*.md',
+      destination: '/md-exports/:path*.md',
+    },
+  ],
   sassOptions: {
     silenceDeprecations: ['legacy-js-api'],
   },

package.json

@@ -18,7 +18,8 @@
     "dev": "yarn enforce-redirects && concurrently \"yarn sidecar\" \"node ./src/hotReloadWatcher.mjs\" \"next dev\"",
     "dev:developer-docs": "yarn enforce-redirects && NEXT_PUBLIC_DEVELOPER_DOCS=1 yarn dev",
     "build:developer-docs": "yarn enforce-redirects && git submodule init && git submodule update && NEXT_PUBLIC_DEVELOPER_DOCS=1 yarn build",
-    "build": "yarn enforce-redirects && next build",
+    "build": "yarn enforce-redirects && next build && yarn generate-md-exports",
+    "generate-md-exports": "node scripts/generate-md-exports.mjs",
     "vercel:build:developer-docs": "yarn enforce-redirects && git submodule init && git submodule update && NEXT_PUBLIC_DEVELOPER_DOCS=1 yarn build",
     "start": "next start",
     "lint": "next lint",
@@ -61,6 +62,7 @@
     "framer-motion": "^10.12.16",
     "github-slugger": "^2.0.0",
     "gray-matter": "^4.0.3",
+    "hast-util-select": "^6.0.4",
     "hast-util-to-jsx-runtime": "^2.3.2",
     "hastscript": "^8.0.0",
     "image-size": "^1.2.1",
@@ -85,21 +87,26 @@
     "react-select": "^5.7.3",
     "react-textarea-autosize": "^8.5.3",
     "rehype-autolink-headings": "^7.1.0",
+    "rehype-parse": "^9.0.1",
     "rehype-preset-minify": "^7.0.0",
     "rehype-prism-diff": "^1.1.2",
     "rehype-prism-plus": "^1.6.3",
+    "rehype-remark": "^10.0.1",
     "rehype-stringify": "^10.0.0",
-    "remark-gfm": "^4.0.0",
+    "remark-gfm": "^4.0.1",
     "remark-mdx-images": "^3.0.0",
     "remark-parse": "^11.0.0",
     "remark-prism": "^1.3.6",
+    "remark-stringify": "^11.0.0",
     "rss": "^1.2.2",
     "sass": "^1.69.5",
     "search-insights": "^2.17.2",
     "server-only": "^0.0.1",
     "sharp": "^0.33.4",
     "tailwindcss-scoped-preflight": "^3.0.4",
-    "textarea-markdown-editor": "^1.0.4"
+    "textarea-markdown-editor": "^1.0.4",
+    "unified": "^11.0.5",
+    "unist-util-remove": "^4.0.0"
   },
   "devDependencies": {
     "@babel/preset-typescript": "^7.15.0",

scripts/generate-md-exports.mjs

@@ -0,0 +1,153 @@
+#!/usr/bin/env node
+
+import {fileURLToPath} from 'url';
+
+import {selectAll} from 'hast-util-select';
+import {existsSync} from 'node:fs';
+import {mkdir, opendir, readFile, rm, writeFile} from 'node:fs/promises';
+import {cpus} from 'node:os';
+import * as path from 'node:path';
+import {isMainThread, parentPort, Worker, workerData} from 'node:worker_threads';
+import rehypeParse from 'rehype-parse';
+import rehypeRemark from 'rehype-remark';
+import remarkGfm from 'remark-gfm';
+import remarkStringify from 'remark-stringify';
+import {unified} from 'unified';
+import {remove} from 'unist-util-remove';
+
+async function createWork() {
+  let root = process.cwd();
+  while (!existsSync(path.join(root, 'package.json'))) {
+    const parent = path.dirname(root);
+    if (parent === root) {
+      throw new Error('Could not find package.json in parent directories');
+    }
+    root = parent;
+  }
+  const INPUT_DIR = path.join(root, '.next', 'server', 'app');
+  const OUTPUT_DIR = path.join(root, 'public', 'md-exports');
+
+  console.log(`🚀 Starting markdown generation from: ${INPUT_DIR}`);
+  console.log(`📁 Output directory: ${OUTPUT_DIR}`);
+
+  // Clear output directory
+  await rm(OUTPUT_DIR, {recursive: true, force: true});
+  await mkdir(OUTPUT_DIR, {recursive: true});
+
+  // On a 16-core machine, 8 workers were optimal (and slightly faster than 16)
+  // Putting 4 as the minimum as Vercel has 4 cores per builder and it may help
+  // us cut down some of the time there.
+  // Source: https://vercel.com/docs/limits#build-container-resources
+  const numWorkers = Math.max(Math.floor(cpus().length / 2), 2);
+  const workerTasks = new Array(numWorkers).fill(null).map(() => []);
+
+  console.log(`🔎 Discovering files to convert...`);
+
+  let numFiles = 0;
+  let workerIdx = 0;
+  // Need a high buffer size here otherwise Node skips some subdirectories!
+  // See https://github.com/nodejs/node/issues/48820
+  const dir = await opendir(INPUT_DIR, {recursive: true, bufferSize: 1024});
+  for await (const dirent of dir) {
+    if (dirent.name.endsWith('.html') && dirent.isFile()) {
+      const sourcePath = path.join(dirent.parentPath || dirent.path, dirent.name);
+      const targetDir = path.join(
+        OUTPUT_DIR,
+        path.relative(INPUT_DIR, dirent.parentPath || dirent.path)
+      );
+      await mkdir(targetDir, {recursive: true});
+      const targetPath = path.join(targetDir, dirent.name.slice(0, -5) + '.md');
+      workerTasks[workerIdx].push({sourcePath, targetPath});
+      workerIdx = (workerIdx + 1) % numWorkers;
+      numFiles++;
+    }
+  }
+
+  console.log(`📄 Converting ${numFiles} files with ${numWorkers} workers...`);
+
+  const selfPath = fileURLToPath(import.meta.url);
+  const workerPromises = new Array(numWorkers - 1).fill(null).map((_, idx) => {
+    return new Promise((resolve, reject) => {
+      const worker = new Worker(selfPath, {workerData: workerTasks[idx]});
+      let hasErrors = false;
+      worker.on('message', data => {
+        if (data.failedTasks.length === 0) {
+          console.log(`✅ Worker[${idx}]: ${data.success} files successfully.`);
+        } else {
+          hasErrors = true;
+          console.error(`❌ Worker[${idx}]: ${data.failedTasks.length} files failed:`);
+          console.error(data.failedTasks);
+        }
+      });
+      worker.on('error', reject);
+      worker.on('exit', code => {
+        if (code !== 0) {
+          reject(new Error(`Worker[${idx}] stopped with exit code ${code}`));
+        } else {
+          hasErrors ? reject(new Error(`Worker[${idx}] had some errors.`)) : resolve();
+        }
+      });
+    });
+  });
+  // The main thread can also process tasks -- That's 65% more bullet per bullet! -Cave Johnson
+  workerPromises.push(processTaskList(workerTasks[workerTasks.length - 1]));
+
+  await Promise.all(workerPromises);
+
+  console.log(`📄 Generated ${numFiles} markdown files from HTML.`);
+  console.log('✅ Markdown export generation complete!');
+}
+
+async function genMDFromHTML(source, target) {
+  const text = await readFile(source, {encoding: 'utf8'});
+  await writeFile(
+    target,
+    String(
+      await unified()
+        .use(rehypeParse)
+        // Need the `main div > hgroup` selector for the headers
+        .use(() => tree => selectAll('main div > hgroup, div#main', tree))
+        // If we don't do this wrapping, rehypeRemark just returns an empty string -- yeah WTF?
+        .use(() => tree => ({
+          type: 'element',
+          tagName: 'div',
+          properties: {},
+          children: tree,
+        }))
+        .use(rehypeRemark, {
+          document: false,
+          handlers: {
+            // Remove buttons as they usually get confusing in markdown, especially since we use them as tab headers
+            button() {},
+          },
+        })
+        // We end up with empty inline code blocks, probably from some tab logic in the HTML, remove them
+        .use(() => tree => remove(tree, {type: 'inlineCode', value: ''}))
+        .use(remarkGfm)
+        .use(remarkStringify)
+        .process(text)
+    )
+  );
+}
+
+async function processTaskList(tasks) {
+  const failedTasks = [];
+  for (const {sourcePath, targetPath} of tasks) {
+    try {
+      await genMDFromHTML(sourcePath, targetPath);
+    } catch (error) {
+      failedTasks.push({sourcePath, targetPath, error});
+    }
+  }
+  return {success: tasks.length - failedTasks.length, failedTasks};
+}
+
+async function doWork(tasks) {
+  parentPort.postMessage(await processTaskList(tasks));
+}
+
+if (isMainThread) {
+  await createWork();
+} else {
+  await doWork(workerData);
+}

src/components/apiExamples/apiExamples.tsx

@@ -129,7 +129,11 @@ export function ApiExamples({api}: Props) {
           </button>
         </div>
         <pre className={`${styles['api-block-example']} relative`}>
-          <div className={codeBlockStyles.copied} style={{opacity: showCopied ? 1 : 0}}>
+          <div
+            data-mdast="ignore"
+            className={codeBlockStyles.copied}
+            style={{opacity: showCopied ? 1 : 0}}
+          >
             Copied
           </div>
           {selectedTabView === 0 &&

src/components/codeBlock/index.tsx

@@ -55,7 +55,11 @@ export function CodeBlock({filename, language, children}: CodeBlockProps) {
           </button>
         )}
       </div>
-      <div className={styles.copied} style={{opacity: showCopied ? 1 : 0}}>
+      <div
+        data-mdast="ignore"
+        className={styles.copied}
+        style={{opacity: showCopied ? 1 : 0}}
+      >
         Copied
       </div>
       <div ref={codeRef}>

src/mdx.ts

@@ -429,9 +429,9 @@ export async function getFileBySlug(slug: string) {
             ],
           },
         ],
-        [rehypePrismPlus, {ignoreMissing: true}],
+        [rehypePrismPlus, {ignoreMissing: true}] as any,
         rehypeOnboardingLines,
-        [rehypePrismDiff, {remove: true}],
+        [rehypePrismDiff, {remove: true}] as any,
         rehypePresetMinify,
       ];
       return options;

src/middleware.ts

@@ -19,6 +19,7 @@ export const config = {
 
 // This function can be marked `async` if using `await` inside
 export function middleware(request: NextRequest) {
+  // Remove the llms.txt handling - it's now handled by Next.js redirects
   return handleRedirects(request);
 }
 
@@ -2346,7 +2347,7 @@ const USER_DOCS_REDIRECTS: Redirect[] = [
     to: '/security-legal-pii/scrubbing/',
   },
   {
-    from: '/data-management/advanced-datascrubbing/',
+    from: '/data-management-settings/advanced-datascrubbing/',
     to: '/security-legal-pii/scrubbing/advanced-datascrubbing/',
   },
   {