[WIP] mdx generator

Author: avivkeller

package-lock.json

@@ -47,6 +47,8 @@
     "hast-util-to-string": "^3.0.1",
     "hastscript": "^9.0.1",
     "html-minifier-terser": "^7.2.0",
+    "mdast-util-to-markdown": "^2.1.2",
+    "mdast-util-mdx-jsx": "^3.2.0",
     "rehype-stringify": "^10.0.1",
     "remark-gfm": "^4.0.1",
     "remark-parse": "^11.0.0",

package.json

@@ -11,6 +11,7 @@ import apiLinks from './api-links/index.mjs';
 import oramaDb from './orama-db/index.mjs';
 import astJs from './ast-js/index.mjs';
 import llmsTxt from './llms-txt/index.mjs';
+import mdx from './mdx/index.mjs';
 
 export const publicGenerators = {
   'json-simple': jsonSimple,
@@ -23,6 +24,7 @@ export const publicGenerators = {
   'api-links': apiLinks,
   'orama-db': oramaDb,
   'llms-txt': llmsTxt,
+  mdx: mdx,
 };
 
 export const allGenerators = {

src/generators/index.mjs

@@ -209,7 +209,7 @@ export default (headNodes, metadataEntries, remark) => {
     // Parses the metadata pieces of each node and the content
     metadataEntries.map(entry => {
       // Deep clones the content nodes to avoid affecting upstream nodes
-      const content = JSON.parse(JSON.stringify(entry.content));
+      const content = structuredClone(entry.content);
 
       // Parses the Heading nodes into Heading elements
       visit(content, createQueries.UNIST.isHeading, buildHeading);

src/generators/legacy-html/utils/buildContent.mjs

@@ -0,0 +1,6 @@
+export const STABILITY_LEVELS = [
+  'danger', // Deprecated
+  'warning', // Experimental
+  'success', // Stable
+  'info', //
+];

src/generators/mdx/constants.mjs

@@ -0,0 +1,41 @@
+import { groupNodesByModule } from '../../utils/generators.mjs';
+import buildContent from './utils/buildContent.mjs';
+import { toMarkdown } from 'mdast-util-to-markdown';
+import { mdxJsxToMarkdown } from 'mdast-util-mdx-jsx';
+
+/**
+ * This generator generates an MDX AST from an input MDAST
+ *
+ * @typedef {Array<ApiDocMetadataEntry>} Input
+ *
+ * @type {GeneratorMetadata<Input, string>}
+ */
+export default {
+  name: 'llms-txt',
+  version: '1.0.0',
+  description: 'Generates a MDX file from the input AST',
+  dependsOn: 'ast',
+
+  /**
+   * Generates a MDX AST
+   *
+   * @param {Input} entries
+   * @returns {Promise<string[]>} Array of generated content
+   */
+  async generate(entries) {
+    // Pre-process all data once
+    const headNodes = entries
+      .filter(node => node.heading.depth === 1)
+      .sort((a, b) => a.heading.data.name.localeCompare(b.heading.data.name));
+
+    const modules = groupNodesByModule(entries);
+
+    // Process nodes in parallel for better performance
+    const results = await Promise.all(
+      headNodes.map(node => buildContent(headNodes, modules.get(node.api)))
+    );
+
+    console.log(toMarkdown(results[0], { extensions: [mdxJsxToMarkdown()] }));
+    return results;
+  },
+};

src/generators/mdx/index.mjs

@@ -0,0 +1,52 @@
+'use strict';
+
+import { u as createTree } from 'unist-builder';
+import { SKIP, visit } from 'unist-util-visit';
+import createQueries from '../../../utils/queries/index.mjs';
+import { createJSXElement } from './mdxUtilities.mjs';
+import { STABILITY_LEVELS } from '../constants.mjs';
+
+/**
+ *
+ * @param {import('@types/mdast').Blockquote} node The HTML AST tree of the Stability Index content
+ * @param {number} index The index of the current node
+ * @param {import('unist').Parent} parent The parent node of the current node
+ */
+const visitStabilityNode = (node, index, parent) => {
+  parent.children.splice(
+    index,
+    1,
+    createJSXElement('AlertBox', {
+      children: node.data.description,
+      level: STABILITY_LEVELS[node.data.index],
+      title: node.data.index,
+    })
+  );
+  return [SKIP];
+};
+
+/**
+ * Process a single content entry by applying all transformations
+ * @param {import('unist').Node} content The content to process
+ * @returns {import('unist').Node} The processed content
+ */
+const processContent = content => {
+  const clonedContent = structuredClone(content);
+  visit(clonedContent, createQueries.UNIST.isStabilityNode, visitStabilityNode);
+  return clonedContent;
+};
+
+/**
+ * Transforms API metadata entries into markdown content with special handling for stability nodes
+ * @param {any} _ Unused parameter
+ * @param {Array<ApiDocMetadataEntry>} metadataEntries Entries to transform
+ * @returns {import('unist').Node} Root node with processed content
+ */
+export default (_, metadataEntries) => {
+  const rootNode = createTree(
+    'root',
+    metadataEntries.map(entry => processContent(entry.content))
+  );
+
+  return rootNode;
+};

src/generators/mdx/utils/buildContent.mjs

@@ -0,0 +1,34 @@
+'use strict';
+
+import { u as createTree } from 'unist-builder';
+
+/**
+ * Creates an MDX JSX element.
+ *
+ * @param {string} name - The name of the JSX element
+ * @param {{
+ * inline?: boolean,
+ * children?: string | import('unist').Node[],
+ * [key: string]: string
+ * }} [options={}] - Options including type, children, and JSX attributes
+ * @returns {import('unist').Node} The created MDX JSX element node
+ */
+export const createJSXElement = (
+  name,
+  { inline = false, children = [], ...attributes } = {}
+) => {
+  const processedChildren =
+    typeof children === 'string'
+      ? [createTree('text', { value: children })]
+      : (children ?? []);
+
+  const attrs = Object.entries(attributes).map(([key, value]) =>
+    createTree('mdxJsxAttribute', { name: key, value: String(value) })
+  );
+
+  return createTree(inline ? 'mdxJsxTextElement' : 'mdxJsxFlowElement', {
+    name,
+    attributes: attrs,
+    children: processedChildren,
+  });
+};