refactor: create metadata generator

araujogui · araujogui · commit 2f1cb757fb02 · 2025-05-28T11:23:12.000-03:00
diff --git a/bin/commands/generate.mjs b/bin/commands/generate.mjs
@@ -135,17 +135,24 @@ export default {
       process.exit(1);
     }
 
-    const { runGenerators } = createGenerator(docs);
     const { getAllMajors } = createNodeReleases(opts.changelog);
 
-    await runGenerators({
-      generators: opts.target,
-      input: opts.input,
-      output: opts.output && resolve(opts.output),
-      version: coerce(opts.version),
-      releases: await getAllMajors(),
-      gitRef: opts.gitRef,
-      threads: parseInt(opts.threads, 10),
-    });
+    const releases = await getAllMajors();
+
+    await Promise.all(
+      docs.map(async doc => {
+        const { runGenerators } = createGenerator(doc);
+
+        await runGenerators({
+          generators: opts.target,
+          input: opts.input,
+          output: opts.output && resolve(opts.output),
+          version: coerce(opts.version),
+          releases,
+          gitRef: opts.gitRef,
+          threads: parseInt(opts.threads, 10),
+        });
+      })
+    );
   },
 };
diff --git a/bin/utils.mjs b/bin/utils.mjs
@@ -24,7 +24,7 @@ const parser = lazy(createMarkdownParser);
  * @param {string[]} input - Glob patterns for input files.
  * @param {string[]} [ignore] - Glob patterns to ignore.
  * @param {import('../src/linter/types').Linter} [linter] - Linter instance
- * @returns {Promise<ApiDocMetadataEntry[]>} - Parsed documentation objects.
+ * @returns {Promise<ParserOutput<import('mdast').Root>[]>}
  */
 export async function loadAndParse(input, ignore, linter) {
   const files = await loader().loadFiles(input, ignore);
diff --git a/src/generators.mjs b/src/generators.mjs
@@ -4,11 +4,6 @@ import { allGenerators } from './generators/index.mjs';
 import WorkerPool from './threading/index.mjs';
 
 /**
- * @typedef {{ ast: GeneratorMetadata<ApiDocMetadataEntry, ApiDocMetadataEntry>}} AstGenerator The AST "generator" is a facade for the AST tree and it isn't really a generator
- * @typedef {AvailableGenerators & AstGenerator} AllGenerators A complete set of the available generators, including the AST one
- * @param markdownInput
- * @param jsInput
- *
  * This method creates a system that allows you to register generators
  * and then execute them in a specific order, keeping track of the
  * generation process, and handling errors that may occur from the
@@ -21,18 +16,20 @@ import WorkerPool from './threading/index.mjs';
  * Generators can also write to files. These would usually be considered
  * the final generators in the chain.
  *
- * @param {ApiDocMetadataEntry} markdownInput The parsed API doc metadata entries
- * @param {Array<import('acorn').Program>} parsedJsFiles
+ * @typedef {{ ast: GeneratorMetadata<ParserOutput, ParserOutput>}} AstGenerator The AST "generator" is a facade for the AST tree and it isn't really a generator
+ * @typedef {AvailableGenerators & AstGenerator} AllGenerators A complete set of the available generators, including the AST one
+ *
+ * @param {ParserOutput} input The API doc AST tree
  */
-const createGenerator = markdownInput => {
+const createGenerator = input => {
   /**
    * We store all the registered generators to be processed
    * within a Record, so we can access their results at any time whenever needed
    * (we store the Promises of the generator outputs)
    *
    * @type {{ [K in keyof AllGenerators]: ReturnType<AllGenerators[K]['generate']> }}
    */
-  const cachedGenerators = { ast: Promise.resolve(markdownInput) };
+  const cachedGenerators = { ast: Promise.resolve(input) };
 
   const threadPool = new WorkerPool();
 
diff --git a/src/generators/addon-verify/index.mjs b/src/generators/addon-verify/index.mjs
@@ -30,7 +30,7 @@ export default {
   description:
     'Generates a file list from code blocks extracted from `doc/api/addons.md` to facilitate C++ compilation and JavaScript runtime validations',
 
-  dependsOn: 'ast',
+  dependsOn: 'metadata',
 
   /**
    * Generates a file list from code blocks.
diff --git a/src/generators/ast-js/index.mjs b/src/generators/ast-js/index.mjs
@@ -20,7 +20,7 @@ export default {
 
   description: 'Parses Javascript source files passed into the input.',
 
-  dependsOn: 'ast',
+  dependsOn: 'metadata',
 
   /**
    * @param {Input} _
diff --git a/src/generators/index.mjs b/src/generators/index.mjs
@@ -11,8 +11,10 @@ import apiLinks from './api-links/index.mjs';
 import oramaDb from './orama-db/index.mjs';
 import astJs from './ast-js/index.mjs';
 import llmsTxt from './llms-txt/index.mjs';
+import metadata from './metadata/index.mjs';
 
 export const publicGenerators = {
+  metadata: metadata,
   'json-simple': jsonSimple,
   'legacy-html': legacyHtml,
   'legacy-html-all': legacyHtmlAll,
diff --git a/src/generators/json-simple/index.mjs b/src/generators/json-simple/index.mjs
@@ -26,7 +26,7 @@ export default {
   description:
     'Generates the simple JSON version of the API docs, and returns it as a string',
 
-  dependsOn: 'ast',
+  dependsOn: 'metadata',
 
   /**
    * Generates the simplified JSON version of the API docs
diff --git a/src/generators/legacy-html/index.mjs b/src/generators/legacy-html/index.mjs
@@ -41,7 +41,7 @@ export default {
   description:
     'Generates the legacy version of the API docs in HTML, with the assets and styles included as files',
 
-  dependsOn: 'ast',
+  dependsOn: 'metadata',
 
   /**
    * Generates the legacy version of the API docs in HTML
diff --git a/src/generators/legacy-json/index.mjs b/src/generators/legacy-json/index.mjs
@@ -25,7 +25,7 @@ export default {
 
   description: 'Generates the legacy version of the JSON API docs.',
 
-  dependsOn: 'ast',
+  dependsOn: 'metadata',
 
   /**
    * Generates a legacy JSON file.
diff --git a/src/generators/llms-txt/index.mjs b/src/generators/llms-txt/index.mjs
@@ -19,7 +19,7 @@ export default {
   description:
     'Generates a llms.txt file to provide information to LLMs at inference time',
 
-  dependsOn: 'ast',
+  dependsOn: 'metadata',
 
   /**
    * Generates a llms.txt file
diff --git a/src/generators/man-page/index.mjs b/src/generators/man-page/index.mjs
@@ -25,7 +25,7 @@ export default {
 
   description: 'Generates the Node.js man-page.',
 
-  dependsOn: 'ast',
+  dependsOn: 'metadata',
 
   /**
    * Generates the Node.js man-page
diff --git a/src/generators/metadata/index.mjs b/src/generators/metadata/index.mjs
@@ -0,0 +1,167 @@
+'use strict';
+
+import { u as createTree } from 'unist-builder';
+import { findAfter } from 'unist-util-find-after';
+import { remove } from 'unist-util-remove';
+import { selectAll } from 'unist-util-select';
+import { SKIP, visit } from 'unist-util-visit';
+
+import { createNodeSlugger } from '../../utils/slugger/index.mjs';
+import { getRemark } from '../../utils/remark.mjs';
+import createQueries from '../../utils/queries/index.mjs';
+import createMetadata from '../../metadata.mjs';
+
+/**
+ * This generator generates a flattened list of metadata entries from a API doc
+ *
+ * @typedef {ParserOutput<import('mdast').Root>} Input
+ *
+ * @type {GeneratorMetadata<Input, ApiDocMetadataEntry[]>}
+ */
+export default {
+  name: 'metadata',
+
+  version: '1.0.0',
+
+  description: 'generates a flattened list of API doc metadata entries',
+
+  dependsOn: 'ast',
+
+  /**
+   * @param {Input} input
+   * @returns {Promise<ApiDocMetadataEntry[]>}
+   */
+  async generate({ file, tree }) {
+    const {
+      setHeadingMetadata,
+      addYAMLMetadata,
+      updateMarkdownLink,
+      updateTypeReference,
+      updateLinkReference,
+      addStabilityMetadata,
+    } = createQueries();
+
+    // Creates an instance of the Remark processor with GFM support
+    // which is used for stringifying the AST tree back to Markdown
+    const remarkProcessor = getRemark();
+
+    /**
+     * This holds references to all the Metadata entries for a given file
+     * this is used so we can traverse the AST tree and keep mutating things
+     * and then stringify the whole api doc file at once without creating sub traversals
+     *
+     * Then once we have the whole file parsed, we can split the resulting string into sections
+     * and seal the Metadata Entries (`.create()`) and return the result to the caller of parae.
+     *
+     * @type {Array<ApiDocMetadataEntry>}
+     */
+    const metadataCollection = [];
+
+    // Creates a new Slugger instance for the current API doc file
+    const nodeSlugger = createNodeSlugger();
+
+    // Get all Markdown Footnote definitions from the tree
+    const markdownDefinitions = selectAll('definition', tree);
+
+    // Get all Markdown Heading entries from the tree
+    const headingNodes = selectAll('heading', tree);
+
+    // Handles Markdown link references and updates them to be plain links
+    visit(tree, createQueries.UNIST.isLinkReference, node =>
+      updateLinkReference(node, markdownDefinitions)
+    );
+
+    // Removes all the original definitions from the tree as they are not needed
+    // anymore, since all link references got updated to be plain links
+    remove(tree, markdownDefinitions);
+
+    // Handles the normalisation URLs that reference to API doc files with .md extension
+    // to replace the .md into .html, since the API doc files get eventually compiled as HTML
+    visit(tree, createQueries.UNIST.isMarkdownUrl, node =>
+      updateMarkdownLink(node)
+    );
+
+    // If the document has no headings but it has content, we add a fake heading to the top
+    // so that our parsing logic can work correctly, and generate content for the whole file
+    if (headingNodes.length === 0 && tree.children.length > 0) {
+      tree.children.unshift(createTree('heading', { depth: 1 }, []));
+    }
+
+    // Handles iterating the tree and creating subtrees for each API doc entry
+    // where an API doc entry is defined by a Heading Node
+    // (so all elements after a Heading until the next Heading)
+    // and then it creates and updates a Metadata entry for each API doc entry
+    // and then generates the final content for each API doc entry and pushes it to the collection
+    visit(tree, createQueries.UNIST.isHeading, (headingNode, index) => {
+      // Creates a new Metadata entry for the current API doc file
+      const apiEntryMetadata = createMetadata(nodeSlugger);
+
+      // Adds the Metadata of the current Heading Node to the Metadata entry
+      setHeadingMetadata(headingNode, apiEntryMetadata);
+
+      // We retrieve the immediate next Heading if it exists
+      // This is used for ensuring that we don't include items that would
+      // belong only to the next heading to the current Heading metadata
+      // Note that if there is no next heading, we use the current node as the next one
+      const nextHeadingNode =
+        findAfter(tree, index, createQueries.UNIST.isHeading) ?? headingNode;
+
+      // This is the cutover index of the subtree that we should get
+      // of all the Nodes within the AST tree that belong to this section
+      // If `next` is equals the current heading, it means there's no next heading
+      // and we are reaching the end of the document, hence the cutover should be the end of
+      // the document itself.
+      const stop =
+        headingNode === nextHeadingNode
+          ? tree.children.length
+          : tree.children.indexOf(nextHeadingNode);
+
+      // Retrieves all the nodes that should belong to the current API docs section
+      // `index + 1` is used to skip the current Heading Node
+      const subTree = createTree('root', tree.children.slice(index, stop));
+
+      // Visits all Stability Index nodes from the current subtree if there's any
+      // and then apply the Stability Index metadata to the current metadata entry
+      visit(subTree, createQueries.UNIST.isStabilityNode, node =>
+        addStabilityMetadata(node, apiEntryMetadata)
+      );
+
+      // Visits all HTML nodes from the current subtree and if there's any that matches
+      // our YAML metadata structure, it transforms into YAML metadata
+      // and then apply the YAML Metadata to the current Metadata entry
+      visit(subTree, createQueries.UNIST.isYamlNode, node => {
+        // TODO: Is there always only one YAML node?
+        apiEntryMetadata.setYamlPosition(node.position);
+        addYAMLMetadata(node, apiEntryMetadata);
+      });
+
+      // Visits all Text nodes from the current subtree and if there's any that matches
+      // any API doc type reference and then updates the type reference to be a Markdown link
+      visit(subTree, createQueries.UNIST.isTextWithType, (node, _, parent) =>
+        updateTypeReference(node, parent)
+      );
+
+      // Removes already parsed items from the subtree so that they aren't included in the final content
+      remove(subTree, [createQueries.UNIST.isYamlNode]);
+
+      // Applies the AST transformations to the subtree based on the API doc entry Metadata
+      // Note that running the transformation on the subtree isn't costly as it is a reduced tree
+      // and the GFM transformations aren't that heavy
+      const parsedSubTree = remarkProcessor.runSync(subTree);
+
+      // We seal and create the API doc entry Metadata and push them to the collection
+      const parsedApiEntryMetadata = apiEntryMetadata.create(
+        file,
+        parsedSubTree
+      );
+
+      // We push the parsed API doc entry Metadata to the collection
+      metadataCollection.push(parsedApiEntryMetadata);
+
+      return SKIP;
+    });
+
+    // Returns the Metadata entries for the given API doc file
+    return metadataCollection;
+  },
+};
diff --git a/src/generators/orama-db/index.mjs b/src/generators/orama-db/index.mjs
@@ -21,7 +21,7 @@ export default {
 
   description: 'Generates the Orama database for the API docs.',
 
-  dependsOn: 'ast',
+  dependsOn: 'metadata',
 
   /**
    * Generates the Orama database.
diff --git a/src/loaders/markdown.mjs b/src/loaders/markdown.mjs
@@ -15,8 +15,8 @@ const createLoader = () => {
   /**
    * Loads API Doc files and transforms it into VFiles
    *
-   * @param {string} searchPath A glob/path for API docs to be loaded
-   * @param {string | undefined} ignorePath A glob/path of files to ignore
+   * @param {string[]} searchPath A glob/path for API docs to be loaded
+   * @param {string[] | undefined} ignorePath A glob/path of files to ignore
    * The input string can be a simple path (relative or absolute)
    * The input string can also be any allowed glob string
    *
diff --git a/src/metadata.mjs b/src/metadata.mjs
@@ -109,7 +109,7 @@ const createMetadata = slugger => {
      * The Navigation entries has a dedicated separate method for retrieval
      * as it can be manipulated outside of the scope of the generation of the content
      *
-     * @param {import('vfile').VFile} apiDoc The API doc file being parsed
+     * @param {{stem?: string, basename?: string}} apiDoc The API doc file being parsed
      * @param {ApiDocMetadataEntry['content']} section An AST tree containing the Nodes of the API doc entry section
      * @returns {ApiDocMetadataEntry} The locally created Metadata entries
      */
diff --git a/src/parsers/markdown.mjs b/src/parsers/markdown.mjs
diff --git a/src/types.d.ts b/src/types.d.ts
