refactor: some improvements

Author: araujogui

src/constants.mjs

@@ -6,3 +6,6 @@ export const DOC_NODE_VERSION = process.version;
 // This is the Node.js CHANGELOG to be consumed to generate a list of all major Node.js versions
 export const DOC_NODE_CHANGELOG_URL =
   'https://raw.githubusercontent.com/nodejs/node/HEAD/CHANGELOG.md';
+
+// The base URL for the latest Node.js documentation
+export const LATEST_DOC_API_BASE_URL = 'https://nodejs.org/docs/latest/';

src/generators/llms-txt/constants.mjs

@@ -1,3 +1,6 @@
-export const ENTRY_IGNORE_LIST = ['doc/api/synopsis.md'];
-
-export const LATEST_DOC_API_BASE_URL = 'https://nodejs.org/docs/latest';
+// These files are not part of the API documentation and are manually included
+// in the llms.txt file
+export const ENTRY_IGNORE_LIST = [
+  'doc/api/synopsis.md',
+  'doc/api/documentation.md',
+];

src/generators/llms-txt/index.mjs

@@ -1,45 +1,62 @@
-'use strict';
-
 import { readFile, writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
-import { generateDocEntry } from './utils/generateDocEntry.mjs';
-import { LATEST_DOC_API_BASE_URL } from './constants.mjs';
+
+import { buildApiDocLink } from './utils/buildApiDocLink.mjs';
+import { ENTRY_IGNORE_LIST } from './constants.mjs';
+import { getIntroLinks } from './utils/getIntroLinks.mjs';
 
 /**
+ * This generator generates a llms.txt file to provide information to LLMs at
+ * inference time
+ *
  * @typedef {Array<ApiDocMetadataEntry>} Input
  *
  * @type {GeneratorMetadata<Input, string>}
  */
 export default {
   name: 'llms-txt',
+
   version: '1.0.0',
-  description: 'Generates a llms.txt file of the API docs',
+
+  description:
+    'Generates a llms.txt file to provide information to LLMs at inference time',
+
   dependsOn: 'ast',
 
   /**
-   * @param {Input} input The API documentation metadata
-   * @param {Partial<GeneratorOptions>} options Generator options
-   * @returns {Promise<string>} The generated documentation text
+   * Generates a llms.txt file
+   *
+   * @param {Input} entries
+   * @param {Partial<GeneratorOptions>} options
+   * @returns {Promise<void>}
    */
-  async generate(input, options) {
+  async generate(entries, { output }) {
     const template = await readFile(
       join(import.meta.dirname, 'template.txt'),
       'utf-8'
     );
 
-    const apiDocEntries = input.map(generateDocEntry).filter(Boolean);
+    const introLinks = getIntroLinks().join('\n');
 
-    const introductionEntries = [
-      `- [About this documentation](${LATEST_DOC_API_BASE_URL}/api/documentation.md)`,
-      `- [Usage and examples](${LATEST_DOC_API_BASE_URL}/api/synopsis.md)`,
-    ];
+    const apiDocsLinks = entries
+      .filter(entry => {
+        // Filter non top-level headings and ignored entries
+        return (
+          entry.heading.depth === 1 || ENTRY_IGNORE_LIST.includes(entry.path)
+        );
+      })
+      .map(entry => {
+        const link = buildApiDocLink(entry);
+        return `- ${link}`;
+      })
+      .join('\n');
 
     const filledTemplate = template
-      .replace('__INTRODUCTION__', introductionEntries.join('\n'))
-      .replace('__API_DOCS__', apiDocEntries.join('\n'));
+      .replace('__INTRODUCTION__', introLinks)
+      .replace('__API_DOCS__', apiDocsLinks);
 
-    if (options.output) {
-      await writeFile(join(options.output, 'llms.txt'), filledTemplate);
+    if (output) {
+      await writeFile(join(output, 'llms.txt'), filledTemplate);
     }
 
     return filledTemplate;

src/generators/llms-txt/utils/buildApiDocLink.mjs

@@ -0,0 +1,33 @@
+import { LATEST_DOC_API_BASE_URL } from '../../../constants.mjs';
+import { transformNodeToString } from '../../../utils/unist.mjs';
+
+/**
+ * Builds a markdown link for an API doc entry
+ *
+ * @param {ApiDocMetadataEntry} entry
+ * @returns {string}
+ */
+export const buildApiDocLink = entry => {
+  const title = entry.heading.data.name;
+
+  // Remove the leading doc/ from the path
+  const path = entry.api_doc_source.replace(/^doc\//, '');
+  const url = new URL(path, LATEST_DOC_API_BASE_URL);
+
+  const link = `[${title}](${url})`;
+
+  // Find the first paragraph in the content
+  const descriptionNode = entry.content.children.find(
+    child => child.type === 'paragraph'
+  );
+
+  if (!descriptionNode) {
+    return link;
+  }
+
+  const description = transformNodeToString(descriptionNode)
+    // Remove newlines and extra spaces
+    .replace(/[\r\n]+/g, ' ');
+
+  return `${link}: ${description}`;
+};

src/generators/llms-txt/utils/generateDocEntry.mjs

@@ -0,0 +1,18 @@
+import { LATEST_DOC_API_BASE_URL } from '../constants.mjs';
+
+/**
+ * Generates a list of introduction links for the llms.txt file
+ *
+ * @returns {string[]}
+ */
+export const getIntroLinks = () => {
+  const aboutDocUrl = new URL('/api/documentation.md', LATEST_DOC_API_BASE_URL);
+  const usageExamplesUrl = new URL('/api/synopsis.md', LATEST_DOC_API_BASE_URL);
+
+  const introLinks = [
+    `- [About this documentation](${aboutDocUrl})`,
+    `- [Usage and examples](${usageExamplesUrl})`,
+  ];
+
+  return introLinks;
+};