feat: create llms.txt generator

src/generators/index.mjs

@@ -10,6 +10,7 @@ import addonVerify from './addon-verify/index.mjs';
 import apiLinks from './api-links/index.mjs';
 import oramaDb from './orama-db/index.mjs';
 import astJs from './ast-js/index.mjs';
+import llmsTxt from './llms-txt/index.mjs';
 
 export const publicGenerators = {
   'json-simple': jsonSimple,
@@ -21,6 +22,7 @@ export const publicGenerators = {
   'addon-verify': addonVerify,
   'api-links': apiLinks,
   'orama-db': oramaDb,
+  'llms-txt': llmsTxt,
 };
 
 export const allGenerators = {

src/generators/llms-txt/constants.mjs

@@ -0,0 +1,3 @@
+export const ENTRY_IGNORE_LIST = ['doc/api/synopsis.md'];
+
+export const LATEST_DOC_API_BASE_URL = 'https://nodejs.org/docs/latest';

src/generators/llms-txt/index.mjs

@@ -0,0 +1,47 @@
+'use strict';
+
+import { readFile, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { generateDocEntry } from './utils/generateDocEntry.mjs';
+import { LATEST_DOC_API_BASE_URL } from './constants.mjs';
+
+/**
+ * @typedef {Array<ApiDocMetadataEntry>} Input
+ *
+ * @type {GeneratorMetadata<Input, string>}
+ */
+export default {
+  name: 'llms-txt',
+  version: '1.0.0',
+  description: 'Generates a llms.txt file of the API docs',
+  dependsOn: 'ast',
+
+  /**
+   * @param {Input} input The API documentation metadata
+   * @param {Partial<GeneratorOptions>} options Generator options
+   * @returns {Promise<string>} The generated documentation text
+   */
+  async generate(input, options) {
+    const template = await readFile(
+      join(import.meta.dirname, 'template.txt'),
+      'utf-8'
+    );
+
+    const apiDocEntries = input.map(generateDocEntry).filter(Boolean);
+
+    const introductionEntries = [
+      `- [About this documentation](${LATEST_DOC_API_BASE_URL}/api/documentation.md)`,
+      `- [Usage and examples](${LATEST_DOC_API_BASE_URL}/api/synopsis.md)`,
+    ];
+
+    const filledTemplate = template
+      .replace('__INTRODUCTION__', introductionEntries.join('\n'))
+      .replace('__API_DOCS__', apiDocEntries.join('\n'));
+
+    if (options.output) {
+      await writeFile(join(options.output, 'llms.txt'), filledTemplate);
+    }
+
+    return filledTemplate;
+  },
+};

src/generators/llms-txt/template.txt

@@ -0,0 +1,9 @@
+# Node.js Documentation
+
+> Node.js is an open-source, cross-platform JavaScript runtime environment that executes JavaScript code outside a web browser. Node.js uses an event-driven, non-blocking I/O model that makes it lightweight and efficient for building scalable network applications.
+
+## Introduction
+__INTRODUCTION__
+
+## API Documentations
+__API_DOCS__

src/generators/llms-txt/utils/generateDocEntry.mjs

@@ -0,0 +1,39 @@
+import { ENTRY_IGNORE_LIST, LATEST_DOC_API_BASE_URL } from '../constants.mjs';
+import { paragraphToString } from './paragraphToString.mjs';
+
+/**
+ * Generates a documentation entry string
+ *
+ * @param {ApiDocMetadataEntry} entry
+ * @returns {string}
+ */
+export function generateDocEntry(entry) {
+  if (ENTRY_IGNORE_LIST.includes(entry.api_doc_source)) {
+    return null;
+  }
+
+  if (entry.heading.depth !== 1) {
+    return null;
+  }
+
+  // Remove the leading /doc of string
+  const path = entry.api_doc_source.replace(/^doc\//, '');
+
+  const entryLink = `[${entry.heading.data.name}](${LATEST_DOC_API_BASE_URL}/${path})`;
+
+  const descriptionNode = entry.content.children.find(
+    child => child.type === 'paragraph'
+  );
+
+  if (!descriptionNode) {
+    console.warn(`No description found for entry: ${entry.api_doc_source}`);
+    return `- ${entryLink}`;
+  }
+
+  const description = paragraphToString(descriptionNode).replace(
+    /[\r\n]+/g,
+    ' '
+  );
+
+  return `- ${entryLink}: ${description}`;
+}

src/generators/llms-txt/utils/paragraphToString.mjs

@@ -0,0 +1,36 @@
+/**
+ * Extracts text content from a node recursively
+ *
+ * @param {import('mdast').Paragraph} node The AST node to extract text from
+ * @returns {string} The extracted text content
+ */
+function extractTextContent(node) {
+  if (!node) {
+    return '';
+  }
+
+  if (node.type === 'text' || node.type === 'inlineCode') {
+    return node.value;
+  }
+
+  if (node.children && Array.isArray(node.children)) {
+    return node.children.map(extractTextContent).join('');
+  }
+
+  return '';
+}
+
+/**
+ * Extracts text from a paragraph node.
+ *
+ * @param {import('mdast').Paragraph} node The paragraph node to extract text from
+ * @returns {string} The extracted text content
+ * @throws {Error} If the node is not a paragraph
+ */
+export function paragraphToString(node) {
+  if (node.type !== 'paragraph') {
+    throw new Error('Node is not a paragraph');
+  }
+
+  return node.children.map(extractTextContent).join('');
+}