feat: create llms.txt generator (#254)

* feat: create llms.txt generator

* refactor: improvements

* fix: typo

* refactor: remove paragraphToString util

* refactor: some improvements

* fix: doc api url path

* fix: doc api url path

* refactor: some changes

* feat: add llm_description prop

* test: add llm_description prop

* refacotr: remove template replace

* feat(linter): create llm description rule

* refactor(linter): remove for-of loop

* refactor(llms-txt): remove docs url suffix

* fix: base url const

Author: araujogui

src/constants.mjs

@@ -6,3 +6,6 @@ export const DOC_NODE_VERSION = process.version;
 // This is the Node.js CHANGELOG to be consumed to generate a list of all major Node.js versions
 export const DOC_NODE_CHANGELOG_URL =
   'https://raw.githubusercontent.com/nodejs/node/HEAD/CHANGELOG.md';
+
+// The base URL for the Node.js website
+export const BASE_URL = 'https://nodejs.org/';

src/generators/index.mjs

@@ -10,6 +10,7 @@ import addonVerify from './addon-verify/index.mjs';
 import apiLinks from './api-links/index.mjs';
 import oramaDb from './orama-db/index.mjs';
 import astJs from './ast-js/index.mjs';
+import llmsTxt from './llms-txt/index.mjs';
 
 export const publicGenerators = {
   'json-simple': jsonSimple,
@@ -21,6 +22,7 @@ export const publicGenerators = {
   'addon-verify': addonVerify,
   'api-links': apiLinks,
   'orama-db': oramaDb,
+  'llms-txt': llmsTxt,
 };
 
 export const allGenerators = {

src/generators/llms-txt/index.mjs

@@ -0,0 +1,51 @@
+import { readFile, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+
+import { buildApiDocLink } from './utils/buildApiDocLink.mjs';
+
+/**
+ * This generator generates a llms.txt file to provide information to LLMs at
+ * inference time
+ *
+ * @typedef {Array<ApiDocMetadataEntry>} Input
+ *
+ * @type {GeneratorMetadata<Input, string>}
+ */
+export default {
+  name: 'llms-txt',
+
+  version: '1.0.0',
+
+  description:
+    'Generates a llms.txt file to provide information to LLMs at inference time',
+
+  dependsOn: 'ast',
+
+  /**
+   * Generates a llms.txt file
+   *
+   * @param {Input} entries
+   * @param {Partial<GeneratorOptions>} options
+   * @returns {Promise<void>}
+   */
+  async generate(entries, { output }) {
+    const template = await readFile(
+      join(import.meta.dirname, 'template.txt'),
+      'utf-8'
+    );
+
+    const apiDocsLinks = entries
+      // Filter non top-level headings
+      .filter(entry => entry.heading.depth === 1)
+      .map(entry => `- ${buildApiDocLink(entry)}`)
+      .join('\n');
+
+    const filledTemplate = `${template}${apiDocsLinks}`;
+
+    if (output) {
+      await writeFile(join(output, 'llms.txt'), filledTemplate);
+    }
+
+    return filledTemplate;
+  },
+};

src/generators/llms-txt/template.txt

@@ -0,0 +1,7 @@
+# Node.js Documentation
+
+> Node.js is an open-source, cross-platform JavaScript runtime environment that executes JavaScript code outside a web browser. Node.js uses an event-driven, non-blocking I/O model that makes it lightweight and efficient for building scalable network applications.
+
+Below are the sections of the API documentation. Look out especially towards the links that point towards guidance/introductioon to the structure of this documentation.
+
+## API Documentations

src/generators/llms-txt/utils/buildApiDocLink.mjs

@@ -0,0 +1,49 @@
+import { BASE_URL } from '../../../constants.mjs';
+import { transformNodeToString } from '../../../utils/unist.mjs';
+
+/**
+ * Retrieves the description of a given API doc entry. It first checks whether
+ * the entry has a llm_description property. If not, it extracts the first
+ * paragraph from the entry's content.
+ *
+ * @param {ApiDocMetadataEntry} entry
+ * @returns {string}
+ */
+const getEntryDescription = entry => {
+  if (entry.llm_description) {
+    return entry.llm_description;
+  }
+
+  const descriptionNode = entry.content.children.find(
+    child => child.type === 'paragraph'
+  );
+
+  if (!descriptionNode) {
+    return '';
+  }
+
+  return (
+    transformNodeToString(descriptionNode)
+      // Remove newlines and extra spaces
+      .replace(/[\r\n]+/g, '')
+  );
+};
+
+/**
+ * Builds a markdown link for an API doc entry
+ *
+ * @param {ApiDocMetadataEntry} entry
+ * @returns {string}
+ */
+export const buildApiDocLink = entry => {
+  const title = entry.heading.data.name;
+
+  const path = entry.api_doc_source.replace(/^doc\//, '/docs/latest/');
+  const url = new URL(path, BASE_URL);
+
+  const link = `[${title}](${url})`;
+
+  const description = getEntryDescription(entry);
+
+  return `${link}: ${description}`;
+};

src/linter/constants.mjs

@@ -5,4 +5,6 @@ export const LINT_MESSAGES = {
   missingChangeVersion: 'Missing version field in the API doc entry',
   invalidChangeVersion: 'Invalid version number: {{version}}',
   duplicateStabilityNode: 'Duplicate stability node',
+  missingLlmDescription:
+    'Missing llm_description field or paragraph node in the API doc entry',
 };

src/linter/rules/index.mjs

@@ -3,6 +3,7 @@
 import { duplicateStabilityNodes } from './duplicate-stability-nodes.mjs';
 import { invalidChangeVersion } from './invalid-change-version.mjs';
 import { missingIntroducedIn } from './missing-introduced-in.mjs';
+import { missingLlmDescription } from './missing-llm-description.mjs';
 
 /**
  * @type {Record<string, import('../types').LintRule>}
@@ -11,4 +12,5 @@ export default {
   'duplicate-stability-nodes': duplicateStabilityNodes,
   'invalid-change-version': invalidChangeVersion,
   'missing-introduced-in': missingIntroducedIn,
+  'missing-llm-description': missingLlmDescription,
 };

src/linter/rules/missing-llm-description.mjs

@@ -0,0 +1,47 @@
+import { LINT_MESSAGES } from '../constants.mjs';
+
+/**
+ * Checks if a top-level entry is missing a llm_description field or a paragraph
+ * node.
+ *
+ * @param {ApiDocMetadataEntry[]} entries
+ * @returns {Array<import('../types.d.ts').LintIssue>}
+ */
+export const missingLlmDescription = entries => {
+  return entries
+    .filter(entry => {
+      // Only process top-level headings
+      if (entry.heading.depth !== 1) {
+        return false;
+      }
+
+      // Skip entries that have an llm_description property
+      if (entry.llm_description !== undefined) {
+        return false;
+      }
+
+      const hasParagraph = entry.content.children.some(
+        node => node.type === 'paragraph'
+      );
+
+      // Skip entries that contain a paragraph that can be used as a fallback.
+      if (hasParagraph) {
+        return false;
+      }
+
+      return true;
+    })
+    .map(entry => mapToMissingEntryWarning(entry));
+};
+
+/**
+ * Maps a entry to a warning for missing llm description.
+ *
+ * @param {ApiDocMetadataEntry} entry
+ * @returns {import('../types.d.ts').LintIssue}
+ */
+const mapToMissingEntryWarning = entry => ({
+  level: 'warn',
+  message: LINT_MESSAGES.missingLlmDescription,
+  location: { path: entry.api_doc_source },
+});

src/metadata.mjs

@@ -131,6 +131,7 @@ const createMetadata = slugger => {
         updates = [],
         changes = [],
         tags = [],
+        llm_description,
       } = internalMetadata.properties;
 
       // Also add the slug to the heading data as it is used to build the heading
@@ -157,6 +158,7 @@ const createMetadata = slugger => {
         content: section,
         tags,
         introduced_in,
+        llm_description,
         yaml_position: internalMetadata.yaml_position,
       };
     },

src/test/metadata.test.mjs

@@ -72,6 +72,7 @@ describe('createMetadata', () => {
       heading,
       n_api_version: undefined,
       introduced_in: undefined,
+      llm_description: undefined,
       removed_in: undefined,
       slug: 'test-heading',
       source_link: 'test.com',