Enhance LLM readable markdown rendering (#1569)

* Add new LLM markdown renderer

* Create LlmStubstitutionLeafRenderer instead of post processing and replacing subs in the LLM markdown output

* Also create zip file with top level llms.txt file

* Cleanup LlmInlineRenderers.cs

* Move to dedicated LlmMarkdown folder

* Cleanup LLmMarkdownRenderer.cs

* Revert changes to MarkdownParser.cs

* Cleanup

* Add ability to serve LLM markdown during local development

* Cleanup legacy LLM text output code

* Move description below the title in the frontmatter output

* Fix index page serving

Otherwise it would serve the markdown file

* Fix zip creation

* Use DocumentationObjectPoolProvider for StringWriters

* Remove unused imports

* Optimize imports

* Fix DocumentationObjectPoolProvider usage

* Cleanup comment

* Use action pattern

* Remove unnecessary "!"

* Reuse LlmMarkdownExporter.ConvertToLlmMarkdown

* Make UseLlmMarkdownRenderer use of static action

* Change position of arguments

Author: reakaleek

src/Elastic.Documentation/Extensions/ReusableStringWriter.cs

@@ -18,6 +18,8 @@ public sealed class ReusableStringWriter : TextWriter
 
 	public void Reset() => _sb = null;
 
+	public override string ToString() => _sb?.ToString() ?? string.Empty;
+
 	public override void Write(char value) => _sb?.Append(value);
 
 	public override void Write(char[] buffer, int index, int count)

src/Elastic.Markdown/DocumentationGenerator.cs

@@ -13,8 +13,11 @@
 using Elastic.Documentation.Site.Navigation;
 using Elastic.Documentation.State;
 using Elastic.Markdown.Exporters;
+using Elastic.Markdown.Helpers;
 using Elastic.Markdown.IO;
 using Elastic.Markdown.Links.CrossLinks;
+using Elastic.Markdown.Myst.Renderers;
+using Elastic.Markdown.Myst.Renderers.LlmMarkdown;
 using Markdig.Syntax;
 using Microsoft.Extensions.Logging;
 
@@ -340,6 +343,13 @@ private async Task GenerateDocumentationState(Cancel ctx)
 		await DocumentationSet.OutputDirectory.FileSystem.File.WriteAllBytesAsync(stateFile.FullName, bytes, ctx);
 	}
 
+	public async Task<string> RenderLlmMarkdown(MarkdownFile markdown, Cancel ctx)
+	{
+		await DocumentationSet.Tree.Resolve(ctx);
+		var document = await markdown.ParseFullAsync(ctx);
+		return LlmMarkdownExporter.ConvertToLlmMarkdown(document, DocumentationSet.Context);
+	}
+
 	public async Task<RenderResult> RenderLayout(MarkdownFile markdown, Cancel ctx)
 	{
 		await DocumentationSet.Tree.Resolve(ctx);

src/Elastic.Markdown/Exporters/LLMTextExporter.cs

@@ -0,0 +1,148 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.IO.Abstractions;
+using System.IO.Compression;
+using System.Text;
+using Elastic.Documentation.Configuration;
+using Elastic.Documentation.Configuration.Builder;
+using Elastic.Markdown.Helpers;
+using Elastic.Markdown.Myst.Renderers.LlmMarkdown;
+using Markdig.Syntax;
+
+namespace Elastic.Markdown.Exporters;
+
+/// <summary>
+/// Exports markdown files as LLM-optimized CommonMark using custom renderers
+/// </summary>
+public class LlmMarkdownExporter : IMarkdownExporter
+{
+
+	public ValueTask StartAsync(Cancel ctx = default) => ValueTask.CompletedTask;
+
+	public ValueTask StopAsync(Cancel ctx = default) => ValueTask.CompletedTask;
+
+	public ValueTask<bool> FinishExportAsync(IDirectoryInfo outputFolder, Cancel ctx)
+	{
+		var outputDirectory = Path.Combine(outputFolder.FullName, "docs");
+		var zipPath = Path.Combine(outputDirectory, "llm.zip");
+		using (var zip = ZipFile.Open(zipPath, ZipArchiveMode.Create))
+		{
+			var llmsTxt = Path.Combine(outputDirectory, "llms.txt");
+			var llmsTxtRelativePath = Path.GetRelativePath(outputDirectory, llmsTxt);
+			_ = zip.CreateEntryFromFile(llmsTxt, llmsTxtRelativePath);
+
+			var markdownFiles = Directory.GetFiles(outputDirectory, "*.md", SearchOption.AllDirectories);
+
+			foreach (var file in markdownFiles)
+			{
+				var relativePath = Path.GetRelativePath(outputDirectory, file);
+				_ = zip.CreateEntryFromFile(file, relativePath);
+			}
+		}
+		return ValueTask.FromResult(true);
+	}
+
+	public async ValueTask<bool> ExportAsync(MarkdownExportFileContext fileContext, Cancel ctx)
+	{
+		var llmMarkdown = ConvertToLlmMarkdown(fileContext.Document, fileContext.BuildContext);
+		var outputFile = GetLlmOutputFile(fileContext);
+		if (outputFile.Directory is { Exists: false })
+			outputFile.Directory.Create();
+		var contentWithMetadata = CreateLlmContentWithMetadata(fileContext, llmMarkdown);
+		await fileContext.SourceFile.SourceFile.FileSystem.File.WriteAllTextAsync(
+			outputFile.FullName,
+			contentWithMetadata,
+			Encoding.UTF8,
+			ctx
+		);
+		return true;
+	}
+
+	public static string ConvertToLlmMarkdown(MarkdownDocument document, BuildContext context) =>
+		DocumentationObjectPoolProvider.UseLlmMarkdownRenderer(context, document, static (renderer, obj) =>
+		{
+			_ = renderer.Render(obj);
+		});
+
+	private static IFileInfo GetLlmOutputFile(MarkdownExportFileContext fileContext)
+	{
+		var source = fileContext.SourceFile.SourceFile;
+		var fs = source.FileSystem;
+		var defaultOutputFile = fileContext.DefaultOutputFile;
+
+		var fileName = Path.GetFileNameWithoutExtension(defaultOutputFile.Name);
+		if (fileName == "index")
+		{
+			var root = fileContext.BuildContext.DocumentationOutputDirectory;
+
+			if (defaultOutputFile.Directory!.FullName == root.FullName)
+				return fs.FileInfo.New(Path.Combine(root.FullName, "llms.txt"));
+
+			// For index files: /docs/section/index.html -> /docs/section.md
+			// This allows users to append .md to any URL path
+			var folderName = defaultOutputFile.Directory!.Name;
+			return fs.FileInfo.New(Path.Combine(
+				defaultOutputFile.Directory!.Parent!.FullName,
+				$"{folderName}.md"
+			));
+		}
+		// Regular files: /docs/section/page.html -> /docs/section/page.llm.md
+		var directory = defaultOutputFile.Directory!.FullName;
+		var baseName = Path.GetFileNameWithoutExtension(defaultOutputFile.Name);
+		return fs.FileInfo.New(Path.Combine(directory, $"{baseName}.md"));
+	}
+
+
+	private string CreateLlmContentWithMetadata(MarkdownExportFileContext context, string llmMarkdown)
+	{
+		var sourceFile = context.SourceFile;
+		var metadata = DocumentationObjectPoolProvider.StringBuilderPool.Get();
+
+		_ = metadata.AppendLine("---");
+		_ = metadata.AppendLine($"title: {sourceFile.Title}");
+
+		if (!string.IsNullOrEmpty(sourceFile.YamlFrontMatter?.Description))
+			_ = metadata.AppendLine($"description: {sourceFile.YamlFrontMatter.Description}");
+		else
+		{
+			var descriptionGenerator = new DescriptionGenerator();
+			var generateDescription = descriptionGenerator.GenerateDescription(context.Document);
+			_ = metadata.AppendLine($"description: {generateDescription}");
+		}
+
+		if (!string.IsNullOrEmpty(sourceFile.Url))
+			_ = metadata.AppendLine($"url: {context.BuildContext.CanonicalBaseUrl?.Scheme}://{context.BuildContext.CanonicalBaseUrl?.Host}{sourceFile.Url}");
+
+		var configProducts = context.BuildContext.Configuration.Products.Select(p =>
+		{
+			if (Products.AllById.TryGetValue(p, out var product))
+				return product;
+			throw new ArgumentException($"Invalid product id: {p}");
+		});
+		var frontMatterProducts = sourceFile.YamlFrontMatter?.Products ?? [];
+		var allProducts = frontMatterProducts
+			.Union(configProducts)
+			.Distinct()
+			.ToList();
+		if (allProducts.Count > 0)
+		{
+			_ = metadata.AppendLine("products:");
+			foreach (var product in allProducts.Select(p => p.DisplayName).Order())
+				_ = metadata.AppendLine($"  - {product}");
+		}
+
+		_ = metadata.AppendLine("---");
+		_ = metadata.AppendLine();
+		_ = metadata.AppendLine($"# {sourceFile.Title}");
+		_ = metadata.Append(llmMarkdown);
+
+		return metadata.ToString();
+	}
+}
+
+public static class LlmMarkdownExporterExtensions
+{
+	public static void AddLlmMarkdownExport(this List<IMarkdownExporter> exporters) => exporters.Add(new LlmMarkdownExporter());
+}