Add llms.txt template and generator

src/Elastic.Markdown/Exporters/LlmMarkdownExporter.cs

@@ -17,6 +17,30 @@ namespace Elastic.Markdown.Exporters;
 /// </summary>
 public class LlmMarkdownExporter : IMarkdownExporter
 {
+	private const string LlmsTxtTemplate = """
+		# Elastic Documentation
+
+		> Elastic provides an open source search, analytics, and AI platform, and out-of-the-box solutions for observability and security. The Search AI platform combines the power of search and generative AI to provide near real-time search and analysis with relevance to reduce your time to value.
+		>
+		>Elastic offers the following solutions or types of projects:
+		>
+		>* [Elasticsearch](/solutions/search.md): Build powerful search and RAG applications using Elasticsearch's vector database, AI toolkit, and advanced retrieval capabilities.  
+		>* [Elastic Observability](/solutions/observability.md): Gain comprehensive visibility into applications, infrastructure, and user experience through logs, metrics, traces, and other telemetry data, all in a single interface.
+		>* [Elastic Security](/solutions/security.md): Combine SIEM, endpoint security, and cloud security to provide comprehensive tools for threat detection and prevention, investigation, and response.
+
+		The documentation is organized to guide you through your journey with Elastic, from learning the basics to deploying and managing complex solutions. Here is a detailed breakdown of the documentation structure:
+
+		* [**Elastic fundamentals**](/get-started.md): Understand the basics about the deployment options, platform, and solutions, and features of the documentation.  
+		* [**Solutions and use cases**](/solutions.md): Learn use cases, evaluate, and implement Elastic's solutions: Observability, Search, and Security.  
+		* [**Manage data**](/manage-data.md): Learn about data store primitives, ingestion and enrichment, managing the data lifecycle, and migrating data.  
+		* [**Explore and analyze**](/explore-analyze.md): Get value from data through querying, visualization, machine learning, and alerting.  
+		* [**Deploy and manage**](/deploy-manage.md): Deploy and manage production-ready clusters. Covers deployment options and maintenance tasks.  
+		* [**Manage your Cloud account**](/cloud-account.md): A dedicated section for user-facing cloud account tasks like resetting passwords.  
+		* [**Troubleshoot**](/troubleshoot.md): Identify and resolve problems.  
+		* [**Extend and contribute**](/extend.md): How to contribute to or integrate with Elastic, from open source to plugins to integrations.  
+		* [**Release notes**](/release-notes.md): Contains release notes and changelogs for each new release.  
+		* [**Reference**](/reference.md): Reference material for core tasks and manuals for optional products.
+		""";
 
 	public ValueTask StartAsync(Cancel ctx = default) => ValueTask.CompletedTask;
 
@@ -49,10 +73,22 @@ public async ValueTask<bool> ExportAsync(MarkdownExportFileContext fileContext,
 		var outputFile = GetLlmOutputFile(fileContext);
 		if (outputFile.Directory is { Exists: false })
 			outputFile.Directory.Create();
-		var contentWithMetadata = CreateLlmContentWithMetadata(fileContext, llmMarkdown);
+
+		string content;
+		if (IsRootIndexFile(fileContext))
+		{
+			// Use template for root llms.txt file
+			content = LlmsTxtTemplate;
+		}
+		else
+		{
+			// Regular markdown files get metadata + content
+			content = CreateLlmContentWithMetadata(fileContext, llmMarkdown);
+		}
+
 		await fileContext.SourceFile.SourceFile.FileSystem.File.WriteAllTextAsync(
 			outputFile.FullName,
-			contentWithMetadata,
+			content,
 			Encoding.UTF8,
 			ctx
 		);
@@ -65,6 +101,17 @@ public static string ConvertToLlmMarkdown(MarkdownDocument document, BuildContex
 			_ = renderer.Render(obj);
 		});
 
+	private static bool IsRootIndexFile(MarkdownExportFileContext fileContext)
+	{
+		var defaultOutputFile = fileContext.DefaultOutputFile;
+		var fileName = Path.GetFileNameWithoutExtension(defaultOutputFile.Name);
+		if (fileName != "index")
+			return false;
+
+		var root = fileContext.BuildContext.OutputDirectory;
+		return defaultOutputFile.Directory!.FullName == root.FullName;
+	}
+
 	private static IFileInfo GetLlmOutputFile(MarkdownExportFileContext fileContext)
 	{
 		var source = fileContext.SourceFile.SourceFile;

src/tooling/docs-assembler/Cli/RepositoryCommands.cs

@@ -5,6 +5,7 @@
 using System.Diagnostics.CodeAnalysis;
 using System.IO.Abstractions;
 using System.Net.Mime;
+using System.Text;
 using Actions.Core.Services;
 using Amazon.S3;
 using Amazon.S3.Model;
@@ -200,6 +201,12 @@ public async Task<int> BuildAll(
 			sitemapBuilder.Generate();
 		}
 
+		if (exporters.Contains(Exporter.LLMText))
+		{
+			var llmsEnhancer = new LlmsNavigationEnhancer();
+			await EnhanceLlmsTxtFile(assembleContext, navigation, llmsEnhancer, ctx);
+		}
+
 		await collector.StopAsync(ctx);
 
 		_log.LogInformation("Finished building and exporting exporters {Exporters}", exporters);
@@ -277,4 +284,20 @@ await Parallel.ForEachAsync(repositories,
 
 		return collector.Errors > 0 ? 1 : 0;
 	}
+
+	private static async Task EnhanceLlmsTxtFile(AssembleContext context, GlobalNavigation navigation, LlmsNavigationEnhancer enhancer, Cancel ctx)
+	{
+		var llmsTxtPath = Path.Combine(context.OutputDirectory.FullName, "docs", "llms.txt");
+
+		if (!File.Exists(llmsTxtPath))
+			return; // No llms.txt file to enhance
+
+		var existingContent = await File.ReadAllTextAsync(llmsTxtPath, ctx);
+		var navigationSections = enhancer.GenerateNavigationSections(navigation);
+
+		// Append the navigation sections to the existing boilerplate
+		var enhancedContent = existingContent + Environment.NewLine + navigationSections;
+
+		await File.WriteAllTextAsync(llmsTxtPath, enhancedContent, Encoding.UTF8, ctx);
+	}
 }

src/tooling/docs-assembler/Navigation/LlmsNavigationEnhancer.cs

@@ -0,0 +1,125 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.Text;
+using Documentation.Assembler;
+using Elastic.Documentation.Site.Navigation;
+using Elastic.Markdown.IO;
+using Elastic.Markdown.IO.Navigation;
+
+namespace Documentation.Assembler.Navigation;
+
+/// <summary>
+/// Generates enhanced navigation sections for the llms.txt file
+/// </summary>
+public class LlmsNavigationEnhancer
+{
+	public string GenerateNavigationSections(GlobalNavigation navigation)
+	{
+		var content = new StringBuilder();
+
+		// Get top-level navigation items (excluding hidden ones)
+		var topLevelItems = navigation.TopLevelItems.Where(item => !item.Hidden).ToList();
+
+		foreach (var topLevelItem in topLevelItems)
+		{
+			if (topLevelItem is not DocumentationGroup group)
+				continue;
+
+			// Create H2 section for the category
+			var categoryTitle = GetCategoryDisplayName(group.NavigationTitle);
+			_ = content.AppendLine($"## {categoryTitle}");
+			_ = content.AppendLine();
+
+			// Get first-level children
+			var firstLevelChildren = GetFirstLevelChildren(group);
+
+			if (firstLevelChildren.Count > 0)
+			{
+				foreach (var child in firstLevelChildren)
+				{
+					var title = child.NavigationTitle;
+					var url = ConvertToAbsoluteMarkdownUrl(child.Url);
+					var description = GetDescription(child);
+
+					_ = !string.IsNullOrEmpty(description)
+						? content.AppendLine($"* [{title}]({url}): {description}")
+						: content.AppendLine($"* [{title}]({url})");
+				}
+				_ = content.AppendLine();
+			}
+		}
+
+		return content.ToString();
+	}
+
+	private static string GetCategoryDisplayName(string navigationTitle) =>
+		// Convert navigation titles to display names
+		navigationTitle switch
+		{
+			"Get started" => "Get started",
+			"Solutions" => "Solutions",
+			"Manage data" => "Manage data",
+			"Explore and analyze" => "Explore and analyze",
+			"Deploy and manage" => "Deploy and manage",
+			"Manage your Cloud account and preferences" => "Manage your Cloud account",
+			"Troubleshoot" => "Troubleshoot",
+			"Extend and contribute" => "Extend and contribute",
+			"Release notes" => "Release notes",
+			"Reference" => "Reference",
+			_ => navigationTitle
+		};
+
+	private static List<INavigationItem> GetFirstLevelChildren(DocumentationGroup group)
+	{
+		var children = new List<INavigationItem>();
+
+		foreach (var item in group.NavigationItems)
+		{
+			// Only include non-hidden items
+			if (item.Hidden)
+				continue;
+
+			// Add the item to our list
+			children.Add(item);
+		}
+
+		return children;
+	}
+
+	private static string ConvertToAbsoluteMarkdownUrl(string url)
+	{
+		// Convert HTML URLs to .md URLs for LLM consumption
+		// e.g., "/docs/solutions/search/" -> "https://www.elastic.co/docs/solutions/search.md"
+		var cleanUrl = url.TrimStart('/');
+
+		// Remove "docs/" prefix if present for the markdown filename
+		var markdownPath = cleanUrl;
+		if (markdownPath.StartsWith("docs/"))
+			markdownPath = markdownPath.Substring(5);
+
+		// Convert directory URLs to .md files
+		if (markdownPath.EndsWith('/'))
+			markdownPath = markdownPath.TrimEnd('/') + ".md";
+		else if (!markdownPath.EndsWith(".md"))
+			markdownPath += ".md";
+
+		// Make absolute URL using the canonical base URL (always https://www.elastic.co for production)
+		var baseUrl = "https://www.elastic.co";
+		return $"{baseUrl}/docs/{markdownPath}";
+	}
+
+	private static string? GetDescription(INavigationItem navigationItem) => navigationItem switch
+	{
+		// For file navigation items, extract from frontmatter
+		FileNavigationItem fileItem when fileItem.Model is MarkdownFile markdownFile
+			=> markdownFile.YamlFrontMatter?.Description,
+
+		// For documentation groups, try to get from index file
+		DocumentationGroup group when group.Index is MarkdownFile indexFile
+			=> indexFile.YamlFrontMatter?.Description,
+
+		_ => null
+	};
+}