Ensure we emit llm data as zip too

Author: Mpdreamz

src/Elastic.Markdown/DocumentationGenerator.cs

@@ -173,6 +173,7 @@ await Parallel.ForEachAsync(DocumentationSet.Files, ctx, async (file, token) =>
 				_logger.LogInformation("-> Processed {ProcessedFiles}/{TotalFileCount} files", processedFiles, totalFileCount);
 		});
 		_logger.LogInformation("-> Processed {ProcessedFileCount}/{TotalFileCount} files", processedFileCount, totalFileCount);
+
 	}
 
 	private void HintUnusedSubstitutionKeys()
@@ -245,7 +246,7 @@ private async Task ProcessFile(HashSet<string> offendingFiles, DocumentationFile
 				foreach (var exporter in _markdownExporters)
 				{
 					var document = context.MarkdownDocument ??= await markdown.ParseFullAsync(ctx);
-					_ = await exporter.ExportAsync(new MarkdownExportContext
+					_ = await exporter.ExportAsync(new MarkdownExportFileContext
 					{
 						BuildContext = Context,
 						Resolvers = DocumentationSet.MarkdownParser.Resolvers,

src/Elastic.Markdown/Exporters/IMarkdownExporter.cs

@@ -10,7 +10,11 @@
 
 namespace Elastic.Markdown.Exporters;
 
-public class MarkdownExportContext
+
+public record MarkdownExportContext
+{
+}
+public record MarkdownExportFileContext
 {
 	public required BuildContext BuildContext { get; init; }
 	public required IParserResolvers Resolvers { get; init; }
@@ -24,5 +28,6 @@ public interface IMarkdownExporter
 {
 	ValueTask StartAsync(Cancel ctx = default);
 	ValueTask StopAsync(Cancel ctx = default);
-	ValueTask<bool> ExportAsync(MarkdownExportContext context, Cancel ctx);
+	ValueTask<bool> ExportAsync(MarkdownExportFileContext fileContext, Cancel ctx);
+	ValueTask<bool> FinishExportAsync(IDirectoryInfo outputFolder, Cancel ctx);
 }

src/Elastic.Markdown/Exporters/LLMTextExporter.cs

@@ -4,12 +4,12 @@
 
 using System.Buffers;
 using System.IO.Abstractions;
+using System.IO.Compression;
 using System.Text;
 using Elastic.Documentation.Configuration;
 using Elastic.Markdown.Helpers;
 using Elastic.Markdown.Myst;
 using Elastic.Markdown.Myst.FrontMatter;
-using Markdig.Syntax;
 
 namespace Elastic.Markdown.Exporters;
 
@@ -19,26 +19,53 @@ public class LLMTextExporter : IMarkdownExporter
 
 	public ValueTask StopAsync(Cancel ctx = default) => ValueTask.CompletedTask;
 
-	public async ValueTask<bool> ExportAsync(MarkdownExportContext context, Cancel ctx)
+	public async ValueTask<bool> ExportAsync(MarkdownExportFileContext fileContext, Cancel ctx)
 	{
-		var source = context.SourceFile.SourceFile;
+		var source = fileContext.SourceFile.SourceFile;
 		var fs = source.FileSystem;
-		var llmText = context.LLMText ??= ToLLMText(context.BuildContext, context.SourceFile.YamlFrontMatter, context.Resolvers, source);
+		var llmText = fileContext.LLMText ??= ToLLMText(fileContext.BuildContext, fileContext.SourceFile.YamlFrontMatter, fileContext.Resolvers, source);
 
 		// write to the output version of the Markdown file directly
-		var outputFile = context.DefaultOutputFile;
+		var outputFile = fileContext.DefaultOutputFile;
 		if (outputFile.Name == "index.md")
 		{
+			var root = fileContext.BuildContext.DocumentationOutputDirectory;
 			// Write to a file named after the parent folder
-			outputFile = fs.FileInfo.New(outputFile.Directory!.FullName + ".md");
+			if (outputFile.Directory!.FullName == root.FullName)
+			{
+				// TODO in FinishExportAsync find a way to generate llms.txt
+				// e.g should it embedd all the links?
+				outputFile = fs.FileInfo.New(Path.Combine(root.FullName, "llms.md"));
+			}
+			else
+				outputFile = fs.FileInfo.New(outputFile.Directory!.FullName + ".md");
 		}
+
 		if (outputFile.Directory is { Exists: false })
 			outputFile.Directory.Create();
 
 		await fs.File.WriteAllTextAsync(outputFile.FullName, llmText, ctx);
 		return true;
 	}
 
+	/// <inheritdoc />
+	public ValueTask<bool> FinishExportAsync(IDirectoryInfo outputFolder, Cancel ctx)
+	{
+		var outputDirectory = Path.Combine(outputFolder.FullName, "docs");
+		var zipPath = Path.Combine(outputDirectory, "llm.zip");
+		using (var zip = ZipFile.Open(zipPath, ZipArchiveMode.Create))
+		{
+			var markdownFiles = Directory.GetFiles(outputDirectory, "*.md", SearchOption.AllDirectories);
+
+			foreach (var file in markdownFiles)
+			{
+				var relativePath = Path.GetRelativePath(outputDirectory, file);
+				_ = zip.CreateEntryFromFile(file, relativePath);
+			}
+		}
+		return ValueTask.FromResult(true);
+	}
+
 	public static string ToLLMText(BuildContext buildContext, YamlFrontMatter? frontMatter, IParserResolvers resolvers, IFileInfo source)
 	{
 		var fs = source.FileSystem;
@@ -56,7 +83,6 @@ public static string ToLLMText(BuildContext buildContext, YamlFrontMatter? front
 		DocumentationObjectPoolProvider.StringBuilderPool.Return(sb);
 		var replaced = full.ReplaceSubstitutions(new ParserContext(state));
 		return replaced;
-
 	}
 
 	private static void Read(IFileInfo source, IFileSystem fs, StringBuilder sb, IDirectoryInfo setDirectory)
@@ -83,6 +109,7 @@ private static void Read(IFileInfo source, IFileSystem fs, StringBuilder sb, IDi
 					includePath = Path.Combine(setDirectory.FullName, relativeFile.TrimStart('/').ToString());
 					includeSource = fs.FileInfo.New(includePath);
 				}
+
 				if (includeSource.Extension == "md" && includePath.Contains("_snippets"))
 					Read(includeSource, fs, sb, setDirectory);
 				startIndex = cursor + relativeFileEnd;
@@ -94,7 +121,7 @@ private static void Read(IFileInfo source, IFileSystem fs, StringBuilder sb, IDi
 				startIndex = Math.Min(text.Length, startIndex);
 			}
 		}
+
 		_ = sb.Append(text[startIndex..]);
 	}
 }
-

src/tooling/docs-assembler/Building/AssemblerBuilder.cs

@@ -84,6 +84,12 @@ public async Task BuildAllAsync(FrozenDictionary<string, AssemblerDocumentationS
 				throw;
 			}
 		}
+		foreach (var exporter in markdownExporters)
+		{
+			_logger.LogInformation("Calling FinishExportAsync on {ExporterName}", exporter.GetType().Name);
+			_ = await exporter.FinishExportAsync(context.OutputDirectory, ctx);
+		}
+
 
 		tasks = markdownExporters.Select(async e => await e.StopAsync(ctx));
 		await Task.WhenAll(tasks);

src/tooling/docs-assembler/Exporters/ElasticsearchMarkdownExporter.cs

@@ -2,6 +2,7 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
+using System.IO.Abstractions;
 using Elastic.Documentation.Diagnostics;
 using Elastic.Documentation.Search;
 using Elastic.Documentation.Serialization;
@@ -108,10 +109,10 @@ private async ValueTask<bool> TryWrite(DocumentationDocument document, Cancel ct
 		return false;
 	}
 
-	public async ValueTask<bool> ExportAsync(MarkdownExportContext context, Cancel ctx)
+	public async ValueTask<bool> ExportAsync(MarkdownExportFileContext fileContext, Cancel ctx)
 	{
-		var file = context.SourceFile;
-		var document = context.Document;
+		var file = fileContext.SourceFile;
+		var document = fileContext.Document;
 		if (file.FileName.EndsWith(".toml", StringComparison.OrdinalIgnoreCase))
 			return true;
 
@@ -121,7 +122,7 @@ public async ValueTask<bool> ExportAsync(MarkdownExportContext context, Cancel c
 			return true;
 
 		// TODO!
-		var body = context.LLMText ??= "string.Empty";
+		var body = fileContext.LLMText ??= "string.Empty";
 		var doc = new DocumentationDocument
 		{
 			Title = file.Title,
@@ -133,4 +134,7 @@ public async ValueTask<bool> ExportAsync(MarkdownExportContext context, Cancel c
 		};
 		return await TryWrite(doc, ctx);
 	}
+
+	/// <inheritdoc />
+	public async ValueTask<bool> FinishExportAsync(IDirectoryInfo outputFolder, Cancel ctx) => await _channel.RefreshAsync(ctx);
 }