Add exporter options to assembler (#1320)

* stage

* stage

* Add exporter options to assembler command line

* ensure minimal parse skips validation again

* Bump to ingest and transport that AOT compile under .NET 9

Author: Mpdreamz

Directory.Packages.props

@@ -16,6 +16,7 @@
     <PackageVersion Include="AWSSDK.Core" Version="4.0.0.2" />
     <PackageVersion Include="AWSSDK.SQS" Version="4.0.0.1" />
     <PackageVersion Include="AWSSDK.S3" Version="4.0.0.1" />
+    <PackageVersion Include="Elastic.Ingest.Elasticsearch" Version="0.11.3" />
   </ItemGroup>
   <!-- Build -->
   <ItemGroup>
@@ -63,4 +64,4 @@
     <PackageVersion Include="xunit.runner.visualstudio" Version="3.0.2" />
     <PackageVersion Include="xunit.v3" Version="1.1.0" />
   </ItemGroup>
-</Project>
+</Project>

src/Elastic.Documentation.Search/Elastic.Documentation.Search.csproj

@@ -0,0 +1,17 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net9.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <PackageReference Include="Elastic.Ingest.Elasticsearch" />
+    </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Elastic.Documentation\Elastic.Documentation.csproj"/>
+  </ItemGroup>
+
+</Project>

src/Elastic.Documentation/Diagnostics/IDiagnosticsCollector.cs

@@ -35,6 +35,18 @@ public static void EmitWarning(this IDiagnosticsCollector collector, IFileInfo f
 
 	public static void EmitHint(this IDiagnosticsCollector collector, IFileInfo file, string message) =>
 		collector.EmitHint(file.FullName, message);
+
+	/// Emit an error not associated with a file
+	public static void EmitGlobalError(this IDiagnosticsCollector collector, string message, Exception? e = null) =>
+		collector.EmitError(string.Empty, message, e);
+
+	/// Emit a warning not associated with a file
+	public static void EmitGlobalWarning(this IDiagnosticsCollector collector, string message) =>
+		collector.EmitWarning(string.Empty, message);
+
+	/// Emit a hint not associated with a file
+	public static void EmitGlobalHint(this IDiagnosticsCollector collector, string message) =>
+		collector.EmitHint(string.Empty, message);
 }
 
 

src/Elastic.Documentation/Search/DocumentationDocument.cs

@@ -0,0 +1,29 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.Text.Json.Serialization;
+
+namespace Elastic.Documentation.Search;
+
+public record DocumentationDocument
+{
+	[JsonPropertyName("title")]
+	public string? Title { get; set; }
+
+	[JsonPropertyName("body")]
+	public string? Body { get; set; }
+
+	[JsonPropertyName("abstract")]
+	public string? Abstract { get; set; }
+
+	[JsonPropertyName("headings")]
+	public string[] Headings { get; set; } = [];
+
+	[JsonPropertyName("links")]
+	public string[] Links { get; set; } = [];
+
+	[JsonPropertyName("url")]
+	public string? Url { get; set; }
+}
+

src/Elastic.Documentation/Serialization/SourceGenerationContext.cs

@@ -4,6 +4,7 @@
 
 using System.Text.Json.Serialization;
 using Elastic.Documentation.Links;
+using Elastic.Documentation.Search;
 using Elastic.Documentation.State;
 
 namespace Elastic.Documentation.Serialization;
@@ -16,4 +17,5 @@ namespace Elastic.Documentation.Serialization;
 [JsonSerializable(typeof(GitCheckoutInformation))]
 [JsonSerializable(typeof(LinkRegistry))]
 [JsonSerializable(typeof(LinkRegistryEntry))]
+[JsonSerializable(typeof(DocumentationDocument))]
 public sealed partial class SourceGenerationContext : JsonSerializerContext;

src/Elastic.Markdown/DocumentationGenerator.cs

@@ -18,6 +18,7 @@
 
 namespace Elastic.Markdown;
 
+/// Used primarily for testing, do not use in production paths since it might keep references alive to long
 public interface IConversionCollector
 {
 	void Collect(MarkdownFile file, MarkdownDocument document, string html);
@@ -40,6 +41,7 @@ public class DocumentationGenerator
 	private readonly ILogger _logger;
 	private readonly IFileSystem _writeFileSystem;
 	private readonly IDocumentationFileExporter _documentationFileExporter;
+	private readonly IMarkdownExporter[] _markdownExporters;
 	private HtmlWriter HtmlWriter { get; }
 
 	public DocumentationSet DocumentationSet { get; }
@@ -51,12 +53,14 @@ public DocumentationGenerator(
 		ILoggerFactory logger,
 		INavigationHtmlWriter? navigationHtmlWriter = null,
 		IDocumentationFileOutputProvider? documentationFileOutputProvider = null,
+		IMarkdownExporter[]? markdownExporters = null,
 		IDocumentationFileExporter? documentationExporter = null,
 		IConversionCollector? conversionCollector = null,
 		ILegacyUrlMapper? legacyUrlMapper = null,
 		IPositionalNavigation? positionalNavigation = null
 	)
 	{
+		_markdownExporters = markdownExporters ?? [];
 		_documentationFileOutputProvider = documentationFileOutputProvider;
 		_conversionCollector = conversionCollector;
 		_writeFileSystem = docSet.Context.WriteFileSystem;
@@ -100,7 +104,7 @@ public async Task<GenerationResult> GenerateAll(Cancel ctx)
 
 		var generationState = Context.SkipDocumentationState ? null : GetPreviousGenerationState();
 
-		// clear output directory if force is true but never for assembler builds since these build multiple times to the output.
+		// clear the output directory if force is true but never for assembler builds since these build multiple times to the output.
 		if (Context is { AssemblerBuild: false, Force: true }
 			// clear the output directory if force is false but generation state is null, except for assembler builds.
 			|| (Context is { AssemblerBuild: false, Force: false } && generationState == null))
@@ -209,7 +213,7 @@ private async Task ExtractEmbeddedStaticResources(Cancel ctx)
 		}
 	}
 
-	private async Task ProcessFile(HashSet<string> offendingFiles, DocumentationFile file, DateTimeOffset outputSeenChanges, Cancel token)
+	private async Task ProcessFile(HashSet<string> offendingFiles, DocumentationFile file, DateTimeOffset outputSeenChanges, Cancel ctx)
 	{
 		if (!Context.Force)
 		{
@@ -220,10 +224,27 @@ private async Task ProcessFile(HashSet<string> offendingFiles, DocumentationFile
 		}
 
 		_logger.LogTrace("--> {FileFullPath}", file.SourceFile.FullName);
-		//TODO send file to OutputFile() so we can validate its scope is defined in navigation.yml
 		var outputFile = OutputFile(file.RelativePath);
 		if (outputFile is not null)
-			await _documentationFileExporter.ProcessFile(Context, file, outputFile, HtmlWriter, _conversionCollector, token);
+		{
+			var context = new ProcessingFileContext
+			{
+				BuildContext = Context,
+				OutputFile = outputFile,
+				ConversionCollector = _conversionCollector,
+				File = file,
+				HtmlWriter = HtmlWriter
+			};
+			await _documentationFileExporter.ProcessFile(context, ctx);
+			if (file is MarkdownFile markdown)
+			{
+				foreach (var exporter in _markdownExporters)
+				{
+					var document = context.MarkdownDocument ??= await markdown.ParseFullAsync(ctx);
+					_ = await exporter.ExportAsync(new MarkdownExportContext { Document = document, File = markdown }, ctx);
+				}
+			}
+		}
 	}
 
 	private IFileInfo? OutputFile(string relativePath)

src/Elastic.Markdown/Exporters/DocumentationFileExporter.cs

@@ -5,16 +5,27 @@
 using System.IO.Abstractions;
 using Elastic.Markdown.IO;
 using Elastic.Markdown.Slices;
+using Markdig.Syntax;
 
 namespace Elastic.Markdown.Exporters;
 
+public class ProcessingFileContext
+{
+	public required BuildContext BuildContext { get; init; }
+	public required DocumentationFile File { get; init; }
+	public required IFileInfo OutputFile { get; init; }
+	public required HtmlWriter HtmlWriter { get; init; }
+	public required IConversionCollector? ConversionCollector { get; init; }
+
+	public MarkdownDocument? MarkdownDocument { get; set; }
+}
+
 public interface IDocumentationFileExporter
 {
-	/// Used in documentation state to ensure we break the build cache if a different exporter is chosen
+	/// Used in the documentation state to ensure we break the build cache if a different exporter is chosen
 	string Name { get; }
 
-	Task ProcessFile(BuildContext context, DocumentationFile file, IFileInfo outputFile, HtmlWriter htmlWriter, IConversionCollector? conversionCollector,
-		Cancel token);
+	ValueTask ProcessFile(ProcessingFileContext context, Cancel ctx);
 
 	Task CopyEmbeddedResource(IFileInfo outputFile, Stream resourceStream, Cancel ctx);
 }
@@ -23,16 +34,14 @@ public abstract class DocumentationFileExporterBase(IFileSystem readFileSystem,
 {
 	public abstract string Name { get; }
 
-	public abstract Task ProcessFile(BuildContext context, DocumentationFile file, IFileInfo outputFile, HtmlWriter htmlWriter,
-		IConversionCollector? conversionCollector,
-		Cancel token);
+	public abstract ValueTask ProcessFile(ProcessingFileContext context, Cancel ctx);
 
 	protected async Task CopyFileFsAware(DocumentationFile file, IFileInfo outputFile, Cancel ctx)
 	{
 		// fast path, normal case.
 		if (readFileSystem == writeFileSystem)
 			readFileSystem.File.Copy(file.SourceFile.FullName, outputFile.FullName, true);
-		//slower when we are mocking the write filesystem
+		//slower when we are mocking the write-filesystem
 		else
 		{
 			var bytes = await file.SourceFile.FileSystem.File.ReadAllBytesAsync(file.SourceFile.FullName, ctx);
@@ -49,26 +58,20 @@ public async Task CopyEmbeddedResource(IFileInfo outputFile, Stream resourceStre
 	}
 }
 
-public class DocumentationFileExporter(
-	IFileSystem readFileSystem,
-	IFileSystem writeFileSystem
-) : DocumentationFileExporterBase(readFileSystem, writeFileSystem)
+public class DocumentationFileExporter(IFileSystem readFileSystem, IFileSystem writeFileSystem)
+	: DocumentationFileExporterBase(readFileSystem, writeFileSystem)
 {
-	public override string Name { get; } = nameof(DocumentationFileExporter);
+	public override string Name => nameof(DocumentationFileExporter);
 
-	public override async Task ProcessFile(BuildContext context, DocumentationFile file,
-		IFileInfo outputFile,
-		HtmlWriter htmlWriter,
-		IConversionCollector? conversionCollector,
-		Cancel token)
+	public override async ValueTask ProcessFile(ProcessingFileContext context, Cancel ctx)
 	{
-		if (file is MarkdownFile markdown)
-			await htmlWriter.WriteAsync(outputFile, markdown, conversionCollector, token);
+		if (context.File is MarkdownFile markdown)
+			context.MarkdownDocument = await context.HtmlWriter.WriteAsync(context.OutputFile, markdown, context.ConversionCollector, ctx);
 		else
 		{
-			if (outputFile.Directory is { Exists: false })
-				outputFile.Directory.Create();
-			await CopyFileFsAware(file, outputFile, token);
+			if (context.OutputFile.Directory is { Exists: false })
+				context.OutputFile.Directory.Create();
+			await CopyFileFsAware(context.File, context.OutputFile, ctx);
 		}
 	}
 }

src/Elastic.Markdown/Exporters/IMarkdownExporter.cs

@@ -0,0 +1,22 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using Elastic.Markdown.IO;
+using Markdig.Syntax;
+
+namespace Elastic.Markdown.Exporters;
+
+public class MarkdownExportContext
+{
+	public required MarkdownDocument Document { get; init; }
+	public required MarkdownFile File { get; init; }
+	public string? LLMText { get; set; }
+}
+
+public interface IMarkdownExporter
+{
+	ValueTask StartAsync(Cancel ctx = default);
+	ValueTask StopAsync(Cancel ctx = default);
+	ValueTask<bool> ExportAsync(MarkdownExportContext context, Cancel ctx);
+}

src/Elastic.Markdown/Exporters/NoopDocumentationFileExporter.cs

@@ -3,18 +3,15 @@
 // See the LICENSE file in the project root for more information
 
 using System.IO.Abstractions;
-using Elastic.Markdown.IO;
-using Elastic.Markdown.Slices;
 
 namespace Elastic.Markdown.Exporters;
 
 public class NoopDocumentationFileExporter : IDocumentationFileExporter
 {
 	public string Name { get; } = nameof(NoopDocumentationFileExporter);
 
-	public Task ProcessFile(BuildContext context, DocumentationFile file, IFileInfo outputFile, HtmlWriter htmlWriter,
-		IConversionCollector? conversionCollector, Cancel token) =>
-		Task.CompletedTask;
+	public ValueTask ProcessFile(ProcessingFileContext context, Cancel ctx) =>
+		ValueTask.CompletedTask;
 
 	public Task CopyEmbeddedResource(IFileInfo outputFile, Stream resourceStream, Cancel ctx) => Task.CompletedTask;
 }

src/Elastic.Markdown/Extensions/DetectionRules/RuleDocumentationFileExporter.cs

@@ -14,18 +14,24 @@ public class RuleDocumentationFileExporter(IFileSystem readFileSystem, IFileSyst
 {
 	public override string Name { get; } = nameof(RuleDocumentationFileExporter);
 
-	public override async Task ProcessFile(BuildContext context, DocumentationFile file, IFileInfo outputFile, HtmlWriter htmlWriter,
-		IConversionCollector? conversionCollector, Cancel token)
+	public override async ValueTask ProcessFile(ProcessingFileContext context, Cancel ctx)
 	{
-		if (file is DetectionRuleFile df)
-			await htmlWriter.WriteAsync(DetectionRuleFile.OutputPath(outputFile, context), df, conversionCollector, token);
-		else if (file is MarkdownFile markdown)
-			await htmlWriter.WriteAsync(outputFile, markdown, conversionCollector, token);
-		else
+		var htmlWriter = context.HtmlWriter;
+		var outputFile = context.OutputFile;
+		var conversionCollector = context.ConversionCollector;
+		switch (context.File)
 		{
-			if (outputFile.Directory is { Exists: false })
-				outputFile.Directory.Create();
-			await CopyFileFsAware(file, outputFile, token);
+			case DetectionRuleFile df:
+				context.MarkdownDocument = await htmlWriter.WriteAsync(DetectionRuleFile.OutputPath(outputFile, context.BuildContext), df, conversionCollector, ctx);
+				break;
+			case MarkdownFile markdown:
+				context.MarkdownDocument = await htmlWriter.WriteAsync(outputFile, markdown, conversionCollector, ctx);
+				break;
+			default:
+				if (outputFile.Directory is { Exists: false })
+					outputFile.Directory.Create();
+				await CopyFileFsAware(context.File, outputFile, ctx);
+				break;
 		}
 	}
 }