LLMText exports resolves includes and substitutions

Mpdreamz · Mpdreamz · commit 72e0dd3e5d58 · 2025-06-23T09:34:58.000+02:00
diff --git a/src/Elastic.Markdown/DocumentationGenerator.cs b/src/Elastic.Markdown/DocumentationGenerator.cs
@@ -245,7 +245,13 @@ private async Task ProcessFile(HashSet<string> offendingFiles, DocumentationFile
 				foreach (var exporter in _markdownExporters)
 				{
 					var document = context.MarkdownDocument ??= await markdown.ParseFullAsync(ctx);
-					_ = await exporter.ExportAsync(new MarkdownExportContext { Document = document, File = markdown }, ctx);
+					_ = await exporter.ExportAsync(new MarkdownExportContext
+					{
+						BuildContext = Context,
+						Resolvers = DocumentationSet.MarkdownParser.Resolvers,
+						Document = document,
+						File = markdown
+					}, ctx);
 				}
 			}
 		}
diff --git a/src/Elastic.Markdown/Exporters/IMarkdownExporter.cs b/src/Elastic.Markdown/Exporters/IMarkdownExporter.cs
@@ -2,13 +2,17 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
+using Elastic.Documentation.Configuration;
 using Elastic.Markdown.IO;
+using Elastic.Markdown.Myst;
 using Markdig.Syntax;
 
 namespace Elastic.Markdown.Exporters;
 
 public class MarkdownExportContext
 {
+	public required BuildContext BuildContext { get; init; }
+	public required IParserResolvers Resolvers { get; init; }
 	public required MarkdownDocument Document { get; init; }
 	public required MarkdownFile File { get; init; }
 	public string? LLMText { get; set; }
diff --git a/src/Elastic.Markdown/Exporters/LLMTextExporter.cs b/src/Elastic.Markdown/Exporters/LLMTextExporter.cs
@@ -0,0 +1,85 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.Buffers;
+using System.IO.Abstractions;
+using System.Text;
+using Elastic.Documentation.Configuration;
+using Elastic.Markdown.Helpers;
+using Elastic.Markdown.Myst;
+using Elastic.Markdown.Myst.FrontMatter;
+using Markdig.Syntax;
+
+namespace Elastic.Markdown.Exporters;
+
+public class LLMTextExporter : IMarkdownExporter
+{
+	public ValueTask StartAsync(CancellationToken ctx = default) => ValueTask.CompletedTask;
+
+	public ValueTask StopAsync(CancellationToken ctx = default) => ValueTask.CompletedTask;
+
+	public ValueTask<bool> ExportAsync(MarkdownExportContext context, CancellationToken ctx)
+	{
+		var source = context.File.SourceFile;
+		var llmText = ToLLMText(context.BuildContext, context.File.YamlFrontMatter, context.Resolvers, source);
+		var fs = source.FileSystem;
+
+		//var llmText = context.LLMText ??= LLMRoundtripRenderer.ToLLMText(context.Document);
+		return ValueTask.FromResult(true);
+	}
+
+
+	public static string ToLLMText(BuildContext buildContext, YamlFrontMatter? frontMatter, IParserResolvers resolvers, IFileInfo source)
+	{
+		var fs = source.FileSystem;
+		var sb = DocumentationObjectPoolProvider.StringBuilderPool.Get();
+
+		Read(source, fs, sb);
+		var full = sb.ToString();
+		var state = new ParserState(buildContext)
+		{
+			YamlFrontMatter = frontMatter,
+			MarkdownSourcePath = source,
+			CrossLinkResolver = resolvers.CrossLinkResolver,
+			DocumentationFileLookup = resolvers.DocumentationFileLookup
+		};
+		DocumentationObjectPoolProvider.StringBuilderPool.Return(sb);
+		var replaced = full.ReplaceSubstitutions(new ParserContext(state));
+		return replaced;
+
+	}
+
+	private static void Read(IFileInfo source, IFileSystem fs, StringBuilder sb, int depth = 0)
+	{
+		var text = fs.File.ReadAllText(source.FullName).AsSpan();
+		var spanStart = ":::{include}".AsSpan();
+		var include = SearchValues.Create([spanStart.ToString(), ":::\n"], StringComparison.OrdinalIgnoreCase);
+		int i;
+		var startIndex = 0;
+		while ((i = text[startIndex..].IndexOfAny(include)) >= 0)
+		{
+			var cursor = startIndex + i;
+			var marker = text[cursor..];
+			if (marker.StartsWith(spanStart))
+			{
+				_ = sb.Append(text.Slice(startIndex, i).TrimEnd('\n'));
+				var relativeFileStart = marker.IndexOf('}') + 1;
+				var relativeFileEnd = marker.IndexOf('\n');
+				var relativeFile = marker[relativeFileStart..relativeFileEnd].Trim();
+				var includePath = Path.GetFullPath(Path.Combine(source.Directory!.FullName, relativeFile.ToString()));
+				var includeSource = fs.FileInfo.New(includePath);
+				Read(includeSource, fs, sb, depth + 1);
+				startIndex = cursor + relativeFileEnd;
+				startIndex = Math.Min(text.Length, startIndex);
+			}
+			else
+			{
+				startIndex += i + 4;
+				startIndex = Math.Min(text.Length, startIndex);
+			}
+		}
+		_ = sb.Append(text[startIndex..]);
+	}
+}
+
diff --git a/src/Elastic.Markdown/IO/MarkdownFile.cs b/src/Elastic.Markdown/IO/MarkdownFile.cs
@@ -185,17 +185,6 @@ public async Task<MarkdownDocument> ParseFullAsync(Cancel ctx)
 		return document;
 	}
 
-	public static string ToLLMText(MarkdownDocument document)
-	{
-		using var sw = new StringWriter();
-		var rr = new RoundtripRenderer(sw);
-		rr.Write(document);
-		var outputMarkdown = sw.ToString();
-
-		return outputMarkdown;
-
-	}
-
 	private IReadOnlyDictionary<string, string> GetSubstitutions()
 	{
 		var globalSubstitutions = _globalSubstitutions;
diff --git a/src/Elastic.Markdown/Myst/Directives/DirectiveHtmlRenderer.cs b/src/Elastic.Markdown/Myst/Directives/DirectiveHtmlRenderer.cs
@@ -15,6 +15,7 @@
 using Elastic.Markdown.Myst.Directives.Tabs;
 using Elastic.Markdown.Myst.Directives.Version;
 using Elastic.Markdown.Myst.InlineParsers.Substitution;
+using Elastic.Markdown.Myst.Roles;
 using Markdig;
 using Markdig.Renderers;
 using Markdig.Renderers.Html;
diff --git a/src/Elastic.Markdown/Myst/Directives/DirectiveMarkdownExtension.cs b/src/Elastic.Markdown/Myst/Directives/DirectiveMarkdownExtension.cs
@@ -3,6 +3,7 @@
 // See the LICENSE file in the project root for more information
 
 using Elastic.Markdown.Myst.Renderers;
+using Elastic.Markdown.Myst.Roles;
 using Markdig;
 using Markdig.Extensions.Tables;
 using Markdig.Parsers;
diff --git a/src/Elastic.Markdown/Myst/MarkdownParser.cs b/src/Elastic.Markdown/Myst/MarkdownParser.cs
@@ -25,7 +25,7 @@ namespace Elastic.Markdown.Myst;
 public class MarkdownParser(BuildContext build, IParserResolvers resolvers)
 {
 	private BuildContext Build { get; } = build;
-	private IParserResolvers Resolvers { get; } = resolvers;
+	public IParserResolvers Resolvers { get; } = resolvers;
 
 	public Task<MarkdownDocument> ParseAsync(IFileInfo path, YamlFrontMatter? matter, Cancel ctx) =>
 		ParseFromFile(path, matter, Pipeline, false, ctx);
diff --git a/src/Elastic.Markdown/Myst/Roles/Role.cs b/src/Elastic.Markdown/Myst/Roles/Role.cs
@@ -4,7 +4,7 @@
 
 using Markdig.Syntax.Inlines;
 
-namespace Elastic.Markdown.Myst.Directives;
+namespace Elastic.Markdown.Myst.Roles;
 
 
 //TODO evaluate if we need this
diff --git a/src/tooling/docs-assembler/Exporters/ElasticsearchMarkdownExporter.cs b/src/tooling/docs-assembler/Exporters/ElasticsearchMarkdownExporter.cs
@@ -120,7 +120,8 @@ public async ValueTask<bool> ExportAsync(MarkdownExportContext context, Cancel c
 		if (url.Contains("/reference/integrations"))
 			return true;
 
-		var body = context.LLMText ??= MarkdownFile.ToLLMText(document);
+		// TODO!
+		var body = context.LLMText ??= "string.Empty";
 		var doc = new DocumentationDocument
 		{
 			Title = file.Title,
diff --git a/src/tooling/docs-assembler/Exporters/LLMTextExporter.cs b/src/tooling/docs-assembler/Exporters/LLMTextExporter.cs
diff --git a/tests/authoring/Blocks/Admonitions.fs b/tests/authoring/Blocks/Admonitions.fs
@@ -36,6 +36,11 @@ type ``admonition in list`` () =
     [<Fact>]
     let ``has no errors`` () = markdown |> hasNoErrors
 
+    [<Fact>]
+    let ``converts to LLM`` () =
+        markdown |> convertsToLLM """
+ """
+
 type ``nested admonition in list`` () =
     static let markdown = Setup.Markdown """
 :::{note}
diff --git a/tests/authoring/Directives/IncludeBlocks.fs b/tests/authoring/Directives/IncludeBlocks.fs
@@ -39,6 +39,15 @@ type ``include hoists anchors and table of contents`` () =
             </div>
         """
 
+    [<Fact>]
+    let ``converts to LLM`` () =
+        generator |> convertsToLLM """
+# A Document that lives at the root
+
+## header from snippet [aa]
+
+ """
+
     [<Fact>]
     let ``validate test-links.md HTML includes snippet`` () =
         generator |> converts "test-links.md" |> toHtml """
@@ -70,6 +79,8 @@ type ``include can contain links to parent page's includes`` () =
 
 :::{include} _snippets/my-other-snippet.md
 :::
+
+Some more content after includes
 """
         Snippet "_snippets/my-snippet.md" """
 ## header from snippet [aa]
@@ -82,3 +93,15 @@ type ``include can contain links to parent page's includes`` () =
 
     [<Fact>]
     let ``has no errors`` () = generator |> hasNoErrors
+
+    [<Fact>]
+    let ``converts to LLM`` () =
+        generator |> convertsToLLM """
+# A Document that lives at the root
+
+## header from snippet [aa]
+
+[link to root with included anchor](../index.md#aa)
+
+Some more content after includes
+ """
diff --git a/tests/authoring/Framework/HtmlAssertions.fs b/tests/authoring/Framework/HtmlAssertions.fs
@@ -11,6 +11,8 @@ open AngleSharp.Diffing
 open AngleSharp.Diffing.Core
 open AngleSharp.Html
 open AngleSharp.Html.Parser
+open Elastic.Markdown.Exporters
+open Elastic.Markdown.Myst
 open JetBrains.Annotations
 open Xunit.Sdk
 
@@ -112,8 +114,8 @@ actual: {actual}
     let private createDiff expected actual =
         let diffs =
             DiffBuilder
-                .Compare(actual)
-                .WithTest(expected)
+                .Compare(expected)
+                .WithTest(actual)
                 .Build()
 
         let deepComparision = htmlDiffString diffs
@@ -143,6 +145,33 @@ actual: {actual}
         let defaultFile = actual.MarkdownResults |> Seq.find (fun r -> r.File.RelativePath = "index.md")
         defaultFile |> toHtml expected
 
+    [<DebuggerStepThrough>]
+    let toLLM ([<LanguageInjection("markdown")>]expected: string) (actual: MarkdownResult) =
+        let actualLLM =
+            let buildContext = actual.Context.Generator.Context
+            let resolvers = actual.Context.Set.MarkdownParser.Resolvers
+            let yamlFrontMatter = actual.File.YamlFrontMatter
+            LLMTextExporter.ToLLMText(buildContext, yamlFrontMatter, resolvers, actual.File.SourceFile)
+        let difference = diff expected actualLLM
+        match difference with
+        | s when String.IsNullOrEmpty s -> ()
+        | d ->
+            let msg = $"""LLM text was not equal
+-- DIFF --
+{d}
+
+"""
+            raise (XunitException(msg))
+
+
+    [<DebuggerStepThrough>]
+    let convertsToLLM ([<LanguageInjection("markdown")>]expected: string) (actual: Lazy<GeneratorResults>) =
+        let actual = actual.Value
+
+        let defaultFile = actual.MarkdownResults |> Seq.find (fun r -> r.File.RelativePath = "index.md")
+        defaultFile |> toLLM expected
+
+
     [<DebuggerStepThrough>]
     let containsHtml ([<LanguageInjection("html")>]expected: string) (actual: MarkdownResult) =
 
diff --git a/tests/authoring/Inline/Substitutions.fs b/tests/authoring/Inline/Substitutions.fs
@@ -23,6 +23,17 @@ not a comment
 not a comment</p>
         """
 
+    [<Fact>]
+    let ``converts to LLM`` () =
+        markdown |> convertsToLLM """
+---
+sub:
+  hello-world: "Hello World!"
+---
+The following should be subbed: Hello World!
+not a comment
+"""
+
 
 type ``requires valid syntax and key to be found`` () =
     static let markdown = Setup.Document """---

Original file line number	Diff line number	Diff line change
`@@ -245,7 +245,13 @@ private async Task ProcessFile(HashSet<string> offendingFiles, DocumentationFile`
`245`	`245`	`foreach (var exporter in _markdownExporters)`
`246`	`246`	`{`
`247`	`247`	`var document = context.MarkdownDocument ??= await markdown.ParseFullAsync(ctx);`
`248`		`- _ = await exporter.ExportAsync(new MarkdownExportContext { Document = document, File = markdown }, ctx);`
	`248`	`+ _ = await exporter.ExportAsync(new MarkdownExportContext`
	`249`	`+ {`
	`250`	`+ BuildContext = Context,`
	`251`	`+ Resolvers = DocumentationSet.MarkdownParser.Resolvers,`
	`252`	`+ Document = document,`
	`253`	`+ File = markdown`
	`254`	`+ }, ctx);`
`249`	`255`	`}`
`250`	`256`	`}`
`251`	`257`	`}`
Original file line number	Diff line number	Diff line change
`@@ -25,7 +25,7 @@ namespace Elastic.Markdown.Myst;`
`25`	`25`	`public class MarkdownParser(BuildContext build, IParserResolvers resolvers)`
`26`	`26`	`{`
`27`	`27`	`private BuildContext Build { get; } = build;`
`28`		`- private IParserResolvers Resolvers { get; } = resolvers;`
	`28`	`+ public IParserResolvers Resolvers { get; } = resolvers;`
`29`	`29`
`30`	`30`	`public Task<MarkdownDocument> ParseAsync(IFileInfo path, YamlFrontMatter? matter, Cancel ctx) =>`
`31`	`31`	`ParseFromFile(path, matter, Pipeline, false, ctx);`
Original file line number	Diff line number	Diff line change
`@@ -120,7 +120,8 @@ public async ValueTask<bool> ExportAsync(MarkdownExportContext context, Cancel c`
`120`	`120`	`if (url.Contains("/reference/integrations"))`
`121`	`121`	`return true;`
`122`	`122`
`123`		`- var body = context.LLMText ??= MarkdownFile.ToLLMText(document);`
	`123`	`+ // TODO!`
	`124`	`+ var body = context.LLMText ??= "string.Empty";`
`124`	`125`	`var doc = new DocumentationDocument`
`125`	`126`	`{`
`126`	`127`	`Title = file.Title,`