Support anchors and table of contents from included files

If an included snippet defines it's own headers and anchors they did not contribute to the pages anchors and headers.

-- page.md
```markdown
:::{include} _snippet/snippet.md
:::
```

This resulted in link check failures to `page.md#included-from-snippet`, as well as the headers from a snippet not appearing on the page's table of contents.

This PR also extends our testing framework by allowing to collect data from the conversion using `IConversionCollector`.

The `authoring` test framework uses the same `DocumentationGenerator` that `docs-builder` uses. The default for `IConversionCollector` is `null` in real world usage as we don't want the added memory pressure.

An upside of this is that we can now really fully test the whole HTML layout vs just the markdown HTML.

Lastly this PR includes a tiny fix to DiagnosticLinkInlineParser.cs to always report the resolved path as the full path. That way folks don't need to parse all the parent up `../` instructions themselves when dealing with this error instance.

Author: Mpdreamz

src/Elastic.Markdown/DocumentationGenerator.cs

@@ -9,12 +9,19 @@
 using Elastic.Markdown.IO;
 using Elastic.Markdown.IO.State;
 using Elastic.Markdown.Slices;
+using Markdig.Syntax;
 using Microsoft.Extensions.Logging;
 
 namespace Elastic.Markdown;
 
+public interface IConversionCollector
+{
+	void Collect(MarkdownFile file, MarkdownDocument document, string html);
+}
+
 public class DocumentationGenerator
 {
+	private readonly IConversionCollector? _conversionCollector;
 	private readonly IFileSystem _readFileSystem;
 	private readonly ILogger _logger;
 	private readonly IFileSystem _writeFileSystem;
@@ -26,9 +33,11 @@ public class DocumentationGenerator
 
 	public DocumentationGenerator(
 		DocumentationSet docSet,
-		ILoggerFactory logger
+		ILoggerFactory logger,
+		IConversionCollector? conversionCollector = null
 	)
 	{
+		_conversionCollector = conversionCollector;
 		_readFileSystem = docSet.Context.ReadFileSystem;
 		_writeFileSystem = docSet.Context.WriteFileSystem;
 		_logger = logger.CreateLogger(nameof(DocumentationGenerator));
@@ -161,7 +170,7 @@ private async Task ProcessFile(HashSet<string> offendingFiles, DocumentationFile
 		_logger.LogTrace("--> {FileFullPath}", file.SourceFile.FullName);
 		var outputFile = OutputFile(file.RelativePath);
 		if (file is MarkdownFile markdown)
-			await HtmlWriter.WriteAsync(outputFile, markdown, token);
+			await HtmlWriter.WriteAsync(outputFile, markdown, _conversionCollector, token);
 		else
 		{
 			if (outputFile.Directory is { Exists: false })

src/Elastic.Markdown/IO/DocumentationFile.cs

@@ -2,6 +2,9 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 using System.IO.Abstractions;
+using Elastic.Markdown.Myst;
+using Elastic.Markdown.Myst.FrontMatter;
+using Elastic.Markdown.Slices;
 
 namespace Elastic.Markdown.IO;
 
@@ -22,4 +25,32 @@ public record ExcludedFile(IFileInfo SourceFile, IDirectoryInfo RootPath)
 	: DocumentationFile(SourceFile, RootPath);
 
 public record SnippetFile(IFileInfo SourceFile, IDirectoryInfo RootPath)
-	: DocumentationFile(SourceFile, RootPath);
+	: DocumentationFile(SourceFile, RootPath)
+{
+	private SnippetAnchors? Anchors { get; set; }
+	private bool _parsed;
+
+	public SnippetAnchors? GetAnchors(
+		DocumentationSet set,
+		MarkdownParser parser,
+		YamlFrontMatter? frontMatter
+	)
+	{
+		if (_parsed)
+			return Anchors;
+		var document = parser.ParseAsync(SourceFile, frontMatter, default).GetAwaiter().GetResult();
+		if (!SourceFile.Exists)
+		{
+			_parsed = true;
+			return null;
+		}
+
+		var toc = MarkdownFile.GetAnchors(set, parser, frontMatter, document, new Dictionary<string, string>(), out var anchors);
+		Anchors = new SnippetAnchors(anchors, toc);
+		_parsed = true;
+		return Anchors;
+	}
+}
+
+public record SnippetAnchors(string[] Anchors, IReadOnlyCollection<PageTocItem> TableOfContentItems);
+

src/Elastic.Markdown/IO/DocumentationSet.cs

@@ -187,16 +187,16 @@ private DocumentationFile CreateMarkDownFile(IFileInfo file, BuildContext contex
 		if (Configuration.Exclude.Any(g => g.IsMatch(relativePath)))
 			return new ExcludedFile(file, SourcePath);
 
-		if (Configuration.Files.Contains(relativePath))
-			return new MarkdownFile(file, SourcePath, MarkdownParser, context);
-
-		if (Configuration.Globs.Any(g => g.IsMatch(relativePath)))
-			return new MarkdownFile(file, SourcePath, MarkdownParser, context);
-
 		// we ignore files in folders that start with an underscore
 		if (relativePath.Contains("_snippets"))
 			return new SnippetFile(file, SourcePath);
 
+		if (Configuration.Files.Contains(relativePath))
+			return new MarkdownFile(file, SourcePath, MarkdownParser, context, this);
+
+		if (Configuration.Globs.Any(g => g.IsMatch(relativePath)))
+			return new MarkdownFile(file, SourcePath, MarkdownParser, context, this);
+
 		// we ignore files in folders that start with an underscore
 		if (relativePath.IndexOf("/_", StringComparison.Ordinal) > 0 || relativePath.StartsWith('_'))
 			return new ExcludedFile(file, SourcePath);

src/Elastic.Markdown/IO/MarkdownFile.cs

@@ -21,7 +21,15 @@ public record MarkdownFile : DocumentationFile
 {
 	private string? _navigationTitle;
 
-	public MarkdownFile(IFileInfo sourceFile, IDirectoryInfo rootPath, MarkdownParser parser, BuildContext context)
+	private readonly DocumentationSet _set;
+
+	public MarkdownFile(
+		IFileInfo sourceFile,
+		IDirectoryInfo rootPath,
+		MarkdownParser parser,
+		BuildContext context,
+		DocumentationSet set
+	)
 		: base(sourceFile, rootPath)
 	{
 		FileName = sourceFile.Name;
@@ -30,6 +38,7 @@ public MarkdownFile(IFileInfo sourceFile, IDirectoryInfo rootPath, MarkdownParse
 		MarkdownParser = parser;
 		Collector = context.Collector;
 		_configurationFile = context.Configuration.SourceFile;
+		_set = set;
 	}
 
 	public string Id { get; } = Guid.NewGuid().ToString("N")[..8];
@@ -191,36 +200,73 @@ private void ReadDocumentInstructions(MarkdownDocument document)
 		else if (Title.AsSpan().ReplaceSubstitutions(subs, out var replacement))
 			Title = replacement;
 
-		var contents = document
+		var toc = GetAnchors(_set, MarkdownParser, YamlFrontMatter, document, subs, out var anchors);
+
+		_tableOfContent.Clear();
+		foreach (var t in toc)
+			_tableOfContent[t.Slug] = t;
+
+
+		foreach (var label in anchors)
+			_ = _anchors.Add(label);
+
+		_instructionsParsed = true;
+	}
+
+	public static List<PageTocItem> GetAnchors(
+		DocumentationSet set,
+		MarkdownParser parser,
+		YamlFrontMatter? frontMatter,
+		MarkdownDocument document,
+		IReadOnlyDictionary<string, string> subs,
+		out string[] anchors)
+	{
+		var includeBlocks = document.Descendants<IncludeBlock>().ToArray();
+		var includes = includeBlocks
+			.Select(i =>
+			{
+				var path = i.IncludePathRelative;
+				if (path is null
+					|| !set.FlatMappedFiles.TryGetValue(path, out var file)
+					|| file is not SnippetFile snippet)
+					return null;
+
+				return snippet.GetAnchors(set, parser, frontMatter);
+			})
+			.Where(i => i is not null)
+			.ToArray();
+
+		var includedTocs = includes.SelectMany(i => i!.TableOfContentItems).ToArray();
+		var toc = document
 			.Descendants<HeadingBlock>()
 			.Where(block => block is { Level: >= 2 })
 			.Select(h => (h.GetData("header") as string, h.GetData("anchor") as string, h.Level))
 			.Select(h =>
 			{
 				var header = h.Item1!.StripMarkdown();
-				if (header.AsSpan().ReplaceSubstitutions(subs, out var replacement))
-					header = replacement;
 				return new PageTocItem { Heading = header, Slug = (h.Item2 ?? header).Slugify(), Level = h.Level };
 			})
+			.Concat(includedTocs)
+			.Select(toc => subs.Count == 0
+				? toc
+				: toc.Heading.AsSpan().ReplaceSubstitutions(subs, out var r)
+					? toc with { Heading = r }
+					: toc)
 			.ToList();
 
-		_tableOfContent.Clear();
-		foreach (var t in contents)
-			_tableOfContent[t.Slug] = t;
-
-		var anchors = document.Descendants<DirectiveBlock>()
-			.Select(b => b.CrossReferenceName)
-			.Where(l => !string.IsNullOrWhiteSpace(l))
-			.Select(s => s.Slugify())
-			.Concat(document.Descendants<InlineAnchor>().Select(a => a.Anchor))
-			.Concat(_tableOfContent.Values.Select(t => t.Slug))
-			.Where(anchor => !string.IsNullOrEmpty(anchor))
-			.ToArray();
-
-		foreach (var label in anchors)
-			_ = _anchors.Add(label);
-
-		_instructionsParsed = true;
+		var includedAnchors = includes.SelectMany(i => i!.Anchors).ToArray();
+		anchors =
+		[
+			..document.Descendants<DirectiveBlock>()
+				.Select(b => b.CrossReferenceName)
+				.Where(l => !string.IsNullOrWhiteSpace(l))
+				.Select(s => s.Slugify())
+				.Concat(document.Descendants<InlineAnchor>().Select(a => a.Anchor))
+				.Concat(toc.Select(t => t.Slug))
+				.Where(anchor => !string.IsNullOrEmpty(anchor))
+				.Concat(includedAnchors)
+		];
+		return toc;
 	}
 
 	private YamlFrontMatter ProcessYamlFrontMatter(MarkdownDocument document)

src/Elastic.Markdown/Myst/Directives/IncludeBlock.cs

@@ -35,6 +35,7 @@ public class IncludeBlock(DirectiveBlockParser parser, ParserContext context) :
 	public YamlFrontMatter? FrontMatter { get; } = context.FrontMatter;
 
 	public string? IncludePath { get; private set; }
+	public string? IncludePathRelative { get; private set; }
 
 	public bool Found { get; private set; }
 
@@ -69,6 +70,7 @@ private void ExtractInclusionPath(ParserContext context)
 			includeFrom = DocumentationSourcePath.FullName;
 
 		IncludePath = Path.Combine(includeFrom, includePath.TrimStart('/'));
+		IncludePathRelative = Path.GetRelativePath(includeFrom, IncludePath);
 		if (FileSystem.File.Exists(IncludePath))
 			Found = true;
 		else

src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs

@@ -190,7 +190,7 @@ private static void ValidateInternalUrl(InlineProcessor processor, string url, s
 		if (string.IsNullOrWhiteSpace(url))
 			return;
 
-		var pathOnDisk = Path.Combine(includeFrom, url.TrimStart('/'));
+		var pathOnDisk = Path.GetFullPath(Path.Combine(includeFrom, url.TrimStart('/')));
 		if (!context.Build.ReadFileSystem.File.Exists(pathOnDisk))
 			processor.EmitError(link, $"`{url}` does not exist. resolved to `{pathOnDisk}");
 	}

src/Elastic.Markdown/Slices/HtmlWriter.cs

@@ -3,6 +3,7 @@
 // See the LICENSE file in the project root for more information
 using System.IO.Abstractions;
 using Elastic.Markdown.IO;
+using Markdig.Syntax;
 using RazorSlices;
 
 namespace Elastic.Markdown.Slices;
@@ -26,6 +27,11 @@ private async Task<string> RenderNavigation(MarkdownFile markdown, Cancel ctx =
 	public async Task<string> RenderLayout(MarkdownFile markdown, Cancel ctx = default)
 	{
 		var document = await markdown.ParseFullAsync(ctx);
+		return await RenderLayout(markdown, document, ctx);
+	}
+
+	public async Task<string> RenderLayout(MarkdownFile markdown, MarkdownDocument document, Cancel ctx = default)
+	{
 		var html = MarkdownFile.CreateHtml(document);
 		await DocumentationSet.Tree.Resolve(ctx);
 		_renderedNavigation ??= await RenderNavigation(markdown, ctx);
@@ -57,7 +63,7 @@ public async Task<string> RenderLayout(MarkdownFile markdown, Cancel ctx = defau
 		return await slice.RenderAsync(cancellationToken: ctx);
 	}
 
-	public async Task WriteAsync(IFileInfo outputFile, MarkdownFile markdown, Cancel ctx = default)
+	public async Task WriteAsync(IFileInfo outputFile, MarkdownFile markdown, IConversionCollector? collector, Cancel ctx = default)
 	{
 		if (outputFile.Directory is { Exists: false })
 			outputFile.Directory.Create();
@@ -79,7 +85,9 @@ public async Task WriteAsync(IFileInfo outputFile, MarkdownFile markdown, Cancel
 				: Path.Combine(dir, "index.html");
 		}
 
-		var rendered = await RenderLayout(markdown, ctx);
+		var document = await markdown.ParseFullAsync(ctx);
+		var rendered = await RenderLayout(markdown, document, ctx);
+		collector?.Collect(markdown, document, rendered);
 		await writeFileSystem.File.WriteAllTextAsync(path, rendered, ctx);
 	}
 

src/Elastic.Markdown/Slices/_ViewModels.cs

@@ -69,7 +69,7 @@ public string Link(string path)
 	}
 }
 
-public class PageTocItem
+public record PageTocItem
 {
 	public required string Heading { get; init; }
 	public required string Slug { get; init; }

tests/authoring/Directives/IncludeBlocks.fs

@@ -0,0 +1,57 @@
+module ``directive elements``.``include directive``
+
+open Swensen.Unquote
+open Xunit
+open authoring
+open authoring.MarkdownDocumentAssertions
+
+type ``include hoists anchors and table of contents`` () =
+
+    static let generator = Setup.Generate [
+        Index """
+# A Document that lives at the root
+
+:::{include} _snippets/my-snippet.md
+:::
+"""
+        Snippet "_snippets/my-snippet.md" """
+## header from snippet [aa]
+
+        """
+        Markdown "test-links.md" """
+# parent.md
+
+## some header
+[link to root with included anchor](index.md#aa)
+        """
+    ]
+
+    [<Fact>]
+    let ``validate index.md HTML includes snippet`` () =
+        generator |> converts "index.md" |> toHtml """
+            <h1>A Document that lives at the root</h1>
+            <div class="heading-wrapper" id="aa">
+                <h2><a class="headerlink" href="#aa">header from snippet</a></h2>
+            </div>
+        """
+
+    [<Fact>]
+    let ``validate test-links.md HTML includes snippet`` () =
+        generator |> converts "test-links.md" |> toHtml """
+            <h1>parent.md</h1>
+            <div class="heading-wrapper" id="some-header">
+                <h2><a class="headerlink" href="#some-header">some header</a></h2>
+            </div>
+            <p><a href="/#aa">link to root with included anchor</a></p>
+       """
+
+    [<Fact>]
+    let ``validate index.md includes table of contents`` () =
+        let page = generator |> converts "index.md" |> markdownFile
+        test <@ page.TableOfContents.Count = 1 @>
+        test <@ page.TableOfContents.ContainsKey("aa") @>
+
+    [<Fact>]
+    let ``has no errors`` () = generator |> hasNoErrors
+
+