Support anchors and table of contents from included files (#585)

* Support anchors and table of contents from included files

If an included snippet defines it's own headers and anchors they did not contribute to the pages anchors and headers.

-- page.md
```markdown
:::{include} _snippet/snippet.md
:::
```

This resulted in link check failures to `page.md#included-from-snippet`, as well as the headers from a snippet not appearing on the page's table of contents.

This PR also extends our testing framework by allowing to collect data from the conversion using `IConversionCollector`.

The `authoring` test framework uses the same `DocumentationGenerator` that `docs-builder` uses. The default for `IConversionCollector` is `null` in real world usage as we don't want the added memory pressure.

An upside of this is that we can now really fully test the whole HTML layout vs just the markdown HTML.

Lastly this PR includes a tiny fix to DiagnosticLinkInlineParser.cs to always report the resolved path as the full path. That way folks don't need to parse all the parent up `../` instructions themselves when dealing with this error instance.

* Add back AngleSharp.Diffing

* ensure we lookup snippets anchored to the root

* fix license headers

* don't lookup anchors in includes that are not found (or are cyclical)

* remove htmx tags from test assertions for now

Author: Mpdreamz

src/Elastic.Markdown/DocumentationGenerator.cs

@@ -9,12 +9,19 @@
 using Elastic.Markdown.IO;
 using Elastic.Markdown.IO.State;
 using Elastic.Markdown.Slices;
+using Markdig.Syntax;
 using Microsoft.Extensions.Logging;
 
 namespace Elastic.Markdown;
 
+public interface IConversionCollector
+{
+	void Collect(MarkdownFile file, MarkdownDocument document, string html);
+}
+
 public class DocumentationGenerator
 {
+	private readonly IConversionCollector? _conversionCollector;
 	private readonly IFileSystem _readFileSystem;
 	private readonly ILogger _logger;
 	private readonly IFileSystem _writeFileSystem;
@@ -26,9 +33,11 @@ public class DocumentationGenerator
 
 	public DocumentationGenerator(
 		DocumentationSet docSet,
-		ILoggerFactory logger
+		ILoggerFactory logger,
+		IConversionCollector? conversionCollector = null
 	)
 	{
+		_conversionCollector = conversionCollector;
 		_readFileSystem = docSet.Context.ReadFileSystem;
 		_writeFileSystem = docSet.Context.WriteFileSystem;
 		_logger = logger.CreateLogger(nameof(DocumentationGenerator));
@@ -161,7 +170,7 @@ private async Task ProcessFile(HashSet<string> offendingFiles, DocumentationFile
 		_logger.LogTrace("--> {FileFullPath}", file.SourceFile.FullName);
 		var outputFile = OutputFile(file.RelativePath);
 		if (file is MarkdownFile markdown)
-			await HtmlWriter.WriteAsync(outputFile, markdown, token);
+			await HtmlWriter.WriteAsync(outputFile, markdown, _conversionCollector, token);
 		else
 		{
 			if (outputFile.Directory is { Exists: false })

src/Elastic.Markdown/IO/DocumentationFile.cs

@@ -2,6 +2,9 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 using System.IO.Abstractions;
+using Elastic.Markdown.Myst;
+using Elastic.Markdown.Myst.FrontMatter;
+using Elastic.Markdown.Slices;
 
 namespace Elastic.Markdown.IO;
 
@@ -22,4 +25,32 @@ public record ExcludedFile(IFileInfo SourceFile, IDirectoryInfo RootPath)
 	: DocumentationFile(SourceFile, RootPath);
 
 public record SnippetFile(IFileInfo SourceFile, IDirectoryInfo RootPath)
-	: DocumentationFile(SourceFile, RootPath);
+	: DocumentationFile(SourceFile, RootPath)
+{
+	private SnippetAnchors? Anchors { get; set; }
+	private bool _parsed;
+
+	public SnippetAnchors? GetAnchors(
+		DocumentationSet set,
+		MarkdownParser parser,
+		YamlFrontMatter? frontMatter
+	)
+	{
+		if (_parsed)
+			return Anchors;
+		if (!SourceFile.Exists)
+		{
+			_parsed = true;
+			return null;
+		}
+
+		var document = parser.MinimalParseAsync(SourceFile, default).GetAwaiter().GetResult();
+		var toc = MarkdownFile.GetAnchors(set, parser, frontMatter, document, new Dictionary<string, string>(), out var anchors);
+		Anchors = new SnippetAnchors(anchors, toc);
+		_parsed = true;
+		return Anchors;
+	}
+}
+
+public record SnippetAnchors(string[] Anchors, IReadOnlyCollection<PageTocItem> TableOfContentItems);
+

src/Elastic.Markdown/IO/DocumentationSet.cs

@@ -187,16 +187,16 @@ private DocumentationFile CreateMarkDownFile(IFileInfo file, BuildContext contex
 		if (Configuration.Exclude.Any(g => g.IsMatch(relativePath)))
 			return new ExcludedFile(file, SourcePath);
 
-		if (Configuration.Files.Contains(relativePath))
-			return new MarkdownFile(file, SourcePath, MarkdownParser, context);
-
-		if (Configuration.Globs.Any(g => g.IsMatch(relativePath)))
-			return new MarkdownFile(file, SourcePath, MarkdownParser, context);
-
 		// we ignore files in folders that start with an underscore
 		if (relativePath.Contains("_snippets"))
 			return new SnippetFile(file, SourcePath);
 
+		if (Configuration.Files.Contains(relativePath))
+			return new MarkdownFile(file, SourcePath, MarkdownParser, context, this);
+
+		if (Configuration.Globs.Any(g => g.IsMatch(relativePath)))
+			return new MarkdownFile(file, SourcePath, MarkdownParser, context, this);
+
 		// we ignore files in folders that start with an underscore
 		if (relativePath.IndexOf("/_", StringComparison.Ordinal) > 0 || relativePath.StartsWith('_'))
 			return new ExcludedFile(file, SourcePath);

src/Elastic.Markdown/IO/MarkdownFile.cs

@@ -21,7 +21,15 @@ public record MarkdownFile : DocumentationFile
 {
 	private string? _navigationTitle;
 
-	public MarkdownFile(IFileInfo sourceFile, IDirectoryInfo rootPath, MarkdownParser parser, BuildContext context)
+	private readonly DocumentationSet _set;
+
+	public MarkdownFile(
+		IFileInfo sourceFile,
+		IDirectoryInfo rootPath,
+		MarkdownParser parser,
+		BuildContext context,
+		DocumentationSet set
+	)
 		: base(sourceFile, rootPath)
 	{
 		FileName = sourceFile.Name;
@@ -30,6 +38,7 @@ public MarkdownFile(IFileInfo sourceFile, IDirectoryInfo rootPath, MarkdownParse
 		MarkdownParser = parser;
 		Collector = context.Collector;
 		_configurationFile = context.Configuration.SourceFile;
+		_set = set;
 	}
 
 	public string Id { get; } = Guid.NewGuid().ToString("N")[..8];
@@ -191,36 +200,78 @@ private void ReadDocumentInstructions(MarkdownDocument document)
 		else if (Title.AsSpan().ReplaceSubstitutions(subs, out var replacement))
 			Title = replacement;
 
-		var contents = document
+		if (RelativePath.Contains("esql-functions-operators"))
+		{
+
+		}
+		var toc = GetAnchors(_set, MarkdownParser, YamlFrontMatter, document, subs, out var anchors);
+
+		_tableOfContent.Clear();
+		foreach (var t in toc)
+			_tableOfContent[t.Slug] = t;
+
+
+		foreach (var label in anchors)
+			_ = _anchors.Add(label);
+
+		_instructionsParsed = true;
+	}
+
+	public static List<PageTocItem> GetAnchors(
+		DocumentationSet set,
+		MarkdownParser parser,
+		YamlFrontMatter? frontMatter,
+		MarkdownDocument document,
+		IReadOnlyDictionary<string, string> subs,
+		out string[] anchors)
+	{
+		var includeBlocks = document.Descendants<IncludeBlock>().ToArray();
+		var includes = includeBlocks
+			.Where(i => i.Found)
+			.Select(i =>
+			{
+				var path = i.IncludePathFromSourceDirectory;
+				if (path is null
+					|| !set.FlatMappedFiles.TryGetValue(path, out var file)
+					|| file is not SnippetFile snippet)
+					return null;
+
+				return snippet.GetAnchors(set, parser, frontMatter);
+			})
+			.Where(i => i is not null)
+			.ToArray();
+
+		var includedTocs = includes.SelectMany(i => i!.TableOfContentItems).ToArray();
+		var toc = document
 			.Descendants<HeadingBlock>()
 			.Where(block => block is { Level: >= 2 })
 			.Select(h => (h.GetData("header") as string, h.GetData("anchor") as string, h.Level))
 			.Select(h =>
 			{
 				var header = h.Item1!.StripMarkdown();
-				if (header.AsSpan().ReplaceSubstitutions(subs, out var replacement))
-					header = replacement;
 				return new PageTocItem { Heading = header, Slug = (h.Item2 ?? header).Slugify(), Level = h.Level };
 			})
+			.Concat(includedTocs)
+			.Select(toc => subs.Count == 0
+				? toc
+				: toc.Heading.AsSpan().ReplaceSubstitutions(subs, out var r)
+					? toc with { Heading = r }
+					: toc)
 			.ToList();
 
-		_tableOfContent.Clear();
-		foreach (var t in contents)
-			_tableOfContent[t.Slug] = t;
-
-		var anchors = document.Descendants<DirectiveBlock>()
-			.Select(b => b.CrossReferenceName)
-			.Where(l => !string.IsNullOrWhiteSpace(l))
-			.Select(s => s.Slugify())
-			.Concat(document.Descendants<InlineAnchor>().Select(a => a.Anchor))
-			.Concat(_tableOfContent.Values.Select(t => t.Slug))
-			.Where(anchor => !string.IsNullOrEmpty(anchor))
-			.ToArray();
-
-		foreach (var label in anchors)
-			_ = _anchors.Add(label);
-
-		_instructionsParsed = true;
+		var includedAnchors = includes.SelectMany(i => i!.Anchors).ToArray();
+		anchors =
+		[
+			..document.Descendants<DirectiveBlock>()
+				.Select(b => b.CrossReferenceName)
+				.Where(l => !string.IsNullOrWhiteSpace(l))
+				.Select(s => s.Slugify())
+				.Concat(document.Descendants<InlineAnchor>().Select(a => a.Anchor))
+				.Concat(toc.Select(t => t.Slug))
+				.Where(anchor => !string.IsNullOrEmpty(anchor))
+				.Concat(includedAnchors)
+		];
+		return toc;
 	}
 
 	private YamlFrontMatter ProcessYamlFrontMatter(MarkdownDocument document)

src/Elastic.Markdown/Myst/Directives/DirectiveHtmlRenderer.cs

@@ -231,8 +231,6 @@ private static void WriteIncludeBlock(HtmlRenderer renderer, IncludeBlock block)
 		var document = parser.ParseAsync(file, block.FrontMatter, default).GetAwaiter().GetResult();
 		var html = document.ToHtml(MarkdownParser.Pipeline);
 		_ = renderer.Write(html);
-		//var slice = Include.Create(new IncludeViewModel { Html = html });
-		//RenderRazorSlice(slice, renderer, block);
 	}
 
 	private static void WriteSettingsBlock(HtmlRenderer renderer, SettingsBlock block)

src/Elastic.Markdown/Myst/Directives/IncludeBlock.cs

@@ -35,6 +35,7 @@ public class IncludeBlock(DirectiveBlockParser parser, ParserContext context) :
 	public YamlFrontMatter? FrontMatter { get; } = context.FrontMatter;
 
 	public string? IncludePath { get; private set; }
+	public string? IncludePathFromSourceDirectory { get; private set; }
 
 	public bool Found { get; private set; }
 
@@ -69,6 +70,7 @@ private void ExtractInclusionPath(ParserContext context)
 			includeFrom = DocumentationSourcePath.FullName;
 
 		IncludePath = Path.Combine(includeFrom, includePath.TrimStart('/'));
+		IncludePathFromSourceDirectory = Path.GetRelativePath(DocumentationSourcePath.FullName, IncludePath);
 		if (FileSystem.File.Exists(IncludePath))
 			Found = true;
 		else

src/Elastic.Markdown/Myst/InlineParsers/DiagnosticLinkInlineParser.cs

@@ -190,7 +190,7 @@ private static void ValidateInternalUrl(InlineProcessor processor, string url, s
 		if (string.IsNullOrWhiteSpace(url))
 			return;
 
-		var pathOnDisk = Path.Combine(includeFrom, url.TrimStart('/'));
+		var pathOnDisk = Path.GetFullPath(Path.Combine(includeFrom, url.TrimStart('/')));
 		if (!context.Build.ReadFileSystem.File.Exists(pathOnDisk))
 			processor.EmitError(link, $"`{url}` does not exist. resolved to `{pathOnDisk}");
 	}

src/Elastic.Markdown/Slices/HtmlWriter.cs

@@ -3,6 +3,7 @@
 // See the LICENSE file in the project root for more information
 using System.IO.Abstractions;
 using Elastic.Markdown.IO;
+using Markdig.Syntax;
 using RazorSlices;
 
 namespace Elastic.Markdown.Slices;
@@ -26,6 +27,11 @@ private async Task<string> RenderNavigation(MarkdownFile markdown, Cancel ctx =
 	public async Task<string> RenderLayout(MarkdownFile markdown, Cancel ctx = default)
 	{
 		var document = await markdown.ParseFullAsync(ctx);
+		return await RenderLayout(markdown, document, ctx);
+	}
+
+	public async Task<string> RenderLayout(MarkdownFile markdown, MarkdownDocument document, Cancel ctx = default)
+	{
 		var html = MarkdownFile.CreateHtml(document);
 		await DocumentationSet.Tree.Resolve(ctx);
 		_renderedNavigation ??= await RenderNavigation(markdown, ctx);
@@ -57,7 +63,7 @@ public async Task<string> RenderLayout(MarkdownFile markdown, Cancel ctx = defau
 		return await slice.RenderAsync(cancellationToken: ctx);
 	}
 
-	public async Task WriteAsync(IFileInfo outputFile, MarkdownFile markdown, Cancel ctx = default)
+	public async Task WriteAsync(IFileInfo outputFile, MarkdownFile markdown, IConversionCollector? collector, Cancel ctx = default)
 	{
 		if (outputFile.Directory is { Exists: false })
 			outputFile.Directory.Create();
@@ -79,7 +85,9 @@ public async Task WriteAsync(IFileInfo outputFile, MarkdownFile markdown, Cancel
 				: Path.Combine(dir, "index.html");
 		}
 
-		var rendered = await RenderLayout(markdown, ctx);
+		var document = await markdown.ParseFullAsync(ctx);
+		var rendered = await RenderLayout(markdown, document, ctx);
+		collector?.Collect(markdown, document, rendered);
 		await writeFileSystem.File.WriteAllTextAsync(path, rendered, ctx);
 	}
 

src/Elastic.Markdown/Slices/_ViewModels.cs

@@ -69,7 +69,7 @@ public string Link(string path)
 	}
 }
 
-public class PageTocItem
+public record PageTocItem
 {
 	public required string Heading { get; init; }
 	public required string Slug { get; init; }