Initial support for cross link resolving (#491)

* Initial support for cross link resolving

* ensure cross_links is pluralized in config everywhere

* use docs-content for cross links for now

* add missing license header

* ensure anchor lookups work and urls uses extensionless paths

* dotnet format

* fix test

* remove TODO

Author: Mpdreamz

docs/docset.yml

@@ -1,4 +1,6 @@
 project: 'doc-builder'
+cross_links:
+  - docs-content
 # docs-builder will warn for links to external hosts not declared here
 external_hosts:
   - slack.com

docs/testing/cross-links.md

@@ -1,7 +1,7 @@
 # Cross Links
 
-[Elasticsearch](elasticsearch://index.md)
+[Elasticsearch](docs-content://index.md)
 
 [Kibana][1]
 
-[1]: kibana://index.md
+[1]: docs-content://index.md

src/Elastic.Markdown/CrossLinks/CrossLinkResolver.cs

@@ -0,0 +1,93 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.Collections.Frozen;
+using System.Diagnostics.CodeAnalysis;
+using System.Text.Json;
+using Elastic.Markdown.IO.Configuration;
+using Elastic.Markdown.IO.State;
+using Microsoft.Extensions.Logging;
+
+namespace Elastic.Markdown.CrossLinks;
+
+public interface ICrossLinkResolver
+{
+	Task FetchLinks();
+	bool TryResolve(Action<string> errorEmitter, Uri crossLinkUri, [NotNullWhen(true)] out Uri? resolvedUri);
+}
+
+public class CrossLinkResolver(ConfigurationFile configuration, ILoggerFactory logger) : ICrossLinkResolver
+{
+	private readonly string[] _links = configuration.CrossLinkRepositories;
+	private FrozenDictionary<string, LinkReference> _linkReferences = new Dictionary<string, LinkReference>().ToFrozenDictionary();
+	private readonly ILogger _logger = logger.CreateLogger(nameof(CrossLinkResolver));
+
+	public static LinkReference Deserialize(string json) =>
+		JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkReference)!;
+
+	public async Task FetchLinks()
+	{
+		using var client = new HttpClient();
+		var dictionary = new Dictionary<string, LinkReference>();
+		foreach (var link in _links)
+		{
+			var url = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/elastic/{link}/main/links.json";
+			_logger.LogInformation($"Fetching {url}");
+			var json = await client.GetStringAsync(url);
+			var linkReference = Deserialize(json);
+			dictionary.Add(link, linkReference);
+		}
+		_linkReferences = dictionary.ToFrozenDictionary();
+	}
+
+	public bool TryResolve(Action<string> errorEmitter, Uri crossLinkUri, [NotNullWhen(true)] out Uri? resolvedUri) =>
+		TryResolve(errorEmitter, _linkReferences, crossLinkUri, out resolvedUri);
+
+	private static Uri BaseUri { get; } = new Uri("https://docs-v3-preview.elastic.dev");
+
+	public static bool TryResolve(Action<string> errorEmitter, IDictionary<string, LinkReference> lookup, Uri crossLinkUri, [NotNullWhen(true)] out Uri? resolvedUri)
+	{
+		resolvedUri = null;
+		if (!lookup.TryGetValue(crossLinkUri.Scheme, out var linkReference))
+		{
+			errorEmitter($"'{crossLinkUri.Scheme}' is not declared as valid cross link repository in docset.yml under cross_links");
+			return false;
+		}
+		var lookupPath = crossLinkUri.AbsolutePath.TrimStart('/');
+		if (string.IsNullOrEmpty(lookupPath) && crossLinkUri.Host.EndsWith(".md"))
+			lookupPath = crossLinkUri.Host;
+
+		if (!linkReference.Links.TryGetValue(lookupPath, out var link))
+		{
+			errorEmitter($"'{lookupPath}' is not a valid link in the '{crossLinkUri.Scheme}' cross link repository.");
+			return false;
+		}
+
+		//https://docs-v3-preview.elastic.dev/elastic/docs-content/tree/main/cloud-account/change-your-password
+		var path = lookupPath.Replace(".md", "");
+		if (path.EndsWith("/index"))
+			path = path.Substring(0, path.Length - 6);
+		if (path == "index")
+			path = string.Empty;
+
+		if (!string.IsNullOrEmpty(crossLinkUri.Fragment))
+		{
+			if (link.Anchors is null)
+			{
+				errorEmitter($"'{lookupPath}' does not have any anchors so linking to '{crossLinkUri.Fragment}' is impossible.");
+				return false;
+			}
+
+			if (!link.Anchors.Contains(crossLinkUri.Fragment.TrimStart('#')))
+			{
+				errorEmitter($"'{lookupPath}' has no anchor named: '{crossLinkUri.Fragment}'.");
+				return false;
+			}
+			path += crossLinkUri.Fragment;
+		}
+
+		resolvedUri = new Uri(BaseUri, $"elastic/{crossLinkUri.Scheme}/tree/main/{path}");
+		return true;
+	}
+}

src/Elastic.Markdown/Diagnostics/ProcessorDiagnosticExtensions.cs

@@ -7,6 +7,7 @@
 using Elastic.Markdown.Myst.Directives;
 using Markdig.Helpers;
 using Markdig.Parsers;
+using Markdig.Syntax.Inlines;
 
 namespace Elastic.Markdown.Diagnostics;
 
@@ -131,4 +132,50 @@ public static void EmitWarning(this IBlockExtension block, string message)
 		};
 		block.Build.Collector.Channel.Write(d);
 	}
+
+
+	public static void EmitError(this InlineProcessor processor, LinkInline inline, string message)
+	{
+		var url = inline.Url;
+		var line = inline.Line + 1;
+		var column = inline.Column;
+		var length = url?.Length ?? 1;
+
+		var context = processor.GetContext();
+		if (context.SkipValidation)
+			return;
+		var d = new Diagnostic
+		{
+			Severity = Severity.Error,
+			File = processor.GetContext().Path.FullName,
+			Column = column,
+			Line = line,
+			Message = message,
+			Length = length
+		};
+		context.Build.Collector.Channel.Write(d);
+	}
+
+
+	public static void EmitWarning(this InlineProcessor processor, LinkInline inline, string message)
+	{
+		var url = inline.Url;
+		var line = inline.Line + 1;
+		var column = inline.Column;
+		var length = url?.Length ?? 1;
+
+		var context = processor.GetContext();
+		if (context.SkipValidation)
+			return;
+		var d = new Diagnostic
+		{
+			Severity = Severity.Warning,
+			File = processor.GetContext().Path.FullName,
+			Column = column,
+			Line = line,
+			Message = message,
+			Length = length
+		};
+		context.Build.Collector.Channel.Write(d);
+	}
 }

src/Elastic.Markdown/DocumentationGenerator.cs

@@ -1,9 +1,11 @@
 // Licensed to Elasticsearch B.V under one or more agreements.
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
+
 using System.IO.Abstractions;
 using System.Reflection;
 using System.Text.Json;
+using Elastic.Markdown.CrossLinks;
 using Elastic.Markdown.IO;
 using Elastic.Markdown.IO.State;
 using Elastic.Markdown.Slices;
@@ -20,6 +22,7 @@ public class DocumentationGenerator
 
 	public DocumentationSet DocumentationSet { get; }
 	public BuildContext Context { get; }
+	public ICrossLinkResolver Resolver { get; }
 
 	public DocumentationGenerator(
 		DocumentationSet docSet,
@@ -32,6 +35,7 @@ ILoggerFactory logger
 
 		DocumentationSet = docSet;
 		Context = docSet.Context;
+		Resolver = docSet.LinkResolver;
 		HtmlWriter = new HtmlWriter(DocumentationSet, _writeFileSystem);
 
 		_logger.LogInformation($"Created documentation set for: {DocumentationSet.Name}");
@@ -66,6 +70,9 @@ public async Task GenerateAll(Cancel ctx)
 		if (CompilationNotNeeded(generationState, out var offendingFiles, out var outputSeenChanges))
 			return;
 
+		_logger.LogInformation($"Fetching external links");
+		await Resolver.FetchLinks();
+
 		await ResolveDirectoryTree(ctx);
 
 		await ProcessDocumentationFiles(offendingFiles, outputSeenChanges, ctx);
@@ -77,6 +84,7 @@ public async Task GenerateAll(Cancel ctx)
 
 		_logger.LogInformation($"Generating documentation compilation state");
 		await GenerateDocumentationState(ctx);
+
 		_logger.LogInformation($"Generating links.json");
 		await GenerateLinkReference(ctx);
 

src/Elastic.Markdown/IO/Configuration/ConfigurationFile.cs

@@ -19,6 +19,8 @@ public record ConfigurationFile : DocumentationFile
 	public string? Project { get; }
 	public Glob[] Exclude { get; } = [];
 
+	public string[] CrossLinkRepositories { get; } = [];
+
 	public IReadOnlyCollection<ITocItem> TableOfContents { get; } = [];
 
 	public HashSet<string> Files { get; } = new(StringComparer.OrdinalIgnoreCase);
@@ -72,6 +74,9 @@ public ConfigurationFile(IFileInfo sourceFile, IDirectoryInfo rootPath, BuildCon
 							.Select(Glob.Parse)
 							.ToArray();
 						break;
+					case "cross_links":
+						CrossLinkRepositories = ReadStringArray(entry).ToArray();
+						break;
 					case "subs":
 						_substitutions = ReadDictionary(entry);
 						break;

src/Elastic.Markdown/IO/DocumentationSet.cs

@@ -4,10 +4,12 @@
 
 using System.Collections.Frozen;
 using System.IO.Abstractions;
+using Elastic.Markdown.CrossLinks;
 using Elastic.Markdown.Diagnostics;
 using Elastic.Markdown.IO.Configuration;
 using Elastic.Markdown.IO.Navigation;
 using Elastic.Markdown.Myst;
+using Microsoft.Extensions.Logging;
 
 namespace Elastic.Markdown.IO;
 
@@ -29,15 +31,18 @@ public class DocumentationSet
 
 	public MarkdownParser MarkdownParser { get; }
 
-	public DocumentationSet(BuildContext context)
+	public ICrossLinkResolver LinkResolver { get; }
+
+	public DocumentationSet(BuildContext context, ILoggerFactory logger, ICrossLinkResolver? linkResolver = null)
 	{
 		Context = context;
 		SourcePath = context.SourcePath;
 		OutputPath = context.OutputPath;
 		RelativeSourcePath = Path.GetRelativePath(Paths.Root.FullName, SourcePath.FullName);
 		Configuration = new ConfigurationFile(context.ConfigurationPath, SourcePath, context);
+		LinkResolver = linkResolver ?? new CrossLinkResolver(Configuration, logger);
 
-		MarkdownParser = new MarkdownParser(SourcePath, context, GetMarkdownFile, Configuration);
+		MarkdownParser = new MarkdownParser(SourcePath, context, GetMarkdownFile, Configuration, LinkResolver);
 
 		Name = SourcePath.FullName;
 		OutputStateFile = OutputPath.FileSystem.FileInfo.New(Path.Combine(OutputPath.FullName, ".doc.state"));

src/Elastic.Markdown/IO/State/LinkReference.cs

@@ -11,11 +11,11 @@ public record LinkMetadata
 {
 	[JsonPropertyName("anchors")]
 	[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
-	public required string[]? Anchors { get; init; } = [];
+	public string[]? Anchors { get; init; } = [];
 
 	[JsonPropertyName("hidden")]
 	[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
-	public required bool Hidden { get; init; }
+	public bool Hidden { get; init; }
 }
 
 public record LinkReference
@@ -26,7 +26,7 @@ public record LinkReference
 	[JsonPropertyName("url_path_prefix")]
 	public required string? UrlPathPrefix { get; init; }
 
-	/// Mapping of relative filepath and all the page's anchors for deeplinks
+	/// Mapping of relative filepath and all the page's anchors for deep links
 	[JsonPropertyName("links")]
 	public required Dictionary<string, LinkMetadata> Links { get; init; } = [];
 

src/Elastic.Markdown/Myst/Directives/DirectiveHtmlRenderer.cs

@@ -225,8 +225,9 @@ private void WriteIncludeBlock(HtmlRenderer renderer, IncludeBlock block)
 		if (!block.Found || block.IncludePath is null)
 			return;
 
-		var parser = new MarkdownParser(block.DocumentationSourcePath, block.Build, block.GetDocumentationFile,
-			block.Configuration);
+		var parser = new MarkdownParser(
+			block.DocumentationSourcePath, block.Build, block.GetDocumentationFile,
+			block.Configuration, block.LinksResolver);
 		var file = block.FileSystem.FileInfo.New(block.IncludePath);
 		var document = parser.ParseAsync(file, block.FrontMatter, default).GetAwaiter().GetResult();
 		var html = document.ToHtml(MarkdownParser.Pipeline);
@@ -240,7 +241,10 @@ private void WriteSettingsBlock(HtmlRenderer renderer, SettingsBlock block)
 		if (!block.Found || block.IncludePath is null)
 			return;
 
-		var parser = new MarkdownParser(block.DocumentationSourcePath, block.Build, block.GetDocumentationFile, block.Configuration);
+		var parser = new MarkdownParser(
+			block.DocumentationSourcePath, block.Build, block.GetDocumentationFile, block.Configuration
+			, block.LinksResolver
+		);
 
 		var file = block.FileSystem.FileInfo.New(block.IncludePath);
 

src/Elastic.Markdown/Myst/Directives/IncludeBlock.cs

@@ -2,6 +2,7 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 using System.IO.Abstractions;
+using Elastic.Markdown.CrossLinks;
 using Elastic.Markdown.Diagnostics;
 using Elastic.Markdown.IO;
 using Elastic.Markdown.IO.Configuration;
@@ -25,6 +26,8 @@ public class IncludeBlock(DirectiveBlockParser parser, ParserContext context) :
 
 	public ConfigurationFile Configuration { get; } = context.Configuration;
 
+	public ICrossLinkResolver LinksResolver { get; } = context.LinksResolver;
+
 	public IFileSystem FileSystem { get; } = context.Build.ReadFileSystem;
 
 	public IDirectoryInfo DocumentationSourcePath { get; } = context.Parser.SourcePath;
@@ -40,7 +43,6 @@ public class IncludeBlock(DirectiveBlockParser parser, ParserContext context) :
 	public string? Caption { get; private set; }
 	public string? Label { get; private set; }
 
-
 	//TODO add all options from
 	//https://mystmd.org/guide/directives#directive-include
 	public override void FinalizeAndValidate(ParserContext context)