Patch CrossLinkFetcher with Diagnostics Collector

src/Elastic.Markdown/IO/DocumentationSet.cs

@@ -10,6 +10,7 @@
 using Elastic.Documentation.Configuration;
 using Elastic.Documentation.Configuration.Builder;
 using Elastic.Documentation.Configuration.TableOfContents;
+using Elastic.Documentation.Diagnostics;
 using Elastic.Documentation.LinkIndex;
 using Elastic.Documentation.Links;
 using Elastic.Documentation.Site.Navigation;
@@ -135,6 +136,7 @@ public class DocumentationSet : INavigationLookups, IPositionalNavigation
 	public DocumentationSet(
 		BuildContext context,
 		ILoggerFactory logFactory,
+		IDiagnosticsCollector collector,
 		ICrossLinkResolver? linkResolver = null,
 		TableOfContentsTreeCollector? treeCollector = null
 	)
@@ -144,7 +146,7 @@ public DocumentationSet(
 		SourceDirectory = context.DocumentationSourceDirectory;
 		OutputDirectory = context.OutputDirectory;
 		LinkResolver =
-			linkResolver ?? new CrossLinkResolver(new ConfigurationCrossLinkFetcher(logFactory, context.Configuration, Aws3LinkIndexReader.CreateAnonymous()));
+			linkResolver ?? new CrossLinkResolver(new ConfigurationCrossLinkFetcher(logFactory, context.Configuration, Aws3LinkIndexReader.CreateAnonymous(), collector));
 		Configuration = context.Configuration;
 		EnabledExtensions = InstantiateExtensions();
 		treeCollector ??= new TableOfContentsTreeCollector();

src/Elastic.Markdown/Links/CrossLinks/ConfigurationCrossLinkFetcher.cs

@@ -4,13 +4,14 @@
 
 using System.Collections.Frozen;
 using Elastic.Documentation.Configuration.Builder;
+using Elastic.Documentation.Diagnostics;
 using Elastic.Documentation.LinkIndex;
 using Elastic.Documentation.Links;
 using Microsoft.Extensions.Logging;
 
 namespace Elastic.Markdown.Links.CrossLinks;
 
-public class ConfigurationCrossLinkFetcher(ILoggerFactory logFactory, ConfigurationFile configuration, ILinkIndexReader linkIndexProvider) : CrossLinkFetcher(logFactory, linkIndexProvider)
+public class ConfigurationCrossLinkFetcher(ILoggerFactory logFactory, ConfigurationFile configuration, ILinkIndexReader linkIndexProvider, IDiagnosticsCollector collector) : CrossLinkFetcher(logFactory, linkIndexProvider, collector)
 {
 	public override async Task<FetchedCrossLinks> Fetch(Cancel ctx)
 	{

src/Elastic.Markdown/Links/CrossLinks/CrossLinkFetcher.cs

@@ -6,6 +6,7 @@
 using System.Collections.Frozen;
 using System.Text.Json;
 using Elastic.Documentation.Configuration;
+using Elastic.Documentation.Diagnostics;
 using Elastic.Documentation.LinkIndex;
 using Elastic.Documentation.Links;
 using Elastic.Documentation.Serialization;
@@ -32,7 +33,7 @@ public record FetchedCrossLinks
 	};
 }
 
-public abstract class CrossLinkFetcher(ILoggerFactory logFactory, ILinkIndexReader linkIndexProvider) : IDisposable
+public abstract class CrossLinkFetcher(ILoggerFactory logFactory, ILinkIndexReader linkIndexProvider, IDiagnosticsCollector collector) : IDisposable
 {
 	private readonly ILogger _logger = logFactory.CreateLogger(nameof(CrossLinkFetcher));
 	private readonly HttpClient _client = new();
@@ -60,32 +61,40 @@ protected async Task<LinkRegistryEntry> GetLinkIndexEntry(string repository, Can
 	{
 		var linkIndex = await FetchLinkIndex(ctx);
 		if (!linkIndex.Repositories.TryGetValue(repository, out var repositoryLinks))
+		{
+			collector.EmitError(repository, $"Repository {repository} not found in link index");
 			throw new Exception($"Repository {repository} not found in link index");
+		}
+
 		return GetNextContentSourceLinkIndexEntry(repositoryLinks, repository);
 	}
 
-	protected static LinkRegistryEntry GetNextContentSourceLinkIndexEntry(IDictionary<string, LinkRegistryEntry> repositoryLinks, string repository)
+	protected LinkRegistryEntry GetNextContentSourceLinkIndexEntry(IDictionary<string, LinkRegistryEntry> repositoryLinks, string repository)
 	{
-		var linkIndexEntry =
-			(repositoryLinks.TryGetValue("main", out var link)
-				? link
-				: repositoryLinks.TryGetValue("master", out link) ? link : null)
-				?? throw new Exception($"Repository {repository} found in link index, but no main or master branch found");
-		return linkIndexEntry;
+		var linkIndexEntry = repositoryLinks.TryGetValue("main", out var link) ? link : repositoryLinks.TryGetValue("master", out link) ? link : null;
+		if (linkIndexEntry is not null)
+			return linkIndexEntry;
+
+		collector.EmitError(repository, $"Repository found in link index however neither 'main' nor 'master' branches found");
+		throw new Exception($"Repository {repository} found in link index, but no main or master branch found");
 	}
 
 	protected async Task<RepositoryLinks> Fetch(string repository, string[] keys, Cancel ctx)
 	{
 		var linkIndex = await FetchLinkIndex(ctx);
 		if (!linkIndex.Repositories.TryGetValue(repository, out var repositoryLinks))
+		{
+			collector.EmitError(repository, $"Repository {repository} not found in link index");
 			throw new Exception($"Repository {repository} not found in link index");
+		}
 
 		foreach (var key in keys)
 		{
 			if (repositoryLinks.TryGetValue(key, out var linkIndexEntry))
 				return await FetchLinkIndexEntry(repository, linkIndexEntry, ctx);
 		}
 
+		collector.EmitError(repository, $"Repository found in link index however none of: '{string.Join(", ", keys)}' branches found");
 		throw new Exception($"Repository found in link index however none of: '{string.Join(", ", keys)}' branches found");
 	}
 
@@ -97,7 +106,17 @@ protected async Task<RepositoryLinks> FetchLinkIndexEntry(string repository, Lin
 
 		var url = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/{linkRegistryEntry.Path}";
 		_logger.LogInformation("Fetching links.json for '{Repository}': {Url}", repository, url);
-		var json = await _client.GetStringAsync(url, ctx);
+		string json;
+		try
+		{
+			json = await _client.GetStringAsync(url, ctx);
+		}
+		catch (Exception e)
+		{
+			collector.EmitError(repository, $"An error occurred fetching links.json for '{repository}' from '{url}': {e.Message}");
+			throw new Exception($"An error occurred fetching links.json for '{repository}' from '{url}': {e.Message}", e);
+		}
+
 		linkReference = Deserialize(json);
 		WriteLinksJsonCachedFile(repository, linkRegistryEntry, json);
 		return linkReference;

src/Elastic.Markdown/Links/InboundLinks/LinkIndexCrossLinkFetcher.cs

@@ -3,14 +3,15 @@
 // See the LICENSE file in the project root for more information
 
 using System.Collections.Frozen;
+using Elastic.Documentation.Diagnostics;
 using Elastic.Documentation.LinkIndex;
 using Elastic.Documentation.Links;
 using Elastic.Markdown.Links.CrossLinks;
 using Microsoft.Extensions.Logging;
 
 namespace Elastic.Markdown.Links.InboundLinks;
 
-public class LinksIndexCrossLinkFetcher(ILoggerFactory logFactory, ILinkIndexReader linkIndexProvider) : CrossLinkFetcher(logFactory, linkIndexProvider)
+public class LinksIndexCrossLinkFetcher(ILoggerFactory logFactory, ILinkIndexReader linkIndexProvider, IDiagnosticsCollector collector) : CrossLinkFetcher(logFactory, linkIndexProvider, collector)
 {
 	public override async Task<FetchedCrossLinks> Fetch(Cancel ctx)
 	{

src/Elastic.Markdown/Links/InboundLinks/LinkIndexLinkChecker.cs

@@ -25,7 +25,7 @@ private sealed record RepositoryFilter
 
 	public async Task CheckAll(IDiagnosticsCollector collector, Cancel ctx)
 	{
-		var fetcher = new LinksIndexCrossLinkFetcher(logFactory, _linkIndexProvider);
+		var fetcher = new LinksIndexCrossLinkFetcher(logFactory, _linkIndexProvider, collector);
 		var resolver = new CrossLinkResolver(fetcher);
 		var crossLinks = await resolver.FetchLinks(ctx);
 
@@ -34,7 +34,7 @@ public async Task CheckAll(IDiagnosticsCollector collector, Cancel ctx)
 
 	public async Task CheckRepository(IDiagnosticsCollector collector, string? toRepository, string? fromRepository, Cancel ctx)
 	{
-		var fetcher = new LinksIndexCrossLinkFetcher(logFactory, _linkIndexProvider);
+		var fetcher = new LinksIndexCrossLinkFetcher(logFactory, _linkIndexProvider, collector);
 		var resolver = new CrossLinkResolver(fetcher);
 		var crossLinks = await resolver.FetchLinks(ctx);
 		var filter = new RepositoryFilter
@@ -48,7 +48,7 @@ public async Task CheckRepository(IDiagnosticsCollector collector, string? toRep
 
 	public async Task CheckWithLocalLinksJson(IDiagnosticsCollector collector, string repository, string localLinksJson, Cancel ctx)
 	{
-		var fetcher = new LinksIndexCrossLinkFetcher(logFactory, _linkIndexProvider);
+		var fetcher = new LinksIndexCrossLinkFetcher(logFactory, _linkIndexProvider, collector);
 		var resolver = new CrossLinkResolver(fetcher);
 		// ReSharper disable once RedundantAssignment
 		var crossLinks = await resolver.FetchLinks(ctx);

src/tooling/docs-assembler/AssembleSources.cs

@@ -80,7 +80,7 @@ IReadOnlySet<Exporter> availableExporters
 		HistoryMappings = GetLegacyUrlMappings(assembleContext);
 		var linkIndexProvider = Aws3LinkIndexReader.CreateAnonymous();
 
-		var crossLinkFetcher = new AssemblerCrossLinkFetcher(logFactory, assembleContext.Configuration, assembleContext.Environment, linkIndexProvider);
+		var crossLinkFetcher = new AssemblerCrossLinkFetcher(logFactory, assembleContext.Configuration, assembleContext.Environment, linkIndexProvider, assembleContext.Collector);
 		UriResolver = new PublishEnvironmentUriResolver(NavigationTocMappings, assembleContext.Environment);
 
 		var crossLinkResolver = new CrossLinkResolver(crossLinkFetcher, UriResolver);

src/tooling/docs-assembler/Building/AssemblerCrossLinkFetcher.cs

@@ -4,15 +4,16 @@
 
 using System.Collections.Frozen;
 using Elastic.Documentation.Configuration.Assembler;
+using Elastic.Documentation.Diagnostics;
 using Elastic.Documentation.LinkIndex;
 using Elastic.Documentation.Links;
 using Elastic.Markdown.Links.CrossLinks;
 using Microsoft.Extensions.Logging;
 
 namespace Documentation.Assembler.Building;
 
-public class AssemblerCrossLinkFetcher(ILoggerFactory logFactory, AssemblyConfiguration configuration, PublishEnvironment publishEnvironment, ILinkIndexReader linkIndexProvider)
-	: CrossLinkFetcher(logFactory, linkIndexProvider)
+public class AssemblerCrossLinkFetcher(ILoggerFactory logFactory, AssemblyConfiguration configuration, PublishEnvironment publishEnvironment, ILinkIndexReader linkIndexProvider, IDiagnosticsCollector collector)
+	: CrossLinkFetcher(logFactory, linkIndexProvider, collector)
 {
 	public override async Task<FetchedCrossLinks> Fetch(Cancel ctx)
 	{

src/tooling/docs-assembler/Cli/ContentSourceCommands.cs

@@ -35,7 +35,7 @@ public async Task<int> Validate(Cancel ctx = default)
 		var fs = new FileSystem();
 		var context = new AssembleContext(configuration, configurationContext, "dev", collector, fs, fs, null, null);
 		ILinkIndexReader linkIndexReader = Aws3LinkIndexReader.CreateAnonymous();
-		var fetcher = new AssemblerCrossLinkFetcher(logFactory, context.Configuration, context.Environment, linkIndexReader);
+		var fetcher = new AssemblerCrossLinkFetcher(logFactory, context.Configuration, context.Environment, linkIndexReader, collector);
 		var links = await fetcher.FetchLinkIndex(ctx);
 		var repositories = context.Configuration.AvailableRepositories;
 

src/tooling/docs-assembler/Cli/RepositoryCommands.cs

@@ -239,7 +239,7 @@ await Parallel.ForEachAsync(repositories,
 						checkout.Directory.FullName,
 						outputPath
 					);
-					var set = new DocumentationSet(context, logFactory);
+					var set = new DocumentationSet(context, logFactory, collector);
 					var generator = new DocumentationGenerator(set, logFactory, null, null, null);
 					_ = await generator.GenerateAll(c);
 

src/tooling/docs-assembler/Links/NavigationPrefixChecker.cs

@@ -82,7 +82,7 @@ public async Task CheckAllPublishedLinks(IDiagnosticsCollector collector, Cancel
 	private async Task FetchAndValidateCrossLinks(IDiagnosticsCollector collector, string? updateRepository, RepositoryLinks? updateReference, Cancel ctx)
 	{
 		var linkIndexProvider = Aws3LinkIndexReader.CreateAnonymous();
-		var fetcher = new LinksIndexCrossLinkFetcher(_logFactoryFactory, linkIndexProvider);
+		var fetcher = new LinksIndexCrossLinkFetcher(_logFactoryFactory, linkIndexProvider, collector);
 		var resolver = new CrossLinkResolver(fetcher);
 		var crossLinks = await resolver.FetchLinks(ctx);
 		var dictionary = new Dictionary<string, SeenPaths>();

src/tooling/docs-assembler/Navigation/AssemblerDocumentationSet.cs

@@ -79,6 +79,6 @@ IReadOnlySet<Exporter> availableExporters
 		};
 		BuildContext = buildContext;
 
-		DocumentationSet = new DocumentationSet(buildContext, logFactory, crossLinkResolver, treeCollector);
+		DocumentationSet = new DocumentationSet(buildContext, logFactory, context.Collector, crossLinkResolver, treeCollector);
 	}
 }

src/tooling/docs-builder/Cli/Commands.cs

@@ -153,7 +153,7 @@ public async Task<int> Generate(
 			await githubActionsService.SetOutputAsync("skip", "false");
 
 		// always delete output folder on CI
-		var set = new DocumentationSet(context, logFactory);
+		var set = new DocumentationSet(context, logFactory, collector);
 		if (runningOnCi)
 			set.ClearOutputDirectory();
 
@@ -228,7 +228,7 @@ public async Task<int> Move(
 		var fileSystem = new FileSystem();
 		await using var collector = new ConsoleDiagnosticsCollector(logFactory, null).StartAsync(ctx);
 		var context = new BuildContext(collector, fileSystem, fileSystem, configurationContext, ExportOptions.MetadataOnly, path, null);
-		var set = new DocumentationSet(context, logFactory);
+		var set = new DocumentationSet(context, logFactory, collector);
 
 		var moveCommand = new Move(logFactory, fileSystem, fileSystem, set);
 		var result = await moveCommand.Execute(source, target, dryRun ?? false, ctx);

src/tooling/docs-builder/Http/ReloadableGeneratorState.cs

@@ -25,14 +25,14 @@ public class ReloadableGeneratorState(
 	private IDirectoryInfo OutputPath { get; } = outputPath;
 	public IDirectoryInfo ApiPath { get; } = context.WriteFileSystem.DirectoryInfo.New(Path.Combine(outputPath.FullName, "api"));
 
-	private DocumentationGenerator _generator = new(new DocumentationSet(context, logFactory), logFactory);
+	private DocumentationGenerator _generator = new(new DocumentationSet(context, logFactory, context.Collector), logFactory);
 	public DocumentationGenerator Generator => _generator;
 
 	public async Task ReloadAsync(Cancel ctx)
 	{
 		SourcePath.Refresh();
 		OutputPath.Refresh();
-		var docSet = new DocumentationSet(context, logFactory);
+		var docSet = new DocumentationSet(context, logFactory, context.Collector);
 		_ = await docSet.LinkResolver.FetchLinks(ctx);
 
 		// Add LLM markdown export for dev server

tests/Elastic.Markdown.Tests/Directives/DirectiveBaseTests.cs

@@ -73,7 +73,7 @@ protected DirectiveTest(ITestOutputHelper output, [LanguageInjection("markdown")
 		var configurationContext = TestHelpers.CreateConfigurationContext(FileSystem);
 		var context = new BuildContext(Collector, FileSystem, configurationContext);
 		var linkResolver = new TestCrossLinkResolver();
-		Set = new DocumentationSet(context, logger, linkResolver);
+		Set = new DocumentationSet(context, logger, Collector, linkResolver);
 		File = Set.DocumentationFileLookup(FileSystem.FileInfo.New("docs/index.md")) as MarkdownFile ?? throw new NullReferenceException();
 		Html = default!; //assigned later
 		Document = default!;

tests/Elastic.Markdown.Tests/DocSet/NavigationTestsBase.cs

@@ -33,7 +33,7 @@ protected NavigationTestsBase(ITestOutputHelper output)
 		};
 
 		var linkResolver = new TestCrossLinkResolver();
-		Set = new DocumentationSet(context, LoggerFactory, linkResolver);
+		Set = new DocumentationSet(context, LoggerFactory, collector, linkResolver);
 
 		Set.Files.Should().HaveCountGreaterThan(10);
 		Generator = new DocumentationGenerator(Set, LoggerFactory);

tests/Elastic.Markdown.Tests/Inline/InlneBaseTests.cs

@@ -120,7 +120,7 @@ protected InlineTest(
 			UrlPathPrefix = "/docs"
 		};
 		var linkResolver = new TestCrossLinkResolver();
-		Set = new DocumentationSet(context, logger, linkResolver);
+		Set = new DocumentationSet(context, logger, Collector, linkResolver);
 		File = Set.DocumentationFileLookup(FileSystem.FileInfo.New("docs/index.md")) as MarkdownFile ?? throw new NullReferenceException();
 		Html = default!; //assigned later
 		Document = default!;

tests/Elastic.Markdown.Tests/OutputDirectoryTests.cs

@@ -36,7 +36,7 @@ public async Task CreatesDefaultOutputDirectory()
 		var configurationContext = TestHelpers.CreateConfigurationContext(fileSystem);
 		var context = new BuildContext(collector, fileSystem, configurationContext);
 		var linkResolver = new TestCrossLinkResolver();
-		var set = new DocumentationSet(context, logger, linkResolver);
+		var set = new DocumentationSet(context, logger, collector, linkResolver);
 		var generator = new DocumentationGenerator(set, logger);
 
 		await generator.GenerateAll(TestContext.Current.CancellationToken);

tests/authoring/Framework/Setup.fs

@@ -226,7 +226,7 @@ type Setup =
         let logger = new TestLoggerFactory()
         let conversionCollector = TestConversionCollector()
         let linkResolver = TestCrossLinkResolver(context.Configuration)
-        let set = DocumentationSet(context, logger, linkResolver)
+        let set = DocumentationSet(context, logger, collector, linkResolver)
 
 
         let generator = DocumentationGenerator(set, logger, null, null, null, conversionCollector)