diff --git a/docs-builder.sln.DotSettings b/docs-builder.sln.DotSettings index 2cc897060..571a4f3ad 100644 --- a/docs-builder.sln.DotSettings +++ b/docs-builder.sln.DotSettings @@ -1,4 +1,5 @@  + True True True True diff --git a/src/Elastic.Documentation.Configuration/Builder/TableOfContentsConfiguration.cs b/src/Elastic.Documentation.Configuration/Builder/TableOfContentsConfiguration.cs index 797c40242..346734fbe 100644 --- a/src/Elastic.Documentation.Configuration/Builder/TableOfContentsConfiguration.cs +++ b/src/Elastic.Documentation.Configuration/Builder/TableOfContentsConfiguration.cs @@ -233,7 +233,10 @@ file is null && crossLink is null && folder is null && toc is null && if (crossLink is not null) { - return [new CrossLinkReference(this, crossLink, title, hiddenFile, children ?? [])]; + if (Uri.TryCreate(crossLink, UriKind.Absolute, out var crossUri) && CrossLinkValidator.IsCrossLink(crossUri)) + return [new CrossLinkReference(this, crossUri, title, hiddenFile, children ?? [])]; + else + reader.EmitError($"Cross-link '{crossLink}' is not a valid absolute URI format", tocEntry); } if (folder is not null) diff --git a/src/Elastic.Documentation.Configuration/TableOfContents/ITocItem.cs b/src/Elastic.Documentation.Configuration/TableOfContents/ITocItem.cs index 29ea93ca4..8303dace5 100644 --- a/src/Elastic.Documentation.Configuration/TableOfContents/ITocItem.cs +++ b/src/Elastic.Documentation.Configuration/TableOfContents/ITocItem.cs @@ -14,7 +14,7 @@ public interface ITocItem public record FileReference(ITableOfContentsScope TableOfContentsScope, string RelativePath, bool Hidden, IReadOnlyCollection Children) : ITocItem; -public record CrossLinkReference(ITableOfContentsScope TableOfContentsScope, string CrossLinkUri, string? Title, bool Hidden, IReadOnlyCollection Children) +public record CrossLinkReference(ITableOfContentsScope TableOfContentsScope, Uri CrossLinkUri, string? Title, bool Hidden, IReadOnlyCollection Children) : ITocItem; public record FolderReference(ITableOfContentsScope TableOfContentsScope, string RelativePath, IReadOnlyCollection Children) diff --git a/src/Elastic.Markdown/DocumentationGenerator.cs b/src/Elastic.Markdown/DocumentationGenerator.cs index bba15ee58..df6275aff 100644 --- a/src/Elastic.Markdown/DocumentationGenerator.cs +++ b/src/Elastic.Markdown/DocumentationGenerator.cs @@ -48,7 +48,7 @@ public class DocumentationGenerator public DocumentationSet DocumentationSet { get; } public BuildContext Context { get; } - public ICrossLinkResolver Resolver { get; } + public ICrossLinkResolver CrossLinkResolver { get; } public IMarkdownStringRenderer MarkdownStringRenderer => HtmlWriter; public DocumentationGenerator( @@ -70,7 +70,7 @@ public DocumentationGenerator( DocumentationSet = docSet; Context = docSet.Context; - Resolver = docSet.LinkResolver; + CrossLinkResolver = docSet.CrossLinkResolver; HtmlWriter = new HtmlWriter(DocumentationSet, _writeFileSystem, new DescriptionGenerator(), navigationHtmlWriter, legacyUrlMapper, positionalNavigation); _documentationFileExporter = @@ -120,9 +120,6 @@ public async Task GenerateAll(Cancel ctx) if (CompilationNotNeeded(generationState, out var offendingFiles, out var outputSeenChanges)) return result; - _logger.LogInformation($"Fetching external links"); - _ = await Resolver.FetchLinks(ctx); - await ResolveDirectoryTree(ctx); await ProcessDocumentationFiles(offendingFiles, outputSeenChanges, ctx); diff --git a/src/Elastic.Markdown/IO/DocumentationSet.cs b/src/Elastic.Markdown/IO/DocumentationSet.cs index 42a9f17c4..e955546c9 100644 --- a/src/Elastic.Markdown/IO/DocumentationSet.cs +++ b/src/Elastic.Markdown/IO/DocumentationSet.cs @@ -10,7 +10,6 @@ using Elastic.Documentation.Configuration; using Elastic.Documentation.Configuration.Builder; using Elastic.Documentation.Configuration.TableOfContents; -using Elastic.Documentation.LinkIndex; using Elastic.Documentation.Links; using Elastic.Documentation.Site.Navigation; using Elastic.Markdown.Extensions; @@ -28,6 +27,7 @@ public interface INavigationLookups IReadOnlyCollection TableOfContents { get; } IReadOnlyCollection EnabledExtensions { get; } FrozenDictionary FilesGroupedByFolder { get; } + ICrossLinkResolver CrossLinkResolver { get; } } public interface IPositionalNavigation @@ -94,10 +94,12 @@ public record NavigationLookups : INavigationLookups public required IReadOnlyCollection TableOfContents { get; init; } public required IReadOnlyCollection EnabledExtensions { get; init; } public required FrozenDictionary FilesGroupedByFolder { get; init; } + public required ICrossLinkResolver CrossLinkResolver { get; init; } } public class DocumentationSet : INavigationLookups, IPositionalNavigation { + private readonly ILogger _logger; public BuildContext Context { get; } public string Name { get; } public IFileInfo OutputStateFile { get; } @@ -112,7 +114,7 @@ public class DocumentationSet : INavigationLookups, IPositionalNavigation public MarkdownParser MarkdownParser { get; } - public ICrossLinkResolver LinkResolver { get; } + public ICrossLinkResolver CrossLinkResolver { get; } public TableOfContentsTree Tree { get; } @@ -135,23 +137,23 @@ public class DocumentationSet : INavigationLookups, IPositionalNavigation public DocumentationSet( BuildContext context, ILoggerFactory logFactory, - ICrossLinkResolver? linkResolver = null, + ICrossLinkResolver linkResolver, TableOfContentsTreeCollector? treeCollector = null ) { + _logger = logFactory.CreateLogger(); Context = context; Source = ContentSourceMoniker.Create(context.Git.RepositoryName, null); SourceDirectory = context.DocumentationSourceDirectory; OutputDirectory = context.OutputDirectory; - LinkResolver = - linkResolver ?? new CrossLinkResolver(new ConfigurationCrossLinkFetcher(logFactory, context.Configuration, Aws3LinkIndexReader.CreateAnonymous())); + CrossLinkResolver = linkResolver; Configuration = context.Configuration; EnabledExtensions = InstantiateExtensions(); treeCollector ??= new TableOfContentsTreeCollector(); var resolver = new ParserResolvers { - CrossLinkResolver = LinkResolver, + CrossLinkResolver = CrossLinkResolver, DocumentationFileLookup = DocumentationFileLookup }; MarkdownParser = new MarkdownParser(context, resolver); @@ -184,7 +186,8 @@ public DocumentationSet( FlatMappedFiles = FlatMappedFiles, TableOfContents = Configuration.TableOfContents, EnabledExtensions = EnabledExtensions, - FilesGroupedByFolder = FilesGroupedByFolder + FilesGroupedByFolder = FilesGroupedByFolder, + CrossLinkResolver = CrossLinkResolver }; Tree = new TableOfContentsTree(Source, Context, lookups, treeCollector, ref fileIndex); @@ -232,7 +235,7 @@ private void UpdateNavigationIndex(IReadOnlyCollection navigati UpdateNavigationIndex(documentationGroup.NavigationItems, ref navigationIndex); break; default: - Context.EmitError(Context.ConfigurationPath, $"Unhandled navigation item type: {item.GetType()}"); + Context.EmitError(Context.ConfigurationPath, $"{nameof(DocumentationSet)}.{nameof(UpdateNavigationIndex)}: Unhandled navigation item type: {item.GetType()}"); break; } } @@ -374,26 +377,7 @@ void ValidateExists(string from, string to, IReadOnlyDictionary return FlatMappedFiles.GetValueOrDefault(relativePath); } - public async Task ResolveDirectoryTree(Cancel ctx) - { - await Tree.Resolve(ctx); - - // Validate cross-repo links in navigation - try - { - await NavigationCrossLinkValidator.ValidateNavigationCrossLinksAsync( - Tree, - LinkResolver, - (msg) => Context.EmitError(Context.ConfigurationPath, msg), - ctx - ); - } - catch (Exception e) - { - // Log the error but don't fail the build - Context.EmitError(Context.ConfigurationPath, $"Error validating cross-links in navigation: {e.Message}"); - } - } + public async Task ResolveDirectoryTree(Cancel ctx) => await Tree.Resolve(ctx); private DocumentationFile CreateMarkDownFile(IFileInfo file, BuildContext context) { @@ -476,6 +460,7 @@ public RepositoryLinks CreateLinkReference() public void ClearOutputDirectory() { + _logger.LogInformation("Clearing output directory {OutputDirectory}", OutputDirectory.Name); if (OutputDirectory.Exists) OutputDirectory.Delete(true); OutputDirectory.Create(); diff --git a/src/Elastic.Markdown/IO/MarkdownFile.cs b/src/Elastic.Markdown/IO/MarkdownFile.cs index b5b738d56..b288d025f 100644 --- a/src/Elastic.Markdown/IO/MarkdownFile.cs +++ b/src/Elastic.Markdown/IO/MarkdownFile.cs @@ -121,13 +121,13 @@ public string Url { if (_url is not null) return _url; - if (_set.LinkResolver.UriResolver is IsolatedBuildEnvironmentUriResolver) + if (_set.CrossLinkResolver.UriResolver is IsolatedBuildEnvironmentUriResolver) { _url = DefaultUrlPath; return _url; } var crossLink = new Uri(CrossLink); - var uri = _set.LinkResolver.UriResolver.Resolve(crossLink, DefaultUrlPathSuffix); + var uri = _set.CrossLinkResolver.UriResolver.Resolve(crossLink, DefaultUrlPathSuffix); _url = uri.AbsolutePath; return _url; diff --git a/src/Elastic.Markdown/IO/Navigation/CrossLinkNavigationItem.cs b/src/Elastic.Markdown/IO/Navigation/CrossLinkNavigationItem.cs index 9168372c4..6dd24c352 100644 --- a/src/Elastic.Markdown/IO/Navigation/CrossLinkNavigationItem.cs +++ b/src/Elastic.Markdown/IO/Navigation/CrossLinkNavigationItem.cs @@ -11,11 +11,10 @@ namespace Elastic.Markdown.IO.Navigation; [DebuggerDisplay("CrossLink: {Url}")] public record CrossLinkNavigationItem : ILeafNavigationItem { - // Override Url accessor to use ResolvedUrl if available - string INavigationItem.Url => ResolvedUrl ?? Url; - public CrossLinkNavigationItem(string url, string title, DocumentationGroup group, bool hidden = false) + public CrossLinkNavigationItem(Uri crossLinkUri, Uri resolvedUrl, string title, DocumentationGroup group, bool hidden = false) { - _url = url; + CrossLink = crossLinkUri; + Url = resolvedUrl.ToString(); NavigationTitle = title; Parent = group; NavigationRoot = group.NavigationRoot; @@ -24,14 +23,10 @@ public CrossLinkNavigationItem(string url, string title, DocumentationGroup grou public INodeNavigationItem? Parent { get; set; } public IRootNavigationItem NavigationRoot { get; } - // Original URL from the cross-link - private readonly string _url; - // Store resolved URL for rendering - public string? ResolvedUrl { get; set; } - - // Implement the INavigationItem.Url property to use ResolvedUrl if available - public string Url => ResolvedUrl ?? _url; public string NavigationTitle { get; } + public Uri CrossLink { get; } + public string Url { get; } + public string NavigationTitle { get; } public int NavigationIndex { get; set; } public bool Hidden { get; } public bool IsCrossLink => true; // This is always a cross-link diff --git a/src/Elastic.Markdown/IO/Navigation/DocumentationGroup.cs b/src/Elastic.Markdown/IO/Navigation/DocumentationGroup.cs index eb208382e..fd121a2c5 100644 --- a/src/Elastic.Markdown/IO/Navigation/DocumentationGroup.cs +++ b/src/Elastic.Markdown/IO/Navigation/DocumentationGroup.cs @@ -7,7 +7,6 @@ using Elastic.Documentation.Configuration; using Elastic.Documentation.Configuration.TableOfContents; using Elastic.Documentation.Extensions; -using Elastic.Documentation.Links; using Elastic.Documentation.Site.Navigation; namespace Elastic.Markdown.IO.Navigation; @@ -124,13 +123,6 @@ void AddToNavigationItems(INavigationItem item, ref int fileIndex) { if (tocItem is CrossLinkReference crossLink) { - // Validate crosslink URI and title - if (!CrossLinkValidator.IsValidCrossLink(crossLink.CrossLinkUri, out var errorMessage)) - { - context.EmitError(context.ConfigurationPath, errorMessage!); - continue; - } - // Validate that cross-link has a title if (string.IsNullOrWhiteSpace(crossLink.Title)) { @@ -139,9 +131,13 @@ void AddToNavigationItems(INavigationItem item, ref int fileIndex) continue; } + if (!lookups.CrossLinkResolver.TryResolve(msg => context.EmitError(context.ConfigurationPath, msg), crossLink.CrossLinkUri, out var resolvedUrl)) + continue; // the crosslink resolver will emit an error already + // Create a special navigation item for cross-repository links - var crossLinkItem = new CrossLinkNavigationItem(crossLink.CrossLinkUri, crossLink.Title, this, crossLink.Hidden); + var crossLinkItem = new CrossLinkNavigationItem(crossLink.CrossLinkUri, resolvedUrl, crossLink.Title, this, crossLink.Hidden); AddToNavigationItems(crossLinkItem, ref fileIndex); + } else if (tocItem is FileReference file) { diff --git a/src/Elastic.Markdown/IO/Navigation/NavigationCrossLinkValidator.cs b/src/Elastic.Markdown/IO/Navigation/NavigationCrossLinkValidator.cs deleted file mode 100644 index 717472576..000000000 --- a/src/Elastic.Markdown/IO/Navigation/NavigationCrossLinkValidator.cs +++ /dev/null @@ -1,86 +0,0 @@ -// Licensed to Elasticsearch B.V under one or more agreements. -// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. -// See the LICENSE file in the project root for more information - -using System; -using System.Collections.Generic; -using System.Threading.Tasks; -using Elastic.Documentation.Links; -using Elastic.Documentation.Site.Navigation; -using Elastic.Markdown.Links.CrossLinks; - -namespace Elastic.Markdown.IO.Navigation; - -public static class NavigationCrossLinkValidator -{ - public static async Task ValidateNavigationCrossLinksAsync( - INavigationItem root, - ICrossLinkResolver crossLinkResolver, - Action errorEmitter, - Cancel ctx = default) - { - // Ensure cross-links are fetched before validation - _ = await crossLinkResolver.FetchLinks(ctx); - // Collect all navigation items that contain cross-repo links - var itemsWithCrossLinks = FindNavigationItemsWithCrossLinks(root); - - foreach (var item in itemsWithCrossLinks) - { - if (item is CrossLinkNavigationItem crossLinkItem) - { - var url = crossLinkItem.Url; - if (url != null && Uri.TryCreate(url, UriKind.Absolute, out var crossUri) && - CrossLinkValidator.IsCrossLink(crossUri)) - { - // Try to resolve the cross-link URL - if (crossLinkResolver.TryResolve(errorEmitter, crossUri, out var resolvedUri)) - { - // If resolved successfully, set the resolved URL - crossLinkItem.ResolvedUrl = resolvedUri.ToString(); - } - else - { - // Error already emitted by CrossLinkResolver - // But we won't fail the build - just display the original URL - } - } - } - else if (item is FileNavigationItem fileItem && - fileItem.Url != null && - Uri.TryCreate(fileItem.Url, UriKind.Absolute, out var fileUri) && - CrossLinkValidator.IsCrossLink(fileUri)) - { - // Cross-link URL detected in a FileNavigationItem, but we're not validating it yet - } - } - - return; - } - - private static List FindNavigationItemsWithCrossLinks(INavigationItem item) - { - var results = new List(); - - // Check if this item has a cross-link - if (item is CrossLinkNavigationItem crossLinkItem) - { - var url = crossLinkItem.Url; - if (url != null && - Uri.TryCreate(url, UriKind.Absolute, out var uri) && - CrossLinkValidator.IsCrossLink(uri)) - { - results.Add(item); - } - } - // Recursively check children if this is a container - if (item is INodeNavigationItem containerItem) - { - foreach (var child in containerItem.NavigationItems) - { - results.AddRange(FindNavigationItemsWithCrossLinks(child)); - } - } - - return results; - } -} diff --git a/src/Elastic.Markdown/Links/CrossLinks/CrossLinkFetcher.cs b/src/Elastic.Markdown/Links/CrossLinks/CrossLinkFetcher.cs index 4392a6480..7b0af06c8 100644 --- a/src/Elastic.Markdown/Links/CrossLinks/CrossLinkFetcher.cs +++ b/src/Elastic.Markdown/Links/CrossLinks/CrossLinkFetcher.cs @@ -19,46 +19,43 @@ public record FetchedCrossLinks public required HashSet DeclaredRepositories { get; init; } - public required bool FromConfiguration { get; init; } - public required FrozenDictionary LinkIndexEntries { get; init; } public static FetchedCrossLinks Empty { get; } = new() { DeclaredRepositories = [], LinkReferences = new Dictionary().ToFrozenDictionary(), - FromConfiguration = false, LinkIndexEntries = new Dictionary().ToFrozenDictionary() }; } public abstract class CrossLinkFetcher(ILoggerFactory logFactory, ILinkIndexReader linkIndexProvider) : IDisposable { - private readonly ILogger _logger = logFactory.CreateLogger(nameof(CrossLinkFetcher)); + protected ILogger Logger { get; } = logFactory.CreateLogger(nameof(CrossLinkFetcher)); private readonly HttpClient _client = new(); private LinkRegistry? _linkIndex; public static RepositoryLinks Deserialize(string json) => JsonSerializer.Deserialize(json, SourceGenerationContext.Default.RepositoryLinks)!; - public abstract Task Fetch(Cancel ctx); + public abstract Task FetchCrossLinks(Cancel ctx); - public async Task FetchLinkIndex(Cancel ctx) + public async Task FetchLinkRegistry(Cancel ctx) { if (_linkIndex is not null) { - _logger.LogTrace("Using cached link index"); + Logger.LogTrace("Using cached link index registry (link-index.json)"); return _linkIndex; } - _logger.LogInformation("Getting link index"); + Logger.LogInformation("Fetching link index registry (link-index.json)"); _linkIndex = await linkIndexProvider.GetRegistry(ctx); return _linkIndex; } protected async Task GetLinkIndexEntry(string repository, Cancel ctx) { - var linkIndex = await FetchLinkIndex(ctx); + var linkIndex = await FetchLinkRegistry(ctx); if (!linkIndex.Repositories.TryGetValue(repository, out var repositoryLinks)) throw new Exception($"Repository {repository} not found in link index"); return GetNextContentSourceLinkIndexEntry(repositoryLinks, repository); @@ -74,9 +71,9 @@ protected static LinkRegistryEntry GetNextContentSourceLinkIndexEntry(IDictionar return linkIndexEntry; } - protected async Task Fetch(string repository, string[] keys, Cancel ctx) + protected async Task FetchCrossLinks(string repository, string[] keys, Cancel ctx) { - var linkIndex = await FetchLinkIndex(ctx); + var linkIndex = await FetchLinkRegistry(ctx); if (!linkIndex.Repositories.TryGetValue(repository, out var repositoryLinks)) throw new Exception($"Repository {repository} not found in link index"); @@ -91,12 +88,15 @@ protected async Task Fetch(string repository, string[] keys, Ca protected async Task FetchLinkIndexEntry(string repository, LinkRegistryEntry linkRegistryEntry, Cancel ctx) { + var url = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/{linkRegistryEntry.Path}"; var linkReference = await TryGetCachedLinkReference(repository, linkRegistryEntry); if (linkReference is not null) + { + Logger.LogInformation("Using locally cached links.json for '{Repository}': {Url}", repository, url); return linkReference; + } - var url = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/{linkRegistryEntry.Path}"; - _logger.LogInformation("Fetching links.json for '{Repository}': {Url}", repository, url); + Logger.LogInformation("Fetching links.json for '{Repository}': {Url}", repository, url); var json = await _client.GetStringAsync(url, ctx); linkReference = Deserialize(json); WriteLinksJsonCachedFile(repository, linkRegistryEntry, json); @@ -116,7 +116,7 @@ private void WriteLinksJsonCachedFile(string repository, LinkRegistryEntry linkR } catch (Exception e) { - _logger.LogError(e, "Failed to write cached link reference {CachedPath}", cachedPath); + Logger.LogError(e, "Failed to write cached link reference {CachedPath}", cachedPath); } } @@ -140,7 +140,7 @@ private void WriteLinksJsonCachedFile(string repository, LinkRegistryEntry linkR } catch (Exception e) { - _logger.LogError(e, "Failed to read cached link reference {CachedPath}", cachedPath); + Logger.LogError(e, "Failed to read cached link reference {CachedPath}", cachedPath); return null; } } diff --git a/src/Elastic.Markdown/Links/CrossLinks/CrossLinkResolver.cs b/src/Elastic.Markdown/Links/CrossLinks/CrossLinkResolver.cs index 72de952cb..eacafcbdf 100644 --- a/src/Elastic.Markdown/Links/CrossLinks/CrossLinkResolver.cs +++ b/src/Elastic.Markdown/Links/CrossLinks/CrossLinkResolver.cs @@ -10,22 +10,33 @@ namespace Elastic.Markdown.Links.CrossLinks; public interface ICrossLinkResolver { - Task FetchLinks(Cancel ctx); bool TryResolve(Action errorEmitter, Uri crossLinkUri, [NotNullWhen(true)] out Uri? resolvedUri); IUriEnvironmentResolver UriResolver { get; } } -public class CrossLinkResolver(CrossLinkFetcher fetcher, IUriEnvironmentResolver? uriResolver = null) : ICrossLinkResolver +public class NoopCrossLinkResolver : ICrossLinkResolver { - private FetchedCrossLinks _crossLinks = FetchedCrossLinks.Empty; - public IUriEnvironmentResolver UriResolver { get; } = uriResolver ?? new IsolatedBuildEnvironmentUriResolver(); + public static NoopCrossLinkResolver Instance { get; } = new(); - public async Task FetchLinks(Cancel ctx) + /// + public bool TryResolve(Action errorEmitter, Uri crossLinkUri, [NotNullWhen(true)] out Uri? resolvedUri) { - _crossLinks = await fetcher.Fetch(ctx); - return _crossLinks; + resolvedUri = null; + return false; } + /// + public IUriEnvironmentResolver UriResolver { get; } = new IsolatedBuildEnvironmentUriResolver(); + + private NoopCrossLinkResolver() { } + +} + +public class CrossLinkResolver(FetchedCrossLinks crossLinks, IUriEnvironmentResolver? uriResolver = null) : ICrossLinkResolver +{ + private FetchedCrossLinks _crossLinks = crossLinks; + public IUriEnvironmentResolver UriResolver { get; } = uriResolver ?? new IsolatedBuildEnvironmentUriResolver(); + public bool TryResolve(Action errorEmitter, Uri crossLinkUri, [NotNullWhen(true)] out Uri? resolvedUri) => TryResolve(errorEmitter, _crossLinks, UriResolver, crossLinkUri, out resolvedUri); @@ -50,7 +61,7 @@ public static bool TryResolve( { resolvedUri = null; - // First check if the repository is in the declared repositories list, even if it's not in the link references + // First, check if the repository is in the declared repositories list, even if it's not in the link references var isDeclaredRepo = fetchedCrossLinks.DeclaredRepositories.Contains(crossLinkUri.Scheme); if (!fetchedCrossLinks.LinkReferences.TryGetValue(crossLinkUri.Scheme, out var sourceLinkReference)) diff --git a/src/Elastic.Markdown/Links/CrossLinks/ConfigurationCrossLinkFetcher.cs b/src/Elastic.Markdown/Links/CrossLinks/DocSetConfigurationCrossLinkFetcher.cs similarity index 74% rename from src/Elastic.Markdown/Links/CrossLinks/ConfigurationCrossLinkFetcher.cs rename to src/Elastic.Markdown/Links/CrossLinks/DocSetConfigurationCrossLinkFetcher.cs index 4ef1ee4b1..5c0c50372 100644 --- a/src/Elastic.Markdown/Links/CrossLinks/ConfigurationCrossLinkFetcher.cs +++ b/src/Elastic.Markdown/Links/CrossLinks/DocSetConfigurationCrossLinkFetcher.cs @@ -11,12 +11,15 @@ namespace Elastic.Markdown.Links.CrossLinks; -public class ConfigurationCrossLinkFetcher(ILoggerFactory logFactory, ConfigurationFile configuration, ILinkIndexReader linkIndexProvider) : CrossLinkFetcher(logFactory, linkIndexProvider) +/// Fetches cross-links from all the declared repositories in the docset.yml configuration see +public class DocSetConfigurationCrossLinkFetcher(ILoggerFactory logFactory, ConfigurationFile configuration, ILinkIndexReader? linkIndexProvider = null) + : CrossLinkFetcher(logFactory, linkIndexProvider ?? Aws3LinkIndexReader.CreateAnonymous()) { - private readonly ILogger _logger = logFactory.CreateLogger(nameof(ConfigurationCrossLinkFetcher)); + private readonly ILogger _logger = logFactory.CreateLogger(nameof(DocSetConfigurationCrossLinkFetcher)); - public override async Task Fetch(Cancel ctx) + public override async Task FetchCrossLinks(Cancel ctx) { + Logger.LogInformation("Fetching cross-links for all repositories defined in docset.yml"); var linkReferences = new Dictionary(); var linkIndexEntries = new Dictionary(); var declaredRepositories = new HashSet(); @@ -26,7 +29,7 @@ public override async Task Fetch(Cancel ctx) _ = declaredRepositories.Add(repository); try { - var linkReference = await Fetch(repository, ["main", "master"], ctx); + var linkReference = await FetchCrossLinks(repository, ["main", "master"], ctx); linkReferences.Add(repository, linkReference); var linkIndexReference = await GetLinkIndexEntry(repository, ctx); @@ -62,9 +65,6 @@ public override async Task Fetch(Cancel ctx) DeclaredRepositories = declaredRepositories, LinkReferences = linkReferences.ToFrozenDictionary(), LinkIndexEntries = linkIndexEntries.ToFrozenDictionary(), - FromConfiguration = true }; } - - } diff --git a/src/Elastic.Markdown/Links/InboundLinks/LinkIndexCrossLinkFetcher.cs b/src/Elastic.Markdown/Links/InboundLinks/LinkIndexCrossLinkFetcher.cs index d6ee71686..3f68adec5 100644 --- a/src/Elastic.Markdown/Links/InboundLinks/LinkIndexCrossLinkFetcher.cs +++ b/src/Elastic.Markdown/Links/InboundLinks/LinkIndexCrossLinkFetcher.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information using System.Collections.Frozen; +using Elastic.Documentation.Configuration.Assembler; using Elastic.Documentation.LinkIndex; using Elastic.Documentation.Links; using Elastic.Markdown.Links.CrossLinks; @@ -10,14 +11,16 @@ namespace Elastic.Markdown.Links.InboundLinks; +/// fetches cross-links for all the repositories defined in the publicized link-index.json file using the content source public class LinksIndexCrossLinkFetcher(ILoggerFactory logFactory, ILinkIndexReader linkIndexProvider) : CrossLinkFetcher(logFactory, linkIndexProvider) { - public override async Task Fetch(Cancel ctx) + public override async Task FetchCrossLinks(Cancel ctx) { + Logger.LogInformation("Fetching cross-links for all repositories defined in publicized link-index.json link index registry"); var linkReferences = new Dictionary(); var linkEntries = new Dictionary(); var declaredRepositories = new HashSet(); - var linkIndex = await FetchLinkIndex(ctx); + var linkIndex = await FetchLinkRegistry(ctx); foreach (var (repository, value) in linkIndex.Repositories) { var linkIndexEntry = GetNextContentSourceLinkIndexEntry(value, repository); @@ -33,7 +36,6 @@ public override async Task Fetch(Cancel ctx) DeclaredRepositories = declaredRepositories, LinkReferences = linkReferences.ToFrozenDictionary(), LinkIndexEntries = linkEntries.ToFrozenDictionary(), - FromConfiguration = false }; } diff --git a/src/Elastic.Markdown/Links/InboundLinks/LinkIndexLinkChecker.cs b/src/Elastic.Markdown/Links/InboundLinks/LinkIndexLinkChecker.cs index ae58eb07b..6d2e630a1 100644 --- a/src/Elastic.Markdown/Links/InboundLinks/LinkIndexLinkChecker.cs +++ b/src/Elastic.Markdown/Links/InboundLinks/LinkIndexLinkChecker.cs @@ -26,8 +26,8 @@ private sealed record RepositoryFilter public async Task CheckAll(IDiagnosticsCollector collector, Cancel ctx) { var fetcher = new LinksIndexCrossLinkFetcher(logFactory, _linkIndexProvider); - var resolver = new CrossLinkResolver(fetcher); - var crossLinks = await resolver.FetchLinks(ctx); + var crossLinks = await fetcher.FetchCrossLinks(ctx); + var resolver = new CrossLinkResolver(crossLinks); ValidateCrossLinks(collector, crossLinks, resolver, RepositoryFilter.None); } @@ -35,8 +35,8 @@ public async Task CheckAll(IDiagnosticsCollector collector, Cancel ctx) public async Task CheckRepository(IDiagnosticsCollector collector, string? toRepository, string? fromRepository, Cancel ctx) { var fetcher = new LinksIndexCrossLinkFetcher(logFactory, _linkIndexProvider); - var resolver = new CrossLinkResolver(fetcher); - var crossLinks = await resolver.FetchLinks(ctx); + var crossLinks = await fetcher.FetchCrossLinks(ctx); + var resolver = new CrossLinkResolver(crossLinks); var filter = new RepositoryFilter { LinksTo = toRepository, @@ -49,9 +49,8 @@ public async Task CheckRepository(IDiagnosticsCollector collector, string? toRep public async Task CheckWithLocalLinksJson(IDiagnosticsCollector collector, string repository, string localLinksJson, Cancel ctx) { var fetcher = new LinksIndexCrossLinkFetcher(logFactory, _linkIndexProvider); - var resolver = new CrossLinkResolver(fetcher); - // ReSharper disable once RedundantAssignment - var crossLinks = await resolver.FetchLinks(ctx); + var crossLinks = await fetcher.FetchCrossLinks(ctx); + var resolver = new CrossLinkResolver(crossLinks); if (string.IsNullOrEmpty(repository)) throw new ArgumentNullException(nameof(repository)); if (string.IsNullOrEmpty(localLinksJson)) @@ -105,8 +104,7 @@ RepositoryFilter filter foreach (var crossLink in linkReference.CrossLinks) { - // if we are filtering we only want errors from inbound links to a certain - // repository + // if we are filtering, we only want errors from inbound links to a certain repository var uri = new Uri(crossLink); if (filter.LinksTo != null && uri.Scheme != filter.LinksTo) continue; diff --git a/src/tooling/docs-assembler/AssembleSources.cs b/src/tooling/docs-assembler/AssembleSources.cs index a57084b2e..835cc9415 100644 --- a/src/tooling/docs-assembler/AssembleSources.cs +++ b/src/tooling/docs-assembler/AssembleSources.cs @@ -42,7 +42,7 @@ public class AssembleSources public FrozenDictionary NavigationTocMappings { get; } - public FrozenDictionary> HistoryMappings { get; } + public FrozenDictionary> LegacyUrlMappings { get; } public FrozenDictionary TocConfigurationMapping { get; } @@ -59,7 +59,26 @@ public static async Task AssembleAsync( Cancel ctx ) { - var sources = new AssembleSources(logFactory, context, checkouts, configurationContext, availableExporters); + var linkIndexProvider = Aws3LinkIndexReader.CreateAnonymous(); + var navigationTocMappings = GetTocMappings(context); + var legacyUrlMappings = GetLegacyUrlMappings(context); + var uriResolver = new PublishEnvironmentUriResolver(navigationTocMappings, context.Environment); + + var crossLinkFetcher = new AssemblerCrossLinkFetcher(logFactory, context.Configuration, context.Environment, linkIndexProvider); + var crossLinks = await crossLinkFetcher.FetchCrossLinks(ctx); + var crossLinkResolver = new CrossLinkResolver(crossLinks, uriResolver); + + var sources = new AssembleSources( + logFactory, + context, + checkouts, + configurationContext, + navigationTocMappings, + legacyUrlMappings, + uriResolver, + crossLinkResolver, + availableExporters + ); foreach (var (_, set) in sources.AssembleSets) await set.DocumentationSet.ResolveDirectoryTree(ctx); return sources; @@ -70,21 +89,21 @@ private AssembleSources( AssembleContext assembleContext, Checkout[] checkouts, IConfigurationContext configurationContext, + FrozenDictionary navigationTocMappings, + FrozenDictionary> legacyUrlMappings, + PublishEnvironmentUriResolver uriResolver, + ICrossLinkResolver crossLinkResolver, IReadOnlySet availableExporters ) { + NavigationTocMappings = navigationTocMappings; + LegacyUrlMappings = legacyUrlMappings; + UriResolver = uriResolver; AssembleContext = assembleContext; - NavigationTocMappings = GetTocMappings(assembleContext); - HistoryMappings = GetLegacyUrlMappings(assembleContext); - var linkIndexProvider = Aws3LinkIndexReader.CreateAnonymous(); - - var crossLinkFetcher = new AssemblerCrossLinkFetcher(logFactory, assembleContext.Configuration, assembleContext.Environment, linkIndexProvider); - UriResolver = new PublishEnvironmentUriResolver(NavigationTocMappings, assembleContext.Environment); - - var crossLinkResolver = new CrossLinkResolver(crossLinkFetcher, UriResolver); AssembleSets = checkouts .Where(c => c.Repository is { Skip: false }) - .Select(c => new AssemblerDocumentationSet(logFactory, assembleContext, c, crossLinkResolver, TreeCollector, configurationContext, availableExporters)) + .Select(c => new AssemblerDocumentationSet(logFactory, assembleContext, c, crossLinkResolver, TreeCollector, configurationContext, + availableExporters)) .ToDictionary(s => s.Checkout.Repository.Name, s => s) .ToFrozenDictionary(); diff --git a/src/tooling/docs-assembler/Building/AssemblerBuilder.cs b/src/tooling/docs-assembler/Building/AssemblerBuilder.cs index 083d11a35..b0771bd16 100644 --- a/src/tooling/docs-assembler/Building/AssemblerBuilder.cs +++ b/src/tooling/docs-assembler/Building/AssemblerBuilder.cs @@ -58,7 +58,7 @@ public async Task BuildAllAsync(FrozenDictionary public class AssemblerCrossLinkFetcher(ILoggerFactory logFactory, AssemblyConfiguration configuration, PublishEnvironment publishEnvironment, ILinkIndexReader linkIndexProvider) : CrossLinkFetcher(logFactory, linkIndexProvider) { - public override async Task Fetch(Cancel ctx) + public override async Task FetchCrossLinks(Cancel ctx) { + Logger.LogInformation("Fetching cross-links for all repositories defined in assembler.yml"); var linkReferences = new Dictionary(); var linkIndexEntries = new Dictionary(); var declaredRepositories = new HashSet(); @@ -38,7 +40,7 @@ public override async Task Fetch(Cancel ctx) var branch = repository.GetBranch(publishEnvironment.ContentSource); - var linkReference = await Fetch(repositoryName, [branch], ctx); + var linkReference = await FetchCrossLinks(repositoryName, [branch], ctx); linkReferences.Add(repositoryName, linkReference); var linkIndexReference = await GetLinkIndexEntry(repositoryName, ctx); linkIndexEntries.Add(repositoryName, linkIndexReference); @@ -49,7 +51,6 @@ public override async Task Fetch(Cancel ctx) DeclaredRepositories = declaredRepositories, LinkIndexEntries = linkIndexEntries.ToFrozenDictionary(), LinkReferences = linkReferences.ToFrozenDictionary(), - FromConfiguration = false }; } } diff --git a/src/tooling/docs-assembler/Building/SitemapBuilder.cs b/src/tooling/docs-assembler/Building/SitemapBuilder.cs index 13423b1c0..508e95175 100644 --- a/src/tooling/docs-assembler/Building/SitemapBuilder.cs +++ b/src/tooling/docs-assembler/Building/SitemapBuilder.cs @@ -38,7 +38,7 @@ public void Generate() { DocumentationGroup group => (group.Index.Url, NavigationItem: group), FileNavigationItem file => (file.Model.Url, NavigationItem: file as INavigationItem), - _ => throw new Exception($"Unhandled navigation item type: {n.GetType()}") + _ => throw new Exception($"{nameof(SitemapBuilder)}.{nameof(Generate)}: Unhandled navigation item type: {n.GetType()}") }) .Select(n => n.Url) .Distinct() @@ -79,7 +79,7 @@ private static IReadOnlyCollection GetNavigationItems(IReadOnly case CrossLinkNavigationItem: continue; // we do not emit cross links in the sitemap default: - throw new Exception($"Unhandled navigation item type: {item.GetType()}"); + throw new Exception($"{nameof(SitemapBuilder)}.{nameof(GetNavigationItems)}: Unhandled navigation item type: {item.GetType()}"); } } diff --git a/src/tooling/docs-assembler/Cli/ContentSourceCommands.cs b/src/tooling/docs-assembler/Cli/ContentSourceCommands.cs index 29293af0c..3df93f9a2 100644 --- a/src/tooling/docs-assembler/Cli/ContentSourceCommands.cs +++ b/src/tooling/docs-assembler/Cli/ContentSourceCommands.cs @@ -36,7 +36,7 @@ public async Task Validate(Cancel ctx = default) var context = new AssembleContext(configuration, configurationContext, "dev", collector, fs, fs, null, null); ILinkIndexReader linkIndexReader = Aws3LinkIndexReader.CreateAnonymous(); var fetcher = new AssemblerCrossLinkFetcher(logFactory, context.Configuration, context.Environment, linkIndexReader); - var links = await fetcher.FetchLinkIndex(ctx); + var links = await fetcher.FetchLinkRegistry(ctx); var repositories = context.Configuration.AvailableRepositories; var reportPath = context.ConfigurationFileProvider.AssemblerFile; diff --git a/src/tooling/docs-assembler/Cli/RepositoryCommands.cs b/src/tooling/docs-assembler/Cli/RepositoryCommands.cs index 1ac7d4726..6ccbbd6cf 100644 --- a/src/tooling/docs-assembler/Cli/RepositoryCommands.cs +++ b/src/tooling/docs-assembler/Cli/RepositoryCommands.cs @@ -21,6 +21,7 @@ using Elastic.Documentation.Tooling.Diagnostics.Console; using Elastic.Markdown; using Elastic.Markdown.IO; +using Elastic.Markdown.Links.CrossLinks; using Microsoft.Extensions.Logging; namespace Documentation.Assembler.Cli; @@ -180,7 +181,7 @@ public async Task BuildAll( var pathProvider = new GlobalNavigationPathProvider(navigationFile, assembleSources, assembleContext); var htmlWriter = new GlobalNavigationHtmlWriter(logFactory, navigation, collector); var legacyPageChecker = new LegacyPageChecker(); - var historyMapper = new PageLegacyUrlMapper(legacyPageChecker, assembleSources.HistoryMappings); + var historyMapper = new PageLegacyUrlMapper(legacyPageChecker, assembleSources.LegacyUrlMappings); var builder = new AssemblerBuilder(logFactory, assembleContext, navigation, htmlWriter, pathProvider, historyMapper); await builder.BuildAllAsync(assembleSources.AssembleSets, exporters, ctx); @@ -242,8 +243,11 @@ await Parallel.ForEachAsync(repositories, checkout.Directory.FullName, outputPath ); - var set = new DocumentationSet(context, logFactory); - var generator = new DocumentationGenerator(set, logFactory, null, null, null); + var crossLinkFetcher = new DocSetConfigurationCrossLinkFetcher(logFactory, context.Configuration); + var crossLinks = await crossLinkFetcher.FetchCrossLinks(c); + var crossLinkResolver = new CrossLinkResolver(crossLinks); + var set = new DocumentationSet(context, logFactory, crossLinkResolver); + var generator = new DocumentationGenerator(set, logFactory); _ = await generator.GenerateAll(c); IAmazonS3 s3Client = new AmazonS3Client(); diff --git a/src/tooling/docs-assembler/Links/NavigationPrefixChecker.cs b/src/tooling/docs-assembler/Links/NavigationPrefixChecker.cs index 14141aa93..2f5173381 100644 --- a/src/tooling/docs-assembler/Links/NavigationPrefixChecker.cs +++ b/src/tooling/docs-assembler/Links/NavigationPrefixChecker.cs @@ -83,11 +83,11 @@ private async Task FetchAndValidateCrossLinks(IDiagnosticsCollector collector, s { var linkIndexProvider = Aws3LinkIndexReader.CreateAnonymous(); var fetcher = new LinksIndexCrossLinkFetcher(_logFactoryFactory, linkIndexProvider); - var resolver = new CrossLinkResolver(fetcher); - var crossLinks = await resolver.FetchLinks(ctx); + var crossLinks = await fetcher.FetchCrossLinks(ctx); + var crossLinkResolver = new CrossLinkResolver(crossLinks); var dictionary = new Dictionary(); if (!string.IsNullOrEmpty(updateRepository) && updateReference is not null) - crossLinks = resolver.UpdateLinkReference(updateRepository, updateReference); + crossLinks = crossLinkResolver.UpdateLinkReference(updateRepository, updateReference); foreach (var (repository, linkReference) in crossLinks.LinkReferences) { if (!_repositories.Contains(repository)) diff --git a/src/tooling/docs-assembler/Navigation/AssemblerDocumentationSet.cs b/src/tooling/docs-assembler/Navigation/AssemblerDocumentationSet.cs index 8968a06c9..0219a1cdb 100644 --- a/src/tooling/docs-assembler/Navigation/AssemblerDocumentationSet.cs +++ b/src/tooling/docs-assembler/Navigation/AssemblerDocumentationSet.cs @@ -27,7 +27,7 @@ public AssemblerDocumentationSet( ILoggerFactory logFactory, AssembleContext context, Checkout checkout, - CrossLinkResolver crossLinkResolver, + ICrossLinkResolver crossLinkResolver, TableOfContentsTreeCollector treeCollector, IConfigurationContext configurationContext, IReadOnlySet availableExporters diff --git a/src/tooling/docs-assembler/Navigation/GlobalNavigation.cs b/src/tooling/docs-assembler/Navigation/GlobalNavigation.cs index 82e98557d..ac3669ffa 100644 --- a/src/tooling/docs-assembler/Navigation/GlobalNavigation.cs +++ b/src/tooling/docs-assembler/Navigation/GlobalNavigation.cs @@ -81,6 +81,11 @@ private void UpdateParent( fileNavigationItem.Model.NavigationRoot = topLevelNavigation; _ = allNavigationItems.Add(fileNavigationItem); break; + case CrossLinkNavigationItem crossLinkNavigationItem: + if (parent is not null) + crossLinkNavigationItem.Parent = parent; + _ = allNavigationItems.Add(crossLinkNavigationItem); + break; case DocumentationGroup documentationGroup: if (parent is not null) documentationGroup.Parent = parent; @@ -90,7 +95,7 @@ private void UpdateParent( UpdateParent(allNavigationItems, documentationGroup.NavigationItems, documentationGroup, topLevelNavigation); break; default: - _navigationFile.EmitError($"Unhandled navigation item type: {item.GetType()}"); + _navigationFile.EmitError($"{nameof(GlobalNavigation)}.{nameof(UpdateParent)}: Unhandled navigation item type: {item.GetType()}"); break; } } @@ -112,8 +117,12 @@ private void UpdateNavigationIndex(IReadOnlyCollection navigati documentationGroup.NavigationIndex = groupIndex; UpdateNavigationIndex(documentationGroup.NavigationItems, ref navigationIndex); break; + case CrossLinkNavigationItem crossLinkNavigationItem: + var crossLinkIndex = Interlocked.Increment(ref navigationIndex); + crossLinkNavigationItem.NavigationIndex = crossLinkIndex; + break; default: - _navigationFile.EmitError($"Unhandled navigation item type: {item.GetType()}"); + _navigationFile.EmitError($"{nameof(GlobalNavigation)}.{nameof(UpdateNavigationIndex)}: Unhandled navigation item type: {item.GetType()}"); break; } } diff --git a/src/tooling/docs-assembler/Navigation/GlobalNavigationFile.cs b/src/tooling/docs-assembler/Navigation/GlobalNavigationFile.cs index b730e0296..2777be7f1 100644 --- a/src/tooling/docs-assembler/Navigation/GlobalNavigationFile.cs +++ b/src/tooling/docs-assembler/Navigation/GlobalNavigationFile.cs @@ -148,7 +148,7 @@ public void EmitWarning(string message) => _context.Collector.EmitWarning(NavigationFile, message); public void EmitError(string message) => - _context.Collector.EmitWarning(NavigationFile, message); + _context.Collector.EmitError(NavigationFile, message); private IReadOnlyCollection Deserialize(string key) { diff --git a/src/tooling/docs-builder/Cli/Commands.cs b/src/tooling/docs-builder/Cli/Commands.cs index 88d4606c3..8ef651223 100644 --- a/src/tooling/docs-builder/Cli/Commands.cs +++ b/src/tooling/docs-builder/Cli/Commands.cs @@ -15,6 +15,7 @@ using Elastic.Documentation.Tooling.Diagnostics.Console; using Elastic.Markdown; using Elastic.Markdown.IO; +using Elastic.Markdown.Links.CrossLinks; using Microsoft.Extensions.Logging; namespace Documentation.Builder.Cli; @@ -147,8 +148,12 @@ public async Task Generate( if (runningOnCi) await githubActionsService.SetOutputAsync("skip", "false"); + var crossLinkFetcher = new DocSetConfigurationCrossLinkFetcher(logFactory, context.Configuration); + var crossLinks = await crossLinkFetcher.FetchCrossLinks(ctx); + var crossLinkResolver = new CrossLinkResolver(crossLinks); + // always delete output folder on CI - var set = new DocumentationSet(context, logFactory); + var set = new DocumentationSet(context, logFactory, crossLinkResolver); if (runningOnCi) set.ClearOutputDirectory(); @@ -223,7 +228,8 @@ public async Task Move( var fileSystem = new FileSystem(); await using var collector = new ConsoleDiagnosticsCollector(logFactory, null).StartAsync(ctx); var context = new BuildContext(collector, fileSystem, fileSystem, configurationContext, ExportOptions.MetadataOnly, path, null); - var set = new DocumentationSet(context, logFactory); + + var set = new DocumentationSet(context, logFactory, NoopCrossLinkResolver.Instance); var moveCommand = new Move(logFactory, fileSystem, fileSystem, set); var result = await moveCommand.Execute(source, target, dryRun ?? false, ctx); diff --git a/src/tooling/docs-builder/Http/DocumentationWebHost.cs b/src/tooling/docs-builder/Http/DocumentationWebHost.cs index 96505615d..5466ee6d2 100644 --- a/src/tooling/docs-builder/Http/DocumentationWebHost.cs +++ b/src/tooling/docs-builder/Http/DocumentationWebHost.cs @@ -58,7 +58,7 @@ IConfigurationContext configurationContext _hostedService = collector; Context = new BuildContext(collector, readFs, writeFs, configurationContext, ExportOptions.Default, path, null) { - CanonicalBaseUrl = new Uri(hostUrl), + CanonicalBaseUrl = new Uri(hostUrl) }; GeneratorState = new ReloadableGeneratorState(logFactory, Context.DocumentationSourceDirectory, Context.OutputDirectory, Context); _ = builder.Services @@ -68,7 +68,7 @@ IConfigurationContext configurationContext s.ClientFileExtensions = ".md,.yml"; }) .AddSingleton(_ => GeneratorState) - .AddHostedService(); + .AddHostedService((_) => new ReloadGeneratorService(GeneratorState, logFactory.CreateLogger())); if (IsDotNetWatchBuild()) _ = builder.Services.AddHostedService(); diff --git a/src/tooling/docs-builder/Http/ReloadGeneratorService.cs b/src/tooling/docs-builder/Http/ReloadGeneratorService.cs index 66235b18e..d1d2d20fa 100644 --- a/src/tooling/docs-builder/Http/ReloadGeneratorService.cs +++ b/src/tooling/docs-builder/Http/ReloadGeneratorService.cs @@ -20,10 +20,8 @@ public static void UpdateApplication(Type[]? _) => Task.Run(async () => var __ = LiveReloadMiddleware.RefreshWebSocketRequest(); Console.WriteLine("UpdateApplication"); }); - } - public sealed class ReloadGeneratorService(ReloadableGeneratorState reloadableGenerator, ILogger logger) : IHostedService, IDisposable { private FileSystemWatcher? _watcher; diff --git a/src/tooling/docs-builder/Http/ReloadableGeneratorState.cs b/src/tooling/docs-builder/Http/ReloadableGeneratorState.cs index 548b7f401..ba955597b 100644 --- a/src/tooling/docs-builder/Http/ReloadableGeneratorState.cs +++ b/src/tooling/docs-builder/Http/ReloadableGeneratorState.cs @@ -8,37 +8,54 @@ using Elastic.Markdown; using Elastic.Markdown.Exporters; using Elastic.Markdown.IO; +using Elastic.Markdown.Links.CrossLinks; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; namespace Documentation.Builder.Http; /// Singleton behavior enforced by registration on -public class ReloadableGeneratorState( - ILoggerFactory logFactory, - IDirectoryInfo sourcePath, - IDirectoryInfo outputPath, - BuildContext context) +public class ReloadableGeneratorState : IDisposable { - private IDirectoryInfo SourcePath { get; } = sourcePath; - private IDirectoryInfo OutputPath { get; } = outputPath; - public IDirectoryInfo ApiPath { get; } = context.WriteFileSystem.DirectoryInfo.New(Path.Combine(outputPath.FullName, "api")); + private IDirectoryInfo SourcePath { get; } + private IDirectoryInfo OutputPath { get; } + public IDirectoryInfo ApiPath { get; } + + private DocumentationGenerator _generator; + private readonly ILoggerFactory _logFactory; + private readonly BuildContext _context; + private readonly DocSetConfigurationCrossLinkFetcher _crossLinkFetcher; + + public ReloadableGeneratorState(ILoggerFactory logFactory, + IDirectoryInfo sourcePath, + IDirectoryInfo outputPath, + BuildContext context) + { + _logFactory = logFactory; + _context = context; + SourcePath = sourcePath; + OutputPath = outputPath; + ApiPath = context.WriteFileSystem.DirectoryInfo.New(Path.Combine(outputPath.FullName, "api")); + _crossLinkFetcher = new DocSetConfigurationCrossLinkFetcher(logFactory, _context.Configuration); + // we pass NoopCrossLinkResolver.Instance here because `ReloadAsync` will always be called when the is started. + _generator = new DocumentationGenerator(new DocumentationSet(context, logFactory, NoopCrossLinkResolver.Instance), logFactory); + } - private DocumentationGenerator _generator = new(new DocumentationSet(context, logFactory), logFactory); public DocumentationGenerator Generator => _generator; public async Task ReloadAsync(Cancel ctx) { SourcePath.Refresh(); OutputPath.Refresh(); - var docSet = new DocumentationSet(context, logFactory); - _ = await docSet.LinkResolver.FetchLinks(ctx); + var crossLinks = await _crossLinkFetcher.FetchCrossLinks(ctx); + var crossLinkResolver = new CrossLinkResolver(crossLinks); + var docSet = new DocumentationSet(_context, _logFactory, crossLinkResolver); // Add LLM markdown export for dev server var markdownExporters = new List(); markdownExporters.AddLlmMarkdownExport(); // Consistent LLM-optimized output - var generator = new DocumentationGenerator(docSet, logFactory, markdownExporters: markdownExporters.ToArray()); + var generator = new DocumentationGenerator(docSet, _logFactory, markdownExporters: markdownExporters.ToArray()); await generator.ResolveDirectoryTree(ctx); _ = Interlocked.Exchange(ref _generator, generator); @@ -52,7 +69,13 @@ private async Task ReloadApiReferences(IMarkdownStringRenderer markdownStringRen if (ApiPath.Exists) ApiPath.Delete(true); ApiPath.Create(); - var generator = new OpenApiGenerator(logFactory, context, markdownStringRenderer); + var generator = new OpenApiGenerator(_logFactory, _context, markdownStringRenderer); await generator.Generate(ctx); } + + public void Dispose() + { + _crossLinkFetcher.Dispose(); + GC.SuppressFinalize(this); + } } diff --git a/tests/Elastic.Markdown.Tests/Directives/DirectiveBaseTests.cs b/tests/Elastic.Markdown.Tests/Directives/DirectiveBaseTests.cs index caf2bde88..788afe3d1 100644 --- a/tests/Elastic.Markdown.Tests/Directives/DirectiveBaseTests.cs +++ b/tests/Elastic.Markdown.Tests/Directives/DirectiveBaseTests.cs @@ -83,7 +83,6 @@ public virtual async ValueTask InitializeAsync() { _ = Collector.StartAsync(TestContext.Current.CancellationToken); - await Set.LinkResolver.FetchLinks(TestContext.Current.CancellationToken); Document = await File.ParseFullAsync(TestContext.Current.CancellationToken); var html = MarkdownFile.CreateHtml(Document).AsSpan(); var find = ""; diff --git a/tests/Elastic.Markdown.Tests/Inline/InlneBaseTests.cs b/tests/Elastic.Markdown.Tests/Inline/InlneBaseTests.cs index 321cf05e1..936cf66de 100644 --- a/tests/Elastic.Markdown.Tests/Inline/InlneBaseTests.cs +++ b/tests/Elastic.Markdown.Tests/Inline/InlneBaseTests.cs @@ -131,7 +131,6 @@ public virtual async ValueTask InitializeAsync() _ = Collector.StartAsync(TestContext.Current.CancellationToken); await Set.ResolveDirectoryTree(TestContext.Current.CancellationToken); - await Set.LinkResolver.FetchLinks(TestContext.Current.CancellationToken); Document = await File.ParseFullAsync(TestContext.Current.CancellationToken); var html = MarkdownFile.CreateHtml(Document).AsSpan(); diff --git a/tests/Elastic.Markdown.Tests/TestCrossLinkResolver.cs b/tests/Elastic.Markdown.Tests/TestCrossLinkResolver.cs index aa8f5e838..00ae80d00 100644 --- a/tests/Elastic.Markdown.Tests/TestCrossLinkResolver.cs +++ b/tests/Elastic.Markdown.Tests/TestCrossLinkResolver.cs @@ -12,17 +12,12 @@ namespace Elastic.Markdown.Tests; public class TestCrossLinkResolver : ICrossLinkResolver { + private readonly FetchedCrossLinks _crossLinks; + public IUriEnvironmentResolver UriResolver { get; } = new IsolatedBuildEnvironmentUriResolver(); - private FetchedCrossLinks _crossLinks = FetchedCrossLinks.Empty; - private Dictionary LinkReferences { get; } = []; - private HashSet DeclaredRepositories { get; } = []; - public Task FetchLinks(Cancel ctx) + public TestCrossLinkResolver() { - // Clear existing entries to prevent duplicate key errors when called multiple times - LinkReferences.Clear(); - DeclaredRepositories.Clear(); - // language=json var json = """ { @@ -49,26 +44,26 @@ public Task FetchLinks(Cancel ctx) } """; var reference = CrossLinkFetcher.Deserialize(json); - LinkReferences.Add("docs-content", reference); - LinkReferences.Add("kibana", reference); - DeclaredRepositories.AddRange(["docs-content", "kibana"]); + var linkReferences = new Dictionary(); + var declaredRepositories = new HashSet(); + linkReferences.Add("docs-content", reference); + linkReferences.Add("kibana", reference); + declaredRepositories.AddRange(["docs-content", "kibana"]); - var indexEntries = LinkReferences.ToDictionary(e => e.Key, e => new LinkRegistryEntry + var indexEntries = linkReferences.ToDictionary(e => e.Key, e => new LinkRegistryEntry { Repository = e.Key, - Path = $"elastic/asciidocalypse/{e.Key}/links.json", + Path = $"elastic/docs-builder-tests/{e.Key}/links.json", Branch = "main", ETag = Guid.NewGuid().ToString(), GitReference = Guid.NewGuid().ToString() }); _crossLinks = new FetchedCrossLinks { - DeclaredRepositories = DeclaredRepositories, - LinkReferences = LinkReferences.ToFrozenDictionary(), - FromConfiguration = true, + DeclaredRepositories = declaredRepositories, + LinkReferences = linkReferences.ToFrozenDictionary(), LinkIndexEntries = indexEntries.ToFrozenDictionary() }; - return Task.FromResult(_crossLinks); } public bool TryResolve(Action errorEmitter, Uri crossLinkUri, [NotNullWhen(true)] out Uri? resolvedUri) => diff --git a/tests/authoring/Framework/CrossLinkResolverAssertions.fs b/tests/authoring/Framework/CrossLinkResolverAssertions.fs index 0f1acd54e..3f11aee02 100644 --- a/tests/authoring/Framework/CrossLinkResolverAssertions.fs +++ b/tests/authoring/Framework/CrossLinkResolverAssertions.fs @@ -60,7 +60,6 @@ module CrossLinkResolverAssertions = FetchedCrossLinks( DeclaredRepositories = declaredRepos, LinkReferences = FrozenDictionary.ToFrozenDictionary(dict [repoName, repositoryLinks]), - FromConfiguration = true, LinkIndexEntries = FrozenDictionary.Empty ) diff --git a/tests/authoring/Framework/TestCrossLinkResolver.fs b/tests/authoring/Framework/TestCrossLinkResolver.fs index e9c7e4679..2691bca69 100644 --- a/tests/authoring/Framework/TestCrossLinkResolver.fs +++ b/tests/authoring/Framework/TestCrossLinkResolver.fs @@ -8,7 +8,6 @@ open System open System.Collections.Generic open System.Collections.Frozen open System.Runtime.InteropServices -open System.Threading.Tasks open System.Linq open Elastic.Documentation.Configuration.Builder open Elastic.Documentation.Links @@ -19,22 +18,12 @@ type TestCrossLinkResolver (config: ConfigurationFile) = let references = Dictionary() let declared = HashSet() let uriResolver = IsolatedBuildEnvironmentUriResolver() - - member this.LinkReferences = references - member this.DeclaredRepositories = declared - - interface ICrossLinkResolver with - - member this.UriResolver = uriResolver - - member this.FetchLinks(ctx) = - // Clear existing entries to prevent duplicate key errors when called multiple times - this.LinkReferences.Clear() - this.DeclaredRepositories.Clear() - - let redirects = RepositoryLinks.SerializeRedirects config.Redirects - // language=json - let json = $$"""{ + let mutable crossLinks = FetchedCrossLinks.Empty + + do + let redirects = RepositoryLinks.SerializeRedirects config.Redirects + // language=json + let json = $$"""{ "origin": { "branch": "main", "remote": " https://github.com/elastic/docs-content", @@ -65,48 +54,39 @@ type TestCrossLinkResolver (config: ConfigurationFile) = } } """ - let reference = CrossLinkFetcher.Deserialize json - this.LinkReferences.Add("docs-content", reference) - this.LinkReferences.Add("kibana", reference) - this.DeclaredRepositories.Add("docs-content") |> ignore; - this.DeclaredRepositories.Add("kibana") |> ignore; + let reference = CrossLinkFetcher.Deserialize json + references.Add("docs-content", reference) + references.Add("kibana", reference) + declared.Add("docs-content") |> ignore; + declared.Add("kibana") |> ignore; - let indexEntries = - this.LinkReferences.ToDictionary(_.Key, fun (e : KeyValuePair) -> LinkRegistryEntry( - Repository = e.Key, - Path = $"elastic/asciidocalypse/{e.Key}/links.json", - Branch = "main", - ETag = Guid.NewGuid().ToString(), - GitReference = Guid.NewGuid().ToString() - )) + let indexEntries = + references.ToDictionary(_.Key, fun (e : KeyValuePair) -> LinkRegistryEntry( + Repository = e.Key, + Path = $"elastic/docs-builder-tests/{e.Key}/links.json", + Branch = "main", + ETag = Guid.NewGuid().ToString(), + GitReference = Guid.NewGuid().ToString() + )) - let crossLinks = - FetchedCrossLinks( - DeclaredRepositories=this.DeclaredRepositories, - LinkReferences=this.LinkReferences.ToFrozenDictionary(), - FromConfiguration=true, - LinkIndexEntries=indexEntries.ToFrozenDictionary() - ) - Task.FromResult crossLinks + let resolvedCrossLinks = + FetchedCrossLinks( + DeclaredRepositories=declared, + LinkReferences=references.ToFrozenDictionary(), + LinkIndexEntries=indexEntries.ToFrozenDictionary() + ) + crossLinks <- resolvedCrossLinks - member this.TryResolve(errorEmitter, crossLinkUri, []resolvedUri : byref) = - let indexEntries = - this.LinkReferences.ToDictionary(_.Key, fun (e : KeyValuePair) -> LinkRegistryEntry( - Repository = e.Key, - Path = $"elastic/asciidocalypse/{e.Key}/links.json", - Branch = "main", - ETag = Guid.NewGuid().ToString(), - GitReference = Guid.NewGuid().ToString() - )); + - let crossLinks = - FetchedCrossLinks( - DeclaredRepositories=this.DeclaredRepositories, - LinkReferences=this.LinkReferences.ToFrozenDictionary(), - FromConfiguration=true, - LinkIndexEntries=indexEntries.ToFrozenDictionary() + member this.LinkReferences = references + member this.DeclaredRepositories = declared + + interface ICrossLinkResolver with - ) - CrossLinkResolver.TryResolve(errorEmitter, crossLinks, uriResolver, crossLinkUri, &resolvedUri); + member this.UriResolver = uriResolver + + member this.TryResolve(errorEmitter, crossLinkUri, []resolvedUri : byref) = + CrossLinkResolver.TryResolve(errorEmitter, crossLinks, uriResolver, crossLinkUri, &resolvedUri)