diff --git a/docs-builder.sln b/docs-builder.sln index 15fada31a..a075d35e1 100644 --- a/docs-builder.sln +++ b/docs-builder.sln @@ -99,6 +99,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "assembler-config-validate", actions\assembler-config-validate\action.yml = actions\assembler-config-validate\action.yml EndProjectSection EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Documentation.LinkIndex", "src\Elastic.Documentation.LinkIndex\Elastic.Documentation.LinkIndex.csproj", "{FD1AC230-798B-4AB9-8CE6-A06264885DBC}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -160,6 +162,10 @@ Global {CD94F9E4-7FCD-4152-81F1-4288C6B75367}.Debug|Any CPU.Build.0 = Debug|Any CPU {CD94F9E4-7FCD-4152-81F1-4288C6B75367}.Release|Any CPU.ActiveCfg = Release|Any CPU {CD94F9E4-7FCD-4152-81F1-4288C6B75367}.Release|Any CPU.Build.0 = Release|Any CPU + {FD1AC230-798B-4AB9-8CE6-A06264885DBC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FD1AC230-798B-4AB9-8CE6-A06264885DBC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FD1AC230-798B-4AB9-8CE6-A06264885DBC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FD1AC230-798B-4AB9-8CE6-A06264885DBC}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(NestedProjects) = preSolution {4D198E25-C211-41DC-9E84-B15E89BD7048} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A} @@ -184,5 +190,6 @@ Global {7D36DDDA-9E0B-4D2C-8033-5D62FF8B6166} = {059E787F-85C1-43BE-9DD6-CE319E106383} {FB1C1954-D8E2-4745-BA62-04DD82FB4792} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7} {E20FEEF9-1D1A-4CDA-A546-7FDC573BE399} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7} + {FD1AC230-798B-4AB9-8CE6-A06264885DBC} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A} EndGlobalSection EndGlobal diff --git a/src/Elastic.Documentation.LinkIndex/Elastic.Documentation.LinkIndex.csproj b/src/Elastic.Documentation.LinkIndex/Elastic.Documentation.LinkIndex.csproj new file mode 100644 index 000000000..4a0d944f1 --- /dev/null +++ b/src/Elastic.Documentation.LinkIndex/Elastic.Documentation.LinkIndex.csproj @@ -0,0 +1,17 @@ + + + + net9.0 + enable + enable + + + + + + + + + + + diff --git a/src/Elastic.Documentation.LinkIndex/ILinkIndexReader.cs b/src/Elastic.Documentation.LinkIndex/ILinkIndexReader.cs new file mode 100644 index 000000000..4b2af660a --- /dev/null +++ b/src/Elastic.Documentation.LinkIndex/ILinkIndexReader.cs @@ -0,0 +1,14 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using Elastic.Documentation.Links; + +namespace Elastic.Documentation.LinkIndex; + +public interface ILinkIndexReader +{ + Task GetRegistry(Cancel cancellationToken = default); + Task GetRepositoryLinks(string key, Cancel cancellationToken = default); + string RegistryUrl { get; } +} diff --git a/src/Elastic.Documentation.LinkIndex/ILinkIndexReaderWriter.cs b/src/Elastic.Documentation.LinkIndex/ILinkIndexReaderWriter.cs new file mode 100644 index 000000000..15f842bc9 --- /dev/null +++ b/src/Elastic.Documentation.LinkIndex/ILinkIndexReaderWriter.cs @@ -0,0 +1,7 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +namespace Elastic.Documentation.LinkIndex; + +public interface ILinkIndexReaderWriter : ILinkIndexReader, ILinkIndexWriter; diff --git a/src/Elastic.Documentation.LinkIndex/ILinkIndexWriter.cs b/src/Elastic.Documentation.LinkIndex/ILinkIndexWriter.cs new file mode 100644 index 000000000..4e321738e --- /dev/null +++ b/src/Elastic.Documentation.LinkIndex/ILinkIndexWriter.cs @@ -0,0 +1,12 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using Elastic.Documentation.Links; + +namespace Elastic.Documentation.LinkIndex; + +public interface ILinkIndexWriter +{ + Task SaveRegistry(LinkRegistry registry, Cancel cancellationToken = default); +} diff --git a/src/Elastic.Documentation.LinkIndex/LinkIndexReader.cs b/src/Elastic.Documentation.LinkIndex/LinkIndexReader.cs new file mode 100644 index 000000000..b66b69aba --- /dev/null +++ b/src/Elastic.Documentation.LinkIndex/LinkIndexReader.cs @@ -0,0 +1,56 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Net; +using Amazon.Runtime; +using Amazon.S3; +using Amazon.S3.Model; +using Elastic.Documentation.Links; + +namespace Elastic.Documentation.LinkIndex; + +public class Aws3LinkIndexReader(IAmazonS3 s3Client, string bucketName = "elastic-docs-link-index", string registryKey = "link-index.json") : ILinkIndexReader +{ + + // + // Using to access the link index + // allows to read from the link index without the need to provide AWS credentials. + // + public static Aws3LinkIndexReader CreateAnonymous() + { + var credentials = new AnonymousAWSCredentials(); + var config = new AmazonS3Config + { + RegionEndpoint = Amazon.RegionEndpoint.USEast2 + }; + var s3Client = new AmazonS3Client(credentials, config); + return new AwsS3LinkIndexReaderWriter(s3Client); + } + + public async Task GetRegistry(Cancel cancellationToken = default) + { + var getObjectRequest = new GetObjectRequest + { + BucketName = bucketName, + Key = registryKey + }; + var getObjectResponse = await s3Client.GetObjectAsync(getObjectRequest, cancellationToken); + await using var stream = getObjectResponse.ResponseStream; + var linkIndex = LinkRegistry.Deserialize(stream); + return linkIndex with { ETag = getObjectResponse.ETag }; + } + public async Task GetRepositoryLinks(string key, Cancel cancellationToken) + { + var getObjectRequest = new GetObjectRequest + { + BucketName = bucketName, + Key = key + }; + var getObjectResponse = await s3Client.GetObjectAsync(getObjectRequest, cancellationToken); + await using var stream = getObjectResponse.ResponseStream; + return RepositoryLinks.Deserialize(stream); + } + + public string RegistryUrl { get; } = $"https://{bucketName}.s3.{s3Client.Config.RegionEndpoint.SystemName}.amazonaws.com/{registryKey}"; +} diff --git a/src/Elastic.Documentation.LinkIndex/LinkIndexReaderWriter.cs b/src/Elastic.Documentation.LinkIndex/LinkIndexReaderWriter.cs new file mode 100644 index 000000000..aee8d47c6 --- /dev/null +++ b/src/Elastic.Documentation.LinkIndex/LinkIndexReaderWriter.cs @@ -0,0 +1,41 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Net; +using Amazon.Runtime; +using Amazon.S3; +using Amazon.S3.Model; +using Elastic.Documentation.Links; + +namespace Elastic.Documentation.LinkIndex; + +public class AwsS3LinkIndexReaderWriter( + IAmazonS3 s3Client, + string bucketName = "elastic-docs-link-index", + string registryKey = "link-index.json" +) : Aws3LinkIndexReader(s3Client, bucketName, registryKey), ILinkIndexReaderWriter +{ + private readonly IAmazonS3 _s3Client = s3Client; + private readonly string _bucketName = bucketName; + private readonly string _registryKey = registryKey; + + public async Task SaveRegistry(LinkRegistry registry, Cancel cancellationToken = default) + { + if (registry.ETag == null) + // The ETag should not be null if the LinkReferenceRegistry was retrieved from GetLinkIndex() + throw new InvalidOperationException($"{nameof(LinkRegistry)}.{nameof(registry.ETag)} cannot be null"); + var json = LinkRegistry.Serialize(registry); + var putObjectRequest = new PutObjectRequest + { + BucketName = _bucketName, + Key = _registryKey, + ContentBody = json, + ContentType = "application/json", + IfMatch = registry.ETag // Only update if the ETag matches. Meaning the object has not been changed in the meantime. + }; + var putResponse = await _s3Client.PutObjectAsync(putObjectRequest, cancellationToken); + if (putResponse.HttpStatusCode != HttpStatusCode.OK) + throw new Exception($"Unable to save {nameof(LinkRegistry)} to s3://{_bucketName}/{_registryKey}"); + } +} diff --git a/src/Elastic.Documentation/Links/LinkReferenceRegistry.cs b/src/Elastic.Documentation/Links/LinkRegistry.cs similarity index 50% rename from src/Elastic.Documentation/Links/LinkReferenceRegistry.cs rename to src/Elastic.Documentation/Links/LinkRegistry.cs index 9a9237c0d..0ff49235f 100644 --- a/src/Elastic.Documentation/Links/LinkReferenceRegistry.cs +++ b/src/Elastic.Documentation/Links/LinkRegistry.cs @@ -8,20 +8,54 @@ namespace Elastic.Documentation.Links; -public record LinkReferenceRegistry +public record LinkRegistry { /// Map of branch to [JsonPropertyName("repositories")] public required Dictionary> Repositories { get; init; } - public static LinkReferenceRegistry Deserialize(Stream json) => - JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkReferenceRegistry)!; + [JsonIgnore] + public string? ETag { get; init; } - public static LinkReferenceRegistry Deserialize(string json) => - JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkReferenceRegistry)!; + public LinkRegistry WithLinkRegistryEntry(LinkRegistryEntry entry) + { + var copiedRepositories = new Dictionary>(Repositories); + var repository = entry.Repository; + var branch = entry.Branch; + // repository already exists in links.json + if (copiedRepositories.TryGetValue(repository, out var existingRepositoryEntry)) + { + // The branch already exists in the repository entry + if (existingRepositoryEntry.TryGetValue(branch, out var existingBranchEntry)) + { + if (entry.UpdatedAt > existingBranchEntry.UpdatedAt) + existingRepositoryEntry[branch] = entry; + } + // branch does not exist in the repository entry + else + { + existingRepositoryEntry[branch] = entry; + } + } + // onboarding new repository + else + { + copiedRepositories.Add(repository, new Dictionary + { + { branch, entry } + }); + } + return this with { Repositories = copiedRepositories }; + } - public static string Serialize(LinkReferenceRegistry referenceRegistry) => - JsonSerializer.Serialize(referenceRegistry, SourceGenerationContext.Default.LinkReferenceRegistry); + public static LinkRegistry Deserialize(Stream json) => + JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkRegistry)!; + + public static LinkRegistry Deserialize(string json) => + JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkRegistry)!; + + public static string Serialize(LinkRegistry registry) => + JsonSerializer.Serialize(registry, SourceGenerationContext.Default.LinkRegistry); } public record LinkRegistryEntry @@ -46,4 +80,3 @@ public record LinkRegistryEntry [JsonPropertyName("updated_at")] public DateTime UpdatedAt { get; init; } = DateTime.MinValue; } - diff --git a/src/Elastic.Documentation/Links/LinkReference.cs b/src/Elastic.Documentation/Links/RepositoryLinks.cs similarity index 89% rename from src/Elastic.Documentation/Links/LinkReference.cs rename to src/Elastic.Documentation/Links/RepositoryLinks.cs index 3008b9bbe..b612424d9 100644 --- a/src/Elastic.Documentation/Links/LinkReference.cs +++ b/src/Elastic.Documentation/Links/RepositoryLinks.cs @@ -39,7 +39,7 @@ public record LinkRedirect : LinkSingleRedirect public LinkSingleRedirect[]? Many { get; init; } } -public record LinkReference +public record RepositoryLinks { [JsonPropertyName("origin")] public required GitCheckoutInformation Origin { get; init; } @@ -61,12 +61,12 @@ public record LinkReference public static string SerializeRedirects(Dictionary? redirects) => JsonSerializer.Serialize(redirects, SourceGenerationContext.Default.DictionaryStringLinkRedirect); - public static LinkReference Deserialize(Stream json) => - JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkReference)!; + public static RepositoryLinks Deserialize(Stream json) => + JsonSerializer.Deserialize(json, SourceGenerationContext.Default.RepositoryLinks)!; - public static LinkReference Deserialize(string json) => - JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkReference)!; + public static RepositoryLinks Deserialize(string json) => + JsonSerializer.Deserialize(json, SourceGenerationContext.Default.RepositoryLinks)!; - public static string Serialize(LinkReference reference) => - JsonSerializer.Serialize(reference, SourceGenerationContext.Default.LinkReference); + public static string Serialize(RepositoryLinks reference) => + JsonSerializer.Serialize(reference, SourceGenerationContext.Default.RepositoryLinks); } diff --git a/src/Elastic.Documentation/Serialization/SourceGenerationContext.cs b/src/Elastic.Documentation/Serialization/SourceGenerationContext.cs index 5e6426a4a..69bab9947 100644 --- a/src/Elastic.Documentation/Serialization/SourceGenerationContext.cs +++ b/src/Elastic.Documentation/Serialization/SourceGenerationContext.cs @@ -12,8 +12,8 @@ namespace Elastic.Documentation.Serialization; [JsonSourceGenerationOptions(WriteIndented = true, UseStringEnumConverter = true)] [JsonSerializable(typeof(GenerationState))] -[JsonSerializable(typeof(LinkReference))] +[JsonSerializable(typeof(RepositoryLinks))] [JsonSerializable(typeof(GitCheckoutInformation))] -[JsonSerializable(typeof(LinkReferenceRegistry))] +[JsonSerializable(typeof(LinkRegistry))] [JsonSerializable(typeof(LinkRegistryEntry))] public sealed partial class SourceGenerationContext : JsonSerializerContext; diff --git a/src/Elastic.Markdown/DocumentationGenerator.cs b/src/Elastic.Markdown/DocumentationGenerator.cs index ccdfadd76..3a91c5f65 100644 --- a/src/Elastic.Markdown/DocumentationGenerator.cs +++ b/src/Elastic.Markdown/DocumentationGenerator.cs @@ -276,11 +276,11 @@ private bool CompilationNotNeeded(GenerationState? generationState, out HashSet< return false; } - private async Task GenerateLinkReference(Cancel ctx) + private async Task GenerateLinkReference(Cancel ctx) { var file = DocumentationSet.LinkReferenceFile; var state = DocumentationSet.CreateLinkReference(); - var bytes = JsonSerializer.SerializeToUtf8Bytes(state, SourceGenerationContext.Default.LinkReference); + var bytes = JsonSerializer.SerializeToUtf8Bytes(state, SourceGenerationContext.Default.RepositoryLinks); await DocumentationSet.OutputDirectory.FileSystem.File.WriteAllBytesAsync(file.FullName, bytes, ctx); return state; } diff --git a/src/Elastic.Markdown/Elastic.Markdown.csproj b/src/Elastic.Markdown/Elastic.Markdown.csproj index f4e330173..f1b0e8dde 100644 --- a/src/Elastic.Markdown/Elastic.Markdown.csproj +++ b/src/Elastic.Markdown/Elastic.Markdown.csproj @@ -64,6 +64,7 @@ + diff --git a/src/Elastic.Markdown/IO/DocumentationSet.cs b/src/Elastic.Markdown/IO/DocumentationSet.cs index ca8fe4093..f3f6b918a 100644 --- a/src/Elastic.Markdown/IO/DocumentationSet.cs +++ b/src/Elastic.Markdown/IO/DocumentationSet.cs @@ -8,8 +8,8 @@ using Elastic.Documentation; using Elastic.Documentation.Configuration.Builder; using Elastic.Documentation.Configuration.TableOfContents; +using Elastic.Documentation.LinkIndex; using Elastic.Documentation.Links; -using Elastic.Markdown.Diagnostics; using Elastic.Markdown.Extensions; using Elastic.Markdown.Extensions.DetectionRules; using Elastic.Markdown.IO.Navigation; @@ -126,7 +126,7 @@ public DocumentationSet( SourceDirectory = context.DocumentationSourceDirectory; OutputDirectory = context.DocumentationOutputDirectory; LinkResolver = - linkResolver ?? new CrossLinkResolver(new ConfigurationCrossLinkFetcher(context.Configuration, logger)); + linkResolver ?? new CrossLinkResolver(new ConfigurationCrossLinkFetcher(context.Configuration, Aws3LinkIndexReader.CreateAnonymous(), logger)); Configuration = context.Configuration; EnabledExtensions = InstantiateExtensions(); treeCollector ??= new TableOfContentsTreeCollector(); @@ -361,7 +361,7 @@ MarkdownFile ExtensionOrDefaultMarkdown() } } - public LinkReference CreateLinkReference() + public RepositoryLinks CreateLinkReference() { var redirects = Configuration.Redirects; var crossLinks = Context.Collector.CrossLinks.ToHashSet().ToArray(); @@ -375,7 +375,7 @@ public LinkReference CreateLinkReference() return new LinkMetadata { Anchors = anchors, Hidden = v.File.Hidden }; }); - return new LinkReference + return new RepositoryLinks { Redirects = redirects, UrlPathPrefix = Context.UrlPathPrefix, diff --git a/src/Elastic.Markdown/Links/CrossLinks/ConfigurationCrossLinkFetcher.cs b/src/Elastic.Markdown/Links/CrossLinks/ConfigurationCrossLinkFetcher.cs index 8106ed2bb..4b9f56971 100644 --- a/src/Elastic.Markdown/Links/CrossLinks/ConfigurationCrossLinkFetcher.cs +++ b/src/Elastic.Markdown/Links/CrossLinks/ConfigurationCrossLinkFetcher.cs @@ -5,16 +5,17 @@ using System.Collections.Frozen; using Elastic.Documentation; using Elastic.Documentation.Configuration.Builder; +using Elastic.Documentation.LinkIndex; using Elastic.Documentation.Links; using Microsoft.Extensions.Logging; namespace Elastic.Markdown.Links.CrossLinks; -public class ConfigurationCrossLinkFetcher(ConfigurationFile configuration, ILoggerFactory logger) : CrossLinkFetcher(logger) +public class ConfigurationCrossLinkFetcher(ConfigurationFile configuration, ILinkIndexReader linkIndexProvider, ILoggerFactory logger) : CrossLinkFetcher(linkIndexProvider, logger) { public override async Task Fetch(Cancel ctx) { - var linkReferences = new Dictionary(); + var linkReferences = new Dictionary(); var linkIndexEntries = new Dictionary(); var declaredRepositories = new HashSet(); foreach (var repository in configuration.CrossLinkRepositories) diff --git a/src/Elastic.Markdown/Links/CrossLinks/CrossLinkFetcher.cs b/src/Elastic.Markdown/Links/CrossLinks/CrossLinkFetcher.cs index e914b49f8..ebc4068c6 100644 --- a/src/Elastic.Markdown/Links/CrossLinks/CrossLinkFetcher.cs +++ b/src/Elastic.Markdown/Links/CrossLinks/CrossLinkFetcher.cs @@ -5,6 +5,7 @@ using System.Collections.Frozen; using System.Text.Json; using Elastic.Documentation; +using Elastic.Documentation.LinkIndex; using Elastic.Documentation.Links; using Elastic.Documentation.Serialization; using Elastic.Markdown.IO; @@ -14,7 +15,7 @@ namespace Elastic.Markdown.Links.CrossLinks; public record FetchedCrossLinks { - public required FrozenDictionary LinkReferences { get; init; } + public required FrozenDictionary LinkReferences { get; init; } public required HashSet DeclaredRepositories { get; init; } @@ -25,25 +26,24 @@ public record FetchedCrossLinks public static FetchedCrossLinks Empty { get; } = new() { DeclaredRepositories = [], - LinkReferences = new Dictionary().ToFrozenDictionary(), + LinkReferences = new Dictionary().ToFrozenDictionary(), FromConfiguration = false, LinkIndexEntries = new Dictionary().ToFrozenDictionary() }; } -public abstract class CrossLinkFetcher(ILoggerFactory logger) : IDisposable +public abstract class CrossLinkFetcher(ILinkIndexReader linkIndexProvider, ILoggerFactory logger) : IDisposable { - public const string RegistryUrl = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/link-index.json"; private readonly ILogger _logger = logger.CreateLogger(nameof(CrossLinkFetcher)); private readonly HttpClient _client = new(); - private LinkReferenceRegistry? _linkIndex; + private LinkRegistry? _linkIndex; - public static LinkReference Deserialize(string json) => - JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkReference)!; + public static RepositoryLinks Deserialize(string json) => + JsonSerializer.Deserialize(json, SourceGenerationContext.Default.RepositoryLinks)!; public abstract Task Fetch(Cancel ctx); - public async Task FetchLinkIndex(Cancel ctx) + public async Task FetchLinkIndex(Cancel ctx) { if (_linkIndex is not null) { @@ -51,9 +51,8 @@ public async Task FetchLinkIndex(Cancel ctx) return _linkIndex; } - _logger.LogInformation("Fetching {Url}", RegistryUrl); - var json = await _client.GetStringAsync(RegistryUrl, ctx); - _linkIndex = LinkReferenceRegistry.Deserialize(json); + _logger.LogInformation("Getting link index"); + _linkIndex = await linkIndexProvider.GetRegistry(ctx); return _linkIndex; } @@ -75,7 +74,7 @@ protected static LinkRegistryEntry GetNextContentSourceLinkIndexEntry(IDictionar return linkIndexEntry; } - protected async Task Fetch(string repository, string[] keys, Cancel ctx) + protected async Task Fetch(string repository, string[] keys, Cancel ctx) { var linkIndex = await FetchLinkIndex(ctx); if (!linkIndex.Repositories.TryGetValue(repository, out var repositoryLinks)) @@ -90,7 +89,7 @@ protected async Task Fetch(string repository, string[] keys, Canc throw new Exception($"Repository found in link index however none of: '{string.Join(", ", keys)}' branches found"); } - protected async Task FetchLinkIndexEntry(string repository, LinkRegistryEntry linkRegistryEntry, Cancel ctx) + protected async Task FetchLinkIndexEntry(string repository, LinkRegistryEntry linkRegistryEntry, Cancel ctx) { var linkReference = await TryGetCachedLinkReference(repository, linkRegistryEntry); if (linkReference is not null) @@ -121,7 +120,7 @@ private void WriteLinksJsonCachedFile(string repository, LinkRegistryEntry linkR } } - private async Task TryGetCachedLinkReference(string repository, LinkRegistryEntry linkRegistryEntry) + private async Task TryGetCachedLinkReference(string repository, LinkRegistryEntry linkRegistryEntry) { var cachedFileName = $"links-elastic-{repository}-main-{linkRegistryEntry.ETag}.json"; var cachedPath = Path.Combine(Paths.ApplicationData.FullName, "links", cachedFileName); diff --git a/src/Elastic.Markdown/Links/CrossLinks/CrossLinkResolver.cs b/src/Elastic.Markdown/Links/CrossLinks/CrossLinkResolver.cs index b623a8237..c9eb1ba40 100644 --- a/src/Elastic.Markdown/Links/CrossLinks/CrossLinkResolver.cs +++ b/src/Elastic.Markdown/Links/CrossLinks/CrossLinkResolver.cs @@ -30,10 +30,10 @@ public async Task FetchLinks(Cancel ctx) public bool TryResolve(Action errorEmitter, Action warningEmitter, Uri crossLinkUri, [NotNullWhen(true)] out Uri? resolvedUri) => TryResolve(errorEmitter, warningEmitter, _crossLinks, UriResolver, crossLinkUri, out resolvedUri); - public FetchedCrossLinks UpdateLinkReference(string repository, LinkReference linkReference) + public FetchedCrossLinks UpdateLinkReference(string repository, RepositoryLinks repositoryLinks) { var dictionary = _crossLinks.LinkReferences.ToDictionary(kvp => kvp.Key, kvp => kvp.Value); - dictionary[repository] = linkReference; + dictionary[repository] = repositoryLinks; _crossLinks = _crossLinks with { LinkReferences = dictionary.ToFrozenDictionary() @@ -80,7 +80,7 @@ public static bool TryResolve( private static bool TryFullyValidate(Action errorEmitter, IUriEnvironmentResolver uriResolver, FetchedCrossLinks fetchedCrossLinks, - LinkReference linkReference, + RepositoryLinks repositoryLinks, Uri crossLinkUri, [NotNullWhen(true)] out Uri? resolvedUri) { @@ -89,7 +89,7 @@ private static bool TryFullyValidate(Action errorEmitter, if (string.IsNullOrEmpty(lookupPath) && crossLinkUri.Host.EndsWith(".md")) lookupPath = crossLinkUri.Host; - if (!LookupLink(errorEmitter, fetchedCrossLinks, linkReference, crossLinkUri, ref lookupPath, out var link, out var lookupFragment)) + if (!LookupLink(errorEmitter, fetchedCrossLinks, repositoryLinks, crossLinkUri, ref lookupPath, out var link, out var lookupFragment)) return false; var path = ToTargetUrlPath(lookupPath); @@ -117,7 +117,7 @@ private static bool TryFullyValidate(Action errorEmitter, private static bool LookupLink(Action errorEmitter, FetchedCrossLinks crossLinks, - LinkReference linkReference, + RepositoryLinks repositoryLinks, Uri crossLinkUri, ref string lookupPath, [NotNullWhen(true)] out LinkMetadata? link, @@ -125,7 +125,7 @@ private static bool LookupLink(Action errorEmitter, { lookupFragment = null; - if (linkReference.Redirects is not null && linkReference.Redirects.TryGetValue(lookupPath, out var redirect)) + if (repositoryLinks.Redirects is not null && repositoryLinks.Redirects.TryGetValue(lookupPath, out var redirect)) { var targets = (redirect.Many ?? []) .Select(r => r) @@ -133,10 +133,10 @@ private static bool LookupLink(Action errorEmitter, .Where(s => !string.IsNullOrEmpty(s.To)) .ToArray(); - return ResolveLinkRedirect(targets, errorEmitter, linkReference, crossLinkUri, ref lookupPath, out link, ref lookupFragment); + return ResolveLinkRedirect(targets, errorEmitter, repositoryLinks, crossLinkUri, ref lookupPath, out link, ref lookupFragment); } - if (linkReference.Links.TryGetValue(lookupPath, out link)) + if (repositoryLinks.Links.TryGetValue(lookupPath, out link)) { lookupFragment = crossLinkUri.Fragment; return true; @@ -153,7 +153,7 @@ private static bool LookupLink(Action errorEmitter, private static bool ResolveLinkRedirect( LinkSingleRedirect[] redirects, Action errorEmitter, - LinkReference linkReference, + RepositoryLinks repositoryLinks, Uri crossLinkUri, ref string lookupPath, out LinkMetadata? link, ref string? lookupFragment) { @@ -163,7 +163,7 @@ private static bool ResolveLinkRedirect( { if (string.IsNullOrEmpty(redirect.To)) continue; - if (!linkReference.Links.TryGetValue(redirect.To, out link)) + if (!repositoryLinks.Links.TryGetValue(redirect.To, out link)) continue; if (string.IsNullOrEmpty(fragment)) diff --git a/src/Elastic.Markdown/Links/InboundLinks/LinkIndexCrossLinkFetcher.cs b/src/Elastic.Markdown/Links/InboundLinks/LinkIndexCrossLinkFetcher.cs index d7a452230..cb04c235b 100644 --- a/src/Elastic.Markdown/Links/InboundLinks/LinkIndexCrossLinkFetcher.cs +++ b/src/Elastic.Markdown/Links/InboundLinks/LinkIndexCrossLinkFetcher.cs @@ -4,17 +4,18 @@ using System.Collections.Frozen; using Elastic.Documentation; +using Elastic.Documentation.LinkIndex; using Elastic.Documentation.Links; using Elastic.Markdown.Links.CrossLinks; using Microsoft.Extensions.Logging; namespace Elastic.Markdown.Links.InboundLinks; -public class LinksIndexCrossLinkFetcher(ILoggerFactory logger) : CrossLinkFetcher(logger) +public class LinksIndexCrossLinkFetcher(ILinkIndexReader linkIndexProvider, ILoggerFactory logger) : CrossLinkFetcher(linkIndexProvider, logger) { public override async Task Fetch(Cancel ctx) { - var linkReferences = new Dictionary(); + var linkReferences = new Dictionary(); var linkEntries = new Dictionary(); var declaredRepositories = new HashSet(); var linkIndex = await FetchLinkIndex(ctx); diff --git a/src/Elastic.Markdown/Links/InboundLinks/LinkIndexLinkChecker.cs b/src/Elastic.Markdown/Links/InboundLinks/LinkIndexLinkChecker.cs index d5a24b0b5..34cdc704a 100644 --- a/src/Elastic.Markdown/Links/InboundLinks/LinkIndexLinkChecker.cs +++ b/src/Elastic.Markdown/Links/InboundLinks/LinkIndexLinkChecker.cs @@ -2,8 +2,8 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information -using Elastic.Documentation; using Elastic.Documentation.Diagnostics; +using Elastic.Documentation.LinkIndex; using Elastic.Documentation.Links; using Elastic.Markdown.IO; using Elastic.Markdown.Links.CrossLinks; @@ -14,7 +14,7 @@ namespace Elastic.Markdown.Links.InboundLinks; public class LinkIndexLinkChecker(ILoggerFactory logger) { private readonly ILogger _logger = logger.CreateLogger(); - + private readonly ILinkIndexReader _linkIndexProvider = Aws3LinkIndexReader.CreateAnonymous(); private sealed record RepositoryFilter { public string? LinksTo { get; init; } @@ -25,7 +25,7 @@ private sealed record RepositoryFilter public async Task CheckAll(IDiagnosticsCollector collector, Cancel ctx) { - var fetcher = new LinksIndexCrossLinkFetcher(logger); + var fetcher = new LinksIndexCrossLinkFetcher(_linkIndexProvider, logger); var resolver = new CrossLinkResolver(fetcher); var crossLinks = await resolver.FetchLinks(ctx); @@ -34,7 +34,7 @@ public async Task CheckAll(IDiagnosticsCollector collector, Cancel ctx) public async Task CheckRepository(IDiagnosticsCollector collector, string? toRepository, string? fromRepository, Cancel ctx) { - var fetcher = new LinksIndexCrossLinkFetcher(logger); + var fetcher = new LinksIndexCrossLinkFetcher(_linkIndexProvider, logger); var resolver = new CrossLinkResolver(fetcher); var crossLinks = await resolver.FetchLinks(ctx); var filter = new RepositoryFilter @@ -48,7 +48,7 @@ public async Task CheckRepository(IDiagnosticsCollector collector, string? toRep public async Task CheckWithLocalLinksJson(IDiagnosticsCollector collector, string repository, string localLinksJson, Cancel ctx) { - var fetcher = new LinksIndexCrossLinkFetcher(logger); + var fetcher = new LinksIndexCrossLinkFetcher(_linkIndexProvider, logger); var resolver = new CrossLinkResolver(fetcher); // ReSharper disable once RedundantAssignment var crossLinks = await resolver.FetchLinks(ctx); @@ -65,7 +65,7 @@ public async Task CheckWithLocalLinksJson(IDiagnosticsCollector collector, strin try { var json = await File.ReadAllTextAsync(localLinksJson, ctx); - var localLinkReference = LinkReference.Deserialize(json); + var localLinkReference = RepositoryLinks.Deserialize(json); crossLinks = resolver.UpdateLinkReference(repository, localLinkReference); } catch (Exception e) diff --git a/src/infra/docs-lambda-index-publisher/LinkIndexProvider.cs b/src/infra/docs-lambda-index-publisher/LinkIndexProvider.cs deleted file mode 100644 index 4cd5780cd..000000000 --- a/src/infra/docs-lambda-index-publisher/LinkIndexProvider.cs +++ /dev/null @@ -1,100 +0,0 @@ -// Licensed to Elasticsearch B.V under one or more agreements. -// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. -// See the LICENSE file in the project root for more information - -using System.Net; -using Amazon.Lambda.Core; -using Amazon.S3; -using Amazon.S3.Model; -using Elastic.Documentation.Links; - -namespace Elastic.Documentation.Lambda.LinkIndexUploader; - -/// -/// Gets the link index from S3 once. -/// You can then update the link index with and save it with . -/// If the link index changed in the meantime, will throw an exception, -/// thus all the messages from the queue will be sent back to the queue. -/// -public class LinkIndexProvider(IAmazonS3 s3Client, ILambdaLogger logger, string bucketName, string key) -{ - private string? _etag; - private LinkReferenceRegistry? _linkIndex; - - private async Task GetLinkIndex() - { - var getObjectRequest = new GetObjectRequest - { - BucketName = bucketName, - Key = key - }; - logger.LogInformation("Getting link index from s3://{bucketName}/{key}", bucketName, key); - var getObjectResponse = await s3Client.GetObjectAsync(getObjectRequest); - await using var stream = getObjectResponse.ResponseStream; - _etag = getObjectResponse.ETag; - logger.LogInformation("Successfully got link index from s3://{bucketName}/{key}", bucketName, key); - _linkIndex = LinkReferenceRegistry.Deserialize(stream); - return _linkIndex; - } - - public async Task UpdateLinkIndexEntry(LinkRegistryEntry newEntry) - { - _linkIndex ??= await GetLinkIndex(); - var repository = newEntry.Repository; - var branch = newEntry.Branch; - // repository already exists in links.json - if (_linkIndex.Repositories.TryGetValue(repository, out var existingRepositoryEntry)) - { - // The branch already exists in the repository entry - if (existingRepositoryEntry.TryGetValue(branch, out var existingBranchEntry)) - { - if (newEntry.UpdatedAt > existingBranchEntry.UpdatedAt) - { - existingRepositoryEntry[branch] = newEntry; - logger.LogInformation("Updated existing entry for {repository}@{branch}", repository, branch); - } - else - logger.LogInformation("Skipping update for {repository}@{branch} because the existing entry is newer or equal", repository, branch); - } - // branch does not exist in the repository entry - else - { - existingRepositoryEntry[branch] = newEntry; - logger.LogInformation("Added new entry '{repository}@{branch}' to existing entry for '{repository}'", repository, branch, repository); - } - } - // onboarding new repository - else - { - _linkIndex.Repositories.Add(repository, new Dictionary - { - { branch, newEntry } - }); - logger.LogInformation("Added new entry for {repository}@{branch}", repository, branch); - } - } - - public async Task Save() - { - if (_etag == null || _linkIndex == null) - throw new InvalidOperationException("You must call UpdateLinkIndexEntry() before Save()"); - var json = LinkReferenceRegistry.Serialize(_linkIndex); - logger.LogInformation("Saving link index to s3://{bucketName}/{key}", bucketName, key); - var putObjectRequest = new PutObjectRequest - { - BucketName = bucketName, - Key = key, - ContentBody = json, - ContentType = "application/json", - IfMatch = _etag // Only update if the ETag matches. Meaning the object has not been changed in the meantime. - }; - var putResponse = await s3Client.PutObjectAsync(putObjectRequest); - if (putResponse.HttpStatusCode == HttpStatusCode.OK) - logger.LogInformation("Successfully saved link index to s3://{bucketName}/{key}", bucketName, key); - else - { - logger.LogError("Unable to save index to s3://{bucketName}/{key}", bucketName, key); - throw new Exception($"Unable to save index to s3://{bucketName}/{key}"); - } - } -} diff --git a/src/infra/docs-lambda-index-publisher/LinkReferenceProvider.cs b/src/infra/docs-lambda-index-publisher/LinkReferenceProvider.cs deleted file mode 100644 index abfdfa8d5..000000000 --- a/src/infra/docs-lambda-index-publisher/LinkReferenceProvider.cs +++ /dev/null @@ -1,27 +0,0 @@ -// Licensed to Elasticsearch B.V under one or more agreements. -// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. -// See the LICENSE file in the project root for more information - -using Amazon.Lambda.Core; -using Amazon.S3; -using Amazon.S3.Model; -using Elastic.Documentation.Links; - -namespace Elastic.Documentation.Lambda.LinkIndexUploader; - -public class LinkReferenceProvider(IAmazonS3 s3Client, ILambdaLogger logger, string bucketName) -{ - public async Task GetLinkReference(string key, Cancel ctx) - { - var getObjectRequest = new GetObjectRequest - { - BucketName = bucketName, - Key = key - }; - logger.LogInformation("Getting object {key} from bucket {bucketName}", key, bucketName); - var getObjectResponse = await s3Client.GetObjectAsync(getObjectRequest, ctx); - await using var stream = getObjectResponse.ResponseStream; - logger.LogInformation("Successfully got object {key} from bucket {bucketName}", key, bucketName); - return LinkReference.Deserialize(stream); - } -} diff --git a/src/infra/docs-lambda-index-publisher/Program.cs b/src/infra/docs-lambda-index-publisher/Program.cs index 5b289ca5a..f67b83dde 100644 --- a/src/infra/docs-lambda-index-publisher/Program.cs +++ b/src/infra/docs-lambda-index-publisher/Program.cs @@ -11,6 +11,7 @@ using Amazon.S3; using Amazon.S3.Util; using Elastic.Documentation.Lambda.LinkIndexUploader; +using Elastic.Documentation.LinkIndex; using Elastic.Documentation.Links; const string bucketName = "elastic-docs-link-index"; @@ -26,19 +27,22 @@ static async Task Handler(SQSEvent ev, ILambdaContext context) { var s3Client = new AmazonS3Client(); - var linkIndexProvider = new LinkIndexProvider(s3Client, context.Logger, bucketName, indexFile); + ILinkIndexReaderWriter linkIndexReaderWriter = new AwsS3LinkIndexReaderWriter(s3Client, bucketName, indexFile); var batchItemFailures = new List(); + + var linkRegistry = await linkIndexReaderWriter.GetRegistry(); + foreach (var message in ev.Records) { context.Logger.LogInformation("Processing message {MessageId}", message.MessageId); context.Logger.LogInformation("Message body: {MessageBody}", message.Body); try { - var s3RecordLinkReferenceTuples = await GetS3RecordLinkReferenceTuples(s3Client, message, context); + var s3RecordLinkReferenceTuples = await GetS3RecordLinkReferenceTuples(linkIndexReaderWriter, message); foreach (var (s3Record, linkReference) in s3RecordLinkReferenceTuples) { var newEntry = ConvertToLinkIndexEntry(s3Record, linkReference); - await linkIndexProvider.UpdateLinkIndexEntry(newEntry); + linkRegistry = linkRegistry.WithLinkRegistryEntry(newEntry); } } catch (Exception e) @@ -53,7 +57,7 @@ static async Task Handler(SQSEvent ev, ILambdaContext context) } try { - await linkIndexProvider.Save(); + await linkIndexReaderWriter.SaveRegistry(linkRegistry); var response = new SQSBatchResponse(batchItemFailures); if (batchItemFailures.Count > 0) context.Logger.LogInformation("Failed to process {batchItemFailuresCount} of {allMessagesCount} messages. Returning them to the queue.", batchItemFailures.Count, ev.Records.Count); @@ -77,7 +81,7 @@ static async Task Handler(SQSEvent ev, ILambdaContext context) } } -static LinkRegistryEntry ConvertToLinkIndexEntry(S3EventNotification.S3EventNotificationRecord record, LinkReference linkReference) +static LinkRegistryEntry ConvertToLinkIndexEntry(S3EventNotification.S3EventNotificationRecord record, RepositoryLinks linkReference) { var s3Object = record.S3.Object; var keyTokens = s3Object.Key.Split('/'); @@ -94,15 +98,14 @@ static LinkRegistryEntry ConvertToLinkIndexEntry(S3EventNotification.S3EventNoti }; } -static async Task> GetS3RecordLinkReferenceTuples(IAmazonS3 s3Client, - SQSEvent.SQSMessage message, ILambdaContext context) +static async Task> GetS3RecordLinkReferenceTuples(ILinkIndexReaderWriter linkIndexReaderWriter, + SQSEvent.SQSMessage message) { var s3Event = S3EventNotification.ParseJson(message.Body); - var recordLinkReferenceTuples = new ConcurrentBag<(S3EventNotification.S3EventNotificationRecord, LinkReference)>(); - var linkReferenceProvider = new LinkReferenceProvider(s3Client, context.Logger, bucketName); + var recordLinkReferenceTuples = new ConcurrentBag<(S3EventNotification.S3EventNotificationRecord, RepositoryLinks)>(); await Parallel.ForEachAsync(s3Event.Records, async (record, ctx) => { - var linkReference = await linkReferenceProvider.GetLinkReference(record.S3.Object.Key, ctx); + var linkReference = await linkIndexReaderWriter.GetRepositoryLinks(record.S3.Object.Key, ctx); recordLinkReferenceTuples.Add((record, linkReference)); }); return recordLinkReferenceTuples; diff --git a/src/infra/docs-lambda-index-publisher/docs-lambda-index-publisher.csproj b/src/infra/docs-lambda-index-publisher/docs-lambda-index-publisher.csproj index 8358b3dbf..e8994b9aa 100644 --- a/src/infra/docs-lambda-index-publisher/docs-lambda-index-publisher.csproj +++ b/src/infra/docs-lambda-index-publisher/docs-lambda-index-publisher.csproj @@ -29,6 +29,7 @@ + diff --git a/src/tooling/docs-assembler/AssembleSources.cs b/src/tooling/docs-assembler/AssembleSources.cs index d719aa26c..1843a7b74 100644 --- a/src/tooling/docs-assembler/AssembleSources.cs +++ b/src/tooling/docs-assembler/AssembleSources.cs @@ -4,6 +4,7 @@ using System.Collections.Frozen; using System.IO.Abstractions; +using Amazon.S3; using Documentation.Assembler.Building; using Documentation.Assembler.Navigation; using Documentation.Assembler.Sourcing; @@ -11,6 +12,7 @@ using Elastic.Documentation.Configuration; using Elastic.Documentation.Configuration.Assembler; using Elastic.Documentation.Configuration.Builder; +using Elastic.Documentation.LinkIndex; using Elastic.Markdown.IO.Navigation; using Elastic.Markdown.Links.CrossLinks; using Microsoft.Extensions.Logging.Abstractions; @@ -60,8 +62,8 @@ private AssembleSources(AssembleContext assembleContext, Checkout[] checkouts) AssembleContext = assembleContext; TocTopLevelMappings = GetConfiguredSources(assembleContext); HistoryMappings = GetHistoryMapping(assembleContext); - - var crossLinkFetcher = new AssemblerCrossLinkFetcher(NullLoggerFactory.Instance, assembleContext.Configuration, assembleContext.Environment); + var linkIndexProvider = Aws3LinkIndexReader.CreateAnonymous(); + var crossLinkFetcher = new AssemblerCrossLinkFetcher(NullLoggerFactory.Instance, assembleContext.Configuration, assembleContext.Environment, linkIndexProvider); UriResolver = new PublishEnvironmentUriResolver(TocTopLevelMappings, assembleContext.Environment); var crossLinkResolver = new CrossLinkResolver(crossLinkFetcher, UriResolver); AssembleSets = checkouts diff --git a/src/tooling/docs-assembler/Building/AssemblerCrossLinkFetcher.cs b/src/tooling/docs-assembler/Building/AssemblerCrossLinkFetcher.cs index b590e8444..38c038654 100644 --- a/src/tooling/docs-assembler/Building/AssemblerCrossLinkFetcher.cs +++ b/src/tooling/docs-assembler/Building/AssemblerCrossLinkFetcher.cs @@ -3,20 +3,20 @@ // See the LICENSE file in the project root for more information using System.Collections.Frozen; -using Elastic.Documentation; using Elastic.Documentation.Configuration.Assembler; +using Elastic.Documentation.LinkIndex; using Elastic.Documentation.Links; using Elastic.Markdown.Links.CrossLinks; using Microsoft.Extensions.Logging; namespace Documentation.Assembler.Building; -public class AssemblerCrossLinkFetcher(ILoggerFactory logger, AssemblyConfiguration configuration, PublishEnvironment publishEnvironment) - : CrossLinkFetcher(logger) +public class AssemblerCrossLinkFetcher(ILoggerFactory logger, AssemblyConfiguration configuration, PublishEnvironment publishEnvironment, ILinkIndexReader linkIndexProvider) + : CrossLinkFetcher(linkIndexProvider, logger) { public override async Task Fetch(Cancel ctx) { - var linkReferences = new Dictionary(); + var linkReferences = new Dictionary(); var linkIndexEntries = new Dictionary(); var declaredRepositories = new HashSet(); var repositories = configuration.ReferenceRepositories.Values.Concat([configuration.Narrative]); diff --git a/src/tooling/docs-assembler/Cli/ContentSourceCommands.cs b/src/tooling/docs-assembler/Cli/ContentSourceCommands.cs index 54c27339e..f7d627684 100644 --- a/src/tooling/docs-assembler/Cli/ContentSourceCommands.cs +++ b/src/tooling/docs-assembler/Cli/ContentSourceCommands.cs @@ -5,9 +5,11 @@ using System.Diagnostics.CodeAnalysis; using System.IO.Abstractions; using Actions.Core.Services; +using Amazon.S3; using ConsoleAppFramework; using Documentation.Assembler.Building; using Elastic.Documentation.Configuration.Assembler; +using Elastic.Documentation.LinkIndex; using Elastic.Documentation.Tooling.Diagnostics.Console; using Elastic.Markdown.Links.CrossLinks; using Microsoft.Extensions.Logging; @@ -41,7 +43,8 @@ public async Task Validate(Cancel ctx = default) Force = false, AllowIndexing = false }; - var fetcher = new AssemblerCrossLinkFetcher(logFactory, context.Configuration, context.Environment); + ILinkIndexReader linkIndexReader = Aws3LinkIndexReader.CreateAnonymous(); + var fetcher = new AssemblerCrossLinkFetcher(logFactory, context.Configuration, context.Environment, linkIndexReader); var links = await fetcher.FetchLinkIndex(ctx); var repositories = context.Configuration.ReferenceRepositories.Values.Concat([context.Configuration.Narrative]).ToList(); @@ -49,7 +52,7 @@ public async Task Validate(Cancel ctx = default) { if (!links.Repositories.TryGetValue(repository.Name, out var registryMapping)) { - collector.EmitError(context.ConfigurationPath, $"'{repository}' does not exist in {CrossLinkFetcher.RegistryUrl}"); + collector.EmitError(context.ConfigurationPath, $"'{repository}' does not exist in link index"); continue; } @@ -58,12 +61,12 @@ public async Task Validate(Cancel ctx = default) if (!registryMapping.TryGetValue(next, out _)) { collector.EmitError(context.ConfigurationPath, - $"'{repository.Name}' has not yet published links.json for configured 'next' content source: '{next}' see {CrossLinkFetcher.RegistryUrl}"); + $"'{repository.Name}' has not yet published links.json for configured 'next' content source: '{next}' see {linkIndexReader.RegistryUrl}"); } if (!registryMapping.TryGetValue(current, out _)) { collector.EmitError(context.ConfigurationPath, - $"'{repository.Name}' has not yet published links.json for configured 'current' content source: '{current}' see {CrossLinkFetcher.RegistryUrl}"); + $"'{repository.Name}' has not yet published links.json for configured 'current' content source: '{current}' see {linkIndexReader.RegistryUrl}"); } } diff --git a/src/tooling/docs-assembler/Links/NavigationPrefixChecker.cs b/src/tooling/docs-assembler/Links/NavigationPrefixChecker.cs index 4e9ed1f1a..ae863401f 100644 --- a/src/tooling/docs-assembler/Links/NavigationPrefixChecker.cs +++ b/src/tooling/docs-assembler/Links/NavigationPrefixChecker.cs @@ -3,10 +3,12 @@ // See the LICENSE file in the project root for more information using System.Collections.Immutable; +using Amazon.S3; using Documentation.Assembler.Building; using Documentation.Assembler.Navigation; using Elastic.Documentation; using Elastic.Documentation.Diagnostics; +using Elastic.Documentation.LinkIndex; using Elastic.Documentation.Links; using Elastic.Markdown.IO; using Elastic.Markdown.Links.CrossLinks; @@ -82,9 +84,10 @@ public async Task CheckWithLocalLinksJson(DiagnosticsCollector collector, string public async Task CheckAllPublishedLinks(DiagnosticsCollector collector, Cancel ctx) => await FetchAndValidateCrossLinks(collector, null, null, ctx); - private async Task FetchAndValidateCrossLinks(DiagnosticsCollector collector, string? updateRepository, LinkReference? updateReference, Cancel ctx) + private async Task FetchAndValidateCrossLinks(DiagnosticsCollector collector, string? updateRepository, RepositoryLinks? updateReference, Cancel ctx) { - var fetcher = new LinksIndexCrossLinkFetcher(_loggerFactory); + var linkIndexProvider = Aws3LinkIndexReader.CreateAnonymous(); + var fetcher = new LinksIndexCrossLinkFetcher(linkIndexProvider, _loggerFactory); var resolver = new CrossLinkResolver(fetcher); var crossLinks = await resolver.FetchLinks(ctx); var dictionary = new Dictionary(); @@ -126,12 +129,12 @@ private async Task FetchAndValidateCrossLinks(DiagnosticsCollector collector, st } } - private async Task ReadLocalLinksJsonAsync(string localLinksJson, Cancel ctx) + private async Task ReadLocalLinksJsonAsync(string localLinksJson, Cancel ctx) { try { var json = await File.ReadAllTextAsync(localLinksJson, ctx); - return LinkReference.Deserialize(json); + return RepositoryLinks.Deserialize(json); } catch (Exception e) { diff --git a/tests/Elastic.Markdown.Tests/DocSet/LinkReferenceTests.cs b/tests/Elastic.Markdown.Tests/DocSet/RepositoryLinksTests.cs similarity index 86% rename from tests/Elastic.Markdown.Tests/DocSet/LinkReferenceTests.cs rename to tests/Elastic.Markdown.Tests/DocSet/RepositoryLinksTests.cs index b2b88bf6f..0d48b25e8 100644 --- a/tests/Elastic.Markdown.Tests/DocSet/LinkReferenceTests.cs +++ b/tests/Elastic.Markdown.Tests/DocSet/RepositoryLinksTests.cs @@ -9,11 +9,11 @@ namespace Elastic.Markdown.Tests.DocSet; -public class LinkReferenceTests : NavigationTestsBase +public class RepositoryLinksTests : NavigationTestsBase { - public LinkReferenceTests(ITestOutputHelper output) : base(output) => Reference = Set.CreateLinkReference(); + public RepositoryLinksTests(ITestOutputHelper output) : base(output) => Reference = Set.CreateLinkReference(); - private LinkReference Reference { get; } + private RepositoryLinks Reference { get; } [Fact] public void ShouldNotBeNull() => @@ -54,7 +54,7 @@ public class LinkReferenceSerializationTests [Fact] public void SerializesCurrent() { - var linkReference = new LinkReference + var linkReference = new RepositoryLinks { Origin = new GitCheckoutInformation { @@ -66,7 +66,7 @@ public void SerializesCurrent() Links = [], CrossLinks = [], }; - var json = LinkReference.Serialize(linkReference); + var json = RepositoryLinks.Serialize(linkReference); // language=json json.Should().Be( """ @@ -104,7 +104,7 @@ public void Deserializes() "redirects": null } """; - var linkReference = LinkReference.Deserialize(json); + var linkReference = RepositoryLinks.Deserialize(json); linkReference.Origin.Ref.Should().Be("ref"); } diff --git a/tests/Elastic.Markdown.Tests/Inline/InlineLinkTests.cs b/tests/Elastic.Markdown.Tests/Inline/InlineLinkTests.cs index 8d44e07ed..b724deaed 100644 --- a/tests/Elastic.Markdown.Tests/Inline/InlineLinkTests.cs +++ b/tests/Elastic.Markdown.Tests/Inline/InlineLinkTests.cs @@ -97,7 +97,7 @@ public void GeneratesHtml() => public void EmitsCrossLink() => Collector.CrossLinks.Should().HaveCount(0); } -public class LinkReferenceTest(ITestOutputHelper output) : LinkTestBase(output, +public class RepositoryLinksTest(ITestOutputHelper output) : LinkTestBase(output, """ [test][test] diff --git a/tests/Elastic.Markdown.Tests/TestCrossLinkResolver.cs b/tests/Elastic.Markdown.Tests/TestCrossLinkResolver.cs index 56a439dfa..85fabd552 100644 --- a/tests/Elastic.Markdown.Tests/TestCrossLinkResolver.cs +++ b/tests/Elastic.Markdown.Tests/TestCrossLinkResolver.cs @@ -15,7 +15,7 @@ public class TestCrossLinkResolver : ICrossLinkResolver { public IUriEnvironmentResolver UriResolver { get; } = new IsolatedBuildEnvironmentUriResolver(); private FetchedCrossLinks _crossLinks = FetchedCrossLinks.Empty; - private Dictionary LinkReferences { get; } = []; + private Dictionary LinkReferences { get; } = []; private HashSet DeclaredRepositories { get; } = []; public Task FetchLinks(Cancel ctx) diff --git a/tests/authoring/Framework/TestCrossLinkResolver.fs b/tests/authoring/Framework/TestCrossLinkResolver.fs index 4b9c9d6d4..511d44a8d 100644 --- a/tests/authoring/Framework/TestCrossLinkResolver.fs +++ b/tests/authoring/Framework/TestCrossLinkResolver.fs @@ -16,7 +16,7 @@ open Elastic.Markdown.Links.CrossLinks type TestCrossLinkResolver (config: ConfigurationFile) = - let references = Dictionary() + let references = Dictionary() let declared = HashSet() let uriResolver = IsolatedBuildEnvironmentUriResolver() @@ -28,7 +28,7 @@ type TestCrossLinkResolver (config: ConfigurationFile) = member this.UriResolver = uriResolver member this.FetchLinks(ctx) = - let redirects = LinkReference.SerializeRedirects config.Redirects + let redirects = RepositoryLinks.SerializeRedirects config.Redirects // language=json let json = $$"""{ "origin": { @@ -69,7 +69,7 @@ type TestCrossLinkResolver (config: ConfigurationFile) = this.DeclaredRepositories.Add("elasticsearch") |> ignore let indexEntries = - this.LinkReferences.ToDictionary(_.Key, fun (e : KeyValuePair) -> LinkRegistryEntry( + this.LinkReferences.ToDictionary(_.Key, fun (e : KeyValuePair) -> LinkRegistryEntry( Repository = e.Key, Path = $"elastic/asciidocalypse/{e.Key}/links.json", Branch = "main", @@ -88,7 +88,7 @@ type TestCrossLinkResolver (config: ConfigurationFile) = member this.TryResolve(errorEmitter, warningEmitter, crossLinkUri, []resolvedUri : byref) = let indexEntries = - this.LinkReferences.ToDictionary(_.Key, fun (e : KeyValuePair) -> LinkRegistryEntry( + this.LinkReferences.ToDictionary(_.Key, fun (e : KeyValuePair) -> LinkRegistryEntry( Repository = e.Key, Path = $"elastic/asciidocalypse/{e.Key}/links.json", Branch = "main",