Refactor LinkIndexProvider into own project

docs-builder.sln

@@ -99,6 +99,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "assembler-config-validate",
 		actions\assembler-config-validate\action.yml = actions\assembler-config-validate\action.yml
 	EndProjectSection
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Documentation.LinkIndex", "src\Elastic.Documentation.LinkIndex\Elastic.Documentation.LinkIndex.csproj", "{FD1AC230-798B-4AB9-8CE6-A06264885DBC}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -160,6 +162,10 @@ Global
 		{CD94F9E4-7FCD-4152-81F1-4288C6B75367}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{CD94F9E4-7FCD-4152-81F1-4288C6B75367}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{CD94F9E4-7FCD-4152-81F1-4288C6B75367}.Release|Any CPU.Build.0 = Release|Any CPU
+		{FD1AC230-798B-4AB9-8CE6-A06264885DBC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{FD1AC230-798B-4AB9-8CE6-A06264885DBC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{FD1AC230-798B-4AB9-8CE6-A06264885DBC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{FD1AC230-798B-4AB9-8CE6-A06264885DBC}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{4D198E25-C211-41DC-9E84-B15E89BD7048} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
@@ -184,5 +190,6 @@ Global
 		{7D36DDDA-9E0B-4D2C-8033-5D62FF8B6166} = {059E787F-85C1-43BE-9DD6-CE319E106383}
 		{FB1C1954-D8E2-4745-BA62-04DD82FB4792} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7}
 		{E20FEEF9-1D1A-4CDA-A546-7FDC573BE399} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7}
+		{FD1AC230-798B-4AB9-8CE6-A06264885DBC} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
 	EndGlobalSection
 EndGlobal

src/Elastic.Documentation.LinkIndex/Elastic.Documentation.LinkIndex.csproj

@@ -0,0 +1,17 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net9.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <PackageReference Include="AWSSDK.S3" />
+    </ItemGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\Elastic.Documentation\Elastic.Documentation.csproj" />
+    </ItemGroup>
+
+</Project>

src/Elastic.Documentation.LinkIndex/LinkIndexProvider.cs

@@ -0,0 +1,84 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using System.Net;
+using Amazon.Runtime;
+using Amazon.S3;
+using Amazon.S3.Model;
+using Elastic.Documentation.Links;
+
+namespace Elastic.Documentation.LinkIndex;
+
+public interface ILinkIndexProvider
+{
+	Task<LinkReferenceRegistry> GetLinkIndex(Cancel cancellationToken = default);
+	Task SaveLinkIndex(LinkReferenceRegistry registry, Cancel cancellationToken = default);
+	Task<LinkReference> GetLinkReference(string key, Cancel cancellationToken = default);
+	string GetLinkIndexPublicUrl();
+}
+
+public class AwsS3LinkIndexProvider(IAmazonS3 s3Client, string bucketName = "elastic-docs-link-index", string registryKey = "link-index.json") : ILinkIndexProvider
+{
+
+	// <summary>
+	// Using <see cref="AnonymousAWSCredentials"/> to access the link index
+	// allows to read from the link index without the need to provide AWS credentials.
+	// </summary>
+	public static AwsS3LinkIndexProvider CreateAnonymous()
+	{
+		var credentials = new AnonymousAWSCredentials();
+		var config = new AmazonS3Config
+		{
+			RegionEndpoint = Amazon.RegionEndpoint.USEast2
+		};
+		var s3Client = new AmazonS3Client(credentials, config);
+		return new AwsS3LinkIndexProvider(s3Client);
+	}
+
+	public async Task<LinkReferenceRegistry> GetLinkIndex(Cancel cancellationToken = default)
+	{
+		var getObjectRequest = new GetObjectRequest
+		{
+			BucketName = bucketName,
+			Key = registryKey
+		};
+		var getObjectResponse = await s3Client.GetObjectAsync(getObjectRequest, cancellationToken);
+		await using var stream = getObjectResponse.ResponseStream;
+		var linkIndex = LinkReferenceRegistry.Deserialize(stream);
+		return linkIndex with { ETag = getObjectResponse.ETag };
+	}
+
+	public async Task SaveLinkIndex(LinkReferenceRegistry registry, Cancel cancellationToken = default)
+	{
+		if (registry.ETag == null)
+			// The ETag should not be null if the LinkReferenceRegistry was retrieved from GetLinkIndex()
+			throw new InvalidOperationException($"{nameof(LinkReferenceRegistry)}.{nameof(registry.ETag)} cannot be null");
+		var json = LinkReferenceRegistry.Serialize(registry);
+		var putObjectRequest = new PutObjectRequest
+		{
+			BucketName = bucketName,
+			Key = registryKey,
+			ContentBody = json,
+			ContentType = "application/json",
+			IfMatch = registry.ETag // Only update if the ETag matches. Meaning the object has not been changed in the meantime.
+		};
+		var putResponse = await s3Client.PutObjectAsync(putObjectRequest, cancellationToken);
+		if (putResponse.HttpStatusCode != HttpStatusCode.OK)
+			throw new Exception($"Unable to save {nameof(LinkReferenceRegistry)} to s3://{bucketName}/{registryKey}");
+	}
+
+	public async Task<LinkReference> GetLinkReference(string key, Cancel cancellationToken)
+	{
+		var getObjectRequest = new GetObjectRequest
+		{
+			BucketName = bucketName,
+			Key = key
+		};
+		var getObjectResponse = await s3Client.GetObjectAsync(getObjectRequest, cancellationToken);
+		await using var stream = getObjectResponse.ResponseStream;
+		return LinkReference.Deserialize(stream);
+	}
+
+	public string GetLinkIndexPublicUrl() => $"https://{bucketName}.s3.{s3Client.Config.RegionEndpoint.SystemName}.amazonaws.com/{registryKey}";
+}

src/Elastic.Documentation/Links/LinkReferenceRegistry.cs

@@ -14,6 +14,40 @@ public record LinkReferenceRegistry
 	[JsonPropertyName("repositories")]
 	public required Dictionary<string, Dictionary<string, LinkRegistryEntry>> Repositories { get; init; }
 
+	[JsonIgnore]
+	public string? ETag { get; init; }
+
+	public LinkReferenceRegistry WithLinkRegistryEntry(LinkRegistryEntry entry)
+	{
+		var copiedRepositories = new Dictionary<string, Dictionary<string, LinkRegistryEntry>>(Repositories);
+		var repository = entry.Repository;
+		var branch = entry.Branch;
+		// repository already exists in links.json
+		if (copiedRepositories.TryGetValue(repository, out var existingRepositoryEntry))
+		{
+			// The branch already exists in the repository entry
+			if (existingRepositoryEntry.TryGetValue(branch, out var existingBranchEntry))
+			{
+				if (entry.UpdatedAt > existingBranchEntry.UpdatedAt)
+					existingRepositoryEntry[branch] = entry;
+			}
+			// branch does not exist in the repository entry
+			else
+			{
+				existingRepositoryEntry[branch] = entry;
+			}
+		}
+		// onboarding new repository
+		else
+		{
+			copiedRepositories.Add(repository, new Dictionary<string, LinkRegistryEntry>
+			{
+				{ branch, entry }
+			});
+		}
+		return this with { Repositories = copiedRepositories };
+	}
+
 	public static LinkReferenceRegistry Deserialize(Stream json) =>
 		JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkReferenceRegistry)!;
 
@@ -46,4 +80,3 @@ public record LinkRegistryEntry
 	[JsonPropertyName("updated_at")]
 	public DateTime UpdatedAt { get; init; } = DateTime.MinValue;
 }
-

src/Elastic.Markdown/Elastic.Markdown.csproj

@@ -64,6 +64,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\Elastic.Documentation.LinkIndex\Elastic.Documentation.LinkIndex.csproj" />
     <ProjectReference Include="..\Elastic.Documentation\Elastic.Documentation.csproj" />
     <ProjectReference Include="..\Elastic.Documentation.Configuration\Elastic.Documentation.Configuration.csproj" />
   </ItemGroup>

src/Elastic.Markdown/IO/DocumentationSet.cs

@@ -8,8 +8,8 @@
 using Elastic.Documentation;
 using Elastic.Documentation.Configuration.Builder;
 using Elastic.Documentation.Configuration.TableOfContents;
+using Elastic.Documentation.LinkIndex;
 using Elastic.Documentation.Links;
-using Elastic.Markdown.Diagnostics;
 using Elastic.Markdown.Extensions;
 using Elastic.Markdown.Extensions.DetectionRules;
 using Elastic.Markdown.IO.Navigation;
@@ -126,7 +126,7 @@ public DocumentationSet(
 		SourceDirectory = context.DocumentationSourceDirectory;
 		OutputDirectory = context.DocumentationOutputDirectory;
 		LinkResolver =
-			linkResolver ?? new CrossLinkResolver(new ConfigurationCrossLinkFetcher(context.Configuration, logger));
+			linkResolver ?? new CrossLinkResolver(new ConfigurationCrossLinkFetcher(context.Configuration, AwsS3LinkIndexProvider.CreateAnonymous(), logger));
 		Configuration = context.Configuration;
 		EnabledExtensions = InstantiateExtensions();
 		treeCollector ??= new TableOfContentsTreeCollector();

src/Elastic.Markdown/Links/CrossLinks/ConfigurationCrossLinkFetcher.cs

@@ -5,12 +5,13 @@
 using System.Collections.Frozen;
 using Elastic.Documentation;
 using Elastic.Documentation.Configuration.Builder;
+using Elastic.Documentation.LinkIndex;
 using Elastic.Documentation.Links;
 using Microsoft.Extensions.Logging;
 
 namespace Elastic.Markdown.Links.CrossLinks;
 
-public class ConfigurationCrossLinkFetcher(ConfigurationFile configuration, ILoggerFactory logger) : CrossLinkFetcher(logger)
+public class ConfigurationCrossLinkFetcher(ConfigurationFile configuration, ILinkIndexProvider linkIndexProvider, ILoggerFactory logger) : CrossLinkFetcher(linkIndexProvider, logger)
 {
 	public override async Task<FetchedCrossLinks> Fetch(Cancel ctx)
 	{

src/Elastic.Markdown/Links/CrossLinks/CrossLinkFetcher.cs

@@ -5,6 +5,7 @@
 using System.Collections.Frozen;
 using System.Text.Json;
 using Elastic.Documentation;
+using Elastic.Documentation.LinkIndex;
 using Elastic.Documentation.Links;
 using Elastic.Documentation.Serialization;
 using Elastic.Markdown.IO;
@@ -31,9 +32,8 @@ public record FetchedCrossLinks
 	};
 }
 
-public abstract class CrossLinkFetcher(ILoggerFactory logger) : IDisposable
+public abstract class CrossLinkFetcher(ILinkIndexProvider linkIndexProvider, ILoggerFactory logger) : IDisposable
 {
-	public const string RegistryUrl = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/link-index.json";
 	private readonly ILogger _logger = logger.CreateLogger(nameof(CrossLinkFetcher));
 	private readonly HttpClient _client = new();
 	private LinkReferenceRegistry? _linkIndex;
@@ -51,9 +51,8 @@ public async Task<LinkReferenceRegistry> FetchLinkIndex(Cancel ctx)
 			return _linkIndex;
 		}
 
-		_logger.LogInformation("Fetching {Url}", RegistryUrl);
-		var json = await _client.GetStringAsync(RegistryUrl, ctx);
-		_linkIndex = LinkReferenceRegistry.Deserialize(json);
+		_logger.LogInformation("Getting link index");
+		_linkIndex = await linkIndexProvider.GetLinkIndex(ctx);
 		return _linkIndex;
 	}
 

src/Elastic.Markdown/Links/InboundLinks/LinkIndexCrossLinkFetcher.cs

@@ -4,13 +4,14 @@
 
 using System.Collections.Frozen;
 using Elastic.Documentation;
+using Elastic.Documentation.LinkIndex;
 using Elastic.Documentation.Links;
 using Elastic.Markdown.Links.CrossLinks;
 using Microsoft.Extensions.Logging;
 
 namespace Elastic.Markdown.Links.InboundLinks;
 
-public class LinksIndexCrossLinkFetcher(ILoggerFactory logger) : CrossLinkFetcher(logger)
+public class LinksIndexCrossLinkFetcher(ILinkIndexProvider linkIndexProvider, ILoggerFactory logger) : CrossLinkFetcher(linkIndexProvider, logger)
 {
 	public override async Task<FetchedCrossLinks> Fetch(Cancel ctx)
 	{

src/Elastic.Markdown/Links/InboundLinks/LinkIndexLinkChecker.cs

@@ -2,8 +2,8 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
-using Elastic.Documentation;
 using Elastic.Documentation.Diagnostics;
+using Elastic.Documentation.LinkIndex;
 using Elastic.Documentation.Links;
 using Elastic.Markdown.IO;
 using Elastic.Markdown.Links.CrossLinks;
@@ -14,7 +14,7 @@ namespace Elastic.Markdown.Links.InboundLinks;
 public class LinkIndexLinkChecker(ILoggerFactory logger)
 {
 	private readonly ILogger _logger = logger.CreateLogger<LinkIndexLinkChecker>();
-
+	private readonly ILinkIndexProvider _linkIndexProvider = AwsS3LinkIndexProvider.CreateAnonymous();
 	private sealed record RepositoryFilter
 	{
 		public string? LinksTo { get; init; }
@@ -25,7 +25,7 @@ private sealed record RepositoryFilter
 
 	public async Task CheckAll(IDiagnosticsCollector collector, Cancel ctx)
 	{
-		var fetcher = new LinksIndexCrossLinkFetcher(logger);
+		var fetcher = new LinksIndexCrossLinkFetcher(_linkIndexProvider, logger);
 		var resolver = new CrossLinkResolver(fetcher);
 		var crossLinks = await resolver.FetchLinks(ctx);
 
@@ -34,7 +34,7 @@ public async Task CheckAll(IDiagnosticsCollector collector, Cancel ctx)
 
 	public async Task CheckRepository(IDiagnosticsCollector collector, string? toRepository, string? fromRepository, Cancel ctx)
 	{
-		var fetcher = new LinksIndexCrossLinkFetcher(logger);
+		var fetcher = new LinksIndexCrossLinkFetcher(_linkIndexProvider, logger);
 		var resolver = new CrossLinkResolver(fetcher);
 		var crossLinks = await resolver.FetchLinks(ctx);
 		var filter = new RepositoryFilter
@@ -48,7 +48,7 @@ public async Task CheckRepository(IDiagnosticsCollector collector, string? toRep
 
 	public async Task CheckWithLocalLinksJson(IDiagnosticsCollector collector, string repository, string localLinksJson, Cancel ctx)
 	{
-		var fetcher = new LinksIndexCrossLinkFetcher(logger);
+		var fetcher = new LinksIndexCrossLinkFetcher(_linkIndexProvider, logger);
 		var resolver = new CrossLinkResolver(fetcher);
 		// ReSharper disable once RedundantAssignment
 		var crossLinks = await resolver.FetchLinks(ctx);