Skip to content

Commit db83acb

Browse files
reakaleekreakaleek
committed
Refactor LinkIndexProvider into own project
1 parent aa713b3 commit db83acb

File tree

18 files changed

+187
-158
lines changed

18 files changed

+187
-158
lines changed

‎docs-builder.sln‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "assembler-config-validate",
9999
actions\assembler-config-validate\action.yml = actions\assembler-config-validate\action.yml
100100
EndProjectSection
101101
EndProject
102+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Documentation.LinkIndex", "src\Elastic.Documentation.LinkIndex\Elastic.Documentation.LinkIndex.csproj", "{FD1AC230-798B-4AB9-8CE6-A06264885DBC}"
103+
EndProject
102104
Global
103105
GlobalSection(SolutionConfigurationPlatforms) = preSolution
104106
Debug|Any CPU = Debug|Any CPU
@@ -160,6 +162,10 @@ Global
160162
{CD94F9E4-7FCD-4152-81F1-4288C6B75367}.Debug|Any CPU.Build.0 = Debug|Any CPU
161163
{CD94F9E4-7FCD-4152-81F1-4288C6B75367}.Release|Any CPU.ActiveCfg = Release|Any CPU
162164
{CD94F9E4-7FCD-4152-81F1-4288C6B75367}.Release|Any CPU.Build.0 = Release|Any CPU
165+
{FD1AC230-798B-4AB9-8CE6-A06264885DBC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
166+
{FD1AC230-798B-4AB9-8CE6-A06264885DBC}.Debug|Any CPU.Build.0 = Debug|Any CPU
167+
{FD1AC230-798B-4AB9-8CE6-A06264885DBC}.Release|Any CPU.ActiveCfg = Release|Any CPU
168+
{FD1AC230-798B-4AB9-8CE6-A06264885DBC}.Release|Any CPU.Build.0 = Release|Any CPU
163169
EndGlobalSection
164170
GlobalSection(NestedProjects) = preSolution
165171
{4D198E25-C211-41DC-9E84-B15E89BD7048} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
@@ -184,5 +190,6 @@ Global
184190
{7D36DDDA-9E0B-4D2C-8033-5D62FF8B6166} = {059E787F-85C1-43BE-9DD6-CE319E106383}
185191
{FB1C1954-D8E2-4745-BA62-04DD82FB4792} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7}
186192
{E20FEEF9-1D1A-4CDA-A546-7FDC573BE399} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7}
193+
{FD1AC230-798B-4AB9-8CE6-A06264885DBC} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
187194
EndGlobalSection
188195
EndGlobal

‎src/Elastic.Documentation.LinkIndex/Elastic.Documentation.LinkIndex.csproj‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<TargetFramework>net9.0</TargetFramework>
5+
<ImplicitUsings>enable</ImplicitUsings>
6+
<Nullable>enable</Nullable>
7+
</PropertyGroup>
8+
9+
<ItemGroup>
10+
<PackageReference Include="AWSSDK.S3" />
11+
</ItemGroup>
12+
13+
<ItemGroup>
14+
<ProjectReference Include="..\Elastic.Documentation\Elastic.Documentation.csproj" />
15+
</ItemGroup>
16+
17+
</Project>

‎src/Elastic.Documentation.LinkIndex/LinkIndexProvider.cs‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
// Licensed to Elasticsearch B.V under one or more agreements.
2+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
3+
// See the LICENSE file in the project root for more information
4+
5+
using System.Net;
6+
using Amazon.Runtime;
7+
using Amazon.S3;
8+
using Amazon.S3.Model;
9+
using Elastic.Documentation.Links;
10+
11+
namespace Elastic.Documentation.LinkIndex;
12+
13+
public interface ILinkIndexProvider
14+
{
15+
Task<LinkReferenceRegistry> GetLinkIndex(Cancel cancellationToken = default);
16+
Task SaveLinkIndex(LinkReferenceRegistry registry, Cancel cancellationToken = default);
17+
Task<LinkReference> GetLinkReference(string key, Cancel cancellationToken = default);
18+
19+
string GetLinkIndexPublicUrl();
20+
}
21+
22+
public class AwsS3LinkIndexProvider(IAmazonS3 s3Client, string bucketName = "elastic-docs-link-index", string registryKey = "link-index.json") : ILinkIndexProvider
23+
{
24+
25+
public static AwsS3LinkIndexProvider CreateAnonymous()
26+
{
27+
var credentials = new AnonymousAWSCredentials();
28+
var config = new AmazonS3Config
29+
{
30+
RegionEndpoint = Amazon.RegionEndpoint.USEast2
31+
};
32+
var s3Client = new AmazonS3Client(credentials, config);
33+
return new AwsS3LinkIndexProvider(s3Client);
34+
}
35+
36+
public async Task<LinkReferenceRegistry> GetLinkIndex(Cancel cancellationToken = default)
37+
{
38+
var getObjectRequest = new GetObjectRequest
39+
{
40+
BucketName = bucketName,
41+
Key = registryKey
42+
};
43+
var getObjectResponse = await s3Client.GetObjectAsync(getObjectRequest, cancellationToken);
44+
await using var stream = getObjectResponse.ResponseStream;
45+
var linkIndex = LinkReferenceRegistry.Deserialize(stream);
46+
return linkIndex with { ETag = getObjectResponse.ETag };
47+
}
48+
49+
public async Task SaveLinkIndex(LinkReferenceRegistry registry, Cancel cancellationToken = default)
50+
{
51+
if (registry.ETag == null)
52+
// The ETag should not be null if the LinkReferenceRegistry was retrieved from GetLinkIndex()
53+
throw new InvalidOperationException($"{nameof(LinkReferenceRegistry)}.{nameof(registry.ETag)} cannot be null");
54+
var json = LinkReferenceRegistry.Serialize(registry);
55+
var putObjectRequest = new PutObjectRequest
56+
{
57+
BucketName = bucketName,
58+
Key = registryKey,
59+
ContentBody = json,
60+
ContentType = "application/json",
61+
IfMatch = registry.ETag // Only update if the ETag matches. Meaning the object has not been changed in the meantime.
62+
};
63+
var putResponse = await s3Client.PutObjectAsync(putObjectRequest, cancellationToken);
64+
if (putResponse.HttpStatusCode != HttpStatusCode.OK)
65+
throw new Exception($"Unable to save {nameof(LinkReferenceRegistry)} to s3://{bucketName}/{registryKey}");
66+
}
67+
68+
public async Task<LinkReference> GetLinkReference(string key, Cancel cancellationToken)
69+
{
70+
var getObjectRequest = new GetObjectRequest
71+
{
72+
BucketName = bucketName,
73+
Key = key
74+
};
75+
var getObjectResponse = await s3Client.GetObjectAsync(getObjectRequest, cancellationToken);
76+
await using var stream = getObjectResponse.ResponseStream;
77+
return LinkReference.Deserialize(stream);
78+
}
79+
80+
public string GetLinkIndexPublicUrl() => $"https://{bucketName}.s3.{s3Client.Config.RegionEndpoint.SystemName}.amazonaws.com/{registryKey}";
81+
}

‎src/Elastic.Documentation/Links/LinkReferenceRegistry.cs‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,40 @@ public record LinkReferenceRegistry
1414
[JsonPropertyName("repositories")]
1515
public required Dictionary<string, Dictionary<string, LinkRegistryEntry>> Repositories { get; init; }
1616

17+
[JsonIgnore]
18+
public string? ETag { get; init; }
19+
20+
public LinkReferenceRegistry WithLinkRegistryEntry(LinkRegistryEntry entry)
21+
{
22+
var copiedRepositories = new Dictionary<string, Dictionary<string, LinkRegistryEntry>>(Repositories);
23+
var repository = entry.Repository;
24+
var branch = entry.Branch;
25+
// repository already exists in links.json
26+
if (copiedRepositories.TryGetValue(repository, out var existingRepositoryEntry))
27+
{
28+
// The branch already exists in the repository entry
29+
if (existingRepositoryEntry.TryGetValue(branch, out var existingBranchEntry))
30+
{
31+
if (entry.UpdatedAt > existingBranchEntry.UpdatedAt)
32+
existingRepositoryEntry[branch] = entry;
33+
}
34+
// branch does not exist in the repository entry
35+
else
36+
{
37+
existingRepositoryEntry[branch] = entry;
38+
}
39+
}
40+
// onboarding new repository
41+
else
42+
{
43+
copiedRepositories.Add(repository, new Dictionary<string, LinkRegistryEntry>
44+
{
45+
{ branch, entry }
46+
});
47+
}
48+
return this with { Repositories = copiedRepositories };
49+
}
50+
1751
public static LinkReferenceRegistry Deserialize(Stream json) =>
1852
JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkReferenceRegistry)!;
1953

@@ -46,4 +80,3 @@ public record LinkRegistryEntry
4680
[JsonPropertyName("updated_at")]
4781
public DateTime UpdatedAt { get; init; } = DateTime.MinValue;
4882
}
49-

‎src/Elastic.Markdown/Elastic.Markdown.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464
</ItemGroup>
6565

6666
<ItemGroup>
67+
<ProjectReference Include="..\Elastic.Documentation.LinkIndex\Elastic.Documentation.LinkIndex.csproj" />
6768
<ProjectReference Include="..\Elastic.Documentation\Elastic.Documentation.csproj" />
6869
<ProjectReference Include="..\Elastic.Documentation.Configuration\Elastic.Documentation.Configuration.csproj" />
6970
</ItemGroup>

‎src/Elastic.Markdown/IO/DocumentationSet.cs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,11 @@
55
using System.Collections.Frozen;
66
using System.IO.Abstractions;
77
using System.Runtime.InteropServices;
8+
using Amazon.S3;
89
using Elastic.Documentation;
910
using Elastic.Documentation.Configuration.Builder;
1011
using Elastic.Documentation.Configuration.TableOfContents;
12+
using Elastic.Documentation.LinkIndex;
1113
using Elastic.Documentation.Links;
1214
using Elastic.Markdown.Diagnostics;
1315
using Elastic.Markdown.Extensions;
@@ -126,7 +128,7 @@ public DocumentationSet(
126128
SourceDirectory = context.DocumentationSourceDirectory;
127129
OutputDirectory = context.DocumentationOutputDirectory;
128130
LinkResolver =
129-
linkResolver ?? new CrossLinkResolver(new ConfigurationCrossLinkFetcher(context.Configuration, logger));
131+
linkResolver ?? new CrossLinkResolver(new ConfigurationCrossLinkFetcher(context.Configuration, AwsS3LinkIndexProvider.CreateAnonymous(), logger));
130132
Configuration = context.Configuration;
131133
EnabledExtensions = InstantiateExtensions();
132134
treeCollector ??= new TableOfContentsTreeCollector();

‎src/Elastic.Markdown/Links/CrossLinks/ConfigurationCrossLinkFetcher.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,13 @@
55
using System.Collections.Frozen;
66
using Elastic.Documentation;
77
using Elastic.Documentation.Configuration.Builder;
8+
using Elastic.Documentation.LinkIndex;
89
using Elastic.Documentation.Links;
910
using Microsoft.Extensions.Logging;
1011

1112
namespace Elastic.Markdown.Links.CrossLinks;
1213

13-
public class ConfigurationCrossLinkFetcher(ConfigurationFile configuration, ILoggerFactory logger) : CrossLinkFetcher(logger)
14+
public class ConfigurationCrossLinkFetcher(ConfigurationFile configuration, ILinkIndexProvider linkIndexProvider, ILoggerFactory logger) : CrossLinkFetcher(linkIndexProvider, logger)
1415
{
1516
public override async Task<FetchedCrossLinks> Fetch(Cancel ctx)
1617
{

‎src/Elastic.Markdown/Links/CrossLinks/CrossLinkFetcher.cs‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
using System.Collections.Frozen;
66
using System.Text.Json;
77
using Elastic.Documentation;
8+
using Elastic.Documentation.LinkIndex;
89
using Elastic.Documentation.Links;
910
using Elastic.Documentation.Serialization;
1011
using Elastic.Markdown.IO;
@@ -31,9 +32,8 @@ public record FetchedCrossLinks
3132
};
3233
}
3334

34-
public abstract class CrossLinkFetcher(ILoggerFactory logger) : IDisposable
35+
public abstract class CrossLinkFetcher(ILinkIndexProvider linkIndexProvider, ILoggerFactory logger) : IDisposable
3536
{
36-
public const string RegistryUrl = $"https://elastic-docs-link-index.s3.us-east-2.amazonaws.com/link-index.json";
3737
private readonly ILogger _logger = logger.CreateLogger(nameof(CrossLinkFetcher));
3838
private readonly HttpClient _client = new();
3939
private LinkReferenceRegistry? _linkIndex;
@@ -51,9 +51,8 @@ public async Task<LinkReferenceRegistry> FetchLinkIndex(Cancel ctx)
5151
return _linkIndex;
5252
}
5353

54-
_logger.LogInformation("Fetching {Url}", RegistryUrl);
55-
var json = await _client.GetStringAsync(RegistryUrl, ctx);
56-
_linkIndex = LinkReferenceRegistry.Deserialize(json);
54+
_logger.LogInformation("Getting link index");
55+
_linkIndex = await linkIndexProvider.GetLinkIndex(ctx);
5756
return _linkIndex;
5857
}
5958

‎src/Elastic.Markdown/Links/InboundLinks/LinkIndexCrossLinkFetcher.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,14 @@
44

55
using System.Collections.Frozen;
66
using Elastic.Documentation;
7+
using Elastic.Documentation.LinkIndex;
78
using Elastic.Documentation.Links;
89
using Elastic.Markdown.Links.CrossLinks;
910
using Microsoft.Extensions.Logging;
1011

1112
namespace Elastic.Markdown.Links.InboundLinks;
1213

13-
public class LinksIndexCrossLinkFetcher(ILoggerFactory logger) : CrossLinkFetcher(logger)
14+
public class LinksIndexCrossLinkFetcher(ILinkIndexProvider linkIndexProvider, ILoggerFactory logger) : CrossLinkFetcher(linkIndexProvider, logger)
1415
{
1516
public override async Task<FetchedCrossLinks> Fetch(Cancel ctx)
1617
{

‎src/Elastic.Markdown/Links/InboundLinks/LinkIndexLinkChecker.cs‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,10 @@
22
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
33
// See the LICENSE file in the project root for more information
44

5+
using Amazon.S3;
56
using Elastic.Documentation;
67
using Elastic.Documentation.Diagnostics;
8+
using Elastic.Documentation.LinkIndex;
79
using Elastic.Documentation.Links;
810
using Elastic.Markdown.IO;
911
using Elastic.Markdown.Links.CrossLinks;
@@ -14,7 +16,7 @@ namespace Elastic.Markdown.Links.InboundLinks;
1416
public class LinkIndexLinkChecker(ILoggerFactory logger)
1517
{
1618
private readonly ILogger _logger = logger.CreateLogger<LinkIndexLinkChecker>();
17-
19+
private readonly ILinkIndexProvider _linkIndexProvider = AwsS3LinkIndexProvider.CreateAnonymous();
1820
private sealed record RepositoryFilter
1921
{
2022
public string? LinksTo { get; init; }
@@ -25,7 +27,7 @@ private sealed record RepositoryFilter
2527

2628
public async Task CheckAll(IDiagnosticsCollector collector, Cancel ctx)
2729
{
28-
var fetcher = new LinksIndexCrossLinkFetcher(logger);
30+
var fetcher = new LinksIndexCrossLinkFetcher(_linkIndexProvider, logger);
2931
var resolver = new CrossLinkResolver(fetcher);
3032
var crossLinks = await resolver.FetchLinks(ctx);
3133

@@ -34,7 +36,7 @@ public async Task CheckAll(IDiagnosticsCollector collector, Cancel ctx)
3436

3537
public async Task CheckRepository(IDiagnosticsCollector collector, string? toRepository, string? fromRepository, Cancel ctx)
3638
{
37-
var fetcher = new LinksIndexCrossLinkFetcher(logger);
39+
var fetcher = new LinksIndexCrossLinkFetcher(_linkIndexProvider, logger);
3840
var resolver = new CrossLinkResolver(fetcher);
3941
var crossLinks = await resolver.FetchLinks(ctx);
4042
var filter = new RepositoryFilter
@@ -48,7 +50,7 @@ public async Task CheckRepository(IDiagnosticsCollector collector, string? toRep
4850

4951
public async Task CheckWithLocalLinksJson(IDiagnosticsCollector collector, string repository, string localLinksJson, Cancel ctx)
5052
{
51-
var fetcher = new LinksIndexCrossLinkFetcher(logger);
53+
var fetcher = new LinksIndexCrossLinkFetcher(_linkIndexProvider, logger);
5254
var resolver = new CrossLinkResolver(fetcher);
5355
// ReSharper disable once RedundantAssignment
5456
var crossLinks = await resolver.FetchLinks(ctx);

0 commit comments

Comments
 (0)