Skip to content

Commit 5b2a62d

Browse files
reakaleekreakaleek
authored
Refactor LinkIndexProvider into own project (#1302)
* Refactor LinkIndexProvider into own project * Fix type * Add comement and cleanup * Refactor
1 parent aee87ea commit 5b2a62d

30 files changed

+294
-220
lines changed

‎docs-builder.sln‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "assembler-config-validate",
9999
actions\assembler-config-validate\action.yml = actions\assembler-config-validate\action.yml
100100
EndProjectSection
101101
EndProject
102+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Elastic.Documentation.LinkIndex", "src\Elastic.Documentation.LinkIndex\Elastic.Documentation.LinkIndex.csproj", "{FD1AC230-798B-4AB9-8CE6-A06264885DBC}"
103+
EndProject
102104
Global
103105
GlobalSection(SolutionConfigurationPlatforms) = preSolution
104106
Debug|Any CPU = Debug|Any CPU
@@ -160,6 +162,10 @@ Global
160162
{CD94F9E4-7FCD-4152-81F1-4288C6B75367}.Debug|Any CPU.Build.0 = Debug|Any CPU
161163
{CD94F9E4-7FCD-4152-81F1-4288C6B75367}.Release|Any CPU.ActiveCfg = Release|Any CPU
162164
{CD94F9E4-7FCD-4152-81F1-4288C6B75367}.Release|Any CPU.Build.0 = Release|Any CPU
165+
{FD1AC230-798B-4AB9-8CE6-A06264885DBC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
166+
{FD1AC230-798B-4AB9-8CE6-A06264885DBC}.Debug|Any CPU.Build.0 = Debug|Any CPU
167+
{FD1AC230-798B-4AB9-8CE6-A06264885DBC}.Release|Any CPU.ActiveCfg = Release|Any CPU
168+
{FD1AC230-798B-4AB9-8CE6-A06264885DBC}.Release|Any CPU.Build.0 = Release|Any CPU
163169
EndGlobalSection
164170
GlobalSection(NestedProjects) = preSolution
165171
{4D198E25-C211-41DC-9E84-B15E89BD7048} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
@@ -184,5 +190,6 @@ Global
184190
{7D36DDDA-9E0B-4D2C-8033-5D62FF8B6166} = {059E787F-85C1-43BE-9DD6-CE319E106383}
185191
{FB1C1954-D8E2-4745-BA62-04DD82FB4792} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7}
186192
{E20FEEF9-1D1A-4CDA-A546-7FDC573BE399} = {245023D2-D3CA-47B9-831D-DAB91A2FFDC7}
193+
{FD1AC230-798B-4AB9-8CE6-A06264885DBC} = {BE6011CC-1200-4957-B01F-FCCA10C5CF5A}
187194
EndGlobalSection
188195
EndGlobal

‎src/Elastic.Documentation.LinkIndex/Elastic.Documentation.LinkIndex.csproj‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<TargetFramework>net9.0</TargetFramework>
5+
<ImplicitUsings>enable</ImplicitUsings>
6+
<Nullable>enable</Nullable>
7+
</PropertyGroup>
8+
9+
<ItemGroup>
10+
<PackageReference Include="AWSSDK.S3" />
11+
</ItemGroup>
12+
13+
<ItemGroup>
14+
<ProjectReference Include="..\Elastic.Documentation\Elastic.Documentation.csproj" />
15+
</ItemGroup>
16+
17+
</Project>

‎src/Elastic.Documentation.LinkIndex/ILinkIndexReader.cs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
// Licensed to Elasticsearch B.V under one or more agreements.
2+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
3+
// See the LICENSE file in the project root for more information
4+
5+
using Elastic.Documentation.Links;
6+
7+
namespace Elastic.Documentation.LinkIndex;
8+
9+
public interface ILinkIndexReader
10+
{
11+
Task<LinkRegistry> GetRegistry(Cancel cancellationToken = default);
12+
Task<RepositoryLinks> GetRepositoryLinks(string key, Cancel cancellationToken = default);
13+
string RegistryUrl { get; }
14+
}

‎src/Elastic.Documentation.LinkIndex/ILinkIndexReaderWriter.cs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
// Licensed to Elasticsearch B.V under one or more agreements.
2+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
3+
// See the LICENSE file in the project root for more information
4+
5+
namespace Elastic.Documentation.LinkIndex;
6+
7+
public interface ILinkIndexReaderWriter : ILinkIndexReader, ILinkIndexWriter;

‎src/Elastic.Documentation.LinkIndex/ILinkIndexWriter.cs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
// Licensed to Elasticsearch B.V under one or more agreements.
2+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
3+
// See the LICENSE file in the project root for more information
4+
5+
using Elastic.Documentation.Links;
6+
7+
namespace Elastic.Documentation.LinkIndex;
8+
9+
public interface ILinkIndexWriter
10+
{
11+
Task SaveRegistry(LinkRegistry registry, Cancel cancellationToken = default);
12+
}

‎src/Elastic.Documentation.LinkIndex/LinkIndexReader.cs‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
// Licensed to Elasticsearch B.V under one or more agreements.
2+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
3+
// See the LICENSE file in the project root for more information
4+
5+
using System.Net;
6+
using Amazon.Runtime;
7+
using Amazon.S3;
8+
using Amazon.S3.Model;
9+
using Elastic.Documentation.Links;
10+
11+
namespace Elastic.Documentation.LinkIndex;
12+
13+
public class Aws3LinkIndexReader(IAmazonS3 s3Client, string bucketName = "elastic-docs-link-index", string registryKey = "link-index.json") : ILinkIndexReader
14+
{
15+
16+
// <summary>
17+
// Using <see cref="AnonymousAWSCredentials"/> to access the link index
18+
// allows to read from the link index without the need to provide AWS credentials.
19+
// </summary>
20+
public static Aws3LinkIndexReader CreateAnonymous()
21+
{
22+
var credentials = new AnonymousAWSCredentials();
23+
var config = new AmazonS3Config
24+
{
25+
RegionEndpoint = Amazon.RegionEndpoint.USEast2
26+
};
27+
var s3Client = new AmazonS3Client(credentials, config);
28+
return new AwsS3LinkIndexReaderWriter(s3Client);
29+
}
30+
31+
public async Task<LinkRegistry> GetRegistry(Cancel cancellationToken = default)
32+
{
33+
var getObjectRequest = new GetObjectRequest
34+
{
35+
BucketName = bucketName,
36+
Key = registryKey
37+
};
38+
var getObjectResponse = await s3Client.GetObjectAsync(getObjectRequest, cancellationToken);
39+
await using var stream = getObjectResponse.ResponseStream;
40+
var linkIndex = LinkRegistry.Deserialize(stream);
41+
return linkIndex with { ETag = getObjectResponse.ETag };
42+
}
43+
public async Task<RepositoryLinks> GetRepositoryLinks(string key, Cancel cancellationToken)
44+
{
45+
var getObjectRequest = new GetObjectRequest
46+
{
47+
BucketName = bucketName,
48+
Key = key
49+
};
50+
var getObjectResponse = await s3Client.GetObjectAsync(getObjectRequest, cancellationToken);
51+
await using var stream = getObjectResponse.ResponseStream;
52+
return RepositoryLinks.Deserialize(stream);
53+
}
54+
55+
public string RegistryUrl { get; } = $"https://{bucketName}.s3.{s3Client.Config.RegionEndpoint.SystemName}.amazonaws.com/{registryKey}";
56+
}

‎src/Elastic.Documentation.LinkIndex/LinkIndexReaderWriter.cs‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
// Licensed to Elasticsearch B.V under one or more agreements.
2+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
3+
// See the LICENSE file in the project root for more information
4+
5+
using System.Net;
6+
using Amazon.Runtime;
7+
using Amazon.S3;
8+
using Amazon.S3.Model;
9+
using Elastic.Documentation.Links;
10+
11+
namespace Elastic.Documentation.LinkIndex;
12+
13+
public class AwsS3LinkIndexReaderWriter(
14+
IAmazonS3 s3Client,
15+
string bucketName = "elastic-docs-link-index",
16+
string registryKey = "link-index.json"
17+
) : Aws3LinkIndexReader(s3Client, bucketName, registryKey), ILinkIndexReaderWriter
18+
{
19+
private readonly IAmazonS3 _s3Client = s3Client;
20+
private readonly string _bucketName = bucketName;
21+
private readonly string _registryKey = registryKey;
22+
23+
public async Task SaveRegistry(LinkRegistry registry, Cancel cancellationToken = default)
24+
{
25+
if (registry.ETag == null)
26+
// The ETag should not be null if the LinkReferenceRegistry was retrieved from GetLinkIndex()
27+
throw new InvalidOperationException($"{nameof(LinkRegistry)}.{nameof(registry.ETag)} cannot be null");
28+
var json = LinkRegistry.Serialize(registry);
29+
var putObjectRequest = new PutObjectRequest
30+
{
31+
BucketName = _bucketName,
32+
Key = _registryKey,
33+
ContentBody = json,
34+
ContentType = "application/json",
35+
IfMatch = registry.ETag // Only update if the ETag matches. Meaning the object has not been changed in the meantime.
36+
};
37+
var putResponse = await _s3Client.PutObjectAsync(putObjectRequest, cancellationToken);
38+
if (putResponse.HttpStatusCode != HttpStatusCode.OK)
39+
throw new Exception($"Unable to save {nameof(LinkRegistry)} to s3://{_bucketName}/{_registryKey}");
40+
}
41+
}

‎src/Elastic.Documentation/Links/LinkReferenceRegistry.cs‎ renamed to ‎src/Elastic.Documentation/Links/LinkRegistry.cs‎

Lines changed: 41 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,20 +8,54 @@
88

99
namespace Elastic.Documentation.Links;
1010

11-
public record LinkReferenceRegistry
11+
public record LinkRegistry
1212
{
1313
/// Map of branch to <see cref="LinkRegistryEntry"/>
1414
[JsonPropertyName("repositories")]
1515
public required Dictionary<string, Dictionary<string, LinkRegistryEntry>> Repositories { get; init; }
1616

17-
public static LinkReferenceRegistry Deserialize(Stream json) =>
18-
JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkReferenceRegistry)!;
17+
[JsonIgnore]
18+
public string? ETag { get; init; }
1919

20-
public static LinkReferenceRegistry Deserialize(string json) =>
21-
JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkReferenceRegistry)!;
20+
public LinkRegistry WithLinkRegistryEntry(LinkRegistryEntry entry)
21+
{
22+
var copiedRepositories = new Dictionary<string, Dictionary<string, LinkRegistryEntry>>(Repositories);
23+
var repository = entry.Repository;
24+
var branch = entry.Branch;
25+
// repository already exists in links.json
26+
if (copiedRepositories.TryGetValue(repository, out var existingRepositoryEntry))
27+
{
28+
// The branch already exists in the repository entry
29+
if (existingRepositoryEntry.TryGetValue(branch, out var existingBranchEntry))
30+
{
31+
if (entry.UpdatedAt > existingBranchEntry.UpdatedAt)
32+
existingRepositoryEntry[branch] = entry;
33+
}
34+
// branch does not exist in the repository entry
35+
else
36+
{
37+
existingRepositoryEntry[branch] = entry;
38+
}
39+
}
40+
// onboarding new repository
41+
else
42+
{
43+
copiedRepositories.Add(repository, new Dictionary<string, LinkRegistryEntry>
44+
{
45+
{ branch, entry }
46+
});
47+
}
48+
return this with { Repositories = copiedRepositories };
49+
}
2250

23-
public static string Serialize(LinkReferenceRegistry referenceRegistry) =>
24-
JsonSerializer.Serialize(referenceRegistry, SourceGenerationContext.Default.LinkReferenceRegistry);
51+
public static LinkRegistry Deserialize(Stream json) =>
52+
JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkRegistry)!;
53+
54+
public static LinkRegistry Deserialize(string json) =>
55+
JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkRegistry)!;
56+
57+
public static string Serialize(LinkRegistry registry) =>
58+
JsonSerializer.Serialize(registry, SourceGenerationContext.Default.LinkRegistry);
2559
}
2660

2761
public record LinkRegistryEntry
@@ -46,4 +80,3 @@ public record LinkRegistryEntry
4680
[JsonPropertyName("updated_at")]
4781
public DateTime UpdatedAt { get; init; } = DateTime.MinValue;
4882
}
49-

‎src/Elastic.Documentation/Links/LinkReference.cs‎ renamed to ‎src/Elastic.Documentation/Links/RepositoryLinks.cs‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ public record LinkRedirect : LinkSingleRedirect
3939
public LinkSingleRedirect[]? Many { get; init; }
4040
}
4141

42-
public record LinkReference
42+
public record RepositoryLinks
4343
{
4444
[JsonPropertyName("origin")]
4545
public required GitCheckoutInformation Origin { get; init; }
@@ -61,12 +61,12 @@ public record LinkReference
6161
public static string SerializeRedirects(Dictionary<string, LinkRedirect>? redirects) =>
6262
JsonSerializer.Serialize(redirects, SourceGenerationContext.Default.DictionaryStringLinkRedirect);
6363

64-
public static LinkReference Deserialize(Stream json) =>
65-
JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkReference)!;
64+
public static RepositoryLinks Deserialize(Stream json) =>
65+
JsonSerializer.Deserialize(json, SourceGenerationContext.Default.RepositoryLinks)!;
6666

67-
public static LinkReference Deserialize(string json) =>
68-
JsonSerializer.Deserialize(json, SourceGenerationContext.Default.LinkReference)!;
67+
public static RepositoryLinks Deserialize(string json) =>
68+
JsonSerializer.Deserialize(json, SourceGenerationContext.Default.RepositoryLinks)!;
6969

70-
public static string Serialize(LinkReference reference) =>
71-
JsonSerializer.Serialize(reference, SourceGenerationContext.Default.LinkReference);
70+
public static string Serialize(RepositoryLinks reference) =>
71+
JsonSerializer.Serialize(reference, SourceGenerationContext.Default.RepositoryLinks);
7272
}

‎src/Elastic.Documentation/Serialization/SourceGenerationContext.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,8 @@ namespace Elastic.Documentation.Serialization;
1212

1313
[JsonSourceGenerationOptions(WriteIndented = true, UseStringEnumConverter = true)]
1414
[JsonSerializable(typeof(GenerationState))]
15-
[JsonSerializable(typeof(LinkReference))]
15+
[JsonSerializable(typeof(RepositoryLinks))]
1616
[JsonSerializable(typeof(GitCheckoutInformation))]
17-
[JsonSerializable(typeof(LinkReferenceRegistry))]
17+
[JsonSerializable(typeof(LinkRegistry))]
1818
[JsonSerializable(typeof(LinkRegistryEntry))]
1919
public sealed partial class SourceGenerationContext : JsonSerializerContext;

0 commit comments

Comments
 (0)