Skip to content

Commit d565b78

Browse files
cotticotti
authored
Refactor communication with AWS for KeyValueStore updates (#1503)
* Refactor communication with AWS for KeyValueStore updates
1 parent 0ff74de commit d565b78

File tree

6 files changed

+171
-82
lines changed

6 files changed

+171
-82
lines changed

‎Directory.Packages.props‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,6 @@
1313
<PackageVersion Include="Amazon.Lambda.S3Events" Version="3.1.0" />
1414
<PackageVersion Include="Amazon.Lambda.Serialization.SystemTextJson" Version="2.4.4" />
1515
<PackageVersion Include="Amazon.Lambda.SQSEvents" Version="2.2.0" />
16-
<PackageVersion Include="AWSSDK.CloudFront" Version="4.0.0.10" />
17-
<PackageVersion Include="AWSSDK.CloudFrontKeyValueStore" Version="4.0.0.9" />
1816
<PackageVersion Include="AWSSDK.Core" Version="4.0.0.2" />
1917
<PackageVersion Include="AWSSDK.SQS" Version="4.0.0.1" />
2018
<PackageVersion Include="AWSSDK.S3" Version="4.0.0.1" />

‎src/tooling/Elastic.Documentation.Tooling/ExternalCommands/ExternalCommandExecutor.cs‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ namespace Elastic.Documentation.Tooling.ExternalCommands;
1212
public abstract class ExternalCommandExecutor(DiagnosticsCollector collector, IDirectoryInfo workingDirectory)
1313
{
1414
protected IDirectoryInfo WorkingDirectory => workingDirectory;
15+
protected DiagnosticsCollector Collector => collector;
1516
protected void ExecIn(Dictionary<string, string> environmentVars, string binary, params string[] args)
1617
{
1718
var arguments = new ExecArguments(binary, args)
@@ -77,13 +78,13 @@ string[] CaptureOutput()
7778
}
7879

7980

80-
protected string Capture(string binary, params string[] args) => Capture(false, binary, args);
81-
82-
protected string Capture(bool muteExceptions, string binary, params string[] args)
81+
protected string Capture(string binary, params string[] args) => Capture(false, 10, binary, args);
82+
protected string Capture(bool muteExceptions, string binary, params string[] args) => Capture(muteExceptions, 10, binary, args);
83+
protected string Capture(bool muteExceptions, int attempts, string binary, params string[] args)
8384
{
8485
// Try 10 times to capture the output of the command, if it fails, we'll throw an exception on the last try
8586
Exception? e = null;
86-
for (var i = 0; i <= 9; i++)
87+
for (var i = 1; i <= attempts; i++)
8788
{
8889
try
8990
{

‎src/tooling/docs-assembler/Cli/DeployCommands.cs‎

Lines changed: 2 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,6 @@
66
using System.IO.Abstractions;
77
using System.Text.Json;
88
using Actions.Core.Services;
9-
using Amazon.CloudFront;
10-
using Amazon.CloudFrontKeyValueStore;
11-
using Amazon.CloudFrontKeyValueStore.Model;
129
using Amazon.S3;
1310
using Amazon.S3.Transfer;
1411
using ConsoleAppFramework;
@@ -20,12 +17,6 @@
2017

2118
namespace Documentation.Assembler.Cli;
2219

23-
internal enum KvsOperation
24-
{
25-
Puts,
26-
Deletes
27-
}
28-
2920
internal sealed class DeployCommands(ILoggerFactory logger, ICoreService githubActionsService)
3021
{
3122
[SuppressMessage("Usage", "CA2254:Template should be a static expression")]
@@ -144,74 +135,11 @@ public async Task<int> UpdateRedirects(
144135
}
145136

146137
var kvsName = $"elastic-docs-v3-{environment}-redirects-kvs";
138+
var cloudFrontClient = new AwsCloudFrontKeyValueStoreProxy(collector, new FileSystem().DirectoryInfo.New(Directory.GetCurrentDirectory()));
147139

148-
var cfClient = new AmazonCloudFrontClient();
149-
var kvsClient = new AmazonCloudFrontKeyValueStoreClient();
150-
151-
ConsoleApp.Log("Describing KVS");
152-
var describeResponse = await cfClient.DescribeKeyValueStoreAsync(new Amazon.CloudFront.Model.DescribeKeyValueStoreRequest { Name = kvsName }, ctx);
153-
154-
var kvsArn = describeResponse.KeyValueStore.ARN;
155-
var eTag = describeResponse.ETag;
156-
var existingRedirects = new HashSet<string>();
157-
158-
var listKeysRequest = new ListKeysRequest { KvsARN = kvsArn };
159-
ListKeysResponse listKeysResponse;
160-
161-
do
162-
{
163-
listKeysResponse = await kvsClient.ListKeysAsync(listKeysRequest, ctx);
164-
foreach (var item in listKeysResponse.Items)
165-
_ = existingRedirects.Add(item.Key);
166-
listKeysRequest.NextToken = listKeysResponse.NextToken;
167-
}
168-
while (!string.IsNullOrEmpty(listKeysResponse.NextToken));
169-
170-
var toPut = sourcedRedirects
171-
.Select(kvp => new PutKeyRequestListItem { Key = kvp.Key, Value = kvp.Value });
172-
var toDelete = existingRedirects
173-
.Except(sourcedRedirects.Keys)
174-
.Select(k => new DeleteKeyRequestListItem { Key = k });
175-
176-
ConsoleApp.Log("Updating redirects in KVS");
177-
const int batchSize = 50;
178-
179-
eTag = await ProcessBatchUpdatesAsync(kvsClient, kvsArn, eTag, toPut, batchSize, KvsOperation.Puts, ctx);
180-
_ = await ProcessBatchUpdatesAsync(kvsClient, kvsArn, eTag, toDelete, batchSize, KvsOperation.Deletes, ctx);
140+
cloudFrontClient.UpdateRedirects(kvsName, sourcedRedirects);
181141

182142
await collector.StopAsync(ctx);
183143
return collector.Errors;
184144
}
185-
186-
private static async Task<string> ProcessBatchUpdatesAsync(
187-
IAmazonCloudFrontKeyValueStore kvsClient,
188-
string kvsArn,
189-
string eTag,
190-
IEnumerable<object> items,
191-
int batchSize,
192-
KvsOperation operation,
193-
Cancel ctx)
194-
{
195-
var enumerable = items.ToList();
196-
for (var i = 0; i < enumerable.Count; i += batchSize)
197-
{
198-
var batch = enumerable.Skip(i).Take(batchSize);
199-
var updateRequest = new UpdateKeysRequest
200-
{
201-
KvsARN = kvsArn,
202-
IfMatch = eTag
203-
};
204-
205-
if (operation is KvsOperation.Puts)
206-
updateRequest.Puts = batch.Cast<PutKeyRequestListItem>().ToList();
207-
else if (operation is KvsOperation.Deletes)
208-
updateRequest.Deletes = batch.Cast<DeleteKeyRequestListItem>().ToList();
209-
210-
var update = await kvsClient.UpdateKeysAsync(updateRequest, ctx);
211-
eTag = update.ETag;
212-
}
213-
214-
return eTag;
215-
}
216-
217145
}

‎src/tooling/docs-assembler/Deploying/AwsCloudFrontKeyValueStoreProxy.cs‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
// Licensed to Elasticsearch B.V under one or more agreements.
2+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
3+
// See the LICENSE file in the project root for more information
4+
5+
using System.IO.Abstractions;
6+
using System.Text.Json;
7+
using ConsoleAppFramework;
8+
using Documentation.Assembler.Deploying.Serialization;
9+
using Elastic.Documentation.Diagnostics;
10+
using Elastic.Documentation.Tooling.ExternalCommands;
11+
12+
namespace Documentation.Assembler.Deploying;
13+
14+
internal enum KvsOperation
15+
{
16+
Puts,
17+
Deletes
18+
}
19+
20+
public class AwsCloudFrontKeyValueStoreProxy(DiagnosticsCollector collector, IDirectoryInfo workingDirectory) : ExternalCommandExecutor(collector, workingDirectory)
21+
{
22+
public void UpdateRedirects(string kvsName, IReadOnlyDictionary<string, string> sourcedRedirects)
23+
{
24+
var (kvsArn, eTag) = DescribeKeyValueStore(kvsName);
25+
if (string.IsNullOrEmpty(kvsArn) || string.IsNullOrEmpty(eTag))
26+
return;
27+
28+
var existingRedirects = ListAllKeys(kvsArn);
29+
30+
var toPut = sourcedRedirects
31+
.Select(kvp => new PutKeyRequestListItem { Key = kvp.Key, Value = kvp.Value });
32+
var toDelete = existingRedirects
33+
.Except(sourcedRedirects.Keys)
34+
.Select(k => new DeleteKeyRequestListItem { Key = k });
35+
36+
eTag = ProcessBatchUpdates(kvsArn, eTag, toPut, KvsOperation.Puts);
37+
_ = ProcessBatchUpdates(kvsArn, eTag, toDelete, KvsOperation.Deletes);
38+
}
39+
40+
private (string? Arn, string? ETag) DescribeKeyValueStore(string kvsName)
41+
{
42+
ConsoleApp.Log("Describing KeyValueStore");
43+
try
44+
{
45+
var json = Capture("aws", "cloudfront", "describe-key-value-store", "--name", kvsName, "|", "jq", "-c");
46+
var describeResponse = JsonSerializer.Deserialize<DescribeKeyValueStoreResponse>(json, AwsCloudFrontKeyValueStoreJsonContext.Default.DescribeKeyValueStoreResponse);
47+
if (describeResponse?.ETag is not null && describeResponse.KeyValueStore is { ARN.Length: > 0 })
48+
return (describeResponse.KeyValueStore.ARN, describeResponse.ETag);
49+
50+
Collector.EmitError("", "Could not deserialize the DescribeKeyValueStoreResponse");
51+
return (null, null);
52+
}
53+
catch (Exception e)
54+
{
55+
Collector.EmitError("", "An error occurred while describing the KeyValueStore", e);
56+
return (null, null);
57+
}
58+
}
59+
60+
private HashSet<string> ListAllKeys(string kvsArn)
61+
{
62+
ConsoleApp.Log("Acquiring existing redirects");
63+
var allKeys = new HashSet<string>();
64+
string[] baseArgs = ["cloudfront-keyvaluestore", "list-keys", "--kvs-arn", kvsArn];
65+
string? nextToken = null;
66+
try
67+
{
68+
do
69+
{
70+
var json = Capture("aws", [.. baseArgs, .. nextToken is not null ? (string[])["--starting-token", nextToken] : [], "|", "jq", "-c"]);
71+
var response = JsonSerializer.Deserialize<ListKeysResponse>(json, AwsCloudFrontKeyValueStoreJsonContext.Default.ListKeysResponse);
72+
73+
if (response?.Items != null)
74+
{
75+
foreach (var item in response.Items)
76+
_ = allKeys.Add(item.Key);
77+
}
78+
79+
nextToken = response?.NextToken;
80+
} while (!string.IsNullOrEmpty(nextToken));
81+
}
82+
catch (Exception e)
83+
{
84+
Collector.EmitError("", "An error occurred while acquiring existing redirects in the KeyValueStore", e);
85+
return [];
86+
}
87+
return allKeys;
88+
}
89+
90+
91+
private string ProcessBatchUpdates(
92+
string kvsArn,
93+
string eTag,
94+
IEnumerable<object> items,
95+
KvsOperation operation)
96+
{
97+
const int batchSize = 50;
98+
ConsoleApp.Log($"Processing {items.Count()} items in batches of {batchSize} for {operation} update operation.");
99+
try
100+
{
101+
foreach (var batch in items.Chunk(batchSize))
102+
{
103+
var payload = operation switch
104+
{
105+
KvsOperation.Puts => JsonSerializer.Serialize(batch.Cast<PutKeyRequestListItem>().ToList(),
106+
AwsCloudFrontKeyValueStoreJsonContext.Default.ListPutKeyRequestListItem),
107+
KvsOperation.Deletes => JsonSerializer.Serialize(batch.Cast<DeleteKeyRequestListItem>().ToList(),
108+
AwsCloudFrontKeyValueStoreJsonContext.Default.ListDeleteKeyRequestListItem),
109+
_ => string.Empty
110+
};
111+
var responseJson = Capture(false, 1, "aws", "cloudfront-keyvaluestore", "update-keys", "--kvs-arn", kvsArn, "--if-match", eTag,
112+
$"--{operation.ToString().ToLowerInvariant()}", "--payload", payload, "|", "jq", "-c");
113+
var updateResponse = JsonSerializer.Deserialize<UpdateKeysResponse>(responseJson, AwsCloudFrontKeyValueStoreJsonContext.Default.UpdateKeysResponse);
114+
115+
if (string.IsNullOrEmpty(updateResponse?.ETag))
116+
throw new Exception("Failed to get new ETag after update operation.");
117+
118+
eTag = updateResponse.ETag;
119+
}
120+
}
121+
catch (Exception e)
122+
{
123+
Collector.EmitError("", $"An error occurred while performing a {operation} update to the KeyValueStore", e);
124+
}
125+
return eTag;
126+
}
127+
}

‎src/tooling/docs-assembler/Deploying/Serialization/AwsCloudFrontKeyValueStoreModels.cs‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
// Licensed to Elasticsearch B.V under one or more agreements.
2+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
3+
// See the LICENSE file in the project root for more information
4+
5+
using System.Text.Json.Serialization;
6+
7+
namespace Documentation.Assembler.Deploying.Serialization;
8+
9+
public record DescribeKeyValueStoreResponse([property: JsonPropertyName("ETag")] string ETag, [property: JsonPropertyName("KeyValueStore")] KeyValueStore KeyValueStore);
10+
public record KeyValueStore([property: JsonPropertyName("ARN")] string ARN);
11+
12+
public record ListKeysResponse([property: JsonPropertyName("NextToken")] string? NextToken, [property: JsonPropertyName("Items")] List<KeyItem> Items);
13+
public record KeyItem([property: JsonPropertyName("Key")] string Key);
14+
15+
public record UpdateKeysResponse([property: JsonPropertyName("ETag")] string ETag);
16+
17+
public record PutKeyRequestListItem
18+
{
19+
[JsonPropertyName("Key")]
20+
public required string Key { get; init; }
21+
[JsonPropertyName("Value")]
22+
public required string Value { get; init; }
23+
}
24+
25+
public record DeleteKeyRequestListItem
26+
{
27+
[JsonPropertyName("Key")]
28+
public required string Key { get; init; }
29+
}
30+
31+
[JsonSourceGenerationOptions(WriteIndented = false, UseStringEnumConverter = true)]
32+
[JsonSerializable(typeof(DescribeKeyValueStoreResponse))]
33+
[JsonSerializable(typeof(ListKeysResponse))]
34+
[JsonSerializable(typeof(UpdateKeysResponse))]
35+
[JsonSerializable(typeof(List<PutKeyRequestListItem>))]
36+
[JsonSerializable(typeof(List<DeleteKeyRequestListItem>))]
37+
internal sealed partial class AwsCloudFrontKeyValueStoreJsonContext : JsonSerializerContext;

‎src/tooling/docs-assembler/docs-assembler.csproj‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,6 @@
1717
</PropertyGroup>
1818

1919
<ItemGroup>
20-
<PackageReference Include="AWSSDK.CloudFront" />
21-
<PackageReference Include="AWSSDK.CloudFrontKeyValueStore" />
2220
<PackageReference Include="AWSSDK.S3"/>
2321
<PackageReference Include="ConsoleAppFramework.Abstractions"/>
2422
<PackageReference Include="ConsoleAppFramework" />

0 commit comments

Comments
 (0)