Skip to content

Commit 64bae54

Browse files
MpdreamzMpdreamz
committed
Ensure apply also validates the plan before calling apply
1 parent b4396ac commit 64bae54

File tree

4 files changed

+80
-48
lines changed

4 files changed

+80
-48
lines changed

‎src/tooling/docs-assembler/Cli/DeployCommands.cs‎

Lines changed: 16 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ public async Task<int> Plan(
4444
string environment,
4545
string s3BucketName,
4646
string @out = "",
47-
float deleteThreshold = 0.2f,
47+
float? deleteThreshold = null,
4848
Cancel ctx = default
4949
)
5050
{
@@ -56,20 +56,21 @@ public async Task<int> Plan(
5656
var fs = new FileSystem();
5757
var assembleContext = new AssembleContext(assemblyConfiguration, configurationContext, environment, collector, fs, fs, null, null);
5858
var s3Client = new AmazonS3Client();
59-
IDocsSyncPlanStrategy planner = new AwsS3SyncPlanStrategy(logFactory, s3Client, s3BucketName, assembleContext);
60-
var plan = await planner.Plan(ctx);
59+
var planner = new AwsS3SyncPlanStrategy(logFactory, s3Client, s3BucketName, assembleContext);
60+
var plan = await planner.Plan(deleteThreshold, ctx);
6161
_logger.LogInformation("Total files to sync: {TotalFiles}", plan.TotalSyncRequests);
6262
_logger.LogInformation("Total files to delete: {DeleteCount}", plan.DeleteRequests.Count);
6363
_logger.LogInformation("Total files to add: {AddCount}", plan.AddRequests.Count);
6464
_logger.LogInformation("Total files to update: {UpdateCount}", plan.UpdateRequests.Count);
6565
_logger.LogInformation("Total files to skip: {SkipCount}", plan.SkipRequests.Count);
6666
_logger.LogInformation("Total local source files: {TotalSourceFiles}", plan.TotalSourceFiles);
6767
_logger.LogInformation("Total remote source files: {TotalSourceFiles}", plan.TotalRemoteFiles);
68-
var validationResult = planner.Validate(plan, deleteThreshold);
68+
var validator = new DocsSyncPlanValidator(logFactory);
69+
var validationResult = validator.Validate(plan);
6970
if (!validationResult.Valid)
7071
{
7172
await githubActionsService.SetOutputAsync("plan-valid", "false");
72-
collector.EmitError(@out, $"Plan is invalid, delete ratio: {validationResult.DeleteRatio}, threshold: {validationResult.DeleteThreshold} over {plan.TotalSyncRequests:N0} files while plan has {plan.DeleteRequests:N0} deletions");
73+
collector.EmitError(@out, $"Plan is invalid, delete ratio: {validationResult.DeleteRatio}, threshold: {validationResult.DeleteThreshold} over {plan.TotalRemoteFiles:N0} remote files while plan has {plan.DeleteRequests:N0} deletions");
7374
await collector.StopAsync(ctx);
7475
return collector.Errors;
7576
}
@@ -92,11 +93,7 @@ public async Task<int> Plan(
9293
/// <param name="s3BucketName">The S3 bucket name to deploy to</param>
9394
/// <param name="planFile">The path to the plan file to apply</param>
9495
/// <param name="ctx"></param>
95-
public async Task<int> Apply(
96-
string environment,
97-
string s3BucketName,
98-
string planFile,
99-
Cancel ctx = default)
96+
public async Task<int> Apply(string environment, string s3BucketName, string planFile, Cancel ctx = default)
10097
{
10198
AssignOutputLogger();
10299
await using var collector = new ConsoleDiagnosticsCollector(logFactory, githubActionsService)
@@ -111,7 +108,6 @@ public async Task<int> Apply(
111108
ConcurrentServiceRequests = Environment.ProcessorCount * 2,
112109
MinSizeBeforePartUpload = S3EtagCalculator.PartSize
113110
});
114-
IDocsSyncApplyStrategy applier = new AwsS3SyncApplyStrategy(logFactory, s3Client, transferUtility, s3BucketName, assembleContext, collector);
115111
if (!File.Exists(planFile))
116112
{
117113
collector.EmitError(planFile, "Plan file does not exist.");
@@ -133,6 +129,15 @@ public async Task<int> Apply(
133129
await collector.StopAsync(ctx);
134130
return collector.Errors;
135131
}
132+
var validator = new DocsSyncPlanValidator(logFactory);
133+
var validationResult = validator.Validate(plan);
134+
if (!validationResult.Valid)
135+
{
136+
collector.EmitError(planFile, $"Plan is invalid, delete ratio: {validationResult.DeleteRatio}, threshold: {validationResult.DeleteThreshold} over {plan.TotalRemoteFiles:N0} remote files while plan has {plan.DeleteRequests:N0} deletions");
137+
await collector.StopAsync(ctx);
138+
return collector.Errors;
139+
}
140+
var applier = new AwsS3SyncApplyStrategy(logFactory, s3Client, transferUtility, s3BucketName, assembleContext, collector);
136141
await applier.Apply(plan, ctx);
137142
await collector.StopAsync(ctx);
138143
return collector.Errors;

‎src/tooling/docs-assembler/Deploying/AwsS3SyncPlanStrategy.cs‎

Lines changed: 2 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ private bool IsSymlink(string path)
9292
return fileInfo.LinkTarget != null;
9393
}
9494

95-
public async Task<SyncPlan> Plan(Cancel ctx = default)
95+
public async Task<SyncPlan> Plan(float? deleteThreshold = null, Cancel ctx = default)
9696
{
9797
var remoteObjects = await ListObjects(ctx);
9898
var localObjects = context.OutputDirectory.GetFiles("*", SearchOption.AllDirectories)
@@ -158,6 +158,7 @@ await Parallel.ForEachAsync(localObjects, ctx, async (localFile, token) =>
158158

159159
return new SyncPlan
160160
{
161+
DeleteThresholdDefault = deleteThreshold,
161162
TotalRemoteFiles = remoteObjects.Count,
162163
TotalSourceFiles = localObjects.Length,
163164
DeleteRequests = deleteRequests.ToList(),
@@ -168,40 +169,6 @@ await Parallel.ForEachAsync(localObjects, ctx, async (localFile, token) =>
168169
};
169170
}
170171

171-
/// <inheritdoc />
172-
public PlanValidationResult Validate(SyncPlan plan, float deleteThreshold)
173-
{
174-
if (plan.TotalSourceFiles == 0)
175-
{
176-
_logger.LogError("No files to sync");
177-
return new(false, 1.0f, deleteThreshold);
178-
}
179-
180-
var deleteRatio = (float)plan.DeleteRequests.Count / plan.TotalRemoteFiles;
181-
if (plan.TotalRemoteFiles == 0)
182-
{
183-
_logger.LogInformation("No files discovered in S3, assuming a clean bucket resetting delete threshold to `0.0' as our plan should not have ANY deletions");
184-
deleteThreshold = 0.0f;
185-
}
186-
// if the total remote files are less than or equal to 100, we enforce a higher ratio of 0.8
187-
// this allows newer assembled documentation to be in a higher state of flux
188-
if (plan.TotalRemoteFiles <= 100)
189-
deleteThreshold = Math.Max(deleteThreshold, 0.8f);
190-
191-
// if the total remote files are less than or equal to 1000, we enforce a higher ratio of 0.5
192-
// this allows newer assembled documentation to be in a higher state of flux
193-
else if (plan.TotalRemoteFiles <= 1000)
194-
deleteThreshold = Math.Max(deleteThreshold, 0.5f);
195-
196-
if (deleteRatio > deleteThreshold)
197-
{
198-
_logger.LogError("Delete ratio is {Ratio} which is greater than the threshold of {Threshold}", deleteRatio, deleteThreshold);
199-
return new(false, deleteRatio, deleteThreshold);
200-
}
201-
202-
return new(true, deleteRatio, deleteThreshold);
203-
}
204-
205172
private async Task<Dictionary<string, S3Object>> ListObjects(Cancel ctx = default)
206173
{
207174
var listBucketRequest = new ListObjectsV2Request

‎src/tooling/docs-assembler/Deploying/DocsSync.cs‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,8 @@ namespace Documentation.Assembler.Deploying;
99

1010
public interface IDocsSyncPlanStrategy
1111
{
12-
Task<SyncPlan> Plan(Cancel ctx = default);
12+
Task<SyncPlan> Plan(float? deleteThreshold = null, Cancel ctx = default);
1313

14-
PlanValidationResult Validate(SyncPlan plan, float deleteThreshold);
1514
}
1615
public record PlanValidationResult(bool Valid, float DeleteRatio, float DeleteThreshold);
1716

@@ -52,6 +51,10 @@ public record SkipRequest : SyncRequest
5251

5352
public record SyncPlan
5453
{
54+
/// The user-specified delete threshold
55+
[JsonPropertyName("deletion_threshold_default")]
56+
public required float? DeleteThresholdDefault { get; init; }
57+
5558
/// The total number of source files that were located in the build output
5659
[JsonPropertyName("total_source_files")]
5760
public required int TotalSourceFiles { get; init; }

‎src/tooling/docs-assembler/Deploying/DocsSyncPlanValidator.cs‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
// Licensed to Elasticsearch B.V under one or more agreements.
2+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
3+
// See the LICENSE file in the project root for more information
4+
5+
using Microsoft.Extensions.Logging;
6+
7+
namespace Documentation.Assembler.Deploying;
8+
9+
public class DocsSyncPlanValidator(ILoggerFactory logFactory)
10+
{
11+
private readonly ILogger<AwsS3SyncPlanStrategy> _logger = logFactory.CreateLogger<AwsS3SyncPlanStrategy>();
12+
13+
public PlanValidationResult Validate(SyncPlan plan)
14+
{
15+
if (plan.DeleteThresholdDefault is not null)
16+
_logger.LogInformation("Using user-specified delete threshold of {Threshold}", plan.DeleteThresholdDefault);
17+
18+
var deleteThreshold = plan.DeleteThresholdDefault ?? 0.2f;
19+
if (plan.TotalSourceFiles == 0)
20+
{
21+
_logger.LogError("No files to sync");
22+
return new(false, 1.0f, deleteThreshold);
23+
}
24+
25+
var deleteRatio = (float)plan.DeleteRequests.Count / plan.TotalRemoteFiles;
26+
if (plan.TotalRemoteFiles == 0)
27+
{
28+
_logger.LogInformation("No files discovered in S3, assuming a clean bucket resetting delete threshold to `0.0' as our plan should not have ANY deletions");
29+
deleteThreshold = 0.0f;
30+
}
31+
// if the total remote files are less than or equal to 100, we enforce a higher ratio of 0.8
32+
// this allows newer assembled documentation to be in a higher state of flux
33+
if (plan.TotalRemoteFiles <= 100)
34+
{
35+
_logger.LogInformation("Plan has less than 100 total remote files ensuring delete threshold is at minimum 0.8");
36+
deleteThreshold = Math.Max(deleteThreshold, 0.8f);
37+
}
38+
39+
// if the total remote files are less than or equal to 1000, we enforce a higher ratio of 0.5
40+
// this allows newer assembled documentation to be in a higher state of flux
41+
else if (plan.TotalRemoteFiles <= 1000)
42+
{
43+
_logger.LogInformation("Plan has less than 1000 but more than a 100 total remote files ensuring delete threshold is at minimum 0.5");
44+
deleteThreshold = Math.Max(deleteThreshold, 0.5f);
45+
}
46+
47+
if (deleteRatio > deleteThreshold)
48+
{
49+
_logger.LogError("Delete ratio is {Ratio} which is greater than the threshold of {Threshold}", deleteRatio, deleteThreshold);
50+
return new(false, deleteRatio, deleteThreshold);
51+
}
52+
53+
return new(true, deleteRatio, deleteThreshold);
54+
}
55+
56+
57+
}

0 commit comments

Comments
 (0)