Skip to content

Commit 2ef1e9c

Browse files
MpdreamzMpdreamz
committed
Adjust delete ratio logic for low sync request thresholds
- Update delete ratio calculation to use `TotalSyncRequests` instead of `TotalFilesToSync`. - Enforce stricter delete thresholds for low `TotalSyncRequests` (<100: 0.8, <1000: 0.5) to accommodate higher flux in new documentation.
1 parent d575d91 commit 2ef1e9c

File tree

4 files changed

+47
-22
lines changed

4 files changed

+47
-22
lines changed

‎src/tooling/docs-assembler/Cli/DeployCommands.cs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -60,21 +60,21 @@ public async Task<int> Plan(
6060
var s3Client = new AmazonS3Client();
6161
IDocsSyncPlanStrategy planner = new AwsS3SyncPlanStrategy(logFactory, s3Client, s3BucketName, assembleContext);
6262
var plan = await planner.Plan(ctx);
63-
ConsoleApp.Log("Total files to sync: " + plan.TotalFilesToSync);
63+
ConsoleApp.Log("Total files to sync: " + plan.TotalSyncRequests);
6464
ConsoleApp.Log("Total files to delete: " + plan.DeleteRequests.Count);
6565
ConsoleApp.Log("Total files to add: " + plan.AddRequests.Count);
6666
ConsoleApp.Log("Total files to update: " + plan.UpdateRequests.Count);
6767
ConsoleApp.Log("Total files to skip: " + plan.SkipRequests.Count);
68-
if (plan.TotalFilesToSync == 0)
68+
if (plan.TotalSyncRequests == 0)
6969
{
7070
collector.EmitError(@out, $"Plan has no files to sync so no plan will be written.");
7171
await collector.StopAsync(ctx);
7272
return collector.Errors;
7373
}
74-
var (valid, deleteRatio) = planner.Validate(plan, deleteThreshold);
75-
if (!valid)
74+
var validationResult = planner.Validate(plan, deleteThreshold);
75+
if (!validationResult.Valid)
7676
{
77-
collector.EmitError(@out, $"Plan is invalid, delete ratio: {deleteRatio}, threshold: {deleteThreshold} over {plan.TotalFilesToSync:N0} files while plan has {plan.DeleteRequests:N0} deletions");
77+
collector.EmitError(@out, $"Plan is invalid, delete ratio: {validationResult.DeleteRatio}, threshold: {validationResult.DeleteThreshold} over {plan.TotalSyncRequests:N0} files while plan has {plan.DeleteRequests:N0} deletions");
7878
await collector.StopAsync(ctx);
7979
return collector.Errors;
8080
}

‎src/tooling/docs-assembler/Deploying/AwsS3SyncPlanStrategy.cs‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -163,27 +163,37 @@ await Parallel.ForEachAsync(localObjects, ctx, async (localFile, token) =>
163163
AddRequests = addRequests.ToList(),
164164
UpdateRequests = updateRequests.ToList(),
165165
SkipRequests = skipRequests.ToList(),
166-
TotalFilesToSync = deleteRequests.Count + addRequests.Count + updateRequests.Count + skipRequests.Count
166+
TotalSyncRequests = deleteRequests.Count + addRequests.Count + updateRequests.Count + skipRequests.Count
167167
};
168168
}
169169

170170
/// <inheritdoc />
171-
public (bool, float) Validate(SyncPlan plan, float deleteThreshold)
171+
public PlanValidationResult Validate(SyncPlan plan, float deleteThreshold)
172172
{
173173
if (plan.TotalSourceFiles == 0)
174174
{
175175
_logger.LogError("No files to sync");
176-
return (false, 1.0f);
176+
return new(false, 1.0f, deleteThreshold);
177177
}
178178

179-
var deleteRatio = (float)plan.DeleteRequests.Count / plan.TotalFilesToSync;
179+
var deleteRatio = (float)plan.DeleteRequests.Count / plan.TotalSyncRequests;
180+
// if the total sync requests are less than 100, we enforce a higher ratio of 0.8
181+
// this allows newer assembled documentation to be in a higher state of flux
182+
if (plan.TotalSyncRequests <= 100)
183+
deleteThreshold = Math.Max(deleteThreshold, 0.8f);
184+
185+
// if the total sync requests are less than 1000, we enforce a higher ratio of 0.5
186+
// this allows newer assembled documentation to be in a higher state of flux
187+
else if (plan.TotalSyncRequests <= 1000)
188+
deleteThreshold = Math.Max(deleteThreshold, 0.5f);
189+
180190
if (deleteRatio > deleteThreshold)
181191
{
182192
_logger.LogError("Delete ratio is {Ratio} which is greater than the threshold of {Threshold}", deleteRatio, deleteThreshold);
183-
return (false, deleteRatio);
193+
return new(false, deleteRatio, deleteThreshold);
184194
}
185195

186-
return (true, deleteRatio);
196+
return new(true, deleteRatio, deleteThreshold);
187197
}
188198

189199
private async Task<Dictionary<string, S3Object>> ListObjects(Cancel ctx = default)

‎src/tooling/docs-assembler/Deploying/DocsSync.cs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,9 @@ public interface IDocsSyncPlanStrategy
1111
{
1212
Task<SyncPlan> Plan(Cancel ctx = default);
1313

14-
(bool, float) Validate(SyncPlan plan, float deleteThreshold);
14+
PlanValidationResult Validate(SyncPlan plan, float deleteThreshold);
1515
}
16+
public record PlanValidationResult(bool Valid, float DeleteRatio, float DeleteThreshold);
1617

1718
public interface IDocsSyncApplyStrategy
1819
{
@@ -54,8 +55,8 @@ public record SyncPlan
5455
[JsonPropertyName("total_source_files")]
5556
public required int TotalSourceFiles { get; init; }
5657

57-
[JsonPropertyName("total_files_to_sync")]
58-
public required int TotalFilesToSync { get; init; }
58+
[JsonPropertyName("total_sync_requests")]
59+
public required int TotalSyncRequests { get; init; }
5960

6061
[JsonPropertyName("delete")]
6162
public required IReadOnlyList<DeleteRequest> DeleteRequests { get; init; }

‎tests/docs-assembler.Tests/src/docs-assembler.Tests/DocsSyncTests.cs‎

Lines changed: 22 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ public async Task TestPlan()
6767
// Assert
6868

6969
plan.TotalSourceFiles.Should().Be(5);
70-
plan.TotalFilesToSync.Should().Be(6); //including skip on server
70+
plan.TotalSyncRequests.Should().Be(6); //including skip on server
7171

7272
plan.AddRequests.Count.Should().Be(3);
7373
plan.AddRequests.Should().Contain(i => i.DestinationPath == "docs/add1.md");
@@ -90,6 +90,12 @@ public async Task TestPlan()
9090
[InlineData(7900, 10_000, 10_000, 0, 2100, 0.2, false)]
9191
[InlineData(10_000, 0, 10_000, 10_000, 0, 0.2, true)]
9292
[InlineData(2000, 0, 2000, 2000, 0, 0.2, true)]
93+
// When total files to sync is lower than 100 we enforce a minimum ratio of 0.8
94+
[InlineData(20, 40, 40, 0, 20, 0.2, true)]
95+
[InlineData(19, 100, 100, 0, 81, 0.2, false)]
96+
// When total files to sync is lower than 1000 we enforce a minimum ratio of 0.5
97+
[InlineData(200, 400, 400, 0, 200, 0.2, true)]
98+
[InlineData(199, 1000, 1000, 0, 801, 0.2, false)]
9399
public async Task ValidateAdditionsPlan(
94100
int localFiles,
95101
int remoteFiles,
@@ -105,14 +111,18 @@ bool valid
105111
// Assert
106112

107113
plan.TotalSourceFiles.Should().Be(localFiles);
108-
plan.TotalFilesToSync.Should().Be(totalFilesToSync);
114+
plan.TotalSyncRequests.Should().Be(totalFilesToSync);
109115

110116
plan.AddRequests.Count.Should().Be(totalFilesToAdd);
111117
plan.DeleteRequests.Count.Should().Be(totalFilesToRemove);
112118

113-
var (validResult, deleteRatio) = planStrategy.Validate(plan, deleteThreshold);
119+
var validationResult = planStrategy.Validate(plan, deleteThreshold);
120+
if (plan.TotalSyncRequests <= 100)
121+
validationResult.DeleteThreshold.Should().Be(Math.Max(deleteThreshold, 0.8f));
122+
else if (plan.TotalSyncRequests <= 1000)
123+
validationResult.DeleteThreshold.Should().Be(Math.Max(deleteThreshold, 0.5f));
114124

115-
validResult.Should().Be(valid, $"Delete ratio is {deleteRatio} when maximum is {deleteThreshold}");
125+
validationResult.Valid.Should().Be(valid, $"Delete ratio is {validationResult.DeleteRatio} when maximum is {validationResult.DeleteThreshold}");
116126
}
117127

118128
[Theory]
@@ -139,14 +149,18 @@ bool valid
139149
// Assert
140150

141151
plan.TotalSourceFiles.Should().Be(localFiles);
142-
plan.TotalFilesToSync.Should().Be(totalFilesToSync);
152+
plan.TotalSyncRequests.Should().Be(totalFilesToSync);
143153

144154
plan.UpdateRequests.Count.Should().Be(totalFilesToUpdate);
145155
plan.DeleteRequests.Count.Should().Be(totalFilesToRemove);
146156

147-
var (validResult, deleteRatio) = planStrategy.Validate(plan, deleteThreshold);
157+
var validationResult = planStrategy.Validate(plan, deleteThreshold);
158+
if (plan.TotalSyncRequests <= 100)
159+
validationResult.DeleteThreshold.Should().Be(Math.Max(deleteThreshold, 0.8f));
160+
else if (plan.TotalSyncRequests <= 1000)
161+
validationResult.DeleteThreshold.Should().Be(Math.Max(deleteThreshold, 0.5f));
148162

149-
validResult.Should().Be(valid, $"Delete ratio is {deleteRatio} when maximum is {deleteThreshold}");
163+
validationResult.Valid.Should().Be(valid, $"Delete ratio is {validationResult.DeleteRatio} when maximum is {validationResult.DeleteThreshold}");
150164
}
151165

152166
private static async Task<(AwsS3SyncPlanStrategy planStrategy, SyncPlan plan)> SetupS3SyncContextSetup(
@@ -218,7 +232,7 @@ public async Task TestApply()
218232
var plan = new SyncPlan
219233
{
220234
TotalSourceFiles = 5,
221-
TotalFilesToSync = 6,
235+
TotalSyncRequests = 6,
222236
AddRequests = [
223237
new AddRequest { LocalPath = "docs/add1.md", DestinationPath = "docs/add1.md" },
224238
new AddRequest { LocalPath = "docs/add2.md", DestinationPath = "docs/add2.md" },

0 commit comments

Comments
 (0)