Adjust delete threshold logic in AwsS3SyncPlanStrategy (#1811)

- Update delete ratio calculation to use `TotalRemoteFiles` instead of `TotalSyncRequests`.
- Add logging and reset delete threshold to `0.0` when no S3 remote files are discovered. Ensuring our plan only contains additions.

Emit `plan-valid` as github actions output.

Author: Mpdreamz

src/tooling/docs-assembler/Cli/DeployCommands.cs

@@ -25,12 +25,13 @@ internal sealed class DeployCommands(
 	ICoreService githubActionsService
 )
 {
+	private readonly ILogger<Program> _logger = logFactory.CreateLogger<Program>();
+
 	[SuppressMessage("Usage", "CA2254:Template should be a static expression")]
 	private void AssignOutputLogger()
 	{
-		var log = logFactory.CreateLogger<Program>();
-		ConsoleApp.Log = msg => log.LogInformation(msg);
-		ConsoleApp.LogError = msg => log.LogError(msg);
+		ConsoleApp.Log = msg => _logger.LogInformation(msg);
+		ConsoleApp.LogError = msg => _logger.LogError(msg);
 	}
 
 	/// <summary> Creates a sync plan </summary>
@@ -57,20 +58,17 @@ public async Task<int> Plan(
 		var s3Client = new AmazonS3Client();
 		IDocsSyncPlanStrategy planner = new AwsS3SyncPlanStrategy(logFactory, s3Client, s3BucketName, assembleContext);
 		var plan = await planner.Plan(ctx);
-		ConsoleApp.Log("Total files to sync: " + plan.TotalSyncRequests);
-		ConsoleApp.Log("Total files to delete: " + plan.DeleteRequests.Count);
-		ConsoleApp.Log("Total files to add: " + plan.AddRequests.Count);
-		ConsoleApp.Log("Total files to update: " + plan.UpdateRequests.Count);
-		ConsoleApp.Log("Total files to skip: " + plan.SkipRequests.Count);
-		if (plan.TotalSyncRequests == 0)
-		{
-			collector.EmitError(@out, $"Plan has no files to sync so no plan will be written.");
-			await collector.StopAsync(ctx);
-			return collector.Errors;
-		}
+		_logger.LogInformation("Total files to sync: {TotalFiles}", plan.TotalSyncRequests);
+		_logger.LogInformation("Total files to delete: {DeleteCount}", plan.DeleteRequests.Count);
+		_logger.LogInformation("Total files to add: {AddCount}", plan.AddRequests.Count);
+		_logger.LogInformation("Total files to update: {UpdateCount}", plan.UpdateRequests.Count);
+		_logger.LogInformation("Total files to skip: {SkipCount}", plan.SkipRequests.Count);
+		_logger.LogInformation("Total local source files: {TotalSourceFiles}", plan.TotalSourceFiles);
+		_logger.LogInformation("Total remote source files: {TotalSourceFiles}", plan.TotalRemoteFiles);
 		var validationResult = planner.Validate(plan, deleteThreshold);
 		if (!validationResult.Valid)
 		{
+			await githubActionsService.SetOutputAsync("plan-valid", "false");
 			collector.EmitError(@out, $"Plan is invalid, delete ratio: {validationResult.DeleteRatio}, threshold: {validationResult.DeleteThreshold} over {plan.TotalSyncRequests:N0} files while plan has {plan.DeleteRequests:N0} deletions");
 			await collector.StopAsync(ctx);
 			return collector.Errors;
@@ -85,6 +83,7 @@ public async Task<int> Plan(
 			ConsoleApp.Log("Plan written to " + @out);
 		}
 		await collector.StopAsync(ctx);
+		await githubActionsService.SetOutputAsync("plan-valid", collector.Errors == 0 ? "true" : "false");
 		return collector.Errors;
 	}
 
@@ -121,6 +120,19 @@ public async Task<int> Apply(
 		}
 		var planJson = await File.ReadAllTextAsync(planFile, ctx);
 		var plan = SyncPlan.Deserialize(planJson);
+		_logger.LogInformation("Total files to sync: {TotalFiles}", plan.TotalSyncRequests);
+		_logger.LogInformation("Total files to delete: {DeleteCount}", plan.DeleteRequests.Count);
+		_logger.LogInformation("Total files to add: {AddCount}", plan.AddRequests.Count);
+		_logger.LogInformation("Total files to update: {UpdateCount}", plan.UpdateRequests.Count);
+		_logger.LogInformation("Total files to skip: {SkipCount}", plan.SkipRequests.Count);
+		_logger.LogInformation("Total local source files: {TotalSourceFiles}", plan.TotalSourceFiles);
+		_logger.LogInformation("Total remote source files: {TotalSourceFiles}", plan.TotalRemoteFiles);
+		if (plan.TotalSyncRequests == 0)
+		{
+			_logger.LogInformation("Plan has no files to sync, skipping incremental synchronization");
+			await collector.StopAsync(ctx);
+			return collector.Errors;
+		}
 		await applier.Apply(plan, ctx);
 		await collector.StopAsync(ctx);
 		return collector.Errors;

src/tooling/docs-assembler/Deploying/AwsS3SyncPlanStrategy.cs

@@ -158,6 +158,7 @@ await Parallel.ForEachAsync(localObjects, ctx, async (localFile, token) =>
 
 		return new SyncPlan
 		{
+			TotalRemoteFiles = remoteObjects.Count,
 			TotalSourceFiles = localObjects.Length,
 			DeleteRequests = deleteRequests.ToList(),
 			AddRequests = addRequests.ToList(),
@@ -176,15 +177,20 @@ public PlanValidationResult Validate(SyncPlan plan, float deleteThreshold)
 			return new(false, 1.0f, deleteThreshold);
 		}
 
-		var deleteRatio = (float)plan.DeleteRequests.Count / plan.TotalSyncRequests;
-		// if the total sync requests are less than 100, we enforce a higher ratio of 0.8
+		var deleteRatio = (float)plan.DeleteRequests.Count / plan.TotalRemoteFiles;
+		if (plan.TotalRemoteFiles == 0)
+		{
+			_logger.LogInformation("No files discovered in S3, assuming a clean bucket resetting delete threshold to `0.0' as our plan should not have ANY deletions");
+			deleteThreshold = 0.0f;
+		}
+		// if the total remote files are less than or equal to 100, we enforce a higher ratio of 0.8
 		// this allows newer assembled documentation to be in a higher state of flux
-		if (plan.TotalSyncRequests <= 100)
+		if (plan.TotalRemoteFiles <= 100)
 			deleteThreshold = Math.Max(deleteThreshold, 0.8f);
 
-		// if the total sync requests are less than 1000, we enforce a higher ratio of 0.5
+		// if the total remote files are less than or equal to 1000, we enforce a higher ratio of 0.5
 		// this allows newer assembled documentation to be in a higher state of flux
-		else if (plan.TotalSyncRequests <= 1000)
+		else if (plan.TotalRemoteFiles <= 1000)
 			deleteThreshold = Math.Max(deleteThreshold, 0.5f);
 
 		if (deleteRatio > deleteThreshold)

src/tooling/docs-assembler/Deploying/DocsSync.cs

@@ -52,9 +52,15 @@ public record SkipRequest : SyncRequest
 
 public record SyncPlan
 {
+	/// The total number of source files that were located in the build output
 	[JsonPropertyName("total_source_files")]
 	public required int TotalSourceFiles { get; init; }
 
+	/// The total number of remote files that were located in the remote location
+	[JsonPropertyName("total_remote_files")]
+	public required int TotalRemoteFiles { get; init; }
+
+	/// The total number of sync requests that were generated (sum of <see cref="AddRequests"/>, <see cref="UpdateRequests"/>, <see cref="DeleteRequests"/>)
 	[JsonPropertyName("total_sync_requests")]
 	public required int TotalSyncRequests { get; init; }
 

tests/docs-assembler.Tests/src/docs-assembler.Tests/DocsSyncTests.cs

@@ -66,6 +66,8 @@ public async Task TestPlan()
 
 		// Assert
 
+		plan.TotalRemoteFiles.Should().Be(3);
+
 		plan.TotalSourceFiles.Should().Be(5);
 		plan.TotalSyncRequests.Should().Be(6); //including skip on server
 
@@ -231,6 +233,7 @@ public async Task TestApply()
 		var context = new AssembleContext(config, configurationContext, "dev", collector, fileSystem, fileSystem, null, checkoutDirectory);
 		var plan = new SyncPlan
 		{
+			TotalRemoteFiles = 0,
 			TotalSourceFiles = 5,
 			TotalSyncRequests = 6,
 			AddRequests = [