Skip to content

Commit 930aed9

Browse files
paul-fresquetpaul-fresquet
authored
Add Timer Function to delete Cloudflare R2 snippets (#162)
1 parent e8d666e commit 930aed9

11 files changed

+353
-14
lines changed

src/ByteSync.Functions/Timer/CleanupBlobStorageSnippetsFunction.cs renamed to src/ByteSync.Functions/Timer/CleanupAzureBlobStorageSnippetsFunction.cs

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,13 @@
66

77
namespace ByteSync.Functions.Timer;
88

9-
public class CleanupBlobStorageSnippetsFunction
9+
public class CleanupAzureBlobStorageSnippetsFunction
1010
{
11-
private readonly ILogger<CleanupBlobStorageSnippetsFunction> _logger;
11+
private readonly ILogger<CleanupAzureBlobStorageSnippetsFunction> _logger;
1212
private readonly IMediator _mediator;
1313

14-
public CleanupBlobStorageSnippetsFunction(IConfiguration configuration, IMediator mediator,
15-
ILogger<CleanupBlobStorageSnippetsFunction> logger)
14+
public CleanupAzureBlobStorageSnippetsFunction(IConfiguration configuration, IMediator mediator,
15+
ILogger<CleanupAzureBlobStorageSnippetsFunction> logger)
1616
{
1717
_mediator = mediator;
1818
_logger = logger;
@@ -25,9 +25,9 @@ public async Task<int> RunAsync([TimerTrigger("0 0 0 * * *"
2525
#endif
2626
)] TimerInfo myTimer)
2727
{
28-
_logger.LogInformation("Cleanup BlobStorage Function started at: {Now}", DateTime.Now);
29-
var deletedBlobsCount = await _mediator.Send(new CleanupBlobStorageSnippetsRequest());
30-
_logger.LogInformation("Cleanup BlobStorage Function - Deletion complete, {Deleted} element(s)", deletedBlobsCount);
28+
_logger.LogInformation("Cleanup Azure BlobStorage Function started at: {Now}", DateTime.Now);
29+
var deletedBlobsCount = await _mediator.Send(new CleanupAzureBlobStorageSnippetsRequest());
30+
_logger.LogInformation("Cleanup Azure BlobStorage Function - Deletion complete, {Deleted} element(s)", deletedBlobsCount);
3131
return deletedBlobsCount;
3232
}
3333
}

‎src/ByteSync.Functions/Timer/CleanupCloudflareR2SnippetsFunction.cs‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
using MediatR;
2+
using Microsoft.Extensions.Configuration;
3+
using Microsoft.Extensions.Logging;
4+
using ByteSync.ServerCommon.Commands.Storage;
5+
using Microsoft.Azure.Functions.Worker;
6+
7+
namespace ByteSync.Functions.Timer;
8+
9+
public class CleanupCloudflareR2SnippetsFunction
10+
{
11+
private readonly ILogger<CleanupCloudflareR2SnippetsFunction> _logger;
12+
private readonly IMediator _mediator;
13+
14+
public CleanupCloudflareR2SnippetsFunction(IConfiguration configuration, IMediator mediator,
15+
ILogger<CleanupCloudflareR2SnippetsFunction> logger)
16+
{
17+
_mediator = mediator;
18+
_logger = logger;
19+
}
20+
21+
[Function("CleanupCloudflareR2FilesFunction")]
22+
public async Task<int> RunAsync([TimerTrigger("0 0 0 * * *"
23+
#if DEBUG
24+
, RunOnStartup= true
25+
#endif
26+
)] TimerInfo myTimer)
27+
{
28+
_logger.LogInformation("Cleanup Cloudflare R2 Function started at: {Now}", DateTime.Now);
29+
var deletedObjectsCount = await _mediator.Send(new CleanupCloudflareR2SnippetsRequest());
30+
_logger.LogInformation("Cleanup Cloudflare R2 Function - Deletion complete, {Deleted} element(s)", deletedObjectsCount);
31+
return deletedObjectsCount;
32+
}
33+
}

‎src/ByteSync.ServerCommon/Commands/Storage/CleanupAzureBlobStorageSnippetsCommandHandler.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77

88
namespace ByteSync.ServerCommon.Commands.Storage;
99

10-
public class CleanupAzureBlobStorageSnippetsCommandHandler : IRequestHandler<CleanupBlobStorageSnippetsRequest, int>
10+
public class CleanupAzureBlobStorageSnippetsCommandHandler : IRequestHandler<CleanupAzureBlobStorageSnippetsRequest, int>
1111
{
1212
private readonly IBlobStorageContainerService _blobStorageContainerService;
1313
private readonly ILogger<CleanupAzureBlobStorageSnippetsCommandHandler> _logger;
@@ -23,7 +23,7 @@ public CleanupAzureBlobStorageSnippetsCommandHandler(
2323
_logger = logger;
2424
}
2525

26-
public async Task<int> Handle(CleanupBlobStorageSnippetsRequest request, CancellationToken cancellationToken)
26+
public async Task<int> Handle(CleanupAzureBlobStorageSnippetsRequest request, CancellationToken cancellationToken)
2727
{
2828
if (_blobStorageSettings.RetentionDurationInDays < 1)
2929
{

‎src/ByteSync.ServerCommon/Commands/Storage/CleanupAzureBlobStorageSnippetsRequest.cs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
using MediatR;
2+
3+
namespace ByteSync.ServerCommon.Commands.Storage;
4+
5+
public class CleanupAzureBlobStorageSnippetsRequest : IRequest<int>
6+
{
7+
}

‎src/ByteSync.ServerCommon/Commands/Storage/CleanupCloudflareR2SnippetsCommandHandler.cs‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
using Amazon.S3;
2+
using Amazon.S3.Model;
3+
using ByteSync.ServerCommon.Business.Settings;
4+
using ByteSync.ServerCommon.Interfaces.Services;
5+
using MediatR;
6+
using Microsoft.Extensions.Logging;
7+
using Microsoft.Extensions.Options;
8+
9+
namespace ByteSync.ServerCommon.Commands.Storage;
10+
11+
public class CleanupCloudflareR2SnippetsCommandHandler : IRequestHandler<CleanupCloudflareR2SnippetsRequest, int>
12+
{
13+
private readonly ICloudflareR2Service _cloudflareR2Service;
14+
private readonly ILogger<CleanupCloudflareR2SnippetsCommandHandler> _logger;
15+
private readonly CloudflareR2Settings _cloudflareR2Settings;
16+
17+
public CleanupCloudflareR2SnippetsCommandHandler(
18+
ICloudflareR2Service cloudflareR2Service,
19+
IOptions<CloudflareR2Settings> cloudflareR2Settings,
20+
ILogger<CleanupCloudflareR2SnippetsCommandHandler> logger)
21+
{
22+
_cloudflareR2Service = cloudflareR2Service;
23+
_cloudflareR2Settings = cloudflareR2Settings.Value;
24+
_logger = logger;
25+
}
26+
27+
public async Task<int> Handle(CleanupCloudflareR2SnippetsRequest request, CancellationToken cancellationToken)
28+
{
29+
if (_cloudflareR2Settings.RetentionDurationInDays < 1)
30+
{
31+
_logger.LogWarning("RetentionDurationInDays is less than 1, no element deleted");
32+
return 0;
33+
}
34+
35+
var listObjectsRequest = new ListObjectsV2Request
36+
{
37+
BucketName = _cloudflareR2Settings.BucketName
38+
};
39+
40+
var deletedObjectsCount = 0;
41+
var cutoffDate = DateTime.UtcNow.AddDays(-_cloudflareR2Settings.RetentionDurationInDays);
42+
43+
try
44+
{
45+
var listObjectsResponse = await _cloudflareR2Service.ListObjectsAsync(listObjectsRequest, cancellationToken);
46+
47+
foreach (var s3Object in listObjectsResponse.S3Objects)
48+
{
49+
if (s3Object.LastModified <= cutoffDate)
50+
{
51+
_logger.LogInformation("Deleting obsolete R2 object {ObjectKey} (LastModified:{LastModified})",
52+
s3Object.Key, s3Object.LastModified);
53+
54+
var deleteRequest = new DeleteObjectRequest
55+
{
56+
BucketName = _cloudflareR2Settings.BucketName,
57+
Key = s3Object.Key
58+
};
59+
60+
await _cloudflareR2Service.DeleteObjectAsync(deleteRequest, cancellationToken);
61+
deletedObjectsCount += 1;
62+
}
63+
}
64+
}
65+
catch (AmazonS3Exception ex)
66+
{
67+
_logger.LogError(ex, "Error listing or deleting objects from R2 bucket {BucketName}", _cloudflareR2Settings.BucketName);
68+
return 0;
69+
}
70+
71+
return deletedObjectsCount;
72+
}
73+
}

src/ByteSync.ServerCommon/Commands/Storage/CleanupBlobStorageSnippetsRequest.cs renamed to src/ByteSync.ServerCommon/Commands/Storage/CleanupCloudflareR2SnippetsRequest.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,6 @@
22

33
namespace ByteSync.ServerCommon.Commands.Storage;
44

5-
public class CleanupBlobStorageSnippetsRequest : IRequest<int>
5+
public class CleanupCloudflareR2SnippetsRequest : IRequest<int>
66
{
77
}

‎src/ByteSync.ServerCommon/Interfaces/Services/ICloudflareR2Service.cs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
using Amazon.S3.Model;
2+
3+
namespace ByteSync.ServerCommon.Interfaces.Services;
4+
5+
public interface ICloudflareR2Service
6+
{
7+
Task<ListObjectsV2Response> ListObjectsAsync(ListObjectsV2Request request, CancellationToken cancellationToken);
8+
9+
Task<DeleteObjectResponse> DeleteObjectAsync(DeleteObjectRequest request, CancellationToken cancellationToken);
10+
}

‎src/ByteSync.ServerCommon/Services/CloudflareR2Service.cs‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
using Amazon.S3;
2+
using Amazon.S3.Model;
3+
using ByteSync.ServerCommon.Business.Settings;
4+
using ByteSync.ServerCommon.Interfaces.Services;
5+
using Microsoft.Extensions.Options;
6+
7+
namespace ByteSync.ServerCommon.Services;
8+
9+
public class CloudflareR2Service : ICloudflareR2Service
10+
{
11+
private readonly CloudflareR2Settings _cloudflareR2Settings;
12+
private AmazonS3Client? _s3Client;
13+
14+
public CloudflareR2Service(IOptions<CloudflareR2Settings> cloudflareR2Settings)
15+
{
16+
_cloudflareR2Settings = cloudflareR2Settings.Value;
17+
}
18+
19+
public async Task<ListObjectsV2Response> ListObjectsAsync(ListObjectsV2Request request, CancellationToken cancellationToken)
20+
{
21+
var s3Client = GetS3Client();
22+
return await s3Client.ListObjectsV2Async(request, cancellationToken);
23+
}
24+
25+
public async Task<DeleteObjectResponse> DeleteObjectAsync(DeleteObjectRequest request, CancellationToken cancellationToken)
26+
{
27+
var s3Client = GetS3Client();
28+
return await s3Client.DeleteObjectAsync(request, cancellationToken);
29+
}
30+
31+
private AmazonS3Client GetS3Client()
32+
{
33+
if (_s3Client == null)
34+
{
35+
_s3Client = new AmazonS3Client(
36+
_cloudflareR2Settings.AccessKeyId,
37+
_cloudflareR2Settings.SecretAccessKey,
38+
new AmazonS3Config
39+
{
40+
ServiceURL = _cloudflareR2Settings.Endpoint,
41+
ForcePathStyle = true
42+
});
43+
}
44+
45+
return _s3Client;
46+
}
47+
}

‎tests/ByteSync.Functions.UnitTests/Timer/CleanupCloudflareR2SnippetsFunctionTests.cs‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
using ByteSync.ServerCommon.Commands.Storage;
2+
using ByteSync.Functions.Timer;
3+
using Moq;
4+
using MediatR;
5+
using Microsoft.Extensions.Configuration;
6+
using Microsoft.Extensions.Logging;
7+
using Microsoft.Azure.Functions.Worker;
8+
9+
namespace ByteSync.Functions.UnitTests.Timer;
10+
11+
[TestFixture]
12+
public class CleanupCloudflareR2SnippetsFunctionTests
13+
{
14+
private Mock<IMediator> _mediator = null!;
15+
private Mock<ILogger<CleanupCloudflareR2SnippetsFunction>> _logger = null!;
16+
private Mock<IConfiguration> _configuration = null!;
17+
private CleanupCloudflareR2SnippetsFunction _function = null!;
18+
19+
[SetUp]
20+
public void Setup()
21+
{
22+
_mediator = new Mock<IMediator>();
23+
_logger = new Mock<ILogger<CleanupCloudflareR2SnippetsFunction>>();
24+
_configuration = new Mock<IConfiguration>();
25+
26+
_function = new CleanupCloudflareR2SnippetsFunction(_configuration.Object, _mediator.Object, _logger.Object);
27+
}
28+
29+
[Test]
30+
public async Task RunAsync_ShouldSendCleanupRequest()
31+
{
32+
// Arrange
33+
var expectedDeletedCount = 5;
34+
_mediator.Setup(m => m.Send(It.IsAny<CleanupCloudflareR2SnippetsRequest>(), It.IsAny<CancellationToken>()))
35+
.ReturnsAsync(expectedDeletedCount);
36+
37+
// Act
38+
var result = await _function.RunAsync(It.IsAny<TimerInfo>());
39+
40+
// Assert
41+
Assert.That(result, Is.EqualTo(expectedDeletedCount));
42+
_mediator.Verify(m => m.Send(It.IsAny<CleanupCloudflareR2SnippetsRequest>(), It.IsAny<CancellationToken>()), Times.Once);
43+
}
44+
}

‎tests/ByteSync.ServerCommon.Tests/Commands/Storage/CleanupAzureBlobStorageSnippetsCommandHandlerTests.cs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ public async Task Handle_DeletesBlobsOlderThanRetention()
6060
.Returns(Task.FromResult(container));
6161

6262
// Act
63-
var result = await _handler.Handle(new CleanupBlobStorageSnippetsRequest(), CancellationToken.None);
63+
var result = await _handler.Handle(new CleanupAzureBlobStorageSnippetsRequest(), CancellationToken.None);
6464

6565
// Assert
6666
result.Should().Be(2);
@@ -76,7 +76,7 @@ public async Task Handle_DoesNothingIfRetentionIsTooLow()
7676
_settings.RetentionDurationInDays = 0;
7777

7878
// Act
79-
var result = await _handler.Handle(new CleanupBlobStorageSnippetsRequest(), CancellationToken.None);
79+
var result = await _handler.Handle(new CleanupAzureBlobStorageSnippetsRequest(), CancellationToken.None);
8080

8181
// Assert
8282
result.Should().Be(0);
@@ -94,7 +94,7 @@ public async Task Handle_DoesNothingIfContainerDoesNotExist()
9494
.Returns(Task.FromResult(container));
9595

9696
// Act
97-
var result = await _handler.Handle(new CleanupBlobStorageSnippetsRequest(), CancellationToken.None);
97+
var result = await _handler.Handle(new CleanupAzureBlobStorageSnippetsRequest(), CancellationToken.None);
9898

9999
// Assert
100100
result.Should().Be(0);
@@ -120,7 +120,7 @@ public async Task Handle_ReturnsZeroIfNoBlobsToDelete()
120120
.Returns(Task.FromResult(container));
121121

122122
// Act
123-
var result = await _handler.Handle(new CleanupBlobStorageSnippetsRequest(), CancellationToken.None);
123+
var result = await _handler.Handle(new CleanupAzureBlobStorageSnippetsRequest(), CancellationToken.None);
124124

125125
// Assert
126126
result.Should().Be(0);

0 commit comments

Comments
 (0)