Skip to content

Commit 3c60561

Browse files
authored
[feature] Enable cleanup in CloudflareR2 post-process (#164)
1 parent 3dd93f6 commit 3c60561

File tree

11 files changed

+106
-45
lines changed

11 files changed

+106
-45
lines changed

src/ByteSync.Client/Services/Communications/Transfers/FileDownloader.cs

Lines changed: 17 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -107,26 +107,27 @@ private async Task DownloadFile()
107107
break;
108108
}
109109

110-
var response = await policy.ExecuteAsync(async () =>
110+
var transferParameters = new TransferParameters
111+
{
112+
SessionId = SharedFileDefinition.SessionId,
113+
SharedFileDefinition = SharedFileDefinition,
114+
PartNumber = partNumber
115+
};
116+
var downloadLocation = await _fileTransferApiClient.GetDownloadFileStorageLocation(transferParameters);
117+
var storageProvider = downloadLocation.StorageProvider;
118+
119+
var downloadResponse = await policy.ExecuteAsync(async () =>
111120
{
112-
var transferParameters = new TransferParameters
113-
{
114-
SessionId = SharedFileDefinition.SessionId,
115-
SharedFileDefinition = SharedFileDefinition,
116-
PartNumber = partNumber
117-
};
118-
var downloadLocation = await _fileTransferApiClient.GetDownloadFileStorageLocation(transferParameters);
119-
120121
var memoryStream = new MemoryStream();
121-
var downloadStrategy = _strategies[downloadLocation.StorageProvider];
122+
var downloadStrategy = _strategies[storageProvider];
122123
var response = await downloadStrategy.DownloadAsync(memoryStream, downloadLocation, CancellationTokenSource.Token);
123124

124125
DownloadTarget.AddOrReplaceMemoryStream(partNumber, memoryStream);
125126
return response;
126127
});
127-
if (response.IsSuccess)
128+
if (downloadResponse.IsSuccess)
128129
{
129-
await AssertFilePartIsDownloaded(partNumber);
130+
await AssertFilePartIsDownloaded(partNumber, storageProvider);
130131
await _semaphoreSlim.WaitAsync();
131132
try
132133
{
@@ -166,14 +167,16 @@ private async Task DownloadFile()
166167
}
167168
}
168169

169-
private async Task AssertFilePartIsDownloaded(int partNumber)
170+
private async Task AssertFilePartIsDownloaded(int partNumber, StorageProvider storageProvider)
170171
{
171172
var transferParameters = new TransferParameters
172173
{
173174
SessionId = SharedFileDefinition.SessionId,
174175
SharedFileDefinition = SharedFileDefinition,
175-
PartNumber = partNumber
176+
PartNumber = partNumber,
177+
StorageProvider = storageProvider
176178
};
179+
177180
await _filePartDownloadAsserter.AssertAsync(transferParameters);
178181
}
179182

src/ByteSync.Common/Business/SharedFiles/TransferParameters.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,6 @@ public class TransferParameters
1313
public int? TotalParts { get; set; }
1414

1515
public List<string>? ActionsGroupIds { get; set; }
16+
17+
public StorageProvider StorageProvider { get; set; }
1618
}

src/ByteSync.ServerCommon/Business/Sessions/SharedFileData.cs

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@ public SharedFileData()
99
{
1010

1111
}
12-
13-
public SharedFileData(SharedFileDefinition sharedFileDefinition, ICollection<string> recipients)
12+
13+
public SharedFileData(SharedFileDefinition sharedFileDefinition, ICollection<string> recipients, StorageProvider storageProvider)
1414
{
1515
SharedFileDefinition = sharedFileDefinition;
1616

@@ -24,11 +24,13 @@ public SharedFileData(SharedFileDefinition sharedFileDefinition, ICollection<str
2424
{
2525
cleanedData = recipient.Substring("CIID_".Length);
2626
}
27-
27+
2828
Recipients.Add(cleanedData);
2929
}
30-
30+
3131
DownloadedBy = new Dictionary<int, HashSet<string>>();
32+
33+
StorageProvider = storageProvider;
3234
}
3335

3436
public SharedFileDefinition SharedFileDefinition { get; set; } = null!;
@@ -78,4 +80,6 @@ public bool IsPartFullyDownloaded(int partNumber)
7880

7981
return false;
8082
}
83+
84+
public StorageProvider StorageProvider { get; set; }
8185
}

src/ByteSync.ServerCommon/Commands/FileTransfers/AssertFilePartIsDownloadedCommandHandler.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ public async Task Handle(AssertFilePartIsDownloadedRequest request, Cancellation
3131

3232
if (_transferLocationService.IsSharedFileDefinitionAllowed(sessionMemberData, sharedFileDefinition))
3333
{
34-
await _sharedFilesService.AssertFilePartIsDownloaded(sharedFileDefinition, request.Client, partNumber);
34+
await _sharedFilesService.AssertFilePartIsDownloaded(request.Client, request.TransferParameters);
3535
}
3636

3737
_logger.LogDebug("File part download asserted for session {SessionId}, file {FileId}, part {PartNumber}",

src/ByteSync.ServerCommon/Commands/FileTransfers/AssertFilePartIsUploadedCommandHandler.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ public async Task Handle(AssertFilePartIsUploadedRequest request, CancellationTo
5151
{
5252
var otherSessionMembers = GetOtherSessionMembers(session!, sessionMemberData);
5353

54-
await _sharedFilesService.AssertFilePartIsUploaded(sharedFileDefinition, partNumber,
54+
await _sharedFilesService.AssertFilePartIsUploaded(request.TransferParameters,
5555
otherSessionMembers.Select(sm => sm.ClientInstanceId).ToList());
5656

5757
var transferPush = new FileTransferPush

src/ByteSync.ServerCommon/Commands/FileTransfers/AssertUploadIsFinishedCommandHandler.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ public async Task Handle(AssertUploadIsFinishedRequest request, CancellationToke
4747
{
4848
var otherSessionMembers = GetOtherSessionMembers(session!, sessionMemberData);
4949

50-
await _sharedFilesService.AssertUploadIsFinished(sharedFileDefinition, totalParts,
50+
await _sharedFilesService.AssertUploadIsFinished(request.TransferParameters,
5151
otherSessionMembers.Select(sm => sm.ClientInstanceId).ToList());
5252

5353
var transferPush = new FileTransferPush

src/ByteSync.ServerCommon/Interfaces/Services/ISharedFilesService.cs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,11 @@ namespace ByteSync.ServerCommon.Interfaces.Services;
55

66
public interface ISharedFilesService
77
{
8-
Task AssertFilePartIsUploaded(SharedFileDefinition sharedFileDefinition, int partNumber, ICollection<string> recipients);
8+
Task AssertFilePartIsUploaded(TransferParameters transferParameters, ICollection<string> recipients);
99

10-
Task AssertUploadIsFinished(SharedFileDefinition sharedFileDefinition, int totalParts, ICollection<string> recipients);
10+
Task AssertUploadIsFinished(TransferParameters transferParameters, ICollection<string> recipients);
1111

12-
Task AssertFilePartIsDownloaded(SharedFileDefinition sharedFileDefinition, Client client, int partNumber);
12+
Task AssertFilePartIsDownloaded(Client client, TransferParameters transferParameters);
1313

1414
// Task AssertDownloadIsFinished(SharedFileDefinition sharedFileDefinition, Client client);
1515

src/ByteSync.ServerCommon/Services/SharedFilesService.cs

Lines changed: 34 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -11,40 +11,56 @@ public class SharedFilesService : ISharedFilesService
1111
{
1212
private readonly ISharedFilesRepository _sharedFilesRepository;
1313
private readonly IBlobUrlService _blobUrlService;
14+
private readonly ICloudflareR2UrlService _cloudflareR2UrlService;
1415
private readonly ILogger<SharedFilesService> _logger;
1516

16-
public SharedFilesService(ISharedFilesRepository sharedFilesRepository, IBlobUrlService blobUrlService, ILogger<SharedFilesService> logger)
17+
public SharedFilesService(
18+
ISharedFilesRepository sharedFilesRepository,
19+
IBlobUrlService blobUrlService,
20+
ICloudflareR2UrlService cloudflareR2UrlService,
21+
ILogger<SharedFilesService> logger)
1722
{
1823
_sharedFilesRepository = sharedFilesRepository;
1924
_blobUrlService = blobUrlService;
25+
_cloudflareR2UrlService = cloudflareR2UrlService;
2026
_logger = logger;
2127
}
22-
public async Task AssertFilePartIsUploaded(SharedFileDefinition sharedFileDefinition, int partNumber, ICollection<string> recipients)
28+
29+
public async Task AssertFilePartIsUploaded(TransferParameters transferParameters, ICollection<string> recipients)
2330
{
31+
var sharedFileDefinition = transferParameters.SharedFileDefinition;
32+
var partNumber = transferParameters.PartNumber!.Value;
33+
var storageProvider = transferParameters.StorageProvider;
34+
2435
await _sharedFilesRepository.AddOrUpdate(sharedFileDefinition, sharedFileData =>
2536
{
26-
sharedFileData ??= new SharedFileData(sharedFileDefinition, recipients);
37+
sharedFileData ??= new SharedFileData(sharedFileDefinition, recipients, storageProvider);
2738

2839
sharedFileData.UploadedPartsNumbers.Add(partNumber);
2940

3041
return sharedFileData;
3142
});
3243
}
3344

34-
public async Task AssertUploadIsFinished(SharedFileDefinition sharedFileDefinition, int totalParts, ICollection<string> recipients)
45+
public async Task AssertUploadIsFinished(TransferParameters transferParameters, ICollection<string> recipients)
3546
{
47+
var sharedFileDefinition = transferParameters.SharedFileDefinition;
48+
var totalParts = transferParameters.TotalParts!.Value;
49+
var storageProvider = transferParameters.StorageProvider;
3650
await _sharedFilesRepository.AddOrUpdate(sharedFileDefinition, sharedFileData =>
3751
{
38-
sharedFileData ??= new SharedFileData(sharedFileDefinition, recipients);
52+
sharedFileData ??= new SharedFileData(sharedFileDefinition, recipients, storageProvider);
3953

4054
sharedFileData.TotalParts = totalParts;
4155

4256
return sharedFileData;
4357
});
4458
}
4559

46-
public async Task AssertFilePartIsDownloaded(SharedFileDefinition sharedFileDefinition, Client downloadedBy, int partNumber)
60+
public async Task AssertFilePartIsDownloaded(Client downloadedBy, TransferParameters transferParameters)
4761
{
62+
var sharedFileDefinition = transferParameters.SharedFileDefinition;
63+
var partNumber = transferParameters.PartNumber!.Value;
4864
bool deleteBlob = false;
4965
bool unregister = false;
5066

@@ -74,7 +90,12 @@ await _sharedFilesRepository.AddOrUpdate(sharedFileDefinition, sharedFileData =>
7490
{
7591
try
7692
{
77-
await _blobUrlService.DeleteBlob(sharedFileDefinition, partNumber);
93+
await (transferParameters.StorageProvider switch
94+
{
95+
StorageProvider.AzureBlobStorage => _blobUrlService.DeleteBlob(sharedFileDefinition, partNumber),
96+
StorageProvider.CloudflareR2 => _cloudflareR2UrlService.DeleteObject(sharedFileDefinition, partNumber),
97+
_ => throw new NotSupportedException($"Storage provider {transferParameters.StorageProvider} is not supported")
98+
});
7899
}
79100
catch (Exception ex)
80101
{
@@ -105,7 +126,12 @@ public async Task ClearSession(string sessionId)
105126
{
106127
try
107128
{
108-
await _blobUrlService.DeleteBlob(sharedFileData.SharedFileDefinition, i);
129+
await (sharedFileData.StorageProvider switch
130+
{
131+
StorageProvider.AzureBlobStorage => _blobUrlService.DeleteBlob(sharedFileData.SharedFileDefinition, i),
132+
StorageProvider.CloudflareR2 => _cloudflareR2UrlService.DeleteObject(sharedFileData.SharedFileDefinition, i),
133+
_ => throw new NotSupportedException($"Storage provider {sharedFileData.StorageProvider} is not supported")
134+
});
109135
}
110136
catch (Exception ex)
111137
{

src/ByteSync.ServerCommon/Services/SynchronizationProgressService.cs

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,13 @@ public async Task InformSynchronizationStarted(SynchronizationEntity synchroniza
5353

5454
public async Task UploadIsFinished(SharedFileDefinition sharedFileDefinition, int totalParts, HashSet<string> targetInstanceIds)
5555
{
56-
await _sharedFilesService.AssertUploadIsFinished(sharedFileDefinition, totalParts, targetInstanceIds);
56+
var transferParameters = new TransferParameters
57+
{
58+
SessionId = sharedFileDefinition.SessionId,
59+
SharedFileDefinition = sharedFileDefinition,
60+
TotalParts = totalParts
61+
};
62+
await _sharedFilesService.AssertUploadIsFinished(transferParameters, targetInstanceIds);
5763

5864
var fileTransferPush = new FileTransferPush
5965
{
@@ -68,7 +74,13 @@ public async Task UploadIsFinished(SharedFileDefinition sharedFileDefinition, in
6874

6975
public async Task FilePartIsUploaded(SharedFileDefinition sharedFileDefinition, int partNumber, HashSet<string> targetInstanceIds)
7076
{
71-
await _sharedFilesService.AssertFilePartIsUploaded(sharedFileDefinition, partNumber, targetInstanceIds);
77+
var transferParameters = new TransferParameters
78+
{
79+
SessionId = sharedFileDefinition.SessionId,
80+
SharedFileDefinition = sharedFileDefinition,
81+
PartNumber = partNumber
82+
};
83+
await _sharedFilesService.AssertFilePartIsUploaded(transferParameters, targetInstanceIds);
7284

7385
var fileTransferPush = new FileTransferPush
7486
{

tests/ByteSync.ServerCommon.Tests/Commands/FileTransfers/AssertFilePartIsDownloadedCommandHandlerTests.cs

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,9 @@ public void Setup()
3434
}
3535

3636
[Test]
37-
public async Task Handle_ValidRequest_AssertsFilePartIsDownloaded()
37+
[TestCase(StorageProvider.AzureBlobStorage)]
38+
[TestCase(StorageProvider.CloudflareR2)]
39+
public async Task Handle_ValidRequest_AssertsFilePartIsDownloaded(StorageProvider storageProvider)
3840
{
3941
// Arrange
4042
var sessionId = "session1";
@@ -46,7 +48,8 @@ public async Task Handle_ValidRequest_AssertsFilePartIsDownloaded()
4648
{
4749
SessionId = sessionId,
4850
SharedFileDefinition = sharedFileDefinition,
49-
PartNumber = partNumber
51+
PartNumber = partNumber,
52+
StorageProvider = storageProvider
5053
};
5154
var request = new AssertFilePartIsDownloadedRequest(sessionId, client, transferParameters);
5255

@@ -59,7 +62,7 @@ public async Task Handle_ValidRequest_AssertsFilePartIsDownloaded()
5962
.Returns(true);
6063

6164
// Mock the shared files service
62-
A.CallTo(() => _mockSharedFilesService.AssertFilePartIsDownloaded(sharedFileDefinition, client, partNumber))
65+
A.CallTo(() => _mockSharedFilesService.AssertFilePartIsDownloaded(client, transferParameters))
6366
.Returns(Task.CompletedTask);
6467

6568
// Act
@@ -70,10 +73,14 @@ public async Task Handle_ValidRequest_AssertsFilePartIsDownloaded()
7073
.MustHaveHappenedOnceExactly();
7174
A.CallTo(() => _mockTransferLocationService.IsSharedFileDefinitionAllowed(mockSessionMember, sharedFileDefinition))
7275
.MustHaveHappenedOnceExactly();
76+
A.CallTo(() => _mockSharedFilesService.AssertFilePartIsDownloaded(client, transferParameters))
77+
.MustHaveHappenedOnceExactly();
7378
}
7479

7580
[Test]
76-
public async Task Handle_WhenServiceThrowsException_PropagatesException()
81+
[TestCase(StorageProvider.AzureBlobStorage)]
82+
[TestCase(StorageProvider.CloudflareR2)]
83+
public async Task Handle_WhenServiceThrowsException_PropagatesException(StorageProvider storageProvider)
7784
{
7885
// Arrange
7986
var sessionId = "session1";
@@ -86,7 +93,8 @@ public async Task Handle_WhenServiceThrowsException_PropagatesException()
8693
{
8794
SessionId = sessionId,
8895
SharedFileDefinition = sharedFileDefinition,
89-
PartNumber = partNumber
96+
PartNumber = partNumber,
97+
StorageProvider = storageProvider
9098
};
9199
var request = new AssertFilePartIsDownloadedRequest(sessionId, client, transferParameters);
92100

0 commit comments

Comments
 (0)