Skip to content

Commit e02fa6a

Browse files
Brennan StehlingbrennanMKE
Brennan Stehling
authored and
brennanMKE
committed
WIP
1 parent eb75a5e commit e02fa6a

File tree

3 files changed

+98
-82
lines changed

3 files changed

+98
-82
lines changed

‎AWSS3/AWSS3TransferUtility.m‎

Lines changed: 29 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -2099,7 +2099,7 @@ - (AWSTask *)callFinishMultiPartForUploadTask:(AWSS3TransferUtilityMultiPartUplo
20992099
return [self.s3 completeMultipartUpload:compReq];
21002100
}
21012101

2102-
- (AWSTask *) callAbortMultiPartForUploadTask:(AWSS3TransferUtilityMultiPartUploadTask *) uploadTask {
2102+
- (AWSTask *)callAbortMultiPartForUploadTask:(AWSS3TransferUtilityMultiPartUploadTask *)uploadTask {
21032103
AWSS3AbortMultipartUploadRequest *abortReq = [AWSS3AbortMultipartUploadRequest new];
21042104
abortReq.bucket = uploadTask.bucket;
21052105
abortReq.uploadId = uploadTask.uploadID;
@@ -2377,61 +2377,35 @@ - (void)URLSession:(NSURLSession *)session
23772377
eTag:subTask.eTag
23782378
status:subTask.status
23792379
retry_count:transferUtilityMultiPartUploadTask.retryCount databaseQueue:self.databaseQueue];
2380+
2381+
[transferUtilityMultiPartUploadTask moveTasksToInProgress];
2382+
[transferUtilityMultiPartUploadTask completeIfDone];
23802383

23812384
//If there are parts waiting to be uploaded, pick from the waiting parts list and move it to inProgress
2382-
if ([transferUtilityMultiPartUploadTask.waitingPartsDictionary count] > 0) {
2383-
long numberOfPartsInProgress = [transferUtilityMultiPartUploadTask.inProgressPartsDictionary count];
2384-
while (numberOfPartsInProgress < [self.transferUtilityConfiguration.multiPartConcurrencyLimit integerValue]) {
2385-
if ([transferUtilityMultiPartUploadTask.waitingPartsDictionary count] > 0) {
2386-
//Get a part from the waitingList
2387-
AWSS3TransferUtilityUploadSubTask *nextSubTask = [[transferUtilityMultiPartUploadTask.waitingPartsDictionary allValues] objectAtIndex:0];
2388-
2389-
//Add to inProgress list
2390-
[transferUtilityMultiPartUploadTask.inProgressPartsDictionary setObject:nextSubTask forKey:@(nextSubTask.taskIdentifier)];
2391-
2392-
//Remove it from the waitingList
2393-
[transferUtilityMultiPartUploadTask.waitingPartsDictionary removeObjectForKey:@(nextSubTask.taskIdentifier)];
2394-
AWSDDLogDebug(@"Moving Task[%@] to progress for Multipart[%@]", @(nextSubTask.taskIdentifier), transferUtilityMultiPartUploadTask.uploadID);
2395-
[nextSubTask.sessionTask resume];
2396-
numberOfPartsInProgress++;
2397-
continue;
2398-
}
2399-
break;
2400-
}
2401-
}
2402-
else if ([transferUtilityMultiPartUploadTask.inProgressPartsDictionary count] == 0) {
2403-
//If there are no more inProgress parts, then we are done.
2404-
2405-
//Validate that all the content has been uploaded.
2406-
int64_t totalBytesSent = 0;
2407-
for (AWSS3TransferUtilityUploadSubTask *aSubTask in transferUtilityMultiPartUploadTask.completedPartsSet) {
2408-
totalBytesSent += aSubTask.totalBytesExpectedToSend;
2409-
}
2410-
2411-
if (totalBytesSent != transferUtilityMultiPartUploadTask.contentLength.longLongValue ) {
2412-
NSString *errorMessage = [NSString stringWithFormat:@"Expected to send [%@], but sent [%@] and there are no remaining parts. Failing transfer ",
2413-
transferUtilityMultiPartUploadTask.contentLength, @(totalBytesSent)];
2414-
AWSDDLogDebug(@"%@", errorMessage);
2415-
NSDictionary *userInfo = [NSDictionary dictionaryWithObject:errorMessage
2416-
forKey:@"Message"];
2417-
2418-
transferUtilityMultiPartUploadTask.error = [NSError errorWithDomain:AWSS3TransferUtilityErrorDomain
2419-
code:AWSS3TransferUtilityErrorClientError
2420-
userInfo:userInfo];
2421-
2422-
//Execute call back if provided.
2423-
[self completeTask:transferUtilityMultiPartUploadTask];
2424-
2425-
//Abort the request, so the server can clean up any partials.
2426-
[self callAbortMultiPartForUploadTask:transferUtilityMultiPartUploadTask];
2427-
2428-
//clean up.
2429-
[self cleanupForMultiPartUploadTask:transferUtilityMultiPartUploadTask];
2430-
return;
2431-
}
2432-
2433-
[self completeMultiPartForUploadTask:transferUtilityMultiPartUploadTask];
2434-
}
2385+
// if ([transferUtilityMultiPartUploadTask.waitingPartsDictionary count] > 0) {
2386+
// long numberOfPartsInProgress = [transferUtilityMultiPartUploadTask.inProgressPartsDictionary count];
2387+
// while (numberOfPartsInProgress < [self.transferUtilityConfiguration.multiPartConcurrencyLimit integerValue]) {
2388+
// if ([transferUtilityMultiPartUploadTask.waitingPartsDictionary count] > 0) {
2389+
// //Get a part from the waitingList
2390+
// AWSS3TransferUtilityUploadSubTask *nextSubTask = [[transferUtilityMultiPartUploadTask.waitingPartsDictionary allValues] objectAtIndex:0];
2391+
//
2392+
// //Add to inProgress list
2393+
// [transferUtilityMultiPartUploadTask.inProgressPartsDictionary setObject:nextSubTask forKey:@(nextSubTask.taskIdentifier)];
2394+
//
2395+
// //Remove it from the waitingList
2396+
// [transferUtilityMultiPartUploadTask.waitingPartsDictionary removeObjectForKey:@(nextSubTask.taskIdentifier)];
2397+
// AWSDDLogDebug(@"Moving Task[%@] to progress for Multipart[%@]", @(nextSubTask.taskIdentifier), transferUtilityMultiPartUploadTask.uploadID);
2398+
// [nextSubTask.sessionTask resume];
2399+
// numberOfPartsInProgress++;
2400+
// continue;
2401+
// }
2402+
// break;
2403+
// }
2404+
// }
2405+
// else
2406+
// if (transferUtilityMultiPartUploadTask.inProgressPartsDictionary.count == 0) {
2407+
//
2408+
// }
24352409
}
24362410
}
24372411
else if ([task isKindOfClass:[NSURLSessionDownloadTask class]]) {
@@ -2580,7 +2554,7 @@ - (void)completeTask:(AWSS3TransferUtilityTask *)task removeCompletedTask:(BOOL)
25802554

25812555
}
25822556

2583-
- (void) cleanupForMultiPartUploadTask: (AWSS3TransferUtilityMultiPartUploadTask *) task {
2557+
- (void)cleanupForMultiPartUploadTask:(AWSS3TransferUtilityMultiPartUploadTask *)task {
25842558

25852559
//Add it to list of completed Tasks
25862560
[self.completedTaskDictionary setObject:task forKey:task.transferID];

‎AWSS3/AWSS3TransferUtilityTasks.m‎

Lines changed: 63 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,10 @@ - (instancetype)init {
133133
return self;
134134
}
135135

136+
- (BOOL)isUnderConcurrencyLimit {
137+
return self.inProgressPartsDictionary.count < [self.transferUtility.transferUtilityConfiguration.multiPartConcurrencyLimit integerValue];
138+
}
139+
136140
- (BOOL)isDone {
137141
return _waitingPartsDictionary.count == 0 && _inProgressPartsDictionary.count == 0;
138142
}
@@ -194,41 +198,73 @@ - (void)resume {
194198
// retry_count:self.retryCount
195199
// databaseQueue:self.databaseQueue];
196200

197-
// move parts from waiting to in progress if under the concurrency limit
198-
if (self.inProgressPartsDictionary.count < [self.transferUtility.transferUtilityConfiguration.multiPartConcurrencyLimit integerValue]) {
199-
long numberOfPartsInProgress = [self.inProgressPartsDictionary count];
200-
while (numberOfPartsInProgress < [self.transferUtility.transferUtilityConfiguration.multiPartConcurrencyLimit integerValue]) {
201-
if ([self.waitingPartsDictionary count] > 0) {
202-
//Get a part from the waitingList
203-
AWSS3TransferUtilityUploadSubTask *nextSubTask = [[self.waitingPartsDictionary allValues] objectAtIndex:0];
204-
205-
//Add to inProgress list
206-
[self.inProgressPartsDictionary setObject:nextSubTask forKey:@(nextSubTask.taskIdentifier)];
207-
208-
//Remove it from the waitingList
209-
[self.waitingPartsDictionary removeObjectForKey:@(nextSubTask.taskIdentifier)];
210-
AWSDDLogDebug(@"Moving Task[%@] to progress for Multipart[%@]", @(nextSubTask.taskIdentifier), self.uploadID);
211-
[nextSubTask.sessionTask resume];
212-
nextSubTask.status = AWSS3TransferUtilityTransferStatusInProgress;
213-
numberOfPartsInProgress++;
214-
continue;
215-
}
216-
break;
217-
}
218-
}
201+
[self moveTasksToInProgress];
219202

220203
// Change status from paused to waiting
221204
for (AWSS3TransferUtilityUploadSubTask * nextSubTask in self.waitingPartsDictionary.allValues) {
222205
nextSubTask.status = AWSS3TransferUtilityTransferStatusWaiting;
223206
}
224207

208+
[self completeIfDone];
209+
}
210+
211+
- (void)moveTasksToInProgress {
212+
// move parts from waiting to in progress if under the concurrency limit
213+
while (self.isUnderConcurrencyLimit && self.waitingPartsDictionary.count > 0) {
214+
//Get a part from the waitingList
215+
AWSS3TransferUtilityUploadSubTask *nextSubTask = [[self.waitingPartsDictionary allValues] objectAtIndex:0];
216+
217+
//Add to inProgress list
218+
[self.inProgressPartsDictionary setObject:nextSubTask forKey:@(nextSubTask.taskIdentifier)];
219+
220+
//Remove it from the waitingList
221+
[self.waitingPartsDictionary removeObjectForKey:@(nextSubTask.taskIdentifier)];
222+
AWSDDLogDebug(@"Moving Task[%@] to progress for Multipart[%@]", @(nextSubTask.taskIdentifier), self.uploadID);
223+
[nextSubTask.sessionTask resume];
224+
nextSubTask.status = AWSS3TransferUtilityTransferStatusInProgress;
225+
}
226+
}
227+
228+
- (void)completeIfDone {
225229
// Complete multipart upload if in progress and waiting tasks are done
226-
if (self.isDone) {
227-
AWSDDLogDebug(@"There are %lu waiting upload parts.", (unsigned long)self.waitingPartsDictionary.count);
228-
AWSDDLogDebug(@"There are %lu in progress upload parts.", (unsigned long)self.inProgressPartsDictionary.count);
229-
AWSDDLogDebug(@"There are %lu completed upload parts.", (unsigned long)self.completedPartsSet.count);
230-
[self.transferUtility completeMultiPartForUploadTask:self];
230+
if (!self.isDone) {
231+
return;
231232
}
233+
234+
//If there are no more inProgress parts, then we are done.
235+
236+
//Validate that all the content has been uploaded.
237+
int64_t totalBytesSent = 0;
238+
for (AWSS3TransferUtilityUploadSubTask *aSubTask in self.completedPartsSet) {
239+
totalBytesSent += aSubTask.totalBytesExpectedToSend;
240+
}
241+
242+
if (totalBytesSent != self.contentLength.longLongValue ) {
243+
NSString *errorMessage = [NSString stringWithFormat:@"Expected to send [%@], but sent [%@] and there are no remaining parts. Failing transfer ",
244+
self.contentLength, @(totalBytesSent)];
245+
AWSDDLogDebug(@"%@", errorMessage);
246+
NSDictionary *userInfo = [NSDictionary dictionaryWithObject:errorMessage
247+
forKey:@"Message"];
248+
249+
self.error = [NSError errorWithDomain:AWSS3TransferUtilityErrorDomain
250+
code:AWSS3TransferUtilityErrorClientError
251+
userInfo:userInfo];
252+
253+
//Execute call back if provided.
254+
[self.transferUtility completeTask:self];
255+
256+
//Abort the request, so the server can clean up any partials.
257+
[self.transferUtility callAbortMultiPartForUploadTask:self];
258+
259+
//clean up.
260+
[self.transferUtility cleanupForMultiPartUploadTask:self];
261+
return;
262+
}
263+
264+
AWSDDLogDebug(@"There are %lu waiting upload parts.", (unsigned long)self.waitingPartsDictionary.count);
265+
AWSDDLogDebug(@"There are %lu in progress upload parts.", (unsigned long)self.inProgressPartsDictionary.count);
266+
AWSDDLogDebug(@"There are %lu completed upload parts.", (unsigned long)self.completedPartsSet.count);
267+
[self.transferUtility completeMultiPartForUploadTask:self];
232268
}
233269

234270
- (void)suspend {

‎AWSS3/AWSS3TransferUtility_private.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@ internalDictionaryToAddSubTaskTo:(NSMutableDictionary *)internalDictionaryToAddS
3131
startTransfer:(BOOL)startTransfer
3232
internalDictionaryToAddSubTaskTo:(NSMutableDictionary *)internalDictionaryToAddSubTaskTo;
3333

34+
- (void)completeTask:(AWSS3TransferUtilityTask *)task;
35+
- (AWSTask *)callAbortMultiPartForUploadTask:(AWSS3TransferUtilityMultiPartUploadTask *)uploadTask;
36+
- (void)cleanupForMultiPartUploadTask:(AWSS3TransferUtilityMultiPartUploadTask *)task;
3437
- (void)completeMultiPartForUploadTask:(AWSS3TransferUtilityMultiPartUploadTask *)transferUtilityMultiPartUploadTask;
3538

3639
@end
@@ -71,6 +74,7 @@ internalDictionaryToAddSubTaskTo:(NSMutableDictionary *)internalDictionaryToAddS
7174
@property (copy) NSString * uploadID;
7275
@property BOOL cancelled;
7376
@property BOOL temporaryFileCreated;
77+
@property (readonly) BOOL isUnderConcurrencyLimit;
7478
@property (readonly) BOOL isDone;
7579
@property (strong, nonatomic) NSMutableDictionary <NSNumber *, AWSS3TransferUtilityUploadSubTask *> *waitingPartsDictionary;
7680
@property (strong, nonatomic) NSMutableDictionary <NSNumber *, AWSS3TransferUtilityUploadSubTask *> *inProgressPartsDictionary;
@@ -81,6 +85,8 @@ internalDictionaryToAddSubTaskTo:(NSMutableDictionary *)internalDictionaryToAddS
8185
@property (weak, nonatomic) AWSS3TransferUtility *transferUtility;
8286

8387
- (void)integrateWithTransferUtility:(AWSS3TransferUtility *)transferUtility;
88+
- (void)moveTasksToInProgress;
89+
- (void)completeIfDone;
8490

8591
@end
8692

0 commit comments

Comments
 (0)