Skip to content

Commit 6895fc3

Browse files
0marperez0marperez
committed
saving work - upload file
1 parent bc2fa0a commit 6895fc3

File tree

8 files changed

+829
-35
lines changed

8 files changed

+829
-35
lines changed

‎hll/s3-transfer-manager/api/s3-transfer-manager.api‎

Lines changed: 294 additions & 0 deletions
Large diffs are not rendered by default.

‎hll/s3-transfer-manager/build.gradle.kts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,8 @@ kotlin {
1212
commonMain {
1313
dependencies {
1414
implementation(project(":aws-runtime:aws-http"))
15-
implementation(project(":services:s3"))
15+
implementation(project(":services:s3")) // TODO: Hardcode an S3 Client version to avoid breakages
1616
}
1717
}
1818
}
19-
}
19+
}

‎hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/BusinessMetricInterceptor.kt‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
16
package aws.sdk.kotlin.hll.s3transfermanager
27

38
import aws.sdk.kotlin.runtime.http.interceptors.businessmetrics.AwsBusinessMetric
@@ -13,4 +18,4 @@ internal object BusinessMetricInterceptor : HttpInterceptor {
1318
context.executionContext.emitBusinessMetric(AwsBusinessMetric.S3_TRANSFER)
1419
return context.request
1520
}
16-
}
21+
}

‎hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/MultiPartDownloadType.kt‎

Lines changed: 0 additions & 16 deletions
This file was deleted.

‎hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/S3TransferManager.kt‎

Lines changed: 104 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -5,18 +5,29 @@
55

66
package aws.sdk.kotlin.hll.s3transfermanager
77

8+
import aws.sdk.kotlin.hll.s3transfermanager.model.MultiPartDownloadType
9+
import aws.sdk.kotlin.hll.s3transfermanager.model.Part
10+
import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileRequest
11+
import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileRequest.Companion.toCreateMultiPartUploadRequest
12+
import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileRequest.Companion.toPutObjectRequest
13+
import aws.sdk.kotlin.hll.s3transfermanager.model.UploadFileResponse
814
import aws.sdk.kotlin.services.s3.S3Client
15+
import aws.sdk.kotlin.services.s3.model.CreateMultipartUploadRequest
16+
import aws.sdk.kotlin.services.s3.model.PutObjectRequest
917
import aws.sdk.kotlin.services.s3.withConfig
18+
import kotlinx.coroutines.Deferred
19+
import kotlinx.coroutines.async
20+
import kotlinx.coroutines.coroutineScope
1021

1122
/**
1223
* High level utility for managing transfers to Amazon S3.
1324
*/
1425
public class S3TransferManager private constructor(
1526
public val client: S3Client,
16-
public val partSize: Long?,
17-
public val multipartUploadThreshold: Long?,
18-
public val multipartDownloadType: MultiPartDownloadType?,
19-
public val interceptors: MutableList<TransferInterceptor>?,
27+
public val partSize: Long,
28+
public val multipartUploadThreshold: Long,
29+
public val multipartDownloadType: MultiPartDownloadType,
30+
public val interceptors: MutableList<TransferInterceptor>,
2031
) {
2132
public companion object {
2233
public operator fun invoke(block: Builder.() -> Unit): S3TransferManager =
@@ -25,20 +36,100 @@ public class S3TransferManager private constructor(
2536

2637
public class Builder {
2738
public var client: S3Client? = null
28-
public var partSize: Long? = 8_000_000L
29-
public var multipartUploadThreshold: Long? = 16_000_000L
30-
public var multipartDownloadType: MultiPartDownloadType? = Part
31-
public var interceptors: MutableList<TransferInterceptor>? = mutableListOf()
39+
public var partSize: Long? = null
40+
public var multipartUploadThreshold: Long? = null
41+
public var multipartDownloadType: MultiPartDownloadType? = null
42+
public var interceptors: MutableList<TransferInterceptor>? = null
3243

3344
internal fun build(): S3TransferManager =
3445
S3TransferManager(
3546
client = client?.withConfig { interceptors += BusinessMetricInterceptor } ?: error("client must be set"),
36-
partSize = partSize,
37-
multipartUploadThreshold = multipartUploadThreshold,
38-
multipartDownloadType = multipartDownloadType,
39-
interceptors = interceptors
47+
partSize = partSize ?: 8_000_000L,
48+
multipartUploadThreshold = multipartUploadThreshold ?: 16_000_000L,
49+
multipartDownloadType = multipartDownloadType ?: Part,
50+
interceptors = interceptors ?: mutableListOf(),
4051
)
4152
}
4253

43-
public fun x(): String = "" // TODO
54+
private var context: TransferContext = TransferContext()
55+
56+
// TODO: Try to find parts of the code you can commonize
57+
/**
58+
* TODO
59+
*/
60+
public suspend fun uploadFile(uploadFileRequest: UploadFileRequest): Deferred<UploadFileResponse> = coroutineScope {
61+
async {
62+
val multiPartUpload = uploadFileRequest.contentLength >= multipartUploadThreshold
63+
64+
context.request = if (multiPartUpload) {
65+
uploadFileRequest.toCreateMultiPartUploadRequest()
66+
} else {
67+
uploadFileRequest.toPutObjectRequest()
68+
}
69+
70+
operationHook(TransferInitiated) {
71+
context.transferredBytes = 0L
72+
context.transferableBytes = uploadFileRequest.contentLength
73+
74+
if (multiPartUpload) {
75+
context.response = client.createMultipartUpload(context.request as CreateMultipartUploadRequest)
76+
}
77+
}
78+
79+
operationHook(BytesTransferred) {
80+
if (multiPartUpload) {
81+
// TODO: MPU logic
82+
// TODO: Update bytes transferred
83+
} else {
84+
context.response = client.putObject(context.request as PutObjectRequest)
85+
context.transferredBytes = context.transferableBytes
86+
}
87+
}
88+
89+
operationHook(TransferCompleted) {
90+
if (multiPartUpload) {
91+
// TODO: MPU logic
92+
// TODO: Update bytes transferred?
93+
}
94+
}
95+
96+
UploadFileResponse.fromS3Response(context.response)
97+
}
98+
}
99+
100+
private suspend fun operationHook(hook: TransferHook, block: suspend () -> Any) {
101+
interceptors.forEach { interceptor ->
102+
when (hook) {
103+
is TransferInitiated -> {
104+
interceptor.readBeforeTransferInitiated(context)
105+
context = interceptor.modifyBeforeTransferInitiated(context)
106+
block.invoke()
107+
interceptor.readAfterTransferInitiated(context)
108+
context = interceptor.modifyAfterTransferInitiated(context)
109+
}
110+
is BytesTransferred -> {
111+
interceptor.readBeforeBytesTransferred(context)
112+
context = interceptor.modifyBeforeBytesTransferred(context)
113+
block.invoke()
114+
interceptor.readAfterBytesTransferred(context)
115+
context = interceptor.modifyAfterBytesTransferred(context)
116+
}
117+
is FileTransferred -> {
118+
interceptor.readBeforeFileTransferred(context)
119+
context = interceptor.modifyBeforeFileTransferred(context)
120+
block.invoke()
121+
interceptor.readAfterFileTransferred(context)
122+
context = interceptor.modifyAfterFileTransferred(context)
123+
}
124+
is TransferCompleted -> {
125+
interceptor.readBeforeTransferCompleted(context)
126+
context = interceptor.modifyBeforeTransferCompleted(context)
127+
block.invoke()
128+
interceptor.readAfterTransferCompleted(context)
129+
context = interceptor.modifyAfterTransferCompleted(context)
130+
}
131+
else -> error("TransferHook not implemented: ${hook::class.simpleName}")
132+
}
133+
}
134+
}
44135
}

‎hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/TransferInterceptor.kt‎

Lines changed: 63 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,65 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
16
package aws.sdk.kotlin.hll.s3transfermanager
27

3-
public class TransferInterceptor {
4-
// TODO
5-
}
8+
// TODO: Add documentation to each thing in the context
9+
/**
10+
* TODO
11+
*/
12+
public data class TransferContext(
13+
// Req/Resp
14+
var request: Any? = null,
15+
var response: Any? = null,
16+
17+
// Byte transfers
18+
var transferableBytes: Long? = null,
19+
var currentBytes: ByteArray? = null,
20+
var transferredBytes: Long? = null,
21+
22+
// File transfers
23+
var transferableFiles: Long? = null,
24+
var currentFile: String? = null,
25+
var transferredFiles: Long? = null,
26+
)
27+
28+
// TODO: Add documentation to each hook
29+
/**
30+
* TODO
31+
*/
32+
public interface TransferInterceptor {
33+
// Transfer initialization hooks
34+
public fun readBeforeTransferInitiated(context: TransferContext) {}
35+
public fun modifyBeforeTransferInitiated(context: TransferContext): TransferContext = context
36+
public fun readAfterTransferInitiated(context: TransferContext) {}
37+
public fun modifyAfterTransferInitiated(context: TransferContext): TransferContext = context
38+
39+
// Byte transferring hooks
40+
public fun readBeforeBytesTransferred(context: TransferContext) {}
41+
public fun modifyBeforeBytesTransferred(context: TransferContext): TransferContext = context
42+
public fun readAfterBytesTransferred(context: TransferContext) {}
43+
public fun modifyAfterBytesTransferred(context: TransferContext): TransferContext = context
44+
45+
// File transfer hooks
46+
public fun readBeforeFileTransferred(context: TransferContext) {}
47+
public fun modifyBeforeFileTransferred(context: TransferContext): TransferContext = context
48+
public fun readAfterFileTransferred(context: TransferContext) {}
49+
public fun modifyAfterFileTransferred(context: TransferContext): TransferContext = context
50+
51+
// Transfer completion hooks
52+
public fun readBeforeTransferCompleted(context: TransferContext) {}
53+
public fun modifyBeforeTransferCompleted(context: TransferContext): TransferContext = context
54+
public fun readAfterTransferCompleted(context: TransferContext) {}
55+
public fun modifyAfterTransferCompleted(context: TransferContext): TransferContext = context
56+
}
57+
58+
/**
59+
* TODO
60+
*/
61+
internal interface TransferHook
62+
internal object TransferInitiated : TransferHook
63+
internal object BytesTransferred : TransferHook
64+
internal object FileTransferred : TransferHook
65+
internal object TransferCompleted : TransferHook

‎hll/s3-transfer-manager/common/src/aws/sdk/kotlin/hll/s3transfermanager/model/MultiPartDownloadType.kt‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package aws.sdk.kotlin.hll.s3transfermanager.model
7+
8+
/**
9+
* TODO
10+
*/
11+
public sealed interface MultiPartDownloadType
12+
13+
/**
14+
* TODO
15+
*/
16+
public object Range : MultiPartDownloadType
17+
18+
/**
19+
* TODO
20+
*/
21+
public object Part : MultiPartDownloadType

0 commit comments

Comments
 (0)