feat: s3 transfer manager

Author: 0marperez

hll/s3-transfer-manager/build.gradle.kts

@@ -0,0 +1,11 @@
+kotlin {
+    sourceSets {
+        commonMain {
+            dependencies {
+                implementation(project(":aws-runtime:aws-http"))
+                implementation(libs.smithy.kotlin.http.client.engine.crt)
+                api(project(":services:s3"))
+            }
+        }
+    }
+}

hll/s3-transfer-manager/common/src/aws.sdk.kotlin.hll.s3transfermanager/S3TransferManagerImplementation.kt

@@ -0,0 +1,72 @@
+package aws.sdk.kotlin.hll.s3transfermanager
+
+import aws.sdk.kotlin.services.s3.S3Client
+import aws.sdk.kotlin.services.s3.model.ChecksumAlgorithm
+import aws.smithy.kotlin.runtime.http.engine.crt.CrtHttpEngine
+import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor
+
+public fun S3TransferManager(
+    config: S3TransferManager.Config.Builder.() -> Unit = { },
+): S3TransferManager = S3TransferManagerImplementation(S3TransferManager.Config(config))
+
+internal data class S3TransferManagerImplementation(
+    override val config: S3TransferManager.Config,
+) : S3TransferManager {
+    override fun uploadFile(input: UploadFileRequest): UploadFileResponse {
+        // TODO: How would this change the config on the fly ?
+        TODO("Not yet implemented")
+    }
+
+    override fun downloadFile(input: DownloadFileRequest): DownloadFileResponse {
+        TODO("Not yet implemented")
+    }
+
+    override fun uploadDirectory(input: UploadDirectoryRequest): UploadDirectoryResponse {
+        TODO("Not yet implemented")
+    }
+
+    override fun downloadDirectory(input: DownloadDirectoryRequest): DownloadDirectoryResponse {
+        TODO("Not yet implemented")
+    }
+
+    override fun trackTransfer(input: TrackTransferRequest): TrackTransferResponse {
+        TODO("Not yet implemented")
+    }
+}
+
+internal data class S3TransferManagerConfigImplementation(
+    override val client: S3Client,
+    override val interceptors: List<HttpInterceptor>,
+    override val targetPartSizeBytes: Long,
+    override val multipartUploadThresholdBytes: Long,
+    override val checksumValidationEnabled: Boolean,
+    override val checksumAlgorithm: ChecksumAlgorithm,
+    override val multipartDownloadType: DownloadType,
+) : S3TransferManager.Config {
+    override fun toBuilder(): S3TransferManager.Config.Builder = S3TransferManager.Config.Builder()
+}
+
+internal class S3TransferManagerConfigBuilderImplementation : S3TransferManager.Config.Builder {
+    override var client: S3Client = S3Client { httpClient = CrtHttpEngine() }
+    override var interceptors: MutableList<HttpInterceptor> = mutableListOf()
+    override var targetPartSizeBytes: Long = 8_000_000L
+    override var multipartUploadThresholdBytes: Long = 16_000_000L
+    override var checksumValidationEnabled: Boolean = true
+    override var checksumAlgorithm: ChecksumAlgorithm = ChecksumAlgorithm.Crc32
+    override var multipartDownloadType: DownloadType = DownloadType.PART
+
+    override fun build() = S3TransferManagerConfigImplementation(
+        client = this.client,
+        interceptors = this.interceptors,
+        targetPartSizeBytes = this.targetPartSizeBytes,
+        multipartUploadThresholdBytes = this.multipartUploadThresholdBytes,
+        checksumValidationEnabled = this.checksumValidationEnabled,
+        checksumAlgorithm = this.checksumAlgorithm,
+        multipartDownloadType = this.multipartDownloadType,
+    )
+}
+
+
+
+
+

hll/s3-transfer-manager/common/src/aws.sdk.kotlin.hll.s3transfermanager/S3TransferManagerInterfaces.kt

@@ -0,0 +1,76 @@
+
+package aws.sdk.kotlin.hll.s3transfermanager
+
+import aws.sdk.kotlin.services.s3.S3Client
+import aws.sdk.kotlin.services.s3.model.ChecksumAlgorithm
+import aws.smithy.kotlin.runtime.http.interceptors.HttpInterceptor
+import aws.smithy.kotlin.runtime.io.Closeable
+
+// TODO: Experimental API ?
+public interface S3TransferManager: Closeable {
+    public val config: Config
+
+    public companion object {
+        public fun Config(config: Config.Builder.() -> Unit = { }): Config =
+            Config.Builder().apply(config).build()
+    }
+
+    public interface Config {
+        public companion object {
+            public fun Builder(): Builder = S3TransferManagerConfigBuilderImplementation()
+        }
+
+        public val client: S3Client
+        public val interceptors: List<HttpInterceptor>
+        public val targetPartSizeBytes: Long
+        public val multipartUploadThresholdBytes: Long
+        public val checksumValidationEnabled: Boolean
+        public val checksumAlgorithm: ChecksumAlgorithm
+        public val multipartDownloadType: DownloadType
+
+        public fun toBuilder(): Builder
+
+        public interface Builder {
+            public var client: S3Client
+            public var interceptors: MutableList<HttpInterceptor>
+            public var targetPartSizeBytes: Long
+            public var multipartUploadThresholdBytes: Long
+            public var checksumValidationEnabled: Boolean
+            public var checksumAlgorithm: ChecksumAlgorithm
+            public var multipartDownloadType: DownloadType
+            public fun build(): Config
+        }
+    }
+
+    // TODO: From environment
+
+    public fun uploadFile(input: UploadFileRequest) : UploadFileResponse
+    public fun downloadFile(input: DownloadFileRequest) : DownloadFileResponse
+    public fun uploadDirectory(input: UploadDirectoryRequest) : UploadDirectoryResponse
+    public fun downloadDirectory(input: DownloadDirectoryRequest) : DownloadDirectoryResponse
+    public fun trackTransfer(input: TrackTransferRequest) : TrackTransferResponse
+
+    override fun close() {
+        config.client.close()
+    }
+}
+
+public enum class DownloadType {
+    PART,
+    RANGE,
+}
+
+public interface UploadFileRequest
+public interface UploadFileResponse
+
+public interface DownloadFileRequest
+public interface DownloadFileResponse
+
+public interface UploadDirectoryRequest
+public interface UploadDirectoryResponse
+
+public interface DownloadDirectoryRequest
+public interface DownloadDirectoryResponse
+
+public interface TrackTransferRequest
+public interface TrackTransferResponse

hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3/transfermanager/S3TransferManagerBenchmarks.kt

@@ -0,0 +1,5 @@
+package aws.sdk.kotlin.hll.s3.transfermanager
+
+class S3TransferManagerBenchmarks {
+    // TODO: Look at existing benchmarks
+}

hll/s3-transfer-manager/common/test/aws/sdk/kotlin/hll/s3/transfermanager/S3TransferManagerTest.kt

@@ -0,0 +1,20 @@
+package aws.sdk.kotlin.hll.s3.transfermanager
+
+import aws.sdk.kotlin.hll.s3transfermanager.DownloadType
+import aws.sdk.kotlin.hll.s3transfermanager.S3TransferManager
+import aws.sdk.kotlin.services.s3.S3Client
+import aws.sdk.kotlin.services.s3.model.ChecksumAlgorithm
+
+class S3TransferManagerTest {
+    private val x = S3TransferManager {
+        client = S3Client {}
+        interceptors = mutableListOf()
+        checksumAlgorithm = ChecksumAlgorithm.Crc32
+        multipartDownloadType = DownloadType.PART
+    }.uploadFile(
+        input = TODO()
+    )
+
+    // TODO: Use different HTTP engines and config options
+    // TODO: Look at tests defined in SEP
+}

settings.gradle.kts

@@ -85,6 +85,12 @@ if ("dynamodb".isBootstrappedService) {
     logger.warn(":services:dynamodb is not bootstrapped, skipping :hll:dynamodb-mapper and subprojects")
 }
 
+if ("s3".isBootstrappedService) {
+    include(":hll:s3-transfer-manager")
+} else {
+    logger.warn(":services:s3 is not bootstrapped, skipping :hll:s3-transfer-manager and subprojects")
+}
+
 // Service benchmarks project
 val benchmarkServices = listOf(
     // keep this list in sync with tests/benchmarks/service-benchmarks/build.gradle.kts