feat: support default checksums (#1191)

Author: 0marperez

codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/core/RuntimeTypes.kt

@@ -81,7 +81,7 @@ object RuntimeTypes {
         object Interceptors : RuntimeTypePackage(KotlinDependency.HTTP, "interceptors") {
             val ContinueInterceptor = symbol("ContinueInterceptor")
             val HttpInterceptor = symbol("HttpInterceptor")
-            val Md5ChecksumInterceptor = symbol("Md5ChecksumInterceptor")
+            val HttpChecksumRequiredInterceptor = symbol("HttpChecksumRequiredInterceptor")
             val FlexibleChecksumsRequestInterceptor = symbol("FlexibleChecksumsRequestInterceptor")
             val FlexibleChecksumsResponseInterceptor = symbol("FlexibleChecksumsResponseInterceptor")
             val ResponseLengthValidationInterceptor = symbol("ResponseLengthValidationInterceptor")
@@ -231,6 +231,9 @@ object RuntimeTypes {
         object Config : RuntimeTypePackage(KotlinDependency.SMITHY_CLIENT, "config") {
             val RequestCompressionConfig = symbol("RequestCompressionConfig")
             val CompressionClientConfig = symbol("CompressionClientConfig")
+            val HttpChecksumConfig = symbol("HttpChecksumConfig")
+            val RequestHttpChecksumConfig = symbol("RequestHttpChecksumConfig")
+            val ResponseHttpChecksumConfig = symbol("ResponseHttpChecksumConfig")
         }
 
         object Endpoints : RuntimeTypePackage(KotlinDependency.SMITHY_CLIENT, "endpoints") {
@@ -395,6 +398,7 @@ object RuntimeTypes {
             val TelemetryContextElement = symbol("TelemetryContextElement", "context")
             val TraceSpan = symbol("TraceSpan", "trace")
             val withSpan = symbol("withSpan", "trace")
+            val warn = symbol("warn", "logging")
         }
         object TelemetryDefaults : RuntimeTypePackage(KotlinDependency.TELEMETRY_DEFAULTS) {
             val Global = symbol("Global")
@@ -409,6 +413,7 @@ object RuntimeTypes {
 
         val CompletableDeferred = "kotlinx.coroutines.CompletableDeferred".toSymbol()
         val job = "kotlinx.coroutines.job".toSymbol()
+        val runBlocking = "kotlinx.coroutines.runBlocking".toSymbol()
 
         object Flow {
             // NOTE: smithy-kotlin core has an API dependency on this already

codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/checksums/HttpChecksumRequiredIntegration.kt

@@ -0,0 +1,66 @@
+package software.amazon.smithy.kotlin.codegen.rendering.checksums
+
+import software.amazon.smithy.aws.traits.HttpChecksumTrait
+import software.amazon.smithy.kotlin.codegen.KotlinSettings
+import software.amazon.smithy.kotlin.codegen.core.KotlinWriter
+import software.amazon.smithy.kotlin.codegen.core.RuntimeTypes
+import software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration
+import software.amazon.smithy.kotlin.codegen.model.hasTrait
+import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolGenerator
+import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolMiddleware
+import software.amazon.smithy.model.Model
+import software.amazon.smithy.model.shapes.OperationShape
+import software.amazon.smithy.model.traits.HttpChecksumRequiredTrait
+
+/**
+ * Handles the `httpChecksumRequired` trait.
+ * See: https://smithy.io/2.0/spec/http-bindings.html#httpchecksumrequired-trait
+ */
+class HttpChecksumRequiredIntegration : KotlinIntegration {
+    override fun enabledForService(model: Model, settings: KotlinSettings): Boolean =
+        model.isTraitApplied(HttpChecksumRequiredTrait::class.java)
+
+    override fun customizeMiddleware(
+        ctx: ProtocolGenerator.GenerationContext,
+        resolved: List<ProtocolMiddleware>,
+    ): List<ProtocolMiddleware> = resolved + httpChecksumRequiredDefaultAlgorithmMiddleware + httpChecksumRequiredMiddleware
+}
+
+/**
+ * Adds default checksum algorithm to the execution context
+ */
+private val httpChecksumRequiredDefaultAlgorithmMiddleware = object : ProtocolMiddleware {
+    override val name: String = "httpChecksumRequiredDefaultAlgorithmMiddleware"
+    override val order: Byte = -2 // Before S3 Express (possibly) changes the default (-1) and before calculating checksum (0)
+
+    override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean =
+        op.hasTrait<HttpChecksumRequiredTrait>() && !op.hasTrait<HttpChecksumTrait>()
+
+    override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) {
+        writer.write(
+            "op.context[#T.DefaultChecksumAlgorithm] = #S",
+            RuntimeTypes.HttpClient.Operation.HttpOperationContext,
+            "MD5",
+        )
+    }
+}
+
+/**
+ * Adds interceptor to calculate request checksums.
+ * The `httpChecksum` trait supersedes the `httpChecksumRequired` trait. If both are applied to an operation use `httpChecksum`.
+ *
+ * See: https://smithy.io/2.0/aws/aws-core.html#behavior-with-httpchecksumrequired
+ */
+private val httpChecksumRequiredMiddleware = object : ProtocolMiddleware {
+    override val name: String = "httpChecksumRequiredMiddleware"
+
+    override fun isEnabledFor(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Boolean =
+        op.hasTrait<HttpChecksumRequiredTrait>() && !op.hasTrait<HttpChecksumTrait>()
+
+    override fun render(ctx: ProtocolGenerator.GenerationContext, op: OperationShape, writer: KotlinWriter) {
+        writer.write(
+            "op.interceptors.add(#T())",
+            RuntimeTypes.HttpClient.Interceptors.HttpChecksumRequiredInterceptor,
+        )
+    }
+}

codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpProtocolClientGenerator.kt

@@ -4,7 +4,6 @@
  */
 package software.amazon.smithy.kotlin.codegen.rendering.protocol
 
-import software.amazon.smithy.aws.traits.HttpChecksumTrait
 import software.amazon.smithy.codegen.core.Symbol
 import software.amazon.smithy.kotlin.codegen.core.*
 import software.amazon.smithy.kotlin.codegen.integration.SectionId
@@ -22,7 +21,6 @@ import software.amazon.smithy.model.knowledge.OperationIndex
 import software.amazon.smithy.model.knowledge.TopDownIndex
 import software.amazon.smithy.model.shapes.OperationShape
 import software.amazon.smithy.model.traits.EndpointTrait
-import software.amazon.smithy.model.traits.HttpChecksumRequiredTrait
 
 /**
  * Renders an implementation of a service interface for HTTP protocol
@@ -318,8 +316,6 @@ open class HttpProtocolClientGenerator(
             .forEach { middleware ->
                 middleware.render(ctx, op, writer)
             }
-
-        op.renderIsMd5ChecksumRequired(writer)
     }
 
     /**
@@ -336,27 +332,6 @@ open class HttpProtocolClientGenerator(
      */
     protected open fun renderAdditionalMethods(writer: KotlinWriter) { }
 
-    /**
-     * Render optionally installing Md5ChecksumMiddleware.
-     * The Md5 middleware will only be installed if the operation requires a checksum and the user has not opted-in to flexible checksums.
-     */
-    private fun OperationShape.renderIsMd5ChecksumRequired(writer: KotlinWriter) {
-        val httpChecksumTrait = getTrait<HttpChecksumTrait>()
-
-        // the checksum requirement can be modeled in either HttpChecksumTrait's `requestChecksumRequired` or the HttpChecksumRequired trait
-        if (!hasTrait<HttpChecksumRequiredTrait>() && httpChecksumTrait == null) {
-            return
-        }
-
-        if (hasTrait<HttpChecksumRequiredTrait>() || httpChecksumTrait?.isRequestChecksumRequired == true) {
-            val interceptorSymbol = RuntimeTypes.HttpClient.Interceptors.Md5ChecksumInterceptor
-            val inputSymbol = ctx.symbolProvider.toSymbol(ctx.model.expectShape(inputShape))
-            writer.withBlock("op.interceptors.add(#T<#T> {", "})", interceptorSymbol, inputSymbol) {
-                writer.write("op.context.getOrNull(#T.ChecksumAlgorithm) == null", RuntimeTypes.HttpClient.Operation.HttpOperationContext)
-            }
-        }
-    }
-
     /**
      * render a utility function to populate an operation's ExecutionContext with defaults from service config, environment, etc
      */

codegen/smithy-kotlin-codegen/src/main/resources/META-INF/services/software.amazon.smithy.kotlin.codegen.integration.KotlinIntegration

@@ -13,3 +13,4 @@ software.amazon.smithy.kotlin.codegen.rendering.endpoints.SdkEndpointBuiltinInte
 software.amazon.smithy.kotlin.codegen.rendering.compression.RequestCompressionIntegration
 software.amazon.smithy.kotlin.codegen.rendering.auth.SigV4AsymmetricAuthSchemeIntegration
 software.amazon.smithy.kotlin.codegen.rendering.smoketests.SmokeTestsIntegration
+software.amazon.smithy.kotlin.codegen.rendering.checksums.HttpChecksumRequiredIntegration

runtime/protocol/http-client/api/http-client.api

@@ -255,7 +255,7 @@ public final class aws/smithy/kotlin/runtime/http/engine/internal/ManagedHttpCli
 	public static final fun manage (Laws/smithy/kotlin/runtime/http/engine/HttpClientEngine;)Laws/smithy/kotlin/runtime/http/engine/HttpClientEngine;
 }
 
-public abstract class aws/smithy/kotlin/runtime/http/interceptors/AbstractChecksumInterceptor : aws/smithy/kotlin/runtime/client/Interceptor {
+public abstract class aws/smithy/kotlin/runtime/http/interceptors/CachingChecksumInterceptor : aws/smithy/kotlin/runtime/client/Interceptor {
 	public fun <init> ()V
 	public abstract fun applyChecksum (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;Ljava/lang/String;)Laws/smithy/kotlin/runtime/http/request/HttpRequest;
 	public abstract fun calculateChecksum (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
@@ -331,19 +331,17 @@ public final class aws/smithy/kotlin/runtime/http/interceptors/DiscoveredEndpoin
 	public fun readBeforeTransmit (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V
 }
 
-public final class aws/smithy/kotlin/runtime/http/interceptors/FlexibleChecksumsRequestInterceptor : aws/smithy/kotlin/runtime/http/interceptors/AbstractChecksumInterceptor {
-	public fun <init> ()V
-	public fun <init> (Lkotlin/jvm/functions/Function1;)V
-	public synthetic fun <init> (Lkotlin/jvm/functions/Function1;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
+public final class aws/smithy/kotlin/runtime/http/interceptors/FlexibleChecksumsRequestInterceptor : aws/smithy/kotlin/runtime/http/interceptors/CachingChecksumInterceptor {
+	public fun <init> (ZLaws/smithy/kotlin/runtime/client/config/RequestHttpChecksumConfig;Ljava/lang/String;)V
 	public fun applyChecksum (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;Ljava/lang/String;)Laws/smithy/kotlin/runtime/http/request/HttpRequest;
 	public fun calculateChecksum (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
 	public fun modifyBeforeSigning (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
-	public fun readAfterSerialization (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;)V
 }
 
-public final class aws/smithy/kotlin/runtime/http/interceptors/FlexibleChecksumsResponseInterceptor : aws/smithy/kotlin/runtime/client/Interceptor {
+public class aws/smithy/kotlin/runtime/http/interceptors/FlexibleChecksumsResponseInterceptor : aws/smithy/kotlin/runtime/client/Interceptor {
 	public static final field Companion Laws/smithy/kotlin/runtime/http/interceptors/FlexibleChecksumsResponseInterceptor$Companion;
-	public fun <init> (Lkotlin/jvm/functions/Function1;)V
+	public fun <init> (ZLaws/smithy/kotlin/runtime/client/config/ResponseHttpChecksumConfig;)V
+	public fun ignoreChecksum (Ljava/lang/String;Laws/smithy/kotlin/runtime/client/ProtocolResponseInterceptorContext;)Z
 	public fun modifyBeforeAttemptCompletion-gIAlu-s (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
 	public fun modifyBeforeCompletion-gIAlu-s (Laws/smithy/kotlin/runtime/client/ResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
 	public fun modifyBeforeDeserialization (Laws/smithy/kotlin/runtime/client/ProtocolResponseInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
@@ -369,10 +367,8 @@ public final class aws/smithy/kotlin/runtime/http/interceptors/FlexibleChecksums
 	public final fun getChecksumHeaderValidated ()Laws/smithy/kotlin/runtime/collections/AttributeKey;
 }
 
-public final class aws/smithy/kotlin/runtime/http/interceptors/Md5ChecksumInterceptor : aws/smithy/kotlin/runtime/http/interceptors/AbstractChecksumInterceptor {
+public final class aws/smithy/kotlin/runtime/http/interceptors/HttpChecksumRequiredInterceptor : aws/smithy/kotlin/runtime/http/interceptors/CachingChecksumInterceptor {
 	public fun <init> ()V
-	public fun <init> (Lkotlin/jvm/functions/Function1;)V
-	public synthetic fun <init> (Lkotlin/jvm/functions/Function1;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
 	public fun applyChecksum (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;Ljava/lang/String;)Laws/smithy/kotlin/runtime/http/request/HttpRequest;
 	public fun calculateChecksum (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
 	public fun modifyBeforeSigning (Laws/smithy/kotlin/runtime/client/ProtocolRequestInterceptorContext;Lkotlin/coroutines/Continuation;)Ljava/lang/Object;
@@ -516,9 +512,9 @@ public abstract interface class aws/smithy/kotlin/runtime/http/operation/HttpDes
 
 public final class aws/smithy/kotlin/runtime/http/operation/HttpOperationContext {
 	public static final field INSTANCE Laws/smithy/kotlin/runtime/http/operation/HttpOperationContext;
-	public final fun getChecksumAlgorithm ()Laws/smithy/kotlin/runtime/collections/AttributeKey;
 	public final fun getClockSkew ()Laws/smithy/kotlin/runtime/collections/AttributeKey;
 	public final fun getClockSkewApproximateSigningTime ()Laws/smithy/kotlin/runtime/collections/AttributeKey;
+	public final fun getDefaultChecksumAlgorithm ()Laws/smithy/kotlin/runtime/collections/AttributeKey;
 	public final fun getHostPrefix ()Laws/smithy/kotlin/runtime/collections/AttributeKey;
 	public final fun getHttpCallList ()Laws/smithy/kotlin/runtime/collections/AttributeKey;
 	public final fun getOperationAttributes ()Laws/smithy/kotlin/runtime/collections/AttributeKey;

runtime/protocol/http-client/common/src/aws/smithy/kotlin/runtime/http/interceptors/AbstractChecksumInterceptor.kt renamed to runtime/protocol/http-client/common/src/aws/smithy/kotlin/runtime/http/interceptors/CachingChecksumInterceptor.kt

@@ -9,13 +9,21 @@ import aws.smithy.kotlin.runtime.InternalApi
 import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext
 import aws.smithy.kotlin.runtime.http.request.HttpRequest
 
+/**
+ * Enables inheriting [HttpInterceptor]s to use checksums caching
+ */
 @InternalApi
-public abstract class AbstractChecksumInterceptor : HttpInterceptor {
+public abstract class CachingChecksumInterceptor : HttpInterceptor {
     private var cachedChecksum: String? = null
 
     override suspend fun modifyBeforeSigning(context: ProtocolRequestInterceptorContext<Any, HttpRequest>): HttpRequest {
-        cachedChecksum ?: calculateChecksum(context).also { cachedChecksum = it }
-        return cachedChecksum?.let { applyChecksum(context, it) } ?: context.protocolRequest
+        cachedChecksum = cachedChecksum ?: calculateChecksum(context)
+
+        return if (cachedChecksum != null) {
+            applyChecksum(context, cachedChecksum!!)
+        } else {
+            context.protocolRequest
+        }
     }
 
     public abstract suspend fun calculateChecksum(context: ProtocolRequestInterceptorContext<Any, HttpRequest>): String?

runtime/protocol/http-client/common/src/aws/smithy/kotlin/runtime/http/interceptors/ChecksumInterceptorUtils.kt

@@ -0,0 +1,44 @@
+package aws.smithy.kotlin.runtime.http.interceptors
+
+import aws.smithy.kotlin.runtime.businessmetrics.emitBusinessMetric
+import aws.smithy.kotlin.runtime.client.ProtocolRequestInterceptorContext
+import aws.smithy.kotlin.runtime.hashing.HashFunction
+import aws.smithy.kotlin.runtime.hashing.resolveChecksumAlgorithmHeaderName
+import aws.smithy.kotlin.runtime.hashing.toBusinessMetric
+import aws.smithy.kotlin.runtime.http.operation.HttpOperationContext
+import aws.smithy.kotlin.runtime.http.request.HttpRequest
+import aws.smithy.kotlin.runtime.http.request.toBuilder
+import aws.smithy.kotlin.runtime.http.toCompletingBody
+import aws.smithy.kotlin.runtime.http.toHashingBody
+import kotlinx.coroutines.CompletableDeferred
+import kotlinx.coroutines.job
+
+/**
+ * Configures [HttpRequest] with AWS chunked streaming to calculate checksum during transmission
+ * @return [HttpRequest]
+ */
+internal fun calculateAwsChunkedStreamingChecksum(
+    context: ProtocolRequestInterceptorContext<Any, HttpRequest>,
+    checksumAlgorithm: HashFunction,
+): HttpRequest {
+    val request = context.protocolRequest.toBuilder()
+    val deferredChecksum = CompletableDeferred<String>(context.executionContext.coroutineContext.job)
+    val checksumHeader = checksumAlgorithm.resolveChecksumAlgorithmHeaderName()
+
+    request.body = request.body
+        .toHashingBody(checksumAlgorithm, request.body.contentLength)
+        .toCompletingBody(deferredChecksum)
+
+    request.headers.append("x-amz-trailer", checksumHeader)
+    request.trailingHeaders.append(checksumHeader, deferredChecksum)
+
+    context.executionContext.emitBusinessMetric(checksumAlgorithm.toBusinessMetric())
+
+    return request.build()
+}
+
+/**
+ * @return The default checksum algorithm name in the execution context, null if default checksums are disabled.
+ */
+internal val ProtocolRequestInterceptorContext<Any, HttpRequest>.defaultChecksumAlgorithmName: String?
+    get() = executionContext.getOrNull(HttpOperationContext.DefaultChecksumAlgorithm)