Skip to content

Commit 354c6cf

Browse files
lauzadislauzadis
authored
feat: implement flexible checksums customization (#772)
1 parent 3326e72 commit 354c6cf

File tree

37 files changed

+1478
-265
lines changed

37 files changed

+1478
-265
lines changed

‎.changes/73375c7c-b802-4878-ae24-15b619c065b3.json‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
{
2+
"id": "73375c7c-b802-4878-ae24-15b619c065b3",
3+
"type": "feature",
4+
"description": "Implement flexible checksums customization",
5+
"issues": [
6+
"https://github.com/awslabs/smithy-kotlin/issues/446"
7+
]
8+
}

‎.changes/af027b16-c6f7-4885-9835-1a75315860cf.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"id": "af027b16-c6f7-4885-9835-1a75315860cf",
3+
"type": "feature",
4+
"description": "Add support for unsigned `aws-chunked` requests"
5+
}

‎runtime/auth/aws-signing-common/common/src/aws/smithy/kotlin/runtime/auth/awssigning/AwsChunkedByteReadChannel.kt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
package aws.smithy.kotlin.runtime.auth.awssigning
77

88
import aws.smithy.kotlin.runtime.auth.awssigning.internal.AwsChunkedReader
9-
import aws.smithy.kotlin.runtime.http.Headers
9+
import aws.smithy.kotlin.runtime.http.DeferredHeaders
1010
import aws.smithy.kotlin.runtime.io.SdkBuffer
1111
import aws.smithy.kotlin.runtime.io.SdkByteReadChannel
1212
import aws.smithy.kotlin.runtime.util.InternalApi
@@ -28,7 +28,7 @@ public class AwsChunkedByteReadChannel(
2828
private val signer: AwsSigner,
2929
private val signingConfig: AwsSigningConfig,
3030
private var previousSignature: ByteArray,
31-
private val trailingHeaders: Headers = Headers.Empty,
31+
private val trailingHeaders: DeferredHeaders = DeferredHeaders.Empty,
3232
) : SdkByteReadChannel by delegate {
3333

3434
private val chunkReader = AwsChunkedReader(

‎runtime/auth/aws-signing-common/common/src/aws/smithy/kotlin/runtime/auth/awssigning/AwsHttpSigner.kt‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -133,15 +133,15 @@ public class AwsHttpSigner(private val config: Config) : HttpSigner {
133133

134134
hashSpecification = when {
135135
contextHashSpecification != null -> contextHashSpecification
136-
config.isUnsignedPayload -> HashSpecification.UnsignedPayload
137136
body is HttpBody.Empty -> HashSpecification.EmptyBody
138137
body.isEligibleForAwsChunkedStreaming -> {
139138
if (request.headers.contains("x-amz-trailer")) {
140-
HashSpecification.StreamingAws4HmacSha256PayloadWithTrailers
139+
if (config.isUnsignedPayload) HashSpecification.StreamingUnsignedPayloadWithTrailers else HashSpecification.StreamingAws4HmacSha256PayloadWithTrailers
141140
} else {
142141
HashSpecification.StreamingAws4HmacSha256Payload
143142
}
144143
}
144+
config.isUnsignedPayload -> HashSpecification.UnsignedPayload
145145
// use the payload to compute the hash
146146
else -> HashSpecification.CalculateFromPayload
147147
}
@@ -160,7 +160,12 @@ public class AwsHttpSigner(private val config: Config) : HttpSigner {
160160
request.update(signedRequest)
161161

162162
if (signingConfig.useAwsChunkedEncoding) {
163-
request.setAwsChunkedBody(checkNotNull(config.signer), signingConfig, signingResult.signature)
163+
request.setAwsChunkedBody(
164+
checkNotNull(config.signer),
165+
signingConfig,
166+
signingResult.signature,
167+
request.trailingHeaders.build(),
168+
)
164169
}
165170
}
166171
}

‎runtime/auth/aws-signing-common/common/src/aws/smithy/kotlin/runtime/auth/awssigning/HashSpecification.kt‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,12 @@ public sealed class HashSpecification {
4040
public object StreamingAws4HmacSha256PayloadWithTrailers : HashLiteral("STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER")
4141

4242
/**
43-
* The hash value should indicate ???
43+
* The hash value used for streaming unsigned requests with trailers
44+
*/
45+
public object StreamingUnsignedPayloadWithTrailers : HashLiteral("STREAMING-UNSIGNED-PAYLOAD-TRAILER")
46+
47+
/**
48+
* The hash value indicates that the streaming request is an event stream
4449
*/
4550
public object StreamingAws4HmacSha256Events : HashLiteral("STREAMING-AWS4-HMAC-SHA256-EVENTS")
4651

‎runtime/auth/aws-signing-common/common/src/aws/smithy/kotlin/runtime/auth/awssigning/internal/AwsChunkedReader.kt‎

Lines changed: 61 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,9 @@ import aws.smithy.kotlin.runtime.auth.awssigning.AwsSignatureType
99
import aws.smithy.kotlin.runtime.auth.awssigning.AwsSigner
1010
import aws.smithy.kotlin.runtime.auth.awssigning.AwsSigningConfig
1111
import aws.smithy.kotlin.runtime.auth.awssigning.HashSpecification
12+
import aws.smithy.kotlin.runtime.http.DeferredHeaders
1213
import aws.smithy.kotlin.runtime.http.Headers
14+
import aws.smithy.kotlin.runtime.http.toHeaders
1315
import aws.smithy.kotlin.runtime.io.SdkBuffer
1416

1517
/**
@@ -27,7 +29,7 @@ internal class AwsChunkedReader(
2729
private val signer: AwsSigner,
2830
private val signingConfig: AwsSigningConfig,
2931
private var previousSignature: ByteArray,
30-
private val trailingHeaders: Headers = Headers.Empty,
32+
private val trailingHeaders: DeferredHeaders,
3133
) {
3234

3335
/**
@@ -69,7 +71,7 @@ internal class AwsChunkedReader(
6971
val nextChunk = when {
7072
stream.isClosedForRead() && hasLastChunkBeenSent -> null
7173
else -> {
72-
var next = getSignedChunk()
74+
var next = if (signingConfig.isUnsigned) getUnsignedChunk() else getSignedChunk()
7375
if (next == null) {
7476
check(stream.isClosedForRead()) { "Expected underlying reader to be closed" }
7577
next = getFinalChunk()
@@ -93,18 +95,39 @@ internal class AwsChunkedReader(
9395
*/
9496
private suspend fun getFinalChunk(): SdkBuffer {
9597
// empty chunk
96-
val lastChunk = checkNotNull(getSignedChunk(SdkBuffer()))
98+
val lastChunk = checkNotNull(if (signingConfig.isUnsigned) getUnsignedChunk(SdkBuffer()) else getSignedChunk(SdkBuffer()))
9799

98100
// + any trailers
99101
if (!trailingHeaders.isEmpty()) {
100-
val trailingHeaderChunk = getTrailingHeadersChunk(trailingHeaders)
102+
val trailingHeaderChunk = getTrailingHeadersChunk(trailingHeaders.toHeaders())
101103
lastChunk.writeAll(trailingHeaderChunk)
102104
}
103105
return lastChunk
104106
}
105107

106108
/**
107-
* Get an aws-chunked encoding of [data].
109+
* Read a chunk from the underlying [stream], suspending until a whole chunk has been read OR the channel is exhausted.
110+
* @return an SdkBuffer containing a chunk of data, or null if the channel is exhausted.
111+
*/
112+
private suspend fun Stream.readChunk(): SdkBuffer? {
113+
val sink = SdkBuffer()
114+
115+
// fill up to chunk size bytes
116+
var remaining = CHUNK_SIZE_BYTES.toLong()
117+
while (remaining > 0L) {
118+
val rc = read(sink, remaining)
119+
if (rc == -1L) break
120+
remaining -= rc
121+
}
122+
123+
return when (sink.size) {
124+
0L -> null // delegate closed without reading any data
125+
else -> sink
126+
}
127+
}
128+
129+
/**
130+
* Get a signed aws-chunked encoding of [data].
108131
* If [data] is not set, read the next chunk from [delegate] and add hex-formatted chunk size and chunk signature to the front.
109132
* Note that this function will suspend until the whole chunk has been read OR the channel is exhausted.
110133
* The chunk structure is: `string(IntHexBase(chunk-size)) + ";chunk-signature=" + signature + \r\n + chunk-data + \r\n`
@@ -114,23 +137,7 @@ internal class AwsChunkedReader(
114137
* @return a buffer containing the chunked data or null if no data is available (channel is closed)
115138
*/
116139
private suspend fun getSignedChunk(data: SdkBuffer? = null): SdkBuffer? {
117-
val bodyBuffer = if (data == null) {
118-
val sink = SdkBuffer()
119-
120-
// fill up to chunk size bytes
121-
var remaining = CHUNK_SIZE_BYTES.toLong()
122-
while (remaining > 0L) {
123-
val rc = stream.read(sink, remaining)
124-
if (rc == -1L) break
125-
remaining -= rc
126-
}
127-
when (sink.size) {
128-
0L -> null // delegate closed without reading any data
129-
else -> sink
130-
}
131-
} else {
132-
data
133-
}
140+
val bodyBuffer = data ?: stream.readChunk()
134141

135142
// signer takes a ByteArray unfortunately...
136143
val chunkBody = bodyBuffer?.readByteArray() ?: return null
@@ -155,6 +162,31 @@ internal class AwsChunkedReader(
155162
return signedChunk
156163
}
157164

165+
/**
166+
* Get an unsigned aws-chunked encoding of [data].
167+
* If [data] is not set, read the next chunk from [delegate] and add hex-formatted chunk size to the front.
168+
* Note that this function will suspend until the whole chunk has been read OR the channel is exhausted.
169+
* The unsigned chunk structure is: `string(IntHexBase(chunk-size)) + \r\n + chunk-data + \r\n`
170+
*
171+
* @param data the data which will be encoded to aws-chunked. if not provided, will default to
172+
* reading up to [CHUNK_SIZE_BYTES] from [delegate].
173+
* @return a buffer containing the chunked data or null if no data is available (channel is closed)
174+
*/
175+
private suspend fun getUnsignedChunk(data: SdkBuffer? = null): SdkBuffer? {
176+
val bodyBuffer = data ?: stream.readChunk() ?: return null
177+
178+
val unsignedChunk = SdkBuffer()
179+
180+
// headers
181+
unsignedChunk.apply {
182+
writeUtf8(bodyBuffer.size.toString(16))
183+
writeUtf8("\r\n")
184+
writeAll(bodyBuffer) // append the body
185+
}
186+
187+
return unsignedChunk
188+
}
189+
158190
/**
159191
* Get the trailing headers chunk. The grammar for trailing headers is:
160192
* trailing-header-A:value CRLF
@@ -170,7 +202,11 @@ internal class AwsChunkedReader(
170202
previousSignature = trailerSignature
171203

172204
val trailerBody = SdkBuffer()
173-
trailerBody.writeTrailers(trailingHeaders, trailerSignature.decodeToString())
205+
trailerBody.writeTrailers(trailingHeaders)
206+
if (!signingConfig.isUnsigned) {
207+
trailerBody.writeTrailerSignature(trailerSignature.decodeToString())
208+
}
209+
174210
return trailerBody
175211
}
176212

@@ -193,4 +229,6 @@ internal class AwsChunkedReader(
193229
signatureType = AwsSignatureType.HTTP_REQUEST_TRAILING_HEADERS // signature is for trailing headers
194230
hashSpecification = HashSpecification.CalculateFromPayload // calculate the hash from the trailing headers payload
195231
}.build()
232+
233+
private val AwsSigningConfig.isUnsigned: Boolean get() = hashSpecification == HashSpecification.StreamingUnsignedPayloadWithTrailers
196234
}

‎runtime/auth/aws-signing-common/common/src/aws/smithy/kotlin/runtime/auth/awssigning/internal/AwsChunkedUtil.kt‎

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import aws.smithy.kotlin.runtime.auth.awssigning.AwsHttpSigner
99
import aws.smithy.kotlin.runtime.auth.awssigning.AwsSigner
1010
import aws.smithy.kotlin.runtime.auth.awssigning.AwsSigningConfig
1111
import aws.smithy.kotlin.runtime.auth.awssigning.HashSpecification
12+
import aws.smithy.kotlin.runtime.http.DeferredHeaders
1213
import aws.smithy.kotlin.runtime.http.Headers
1314
import aws.smithy.kotlin.runtime.http.HttpBody
1415
import aws.smithy.kotlin.runtime.http.request.HttpRequestBuilder
@@ -19,10 +20,7 @@ import aws.smithy.kotlin.runtime.io.SdkBuffer
1920
*/
2021
public const val CHUNK_SIZE_BYTES: Int = 65_536
2122

22-
internal fun SdkBuffer.writeTrailers(
23-
trailers: Headers,
24-
signature: String,
25-
) {
23+
internal fun SdkBuffer.writeTrailers(trailers: Headers) {
2624
trailers
2725
.entries()
2826
.sortedBy { entry -> entry.key.lowercase() }
@@ -32,6 +30,9 @@ internal fun SdkBuffer.writeTrailers(
3230
writeUtf8(entry.value.joinToString(",") { v -> v.trim() })
3331
writeUtf8("\r\n")
3432
}
33+
}
34+
35+
internal fun SdkBuffer.writeTrailerSignature(signature: String) {
3536
writeUtf8("x-amz-trailer-signature:${signature}\r\n")
3637
}
3738

@@ -47,20 +48,28 @@ internal val HttpBody.isEligibleForAwsChunkedStreaming: Boolean
4748
*/
4849
internal val AwsSigningConfig.useAwsChunkedEncoding: Boolean
4950
get() = when (hashSpecification) {
50-
is HashSpecification.StreamingAws4HmacSha256Payload, is HashSpecification.StreamingAws4HmacSha256PayloadWithTrailers -> true
51+
is HashSpecification.StreamingAws4HmacSha256Payload,
52+
is HashSpecification.StreamingAws4HmacSha256PayloadWithTrailers,
53+
is HashSpecification.StreamingUnsignedPayloadWithTrailers,
54+
-> true
5155
else -> false
5256
}
5357

5458
/**
5559
* Set the HTTP headers required for the aws-chunked content encoding
5660
*/
5761
internal fun HttpRequestBuilder.setAwsChunkedHeaders() {
58-
headers.setMissing("Content-Encoding", "aws-chunked")
59-
headers.setMissing("Transfer-Encoding", "chunked")
60-
headers.setMissing("X-Amz-Decoded-Content-Length", body.contentLength!!.toString())
62+
headers.append("Content-Encoding", "aws-chunked")
63+
headers["Transfer-Encoding"] = "chunked"
64+
headers["X-Amz-Decoded-Content-Length"] = body.contentLength!!.toString()
6165
}
6266

6367
/**
6468
* Update the HTTP body to use aws-chunked content encoding
6569
*/
66-
internal expect fun HttpRequestBuilder.setAwsChunkedBody(signer: AwsSigner, signingConfig: AwsSigningConfig, signature: ByteArray)
70+
internal expect fun HttpRequestBuilder.setAwsChunkedBody(
71+
signer: AwsSigner,
72+
signingConfig: AwsSigningConfig,
73+
signature: ByteArray,
74+
trailingHeaders: DeferredHeaders,
75+
)

‎runtime/auth/aws-signing-common/jvm/src/aws/smithy/kotlin/runtime/auth/awssigning/AwsChunkedSource.kt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
package aws.smithy.kotlin.runtime.auth.awssigning
77

88
import aws.smithy.kotlin.runtime.auth.awssigning.internal.AwsChunkedReader
9-
import aws.smithy.kotlin.runtime.http.Headers
9+
import aws.smithy.kotlin.runtime.http.DeferredHeaders
1010
import aws.smithy.kotlin.runtime.io.SdkBuffer
1111
import aws.smithy.kotlin.runtime.io.SdkSource
1212
import aws.smithy.kotlin.runtime.io.buffer
@@ -33,7 +33,7 @@ public class AwsChunkedSource(
3333
signer: AwsSigner,
3434
signingConfig: AwsSigningConfig,
3535
previousSignature: ByteArray,
36-
trailingHeaders: Headers = Headers.Empty,
36+
trailingHeaders: DeferredHeaders = DeferredHeaders.Empty,
3737
) : SdkSource {
3838
private val chunkReader = AwsChunkedReader(
3939
delegate.asStream(),

‎runtime/auth/aws-signing-common/jvm/src/aws/smithy/kotlin/runtime/auth/awssigning/internal/AwsChunkedUtilJVM.kt‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,15 +9,27 @@ import aws.smithy.kotlin.runtime.auth.awssigning.AwsChunkedByteReadChannel
99
import aws.smithy.kotlin.runtime.auth.awssigning.AwsChunkedSource
1010
import aws.smithy.kotlin.runtime.auth.awssigning.AwsSigner
1111
import aws.smithy.kotlin.runtime.auth.awssigning.AwsSigningConfig
12-
import aws.smithy.kotlin.runtime.http.HttpBody
12+
import aws.smithy.kotlin.runtime.http.*
1313
import aws.smithy.kotlin.runtime.http.request.HttpRequestBuilder
14-
import aws.smithy.kotlin.runtime.http.toHttpBody
15-
import aws.smithy.kotlin.runtime.http.toSdkByteReadChannel
1614

17-
internal actual fun HttpRequestBuilder.setAwsChunkedBody(signer: AwsSigner, signingConfig: AwsSigningConfig, signature: ByteArray) {
15+
internal actual fun HttpRequestBuilder.setAwsChunkedBody(signer: AwsSigner, signingConfig: AwsSigningConfig, signature: ByteArray, trailingHeaders: DeferredHeaders) {
1816
body = when (body) {
19-
is HttpBody.ChannelContent -> AwsChunkedByteReadChannel(checkNotNull(body.toSdkByteReadChannel()), signer, signingConfig, signature).toHttpBody(-1)
20-
is HttpBody.SourceContent -> AwsChunkedSource((body as HttpBody.SourceContent).readFrom(), signer, signingConfig, signature).toHttpBody(-1)
17+
is HttpBody.ChannelContent -> AwsChunkedByteReadChannel(
18+
checkNotNull(body.toSdkByteReadChannel()),
19+
signer,
20+
signingConfig,
21+
signature,
22+
trailingHeaders,
23+
).toHttpBody(-1)
24+
25+
is HttpBody.SourceContent -> AwsChunkedSource(
26+
(body as HttpBody.SourceContent).readFrom(),
27+
signer,
28+
signingConfig,
29+
signature,
30+
trailingHeaders,
31+
).toHttpBody(-1)
32+
2133
else -> throw ClientException("HttpBody type is not supported")
2234
}
2335
}

0 commit comments

Comments
 (0)