Skip to content

Commit 4050ed3

Browse files
dagnirdagnir
committed
Support trailing checksum in DefaultAwsV4HttpSigner
This commit moves the support for trailing checksums from HttpChecksumStage to the V4 signer signer implementation; this puts it in line with how sync chunked bodies are already handled.
1 parent d8ed0bb commit 4050ed3

File tree

9 files changed

+420
-44
lines changed

9 files changed

+420
-44
lines changed

‎core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/AwsChunkedV4PayloadSigner.java

Lines changed: 104 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@
3131
import java.util.ArrayList;
3232
import java.util.Collections;
3333
import java.util.List;
34+
import java.util.concurrent.CompletableFuture;
35+
import java.util.stream.Collectors;
3436
import org.reactivestreams.Publisher;
3537
import software.amazon.awssdk.annotations.SdkInternalApi;
3638
import software.amazon.awssdk.checksums.SdkChecksum;
@@ -40,11 +42,13 @@
4042
import software.amazon.awssdk.http.SdkHttpRequest;
4143
import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.ChecksumTrailerProvider;
4244
import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.ChunkedEncodedInputStream;
45+
import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.ChunkedEncodedPublisher;
4346
import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.SigV4ChunkExtensionProvider;
4447
import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.SigV4TrailerProvider;
4548
import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.TrailerProvider;
4649
import software.amazon.awssdk.http.auth.aws.internal.signer.io.ChecksumInputStream;
4750
import software.amazon.awssdk.http.auth.aws.internal.signer.io.ResettableContentStreamProvider;
51+
import software.amazon.awssdk.http.auth.aws.internal.signer.io.UnbufferedChecksumSubscriber;
4852
import software.amazon.awssdk.utils.BinaryUtils;
4953
import software.amazon.awssdk.utils.Pair;
5054
import software.amazon.awssdk.utils.Validate;
@@ -116,8 +120,44 @@ public ContentStreamProvider sign(ContentStreamProvider payload, V4RequestSignin
116120

117121
@Override
118122
public Publisher<ByteBuffer> signAsync(Publisher<ByteBuffer> payload, V4RequestSigningResult requestSigningResult) {
119-
// TODO(sra-identity-and-auth): implement this first and remove addFlexibleChecksumInTrailer logic in HttpChecksumStage
120-
throw new UnsupportedOperationException();
123+
ChunkedEncodedPublisher.Builder chunkedStreamBuilder = ChunkedEncodedPublisher.builder()
124+
.publisher(payload)
125+
.chunkSize(chunkSize)
126+
.addEmptyTrailingChunk(true);
127+
128+
preExistingTrailers.forEach(t -> chunkedStreamBuilder.addTrailer(() -> t));
129+
130+
SdkHttpRequest.Builder request = requestSigningResult.getSignedRequest();
131+
132+
String checksum = request.firstMatchingHeader(X_AMZ_CONTENT_SHA256).orElseThrow(
133+
() -> new IllegalArgumentException(X_AMZ_CONTENT_SHA256 + " must be set!")
134+
);
135+
136+
switch (checksum) {
137+
case STREAMING_SIGNED_PAYLOAD: {
138+
RollingSigner rollingSigner = new RollingSigner(requestSigningResult.getSigningKey(),
139+
requestSigningResult.getSignature());
140+
chunkedStreamBuilder.addExtension(new SigV4ChunkExtensionProvider(rollingSigner, credentialScope));
141+
break;
142+
}
143+
case STREAMING_UNSIGNED_PAYLOAD_TRAILER:
144+
setupChecksumTrailerIfNeeded(chunkedStreamBuilder);
145+
break;
146+
case STREAMING_SIGNED_PAYLOAD_TRAILER: {
147+
setupChecksumTrailerIfNeeded(chunkedStreamBuilder);
148+
RollingSigner rollingSigner = new RollingSigner(requestSigningResult.getSigningKey(),
149+
requestSigningResult.getSignature());
150+
chunkedStreamBuilder.addExtension(new SigV4ChunkExtensionProvider(rollingSigner, credentialScope));
151+
chunkedStreamBuilder.addTrailer(
152+
new SigV4TrailerProvider(chunkedStreamBuilder.trailers(), rollingSigner, credentialScope)
153+
);
154+
break;
155+
}
156+
default:
157+
throw new UnsupportedOperationException();
158+
}
159+
160+
return chunkedStreamBuilder.build();
121161
}
122162

123163
@Override
@@ -127,27 +167,66 @@ public void beforeSigning(SdkHttpRequest.Builder request, ContentStreamProvider
127167
setupPreExistingTrailers(request);
128168

129169
// pre-existing trailers
170+
encodedContentLength = calculateEncodedContentLength(request, contentLength);
171+
172+
if (checksumAlgorithm != null) {
173+
String checksumHeaderName = checksumHeaderName(checksumAlgorithm);
174+
request.appendHeader(X_AMZ_TRAILER, checksumHeaderName);
175+
}
176+
request.putHeader(Header.CONTENT_LENGTH, Long.toString(encodedContentLength));
177+
request.appendHeader(CONTENT_ENCODING, AWS_CHUNKED);
178+
}
179+
180+
@Override
181+
public CompletableFuture<Pair<SdkHttpRequest.Builder, Publisher<ByteBuffer>>> beforeSigningAsync(
182+
SdkHttpRequest.Builder request, Publisher<ByteBuffer> payload) {
183+
return moveContentLength(request, payload)
184+
.thenApply(p -> {
185+
SdkHttpRequest.Builder requestBuilder = p.left();
186+
setupPreExistingTrailers(requestBuilder);
187+
188+
long decodedContentLength = requestBuilder.firstMatchingHeader("x-amz-decoded-content-length")
189+
.map(Long::parseLong)
190+
// should not happen, this header is added by moveContentLength
191+
.orElseThrow(() -> new RuntimeException("x-amz-decoded-content-length "
192+
+ "header not present"));
193+
194+
long encodedContentLength = calculateEncodedContentLength(request, decodedContentLength);
195+
196+
if (checksumAlgorithm != null) {
197+
String checksumHeaderName = checksumHeaderName(checksumAlgorithm);
198+
request.appendHeader(X_AMZ_TRAILER, checksumHeaderName);
199+
}
200+
request.putHeader(Header.CONTENT_LENGTH, Long.toString(encodedContentLength));
201+
request.appendHeader(CONTENT_ENCODING, AWS_CHUNKED);
202+
return Pair.of(requestBuilder, p.right());
203+
});
204+
}
205+
206+
private long calculateEncodedContentLength(SdkHttpRequest.Builder requestBuilder, long decodedContentLength) {
207+
long encodedContentLength = 0;
208+
130209
encodedContentLength += calculateExistingTrailersLength();
131210

132-
String checksum = request.firstMatchingHeader(X_AMZ_CONTENT_SHA256).orElseThrow(
211+
String checksum = requestBuilder.firstMatchingHeader(X_AMZ_CONTENT_SHA256).orElseThrow(
133212
() -> new IllegalArgumentException(X_AMZ_CONTENT_SHA256 + " must be set!")
134213
);
135214

136215
switch (checksum) {
137216
case STREAMING_SIGNED_PAYLOAD: {
138217
long extensionsLength = 81; // ;chunk-signature:<sigv4 hex signature, 64 bytes>
139-
encodedContentLength += calculateChunksLength(contentLength, extensionsLength);
218+
encodedContentLength += calculateChunksLength(decodedContentLength, extensionsLength);
140219
break;
141220
}
142221
case STREAMING_UNSIGNED_PAYLOAD_TRAILER:
143222
if (checksumAlgorithm != null) {
144223
encodedContentLength += calculateChecksumTrailerLength(checksumHeaderName(checksumAlgorithm));
145224
}
146-
encodedContentLength += calculateChunksLength(contentLength, 0);
225+
encodedContentLength += calculateChunksLength(decodedContentLength, 0);
147226
break;
148227
case STREAMING_SIGNED_PAYLOAD_TRAILER: {
149228
long extensionsLength = 81; // ;chunk-signature:<sigv4 hex signature, 64 bytes>
150-
encodedContentLength += calculateChunksLength(contentLength, extensionsLength);
229+
encodedContentLength += calculateChunksLength(decodedContentLength, extensionsLength);
151230
if (checksumAlgorithm != null) {
152231
encodedContentLength += calculateChecksumTrailerLength(checksumHeaderName(checksumAlgorithm));
153232
}
@@ -161,12 +240,7 @@ public void beforeSigning(SdkHttpRequest.Builder request, ContentStreamProvider
161240
// terminating \r\n
162241
encodedContentLength += 2;
163242

164-
if (checksumAlgorithm != null) {
165-
String checksumHeaderName = checksumHeaderName(checksumAlgorithm);
166-
request.appendHeader(X_AMZ_TRAILER, checksumHeaderName);
167-
}
168-
request.putHeader(Header.CONTENT_LENGTH, Long.toString(encodedContentLength));
169-
request.appendHeader(CONTENT_ENCODING, AWS_CHUNKED);
243+
return encodedContentLength;
170244
}
171245

172246
/**
@@ -271,6 +345,24 @@ private void setupChecksumTrailerIfNeeded(ChunkedEncodedInputStream.Builder buil
271345
builder.inputStream(checksumInputStream).addTrailer(checksumTrailer);
272346
}
273347

348+
private void setupChecksumTrailerIfNeeded(ChunkedEncodedPublisher.Builder builder) {
349+
if (checksumAlgorithm != null) {
350+
String checksumHeaderName = checksumHeaderName(checksumAlgorithm);
351+
SdkChecksum sdkChecksum = fromChecksumAlgorithm(checksumAlgorithm);
352+
Publisher<ByteBuffer> checksummedPayload = computeChecksum(builder.publisher(), sdkChecksum);
353+
354+
builder.publisher(checksummedPayload);
355+
356+
TrailerProvider checksumTrailer = new ChecksumTrailerProvider(sdkChecksum, checksumHeaderName);
357+
builder.addTrailer(checksumTrailer);
358+
}
359+
}
360+
361+
private Publisher<ByteBuffer> computeChecksum(Publisher<ByteBuffer> publisher, SdkChecksum checksum) {
362+
return subscriber -> publisher.subscribe(
363+
new UnbufferedChecksumSubscriber(Collections.singletonList(checksum), subscriber));
364+
}
365+
274366
static class Builder {
275367
private CredentialScope credentialScope;
276368
private Integer chunkSize;

‎core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSigner.java

Lines changed: 30 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,13 @@
2525
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.PRESIGN_URL_MAX_EXPIRATION_DURATION;
2626
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.X_AMZ_TRAILER;
2727

28+
import java.nio.ByteBuffer;
2829
import java.time.Clock;
2930
import java.time.Duration;
3031
import java.time.Instant;
3132
import java.util.concurrent.CompletableFuture;
3233
import java.util.function.Function;
34+
import org.reactivestreams.Publisher;
3335
import software.amazon.awssdk.annotations.SdkInternalApi;
3436
import software.amazon.awssdk.http.ContentStreamProvider;
3537
import software.amazon.awssdk.http.SdkHttpRequest;
@@ -70,7 +72,7 @@ public CompletableFuture<AsyncSignedRequest> signAsync(AsyncSignRequest<? extend
7072
V4RequestSigner v4RequestSigner = v4RequestSigner(request, v4Properties);
7173
V4PayloadSigner payloadSigner = v4PayloadAsyncSigner(request, v4Properties);
7274

73-
return doSign(request, checksummer, v4RequestSigner, payloadSigner);
75+
return doSignAsync(request, checksummer, v4RequestSigner, payloadSigner);
7476
}
7577

7678
private static V4Properties v4Properties(BaseSignRequest<?, ? extends AwsCredentialsIdentity> request) {
@@ -182,6 +184,8 @@ private static V4PayloadSigner v4PayloadAsyncSigner(
182184
boolean isPayloadSigning = request.requireProperty(PAYLOAD_SIGNING_ENABLED, true);
183185
boolean isEventStreaming = isEventStreaming(request.request());
184186
boolean isChunkEncoding = request.requireProperty(CHUNK_ENCODING_ENABLED, false);
187+
boolean isTrailing = request.request().firstMatchingHeader(X_AMZ_TRAILER).isPresent();
188+
boolean isFlexible = request.hasProperty(CHECKSUM_ALGORITHM) && !hasChecksumHeader(request);
185189

186190
if (isEventStreaming) {
187191
if (isPayloadSigning) {
@@ -194,12 +198,15 @@ private static V4PayloadSigner v4PayloadAsyncSigner(
194198
throw new UnsupportedOperationException("Unsigned payload is not supported with event-streaming.");
195199
}
196200

197-
if (isChunkEncoding && isPayloadSigning) {
198-
// TODO(sra-identity-and-auth): We need to implement aws-chunk content-encoding for async.
199-
// For now, we basically have to treat this as an unsigned case because there are existing s3 use-cases for
200-
// Unsigned-payload + HTTP. These requests SHOULD be signed-payload, but are not pre-SRA, hence the problem. This
201-
// will be taken care of in HttpChecksumStage for now, so we shouldn't throw an unsupported exception here, we
202-
// should just fall through to the default since it will already encoded by the time it gets here.
201+
if (isChunkEncoding) {
202+
if (!isPayloadSigning) {
203+
return AwsChunkedV4PayloadSigner.builder()
204+
.credentialScope(properties.getCredentialScope())
205+
.chunkSize(DEFAULT_CHUNK_SIZE_IN_BYTES)
206+
.checksumAlgorithm(request.property(CHECKSUM_ALGORITHM))
207+
.build();
208+
}
209+
// TODO: support payload signing with chunked encoding
203210
return V4PayloadSigner.create();
204211
}
205212

@@ -230,19 +237,26 @@ private static SignedRequest doSign(SignRequest<? extends AwsCredentialsIdentity
230237
.build();
231238
}
232239

233-
private static CompletableFuture<AsyncSignedRequest> doSign(AsyncSignRequest<? extends AwsCredentialsIdentity> request,
234-
Checksummer checksummer,
235-
V4RequestSigner requestSigner,
236-
V4PayloadSigner payloadSigner) {
240+
private static CompletableFuture<AsyncSignedRequest> doSignAsync(AsyncSignRequest<? extends AwsCredentialsIdentity> request,
241+
Checksummer checksummer,
242+
V4RequestSigner requestSigner,
243+
V4PayloadSigner payloadSigner) {
237244

238245
SdkHttpRequest.Builder requestBuilder = request.request().toBuilder();
239246

240-
return checksummer.checksum(request.payload().orElse(null), requestBuilder)
241-
.thenApply(payload -> {
242-
V4RequestSigningResult requestSigningResultFuture = requestSigner.sign(requestBuilder);
247+
Publisher<ByteBuffer> payload = request.payload().orElse(null);
248+
249+
return checksummer.checksum(payload, requestBuilder)
250+
.thenCompose(checksummedPayload ->
251+
payloadSigner.beforeSigningAsync(requestBuilder, checksummedPayload))
252+
.thenApply(p -> {
253+
SdkHttpRequest.Builder requestToSign = p.left();
254+
Publisher<ByteBuffer> payloadToSign = p.right();
255+
256+
V4RequestSigningResult requestSigningResult = requestSigner.sign(requestToSign);
243257
return AsyncSignedRequest.builder()
244-
.request(requestSigningResultFuture.getSignedRequest().build())
245-
.payload(payloadSigner.signAsync(payload, requestSigningResultFuture))
258+
.request(requestSigningResult.getSignedRequest().build())
259+
.payload(payloadSigner.signAsync(payloadToSign, requestSigningResult))
246260
.build();
247261
});
248262
}

‎core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/V4PayloadSigner.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,12 @@
1616
package software.amazon.awssdk.http.auth.aws.internal.signer;
1717

1818
import java.nio.ByteBuffer;
19+
import java.util.concurrent.CompletableFuture;
1920
import org.reactivestreams.Publisher;
2021
import software.amazon.awssdk.annotations.SdkInternalApi;
2122
import software.amazon.awssdk.http.ContentStreamProvider;
2223
import software.amazon.awssdk.http.SdkHttpRequest;
24+
import software.amazon.awssdk.utils.Pair;
2325

2426
/**
2527
* An interface for defining how to sign a payload via SigV4.
@@ -48,4 +50,9 @@ static V4PayloadSigner create() {
4850
*/
4951
default void beforeSigning(SdkHttpRequest.Builder request, ContentStreamProvider payload) {
5052
}
53+
54+
default CompletableFuture<Pair<SdkHttpRequest.Builder, Publisher<ByteBuffer>>> beforeSigningAsync(
55+
SdkHttpRequest.Builder request, Publisher<ByteBuffer> payload) {
56+
return CompletableFuture.completedFuture(Pair.of(request, payload));
57+
}
5158
}

‎core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/chunkedencoding/ChunkedEncodedPublisher.java

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333

3434
/**
3535
* An implementation of chunk-transfer encoding, but by wrapping a {@link Publisher} of {@link ByteBuffer}. This implementation
36-
* supports chunk-headers, chunk-extensions.
36+
* supports chunk-headers, chunk-extensions, and trailer-part.
3737
* <p>
3838
* Per <a href="https://datatracker.ietf.org/doc/html/rfc7230#section-4.1">RFC-7230</a>, a chunk-transfer encoded message is
3939
* defined as:
@@ -153,8 +153,7 @@ private ByteBuffer encodeChunk(ByteBuffer byteBuffer) {
153153
}
154154

155155
int trailerLen = trailerData.stream()
156-
// + 2 for each CRLF that ends the header-field
157-
.mapToInt(t -> t.remaining() + 2)
156+
.mapToInt(t -> t.remaining() + CRLF.length)
158157
.sum();
159158

160159
int encodedLen = chunkSizeHex.length + extensionsLength + CRLF.length + contentLen + trailerLen + CRLF.length;
@@ -188,6 +187,7 @@ private ByteBuffer encodeChunk(ByteBuffer byteBuffer) {
188187
encoded.put(t);
189188
encoded.put(CRLF);
190189
});
190+
// empty line ends the request body
191191
encoded.put(CRLF);
192192
}
193193

@@ -304,6 +304,10 @@ public Builder publisher(Publisher<ByteBuffer> publisher) {
304304
return this;
305305
}
306306

307+
public Publisher<ByteBuffer> publisher() {
308+
return publisher;
309+
}
310+
307311
public Builder chunkSize(int chunkSize) {
308312
this.chunkSize = chunkSize;
309313
return this;
@@ -324,6 +328,10 @@ public Builder addTrailer(TrailerProvider trailerProvider) {
324328
return this;
325329
}
326330

331+
public List<TrailerProvider> trailers() {
332+
return trailers;
333+
}
334+
327335
public ChunkedEncodedPublisher build() {
328336
return new ChunkedEncodedPublisher(this);
329337
}

0 commit comments

Comments
 (0)