Skip to content

Support async V4 payload signing #6314

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
+982 −318
Open

Support async V4 payload signing #6314

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7c7941c
Support async V4 payload signing
dagnir Jul 21, 2025
a222719
Review comments
dagnir Aug 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .changes/next-release/feature-AWSSDKForJavav2-3b0176a.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
"type": "feature",
"category": "AWS SDK For Java v2",
"contributor": "",
"description": "Add support for payload signing of async streaming requests signed with SigV4. This brings the SigV4 signing behavior for async streaming request bodies in line with the sync clients. In particular, this means that streaming requests made over plaintext HTTP (i.e. not HTTPS) will have signed payloads."
}
4 changes: 2 additions & 2 deletions .../software/amazon/awssdk/http/auth/aws/crt/internal/signer/AwsChunkedV4aPayloadSigner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.STREAMING_ECDSA_SIGNED_PAYLOAD_TRAILER;
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.STREAMING_UNSIGNED_PAYLOAD_TRAILER;
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.X_AMZ_TRAILER;
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerUtils.moveContentLength;

import java.io.InputStream;
import java.nio.charset.StandardCharsets;
Expand All @@ -40,6 +39,7 @@
import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.TrailerProvider;
import software.amazon.awssdk.http.auth.aws.internal.signer.io.ChecksumInputStream;
import software.amazon.awssdk.http.auth.aws.internal.signer.io.ResettableContentStreamProvider;
import software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerUtils;
import software.amazon.awssdk.utils.BinaryUtils;
import software.amazon.awssdk.utils.Pair;
import software.amazon.awssdk.utils.StringInputStream;
Expand Down Expand Up @@ -108,7 +108,7 @@ public ContentStreamProvider sign(ContentStreamProvider payload, V4aRequestSigni
@Override
public void beforeSigning(SdkHttpRequest.Builder request, ContentStreamProvider payload, String checksum) {
long encodedContentLength = 0;
long contentLength = moveContentLength(request, payload);
long contentLength = SignerUtils.computeAndMoveContentLength(request, payload);
setupPreExistingTrailers(request);

// pre-existing trailers
Expand Down
140 changes: 94 additions & 46 deletions .../java/software/amazon/awssdk/http/auth/aws/internal/signer/AwsChunkedV4PayloadSigner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,28 +23,34 @@
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.STREAMING_SIGNED_PAYLOAD_TRAILER;
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.STREAMING_UNSIGNED_PAYLOAD_TRAILER;
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.X_AMZ_CONTENT_SHA256;
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.X_AMZ_DECODED_CONTENT_LENGTH;
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.X_AMZ_TRAILER;
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerUtils.moveContentLength;
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerUtils.computeAndMoveContentLength;

import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import org.reactivestreams.Publisher;
import software.amazon.awssdk.annotations.SdkInternalApi;
import software.amazon.awssdk.checksums.SdkChecksum;
import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm;
import software.amazon.awssdk.http.ContentStreamProvider;
import software.amazon.awssdk.http.Header;
import software.amazon.awssdk.http.SdkHttpRequest;
import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.AsyncChunkEncodedPayload;
import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.ChecksumTrailerProvider;
import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.ChunkedEncodedInputStream;
import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.ChunkedEncodedPayload;
import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.ChunkedEncodedPublisher;
import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.SigV4ChunkExtensionProvider;
import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.SigV4TrailerProvider;
import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.SyncChunkEncodedPayload;
import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.TrailerProvider;
import software.amazon.awssdk.http.auth.aws.internal.signer.io.ChecksumInputStream;
import software.amazon.awssdk.http.auth.aws.internal.signer.io.ResettableContentStreamProvider;
import software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerUtils;
import software.amazon.awssdk.utils.BinaryUtils;
import software.amazon.awssdk.utils.Pair;
import software.amazon.awssdk.utils.Validate;
Expand Down Expand Up @@ -73,81 +79,136 @@ public static Builder builder() {

@Override
public ContentStreamProvider sign(ContentStreamProvider payload, V4RequestSigningResult requestSigningResult) {
SdkHttpRequest.Builder request = requestSigningResult.getSignedRequest();

String checksum = request.firstMatchingHeader(X_AMZ_CONTENT_SHA256).orElseThrow(
() -> new IllegalArgumentException(X_AMZ_CONTENT_SHA256 + " must be set!")
);

ChunkedEncodedInputStream.Builder chunkedEncodedInputStreamBuilder = ChunkedEncodedInputStream
.builder()
.inputStream(payload.newStream())
.chunkSize(chunkSize)
.header(chunk -> Integer.toHexString(chunk.remaining()).getBytes(StandardCharsets.UTF_8));

preExistingTrailers.forEach(trailer -> chunkedEncodedInputStreamBuilder.addTrailer(() -> trailer));
SyncChunkEncodedPayload chunkedPayload = new SyncChunkEncodedPayload(chunkedEncodedInputStreamBuilder);
signCommon(chunkedPayload, requestSigningResult);

return new ResettableContentStreamProvider(chunkedEncodedInputStreamBuilder::build);
}

@Override
public Publisher<ByteBuffer> signAsync(Publisher<ByteBuffer> payload, V4RequestSigningResult requestSigningResult) {
ChunkedEncodedPublisher.Builder chunkedStreamBuilder = ChunkedEncodedPublisher.builder()
.publisher(payload)
.chunkSize(chunkSize)
.addEmptyTrailingChunk(true);

AsyncChunkEncodedPayload chunkedPayload = new AsyncChunkEncodedPayload(chunkedStreamBuilder);
signCommon(chunkedPayload, requestSigningResult);

return chunkedStreamBuilder.build();
}

private void signCommon(ChunkedEncodedPayload payload, V4RequestSigningResult requestSigningResult) {
preExistingTrailers.forEach(t -> payload.addTrailer(() -> t));

SdkHttpRequest.Builder request = requestSigningResult.getSignedRequest();

payload.decodedContentLength(request.firstMatchingHeader(X_AMZ_DECODED_CONTENT_LENGTH)
.map(Long::parseLong)
.orElse(0L));

String checksum = request.firstMatchingHeader(X_AMZ_CONTENT_SHA256).orElseThrow(
() -> new IllegalArgumentException(X_AMZ_CONTENT_SHA256 + " must be set!")
);

switch (checksum) {
case STREAMING_SIGNED_PAYLOAD: {
RollingSigner rollingSigner = new RollingSigner(requestSigningResult.getSigningKey(),
requestSigningResult.getSignature());
chunkedEncodedInputStreamBuilder.addExtension(new SigV4ChunkExtensionProvider(rollingSigner, credentialScope));
payload.addExtension(new SigV4ChunkExtensionProvider(rollingSigner, credentialScope));
break;
}
case STREAMING_UNSIGNED_PAYLOAD_TRAILER:
setupChecksumTrailerIfNeeded(chunkedEncodedInputStreamBuilder);
setupChecksumTrailerIfNeeded(payload);
break;
case STREAMING_SIGNED_PAYLOAD_TRAILER: {
setupChecksumTrailerIfNeeded(payload);
RollingSigner rollingSigner = new RollingSigner(requestSigningResult.getSigningKey(),
requestSigningResult.getSignature());
chunkedEncodedInputStreamBuilder.addExtension(new SigV4ChunkExtensionProvider(rollingSigner, credentialScope));
setupChecksumTrailerIfNeeded(chunkedEncodedInputStreamBuilder);
chunkedEncodedInputStreamBuilder.addTrailer(
new SigV4TrailerProvider(chunkedEncodedInputStreamBuilder.trailers(), rollingSigner, credentialScope)
payload.addExtension(new SigV4ChunkExtensionProvider(rollingSigner, credentialScope));
payload.addTrailer(
new SigV4TrailerProvider(payload.trailers(), rollingSigner, credentialScope)
);
break;
}
default:
throw new UnsupportedOperationException();
}

return new ResettableContentStreamProvider(chunkedEncodedInputStreamBuilder::build);
}

@Override
public Publisher<ByteBuffer> signAsync(Publisher<ByteBuffer> payload, V4RequestSigningResult requestSigningResult) {
// TODO(sra-identity-and-auth): implement this first and remove addFlexibleChecksumInTrailer logic in HttpChecksumStage
throw new UnsupportedOperationException();
}

@Override
public void beforeSigning(SdkHttpRequest.Builder request, ContentStreamProvider payload) {
long encodedContentLength = 0;
long contentLength = moveContentLength(request, payload);
long contentLength = SignerUtils.computeAndMoveContentLength(request, payload);
setupPreExistingTrailers(request);

// pre-existing trailers
encodedContentLength = calculateEncodedContentLength(request, contentLength);

if (checksumAlgorithm != null) {
String checksumHeaderName = checksumHeaderName(checksumAlgorithm);
request.appendHeader(X_AMZ_TRAILER, checksumHeaderName);
}
request.putHeader(Header.CONTENT_LENGTH, Long.toString(encodedContentLength));
request.appendHeader(CONTENT_ENCODING, AWS_CHUNKED);
}

@Override
public CompletableFuture<Pair<SdkHttpRequest.Builder, Optional<Publisher<ByteBuffer>>>> beforeSigningAsync(
SdkHttpRequest.Builder request, Publisher<ByteBuffer> payload) {
return computeAndMoveContentLength(request, payload)
.thenApply(p -> {
SdkHttpRequest.Builder requestBuilder = p.left();
setupPreExistingTrailers(requestBuilder);

long decodedContentLength = requestBuilder.firstMatchingHeader(X_AMZ_DECODED_CONTENT_LENGTH)
.map(Long::parseLong)
// should not happen, this header is added by moveContentLength
.orElseThrow(() -> new RuntimeException(X_AMZ_DECODED_CONTENT_LENGTH
+ " header not present"));

long encodedContentLength = calculateEncodedContentLength(request, decodedContentLength);

if (checksumAlgorithm != null) {
String checksumHeaderName = checksumHeaderName(checksumAlgorithm);
request.appendHeader(X_AMZ_TRAILER, checksumHeaderName);
}
request.putHeader(Header.CONTENT_LENGTH, Long.toString(encodedContentLength));
request.appendHeader(CONTENT_ENCODING, AWS_CHUNKED);
return Pair.of(requestBuilder, p.right());
});
}

private long calculateEncodedContentLength(SdkHttpRequest.Builder requestBuilder, long decodedContentLength) {
long encodedContentLength = 0;

encodedContentLength += calculateExistingTrailersLength();

String checksum = request.firstMatchingHeader(X_AMZ_CONTENT_SHA256).orElseThrow(
String checksum = requestBuilder.firstMatchingHeader(X_AMZ_CONTENT_SHA256).orElseThrow(
() -> new IllegalArgumentException(X_AMZ_CONTENT_SHA256 + " must be set!")
);

switch (checksum) {
case STREAMING_SIGNED_PAYLOAD: {
long extensionsLength = 81; // ;chunk-signature:<sigv4 hex signature, 64 bytes>
encodedContentLength += calculateChunksLength(contentLength, extensionsLength);
encodedContentLength += calculateChunksLength(decodedContentLength, extensionsLength);
break;
}
case STREAMING_UNSIGNED_PAYLOAD_TRAILER:
if (checksumAlgorithm != null) {
encodedContentLength += calculateChecksumTrailerLength(checksumHeaderName(checksumAlgorithm));
}
encodedContentLength += calculateChunksLength(contentLength, 0);
encodedContentLength += calculateChunksLength(decodedContentLength, 0);
break;
case STREAMING_SIGNED_PAYLOAD_TRAILER: {
long extensionsLength = 81; // ;chunk-signature:<sigv4 hex signature, 64 bytes>
encodedContentLength += calculateChunksLength(contentLength, extensionsLength);
encodedContentLength += calculateChunksLength(decodedContentLength, extensionsLength);
if (checksumAlgorithm != null) {
encodedContentLength += calculateChecksumTrailerLength(checksumHeaderName(checksumAlgorithm));
}
Expand All @@ -161,12 +222,7 @@ public void beforeSigning(SdkHttpRequest.Builder request, ContentStreamProvider
// terminating \r\n
encodedContentLength += 2;

if (checksumAlgorithm != null) {
String checksumHeaderName = checksumHeaderName(checksumAlgorithm);
request.appendHeader(X_AMZ_TRAILER, checksumHeaderName);
}
request.putHeader(Header.CONTENT_LENGTH, Long.toString(encodedContentLength));
request.appendHeader(CONTENT_ENCODING, AWS_CHUNKED);
return encodedContentLength;
}

/**
Expand Down Expand Up @@ -250,25 +306,17 @@ private long calculateChecksumTrailerLength(String checksumHeaderName) {
return lengthInBytes + 2;
}

/**
* Add the checksum as a trailer to the chunk-encoded stream.
* <p>
* If the checksum-algorithm is not present, then nothing is done.
*/
private void setupChecksumTrailerIfNeeded(ChunkedEncodedInputStream.Builder builder) {
private void setupChecksumTrailerIfNeeded(ChunkedEncodedPayload payload) {
if (checksumAlgorithm == null) {
return;
}
String checksumHeaderName = checksumHeaderName(checksumAlgorithm);
SdkChecksum sdkChecksum = fromChecksumAlgorithm(checksumAlgorithm);
ChecksumInputStream checksumInputStream = new ChecksumInputStream(
builder.inputStream(),
Collections.singleton(sdkChecksum)
);

TrailerProvider checksumTrailer = new ChecksumTrailerProvider(sdkChecksum, checksumHeaderName);

builder.inputStream(checksumInputStream).addTrailer(checksumTrailer);
payload.checksumPayload(sdkChecksum);
payload.addTrailer(checksumTrailer);
}

static class Builder {
Expand Down
Loading
Loading