Skip to content

Commit 8915ee7

Browse files
dagnirdavidh44
dagnir
and
davidh44
authored
Reuse computed checksums across retries (#6413)
* Reuse computed checksums across retries This commit adds the ability to reuse previously computed checksums for a request across retries. This ensures that if a request data stream is modified between attempts that the server will reject the request. As part of this change, the `http-auth-spi` package has been updated to expose a new interface: `PayloadChecksumStore`. This is a simple storage interface that allows signers to store and retrieve computed checksums. Additionally, a new `SignerProperty` is introduced, `SdkInternalHttpSignerProperty.CHECKSUM_CACHE` so that signers and their callers can access this cache. Note that both the interface and associated signer property are `@SdkProtectedApi` and not intended to be used by non-SDK consumers of `http-auth-spi`. Finally, this adds a dependency on `checksums-spi` for `http-auth-spi`. * Update core/sdk-core/src/main/java/software/amazon/awssdk/core/internal/http/pipeline/stages/SigningStage.java Co-authored-by: David Ho <[email protected]> * Review comments --------- Co-authored-by: David Ho <[email protected]>
1 parent b4801aa commit 8915ee7

File tree

28 files changed

+1237
-88
lines changed

28 files changed

+1237
-88
lines changed

‎core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/crt/internal/signer/AwsChunkedV4aPayloadSigner.java‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,12 +35,15 @@
3535
import software.amazon.awssdk.http.Header;
3636
import software.amazon.awssdk.http.SdkHttpRequest;
3737
import software.amazon.awssdk.http.auth.aws.internal.signer.CredentialScope;
38+
import software.amazon.awssdk.http.auth.aws.internal.signer.NoOpPayloadChecksumStore;
3839
import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.ChecksumTrailerProvider;
3940
import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.ChunkedEncodedInputStream;
4041
import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.TrailerProvider;
4142
import software.amazon.awssdk.http.auth.aws.internal.signer.io.ChecksumInputStream;
4243
import software.amazon.awssdk.http.auth.aws.internal.signer.io.ResettableContentStreamProvider;
44+
import software.amazon.awssdk.http.auth.spi.signer.PayloadChecksumStore;
4345
import software.amazon.awssdk.utils.BinaryUtils;
46+
import software.amazon.awssdk.utils.Logger;
4447
import software.amazon.awssdk.utils.Pair;
4548
import software.amazon.awssdk.utils.StringInputStream;
4649
import software.amazon.awssdk.utils.Validate;
@@ -51,16 +54,20 @@
5154
*/
5255
@SdkInternalApi
5356
public final class AwsChunkedV4aPayloadSigner implements V4aPayloadSigner {
57+
private static final Logger LOG = Logger.loggerFor(AwsChunkedV4aPayloadSigner.class);
5458

5559
private final CredentialScope credentialScope;
5660
private final int chunkSize;
5761
private final ChecksumAlgorithm checksumAlgorithm;
62+
private final PayloadChecksumStore payloadChecksumStore;
5863
private final List<Pair<String, List<String>>> preExistingTrailers = new ArrayList<>();
5964

6065
private AwsChunkedV4aPayloadSigner(Builder builder) {
6166
this.credentialScope = Validate.paramNotNull(builder.credentialScope, "CredentialScope");
6267
this.chunkSize = Validate.isPositive(builder.chunkSize, "ChunkSize");
6368
this.checksumAlgorithm = builder.checksumAlgorithm;
69+
this.payloadChecksumStore = builder.payloadChecksumStore == null ? NoOpPayloadChecksumStore.create() :
70+
builder.payloadChecksumStore;
6471
}
6572

6673
public static Builder builder() {
@@ -241,21 +248,41 @@ private void setupChecksumTrailerIfNeeded(ChunkedEncodedInputStream.Builder buil
241248
return;
242249
}
243250
String checksumHeaderName = checksumHeaderName(checksumAlgorithm);
251+
252+
String cachedChecksum = getCachedChecksum();
253+
254+
if (cachedChecksum != null) {
255+
LOG.debug(() -> String.format("Cached payload checksum available for algorithm %s: %s. Using cached value",
256+
checksumAlgorithm.algorithmId(), checksumHeaderName));
257+
builder.addTrailer(() -> Pair.of(checksumHeaderName, Collections.singletonList(cachedChecksum)));
258+
return;
259+
}
260+
244261
SdkChecksum sdkChecksum = fromChecksumAlgorithm(checksumAlgorithm);
245262
ChecksumInputStream checksumInputStream = new ChecksumInputStream(
246263
builder.inputStream(),
247264
Collections.singleton(sdkChecksum)
248265
);
249266

250-
TrailerProvider checksumTrailer = new ChecksumTrailerProvider(sdkChecksum, checksumHeaderName);
267+
TrailerProvider checksumTrailer =
268+
new ChecksumTrailerProvider(sdkChecksum, checksumHeaderName, checksumAlgorithm, payloadChecksumStore);
251269

252270
builder.inputStream(checksumInputStream).addTrailer(checksumTrailer);
253271
}
254272

273+
private String getCachedChecksum() {
274+
byte[] checksumBytes = payloadChecksumStore.getChecksumValue(checksumAlgorithm);
275+
if (checksumBytes != null) {
276+
return BinaryUtils.toBase64(checksumBytes);
277+
}
278+
return null;
279+
}
280+
255281
static final class Builder {
256282
private CredentialScope credentialScope;
257283
private Integer chunkSize;
258284
private ChecksumAlgorithm checksumAlgorithm;
285+
private PayloadChecksumStore payloadChecksumStore;
259286

260287
public Builder credentialScope(CredentialScope credentialScope) {
261288
this.credentialScope = credentialScope;
@@ -272,6 +299,11 @@ public Builder checksumAlgorithm(ChecksumAlgorithm checksumAlgorithm) {
272299
return this;
273300
}
274301

302+
public Builder checksumCache(PayloadChecksumStore checksumCache) {
303+
this.payloadChecksumStore = checksumCache;
304+
return this;
305+
}
306+
275307
public AwsChunkedV4aPayloadSigner build() {
276308
return new AwsChunkedV4aPayloadSigner(this);
277309
}

‎core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/crt/internal/signer/DefaultAwsCrtV4aHttpSigner.java‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.CredentialUtils.sanitizeCredentials;
3434
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.PRESIGN_URL_MAX_EXPIRATION_DURATION;
3535
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.X_AMZ_TRAILER;
36+
import static software.amazon.awssdk.http.auth.spi.signer.SdkInternalHttpSignerProperty.CHECKSUM_CACHE;
3637

3738
import java.time.Clock;
3839
import java.time.Duration;
@@ -47,11 +48,13 @@
4748
import software.amazon.awssdk.http.SdkHttpRequest;
4849
import software.amazon.awssdk.http.auth.aws.internal.signer.Checksummer;
4950
import software.amazon.awssdk.http.auth.aws.internal.signer.CredentialScope;
51+
import software.amazon.awssdk.http.auth.aws.internal.signer.NoOpPayloadChecksumStore;
5052
import software.amazon.awssdk.http.auth.aws.signer.AwsV4aHttpSigner;
5153
import software.amazon.awssdk.http.auth.aws.signer.RegionSet;
5254
import software.amazon.awssdk.http.auth.spi.signer.AsyncSignRequest;
5355
import software.amazon.awssdk.http.auth.spi.signer.AsyncSignedRequest;
5456
import software.amazon.awssdk.http.auth.spi.signer.BaseSignRequest;
57+
import software.amazon.awssdk.http.auth.spi.signer.PayloadChecksumStore;
5558
import software.amazon.awssdk.http.auth.spi.signer.SignRequest;
5659
import software.amazon.awssdk.http.auth.spi.signer.SignedRequest;
5760
import software.amazon.awssdk.identity.spi.AwsCredentialsIdentity;
@@ -70,7 +73,7 @@ public final class DefaultAwsCrtV4aHttpSigner implements AwsV4aHttpSigner {
7073

7174
@Override
7275
public SignedRequest sign(SignRequest<? extends AwsCredentialsIdentity> request) {
73-
Checksummer checksummer = checksummer(request, null);
76+
Checksummer checksummer = checksummer(request, null, checksumCache(request));
7477
V4aProperties v4aProperties = v4aProperties(request);
7578
AwsSigningConfig signingConfig = signingConfig(request, v4aProperties);
7679
V4aPayloadSigner payloadSigner = v4aPayloadSigner(request, v4aProperties);
@@ -104,7 +107,7 @@ private static V4aProperties v4aProperties(BaseSignRequest<?, ? extends AwsCrede
104107
}
105108

106109
private static V4aPayloadSigner v4aPayloadSigner(
107-
BaseSignRequest<?, ? extends AwsCredentialsIdentity> request,
110+
SignRequest<? extends AwsCredentialsIdentity> request,
108111
V4aProperties v4aProperties) {
109112

110113
boolean isPayloadSigning = isPayloadSigning(request);
@@ -117,6 +120,7 @@ private static V4aPayloadSigner v4aPayloadSigner(
117120
.credentialScope(v4aProperties.getCredentialScope())
118121
.chunkSize(DEFAULT_CHUNK_SIZE_IN_BYTES)
119122
.checksumAlgorithm(request.property(CHECKSUM_ALGORITHM))
123+
.checksumCache(checksumCache(request))
120124
.build();
121125
}
122126

@@ -252,4 +256,12 @@ private static V4aRequestSigningResult sign(SdkHttpRequest request, HttpRequest
252256
signingResult.getSignature(),
253257
signingConfig);
254258
}
259+
260+
private static PayloadChecksumStore checksumCache(SignRequest<? extends AwsCredentialsIdentity> request) {
261+
PayloadChecksumStore cache = request.property(CHECKSUM_CACHE);
262+
if (cache == null) {
263+
return NoOpPayloadChecksumStore.create();
264+
}
265+
return cache;
266+
}
255267
}

‎core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/AwsChunkedV4PayloadSigner.java‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,9 @@
4545
import software.amazon.awssdk.http.auth.aws.internal.signer.chunkedencoding.TrailerProvider;
4646
import software.amazon.awssdk.http.auth.aws.internal.signer.io.ChecksumInputStream;
4747
import software.amazon.awssdk.http.auth.aws.internal.signer.io.ResettableContentStreamProvider;
48+
import software.amazon.awssdk.http.auth.spi.signer.PayloadChecksumStore;
4849
import software.amazon.awssdk.utils.BinaryUtils;
50+
import software.amazon.awssdk.utils.Logger;
4951
import software.amazon.awssdk.utils.Pair;
5052
import software.amazon.awssdk.utils.Validate;
5153

@@ -55,16 +57,20 @@
5557
*/
5658
@SdkInternalApi
5759
public final class AwsChunkedV4PayloadSigner implements V4PayloadSigner {
60+
private static final Logger LOG = Logger.loggerFor(AwsChunkedV4PayloadSigner.class);
5861

5962
private final CredentialScope credentialScope;
6063
private final int chunkSize;
6164
private final ChecksumAlgorithm checksumAlgorithm;
65+
private final PayloadChecksumStore payloadChecksumStore;
6266
private final List<Pair<String, List<String>>> preExistingTrailers = new ArrayList<>();
6367

6468
private AwsChunkedV4PayloadSigner(Builder builder) {
6569
this.credentialScope = Validate.paramNotNull(builder.credentialScope, "CredentialScope");
6670
this.chunkSize = Validate.isPositive(builder.chunkSize, "ChunkSize");
6771
this.checksumAlgorithm = builder.checksumAlgorithm;
72+
this.payloadChecksumStore = builder.payloadChecksumStore == null ? NoOpPayloadChecksumStore.create() :
73+
builder.payloadChecksumStore;
6874
}
6975

7076
public static Builder builder() {
@@ -259,22 +265,43 @@ private void setupChecksumTrailerIfNeeded(ChunkedEncodedInputStream.Builder buil
259265
if (checksumAlgorithm == null) {
260266
return;
261267
}
268+
262269
String checksumHeaderName = checksumHeaderName(checksumAlgorithm);
270+
271+
String cachedChecksum = getCachedChecksum();
272+
273+
if (cachedChecksum != null) {
274+
LOG.debug(() -> String.format("Cached payload checksum available for algorithm %s: %s. Using cached value",
275+
checksumAlgorithm.algorithmId(), checksumHeaderName));
276+
builder.addTrailer(() -> Pair.of(checksumHeaderName, Collections.singletonList(cachedChecksum)));
277+
return;
278+
}
279+
263280
SdkChecksum sdkChecksum = fromChecksumAlgorithm(checksumAlgorithm);
264281
ChecksumInputStream checksumInputStream = new ChecksumInputStream(
265282
builder.inputStream(),
266283
Collections.singleton(sdkChecksum)
267284
);
268285

269-
TrailerProvider checksumTrailer = new ChecksumTrailerProvider(sdkChecksum, checksumHeaderName);
286+
TrailerProvider checksumTrailer =
287+
new ChecksumTrailerProvider(sdkChecksum, checksumHeaderName, checksumAlgorithm, payloadChecksumStore);
270288

271289
builder.inputStream(checksumInputStream).addTrailer(checksumTrailer);
272290
}
273291

292+
private String getCachedChecksum() {
293+
byte[] checksumBytes = payloadChecksumStore.getChecksumValue(checksumAlgorithm);
294+
if (checksumBytes != null) {
295+
return BinaryUtils.toBase64(checksumBytes);
296+
}
297+
return null;
298+
}
299+
274300
static class Builder {
275301
private CredentialScope credentialScope;
276302
private Integer chunkSize;
277303
private ChecksumAlgorithm checksumAlgorithm;
304+
private PayloadChecksumStore payloadChecksumStore;
278305

279306
public Builder credentialScope(CredentialScope credentialScope) {
280307
this.credentialScope = credentialScope;
@@ -291,6 +318,11 @@ public Builder checksumAlgorithm(ChecksumAlgorithm checksumAlgorithm) {
291318
return this;
292319
}
293320

321+
public Builder checksumCache(PayloadChecksumStore payloadChecksumStore) {
322+
this.payloadChecksumStore = payloadChecksumStore;
323+
return this;
324+
}
325+
294326
public AwsChunkedV4PayloadSigner build() {
295327
return new AwsChunkedV4PayloadSigner(this);
296328
}

‎core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/Checksummer.java‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import software.amazon.awssdk.checksums.spi.ChecksumAlgorithm;
3030
import software.amazon.awssdk.http.ContentStreamProvider;
3131
import software.amazon.awssdk.http.SdkHttpRequest;
32+
import software.amazon.awssdk.http.auth.spi.signer.PayloadChecksumStore;
3233
import software.amazon.awssdk.utils.BinaryUtils;
3334

3435
/**
@@ -47,8 +48,9 @@ public interface Checksummer {
4748
* Get a default implementation of a checksummer, which calculates the SHA-256 checksum and places it in the
4849
* x-amz-content-sha256 header.
4950
*/
50-
static Checksummer create() {
51+
static Checksummer create(PayloadChecksumStore cache) {
5152
return new FlexibleChecksummer(
53+
cache,
5254
option().headerName(X_AMZ_CONTENT_SHA256).algorithm(SHA256).formatter(BinaryUtils::toHex).build()
5355
);
5456
}
@@ -57,9 +59,10 @@ static Checksummer create() {
5759
* Get a flexible checksummer that performs two checksums: the given checksum-algorithm and the SHA-256 checksum. It places
5860
* the SHA-256 checksum in x-amz-content-sha256 header, and the given checksum-algorithm in the x-amz-checksum-[name] header.
5961
*/
60-
static Checksummer forFlexibleChecksum(ChecksumAlgorithm checksumAlgorithm) {
62+
static Checksummer forFlexibleChecksum(ChecksumAlgorithm checksumAlgorithm, PayloadChecksumStore cache) {
6163
if (checksumAlgorithm != null) {
6264
return new FlexibleChecksummer(
65+
cache,
6366
option().headerName(X_AMZ_CONTENT_SHA256).algorithm(SHA256).formatter(BinaryUtils::toHex)
6467
.build(),
6568
option().headerName(checksumHeaderName(checksumAlgorithm)).algorithm(checksumAlgorithm)
@@ -82,9 +85,12 @@ static Checksummer forPrecomputed256Checksum(String precomputedSha256) {
8285
* given checksum string. It places the precomputed checksum in x-amz-content-sha256 header, and the given checksum-algorithm
8386
* in the x-amz-checksum-[name] header.
8487
*/
85-
static Checksummer forFlexibleChecksum(String precomputedSha256, ChecksumAlgorithm checksumAlgorithm) {
88+
static Checksummer forFlexibleChecksum(String precomputedSha256,
89+
ChecksumAlgorithm checksumAlgorithm,
90+
PayloadChecksumStore cache) {
8691
if (checksumAlgorithm != null) {
8792
return new FlexibleChecksummer(
93+
cache,
8894
option().headerName(X_AMZ_CONTENT_SHA256).algorithm(new ConstantChecksumAlgorithm(precomputedSha256))
8995
.formatter(b -> new String(b, StandardCharsets.UTF_8)).build(),
9096
option().headerName(checksumHeaderName(checksumAlgorithm)).algorithm(checksumAlgorithm)
@@ -96,7 +102,7 @@ static Checksummer forFlexibleChecksum(String precomputedSha256, ChecksumAlgorit
96102
}
97103

98104
static Checksummer forNoOp() {
99-
return new FlexibleChecksummer();
105+
return new FlexibleChecksummer(NoOpPayloadChecksumStore.create());
100106
}
101107

102108
/**

‎core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/DefaultAwsV4HttpSigner.java‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.OptionalDependencyLoaderUtil.getEventStreamV4PayloadSigner;
2525
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.PRESIGN_URL_MAX_EXPIRATION_DURATION;
2626
import static software.amazon.awssdk.http.auth.aws.internal.signer.util.SignerConstant.X_AMZ_TRAILER;
27+
import static software.amazon.awssdk.http.auth.spi.signer.SdkInternalHttpSignerProperty.CHECKSUM_CACHE;
2728

2829
import java.time.Clock;
2930
import java.time.Duration;
@@ -38,6 +39,7 @@
3839
import software.amazon.awssdk.http.auth.spi.signer.AsyncSignRequest;
3940
import software.amazon.awssdk.http.auth.spi.signer.AsyncSignedRequest;
4041
import software.amazon.awssdk.http.auth.spi.signer.BaseSignRequest;
42+
import software.amazon.awssdk.http.auth.spi.signer.PayloadChecksumStore;
4143
import software.amazon.awssdk.http.auth.spi.signer.SignRequest;
4244
import software.amazon.awssdk.http.auth.spi.signer.SignedRequest;
4345
import software.amazon.awssdk.identity.spi.AwsCredentialsIdentity;
@@ -55,7 +57,7 @@ public final class DefaultAwsV4HttpSigner implements AwsV4HttpSigner {
5557

5658
@Override
5759
public SignedRequest sign(SignRequest<? extends AwsCredentialsIdentity> request) {
58-
Checksummer checksummer = checksummer(request, null);
60+
Checksummer checksummer = checksummer(request, null, checksumCache(request));
5961
V4Properties v4Properties = v4Properties(request);
6062
V4RequestSigner v4RequestSigner = v4RequestSigner(request, v4Properties);
6163
V4PayloadSigner payloadSigner = v4PayloadSigner(request, v4Properties);
@@ -140,7 +142,7 @@ private static Checksummer asyncChecksummer(BaseSignRequest<?, ? extends AwsCred
140142
// check for payload signing is done.
141143
Boolean overridePayloadSigning = shouldTreatAsUnsigned ? false : null;
142144

143-
return checksummer(request, overridePayloadSigning);
145+
return checksummer(request, overridePayloadSigning, PayloadChecksumStore.create());
144146
}
145147

146148
private static V4PayloadSigner v4PayloadSigner(
@@ -168,6 +170,7 @@ private static V4PayloadSigner v4PayloadSigner(
168170
return AwsChunkedV4PayloadSigner.builder()
169171
.credentialScope(properties.getCredentialScope())
170172
.chunkSize(DEFAULT_CHUNK_SIZE_IN_BYTES)
173+
.checksumCache(checksumCache(request))
171174
.checksumAlgorithm(request.property(CHECKSUM_ALGORITHM))
172175
.build();
173176
}
@@ -261,4 +264,12 @@ private static Duration validateExpirationDuration(Duration expirationDuration)
261264
private static boolean isBetweenInclusive(Duration start, Duration x, Duration end) {
262265
return start.compareTo(x) <= 0 && x.compareTo(end) <= 0;
263266
}
267+
268+
private static PayloadChecksumStore checksumCache(SignRequest<? extends AwsCredentialsIdentity> request) {
269+
PayloadChecksumStore cache = request.property(CHECKSUM_CACHE);
270+
if (cache == null) {
271+
return NoOpPayloadChecksumStore.create();
272+
}
273+
return cache;
274+
}
264275
}

‎core/http-auth-aws/src/main/java/software/amazon/awssdk/http/auth/aws/internal/signer/FlexibleChecksummer.java‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import software.amazon.awssdk.http.SdkHttpRequest;
3636
import software.amazon.awssdk.http.auth.aws.internal.signer.io.ChecksumInputStream;
3737
import software.amazon.awssdk.http.auth.aws.internal.signer.io.ChecksumSubscriber;
38+
import software.amazon.awssdk.http.auth.spi.signer.PayloadChecksumStore;
3839
import software.amazon.awssdk.utils.Validate;
3940

4041
/**
@@ -44,10 +45,12 @@
4445
*/
4546
@SdkInternalApi
4647
public final class FlexibleChecksummer implements Checksummer {
48+
private final PayloadChecksumStore cache;
4749
private final Collection<Option> options;
4850
private final Map<Option, SdkChecksum> optionToSdkChecksum;
4951

50-
public FlexibleChecksummer(Option... options) {
52+
public FlexibleChecksummer(PayloadChecksumStore cache, Option... options) {
53+
this.cache = cache;
5154
this.options = Arrays.asList(options);
5255
this.optionToSdkChecksum = this.options.stream().collect(
5356
Collectors.toMap(Function.identity(), o -> fromChecksumAlgorithm(o.algorithm))
@@ -85,9 +88,14 @@ public CompletableFuture<Publisher<ByteBuffer>> checksum(Publisher<ByteBuffer> p
8588

8689
private void addChecksums(SdkHttpRequest.Builder request) {
8790
optionToSdkChecksum.forEach(
88-
(option, sdkChecksum) -> request.putHeader(
89-
option.headerName,
90-
option.formatter.apply(sdkChecksum.getChecksumBytes()))
91+
(option, sdkChecksum) -> {
92+
byte[] checksumValue = cache.getChecksumValue(option.algorithm);
93+
if (checksumValue == null) {
94+
checksumValue = sdkChecksum.getChecksumBytes();
95+
cache.putChecksumValue(option.algorithm, checksumValue);
96+
}
97+
request.putHeader(option.headerName, option.formatter.apply(checksumValue));
98+
}
9199
);
92100
}
93101

0 commit comments

Comments
 (0)