Skip to content

Commit aab40b1

Browse files
DaveCTurnerDaveCTurner
authored
Introduce TestBlobContainerBuilder (#126445)
The mostly-optional parameters to `createBlobContainer` are getting rather numerous in this test harness which makes the tests hard to read. This commit introduces a builder to help name the provided parameters and skip the omitted ones.
1 parent 450516d commit aab40b1

File tree

3 files changed

+155
-66
lines changed

3 files changed

+155
-66
lines changed

‎modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -229,7 +229,7 @@ public void testShouldRetryOnUnresolvableHost() {
229229

230230
private void executeListBlobsAndAssertRetries() {
231231
final int maxRetries = randomIntBetween(3, 5);
232-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null, null, null);
232+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(maxRetries).build();
233233
expectThrows(StorageException.class, () -> blobContainer.listBlobs(randomPurpose()));
234234
assertEquals(maxRetries + 1, requestCounters.get("/storage/v1/b/bucket/o").get());
235235
}
@@ -238,7 +238,7 @@ public void testReadLargeBlobWithRetries() throws Exception {
238238
final int maxRetries = randomIntBetween(2, 10);
239239
final AtomicInteger countDown = new AtomicInteger(maxRetries);
240240

241-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null, null, null);
241+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(maxRetries).build();
242242

243243
// SDK reads in 2 MB chunks so we use twice that to simulate 2 chunks
244244
final byte[] bytes = randomBytes(1 << 22);
@@ -267,7 +267,7 @@ public void testWriteBlobWithRetries() throws Exception {
267267
final int maxRetries = randomIntBetween(2, 10);
268268
final CountDown countDown = new CountDown(maxRetries);
269269

270-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null, null, null);
270+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(maxRetries).build();
271271
final byte[] bytes = randomBlobContent();
272272
httpServer.createContext("/upload/storage/v1/b/bucket/o", safeHandler(exchange -> {
273273
assertThat(exchange.getRequestURI().getQuery(), containsString("uploadType=multipart"));
@@ -309,7 +309,7 @@ public void testWriteBlobWithRetries() throws Exception {
309309
public void testWriteBlobWithReadTimeouts() {
310310
final byte[] bytes = randomByteArrayOfLength(randomIntBetween(10, 128));
311311
final TimeValue readTimeout = TimeValue.timeValueMillis(randomIntBetween(100, 500));
312-
final BlobContainer blobContainer = createBlobContainer(1, readTimeout, null, null, null, null, null);
312+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(1).readTimeout(readTimeout).build();
313313

314314
// HTTP server does not send a response
315315
httpServer.createContext("/upload/storage/v1/b/bucket/o", exchange -> {
@@ -361,7 +361,7 @@ public void testWriteLargeBlob() throws IOException {
361361
logger.debug("starting with resumable upload id [{}]", sessionUploadId.get());
362362

363363
final TimeValue readTimeout = allowReadTimeout.get() ? TimeValue.timeValueSeconds(3) : null;
364-
final BlobContainer blobContainer = createBlobContainer(nbErrors + 1, readTimeout, null, null, null, null, null);
364+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(nbErrors + 1).readTimeout(readTimeout).build();
365365

366366
httpServer.createContext("/upload/storage/v1/b/bucket/o", safeHandler(exchange -> {
367367
final BytesReference requestBody = Streams.readFully(exchange.getRequestBody());
@@ -508,7 +508,7 @@ public String next() {
508508
return Integer.toString(totalDeletesSent++);
509509
}
510510
};
511-
final BlobContainer blobContainer = createBlobContainer(1, null, null, null, null, null, null);
511+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(1).build();
512512
httpServer.createContext("/batch/storage/v1", safeHandler(exchange -> {
513513
assert pendingDeletes.get() <= MAX_DELETES_PER_BATCH;
514514

@@ -544,7 +544,7 @@ public void testCompareAndExchangeWhenThrottled() throws IOException {
544544
httpServer.createContext("/", new ResponseInjectingHttpHandler(requestHandlers, new GoogleCloudStorageHttpHandler("bucket")));
545545

546546
final int maxRetries = randomIntBetween(1, 3);
547-
final BlobContainer container = createBlobContainer(maxRetries, null, null, null, null, null, null);
547+
final BlobContainer container = blobContainerBuilder().maxRetries(maxRetries).build();
548548
final byte[] data = randomBytes(randomIntBetween(1, BlobContainerUtils.MAX_REGISTER_CONTENT_LENGTH));
549549
final String key = randomIdentifier();
550550

‎modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java‎

Lines changed: 57 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -270,7 +270,7 @@ public void testWriteBlobWithRetries() throws Exception {
270270
final int maxRetries = randomInt(5);
271271
final CountDown countDown = new CountDown(maxRetries + 1);
272272

273-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, null, null, null, null);
273+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(maxRetries).disableChunkedEncoding(true).build();
274274

275275
final byte[] bytes = randomBlobContent();
276276
httpServer.createContext(downloadStorageEndpoint(blobContainer, "write_blob_max_retries"), exchange -> {
@@ -318,7 +318,10 @@ public void testWriteBlobWithRetries() throws Exception {
318318
public void testWriteBlobWithReadTimeouts() {
319319
final byte[] bytes = randomByteArrayOfLength(randomIntBetween(10, 128));
320320
final TimeValue readTimeout = TimeValue.timeValueMillis(randomIntBetween(100, 500));
321-
final BlobContainer blobContainer = createBlobContainer(1, readTimeout, true, null, null, null, null);
321+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(1)
322+
.readTimeout(readTimeout)
323+
.disableChunkedEncoding(true)
324+
.build();
322325

323326
// HTTP server does not send a response
324327
httpServer.createContext(downloadStorageEndpoint(blobContainer, "write_blob_timeout"), exchange -> {
@@ -352,7 +355,10 @@ public void testWriteLargeBlob() throws Exception {
352355
final boolean useTimeout = rarely();
353356
final TimeValue readTimeout = useTimeout ? TimeValue.timeValueMillis(randomIntBetween(100, 500)) : null;
354357
final ByteSizeValue bufferSize = ByteSizeValue.of(5, ByteSizeUnit.MB);
355-
final BlobContainer blobContainer = createBlobContainer(null, readTimeout, true, null, bufferSize, null, null);
358+
final BlobContainer blobContainer = blobContainerBuilder().readTimeout(readTimeout)
359+
.disableChunkedEncoding(true)
360+
.bufferSize(bufferSize)
361+
.build();
356362

357363
final int parts = randomIntBetween(1, 5);
358364
final long lastPartSize = randomLongBetween(10, 512);
@@ -448,7 +454,10 @@ public void testWriteLargeBlobStreaming() throws Exception {
448454
final boolean useTimeout = rarely();
449455
final TimeValue readTimeout = useTimeout ? TimeValue.timeValueMillis(randomIntBetween(100, 500)) : null;
450456
final ByteSizeValue bufferSize = ByteSizeValue.of(5, ByteSizeUnit.MB);
451-
final BlobContainer blobContainer = createBlobContainer(null, readTimeout, true, null, bufferSize, null, null);
457+
final BlobContainer blobContainer = blobContainerBuilder().readTimeout(readTimeout)
458+
.disableChunkedEncoding(true)
459+
.bufferSize(bufferSize)
460+
.build();
452461

453462
final int parts = randomIntBetween(1, 5);
454463
final long lastPartSize = randomLongBetween(10, 512);
@@ -557,15 +566,10 @@ public void testReadRetriesAfterMeaningfulProgress() throws Exception {
557566
0,
558567
randomFrom(1000, Math.toIntExact(S3Repository.BUFFER_SIZE_SETTING.get(Settings.EMPTY).getBytes()))
559568
);
560-
final BlobContainer blobContainer = createBlobContainer(
561-
maxRetries,
562-
null,
563-
true,
564-
null,
565-
ByteSizeValue.ofBytes(bufferSizeBytes),
566-
null,
567-
null
568-
);
569+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(maxRetries)
570+
.disableChunkedEncoding(true)
571+
.bufferSize(ByteSizeValue.ofBytes(bufferSizeBytes))
572+
.build();
569573
final int meaningfulProgressBytes = Math.max(1, bufferSizeBytes / 100);
570574

571575
final byte[] bytes = randomBlobContent();
@@ -638,15 +642,10 @@ public void testReadDoesNotRetryForRepositoryAnalysis() {
638642
0,
639643
randomFrom(1000, Math.toIntExact(S3Repository.BUFFER_SIZE_SETTING.get(Settings.EMPTY).getBytes()))
640644
);
641-
final BlobContainer blobContainer = createBlobContainer(
642-
maxRetries,
643-
null,
644-
true,
645-
null,
646-
ByteSizeValue.ofBytes(bufferSizeBytes),
647-
null,
648-
null
649-
);
645+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(maxRetries)
646+
.disableChunkedEncoding(true)
647+
.bufferSize(ByteSizeValue.ofBytes(bufferSizeBytes))
648+
.build();
650649

651650
final byte[] bytes = randomBlobContent();
652651

@@ -684,15 +683,10 @@ public void testReadWithIndicesPurposeRetriesForever() throws IOException {
684683
0,
685684
randomFrom(1000, Math.toIntExact(S3Repository.BUFFER_SIZE_SETTING.get(Settings.EMPTY).getBytes()))
686685
);
687-
final BlobContainer blobContainer = createBlobContainer(
688-
maxRetries,
689-
null,
690-
true,
691-
null,
692-
ByteSizeValue.ofBytes(bufferSizeBytes),
693-
null,
694-
null
695-
);
686+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(maxRetries)
687+
.disableChunkedEncoding(true)
688+
.bufferSize(ByteSizeValue.ofBytes(bufferSizeBytes))
689+
.build();
696690
final int meaningfulProgressBytes = Math.max(1, bufferSizeBytes / 100);
697691

698692
final byte[] bytes = randomBlobContent(512);
@@ -785,7 +779,7 @@ public void handle(HttpExchange exchange) throws IOException {
785779

786780
public void testDoesNotRetryOnNotFound() {
787781
final int maxRetries = between(3, 5);
788-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, null, null, null, null);
782+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(maxRetries).disableChunkedEncoding(true).build();
789783

790784
final AtomicInteger numberOfReads = new AtomicInteger(0);
791785
@SuppressForbidden(reason = "use a http server")
@@ -816,8 +810,11 @@ public void handle(HttpExchange exchange) throws IOException {
816810
}
817811

818812
public void testSnapshotDeletesRetryOnThrottlingError() throws IOException {
819-
// disable AWS-client retries
820-
final BlobContainer blobContainer = createBlobContainer(0, null, true, null, null, null, null);
813+
final BlobContainer blobContainer = blobContainerBuilder()
814+
// disable AWS-client retries
815+
.maxRetries(0)
816+
.disableChunkedEncoding(true)
817+
.build();
821818

822819
int numBlobsToDelete = randomIntBetween(500, 3000);
823820
List<String> blobsToDelete = new ArrayList<>();
@@ -836,8 +833,11 @@ public void testSnapshotDeletesRetryOnThrottlingError() throws IOException {
836833
}
837834

838835
public void testSnapshotDeletesAbortRetriesWhenThreadIsInterrupted() {
839-
// disable AWS-client retries
840-
final BlobContainer blobContainer = createBlobContainer(0, null, true, null, null, null, null);
836+
final BlobContainer blobContainer = blobContainerBuilder()
837+
// disable AWS-client retries
838+
.maxRetries(0)
839+
.disableChunkedEncoding(true)
840+
.build();
841841

842842
int numBlobsToDelete = randomIntBetween(500, 3000);
843843
List<String> blobsToDelete = new ArrayList<>();
@@ -873,8 +873,11 @@ public void testSnapshotDeletesAbortRetriesWhenThreadIsInterrupted() {
873873
}
874874

875875
public void testNonSnapshotDeletesAreNotRetried() {
876-
// disable AWS-client retries
877-
final BlobContainer blobContainer = createBlobContainer(0, null, true, null, null, null, null);
876+
final BlobContainer blobContainer = blobContainerBuilder()
877+
// disable AWS-client retries
878+
.maxRetries(0)
879+
.disableChunkedEncoding(true)
880+
.build();
878881

879882
int numBlobsToDelete = randomIntBetween(500, 3000);
880883
List<String> blobsToDelete = new ArrayList<>();
@@ -902,8 +905,11 @@ public void testNonSnapshotDeletesAreNotRetried() {
902905
}
903906

904907
public void testNonThrottlingErrorsAreNotRetried() {
905-
// disable AWS-client retries
906-
final BlobContainer blobContainer = createBlobContainer(0, null, true, null, null, null, null);
908+
final BlobContainer blobContainer = blobContainerBuilder()
909+
// disable AWS-client retries
910+
.maxRetries(0)
911+
.disableChunkedEncoding(true)
912+
.build();
907913

908914
int numBlobsToDelete = randomIntBetween(500, 3000);
909915
List<String> blobsToDelete = new ArrayList<>();
@@ -982,7 +988,7 @@ private Set<OperationPurpose> operationPurposesThatRetryOnDelete() {
982988

983989
public void testGetRegisterRetries() {
984990
final var maxRetries = between(0, 3);
985-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null, null, null);
991+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(maxRetries).build();
986992

987993
interface FailingHandlerFactory {
988994
void addHandler(String blobName, Integer... responseCodes);
@@ -1052,7 +1058,11 @@ interface FailingHandlerFactory {
10521058
public void testSuppressedDeletionErrorsAreCapped() {
10531059
final TimeValue readTimeout = TimeValue.timeValueMillis(randomIntBetween(100, 500));
10541060
int maxBulkDeleteSize = randomIntBetween(1, 10);
1055-
final BlobContainer blobContainer = createBlobContainer(1, readTimeout, true, null, null, maxBulkDeleteSize, null);
1061+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(1)
1062+
.readTimeout(readTimeout)
1063+
.disableChunkedEncoding(true)
1064+
.maxBulkDeletes(maxBulkDeleteSize)
1065+
.build();
10561066
httpServer.createContext("/", exchange -> {
10571067
if (isMultiDeleteRequest(exchange)) {
10581068
exchange.sendResponseHeaders(
@@ -1084,7 +1094,11 @@ public void testSuppressedDeletionErrorsAreCapped() {
10841094
public void testTrimmedLogAndCappedSuppressedErrorOnMultiObjectDeletionException() {
10851095
final TimeValue readTimeout = TimeValue.timeValueMillis(randomIntBetween(100, 500));
10861096
int maxBulkDeleteSize = randomIntBetween(10, 30);
1087-
final BlobContainer blobContainer = createBlobContainer(1, readTimeout, true, null, null, maxBulkDeleteSize, null);
1097+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(1)
1098+
.readTimeout(readTimeout)
1099+
.disableChunkedEncoding(true)
1100+
.maxBulkDeletes(maxBulkDeleteSize)
1101+
.build();
10881102

10891103
final Pattern pattern = Pattern.compile("<Key>(.+?)</Key>");
10901104
httpServer.createContext("/", exchange -> {

0 commit comments

Comments
 (0)