Skip to content

Commit 2764be3

Browse files
DaveCTurnerDaveCTurner
committed
Introduce TestBlobContainerBuilder
The mostly-optional parameters to `createBlobContainer` are getting rather numerous in this test which makes the tests hard to read. This commit introduces a builder to help name the provided parameters and skip the omitted ones.
1 parent 47e352f commit 2764be3

File tree

3 files changed

+141
-41
lines changed

3 files changed

+141
-41
lines changed

‎modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -228,7 +228,7 @@ public void testShouldRetryOnUnresolvableHost() {
228228

229229
private void executeListBlobsAndAssertRetries() {
230230
final int maxRetries = randomIntBetween(3, 5);
231-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null, null);
231+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(maxRetries).build();
232232
expectThrows(StorageException.class, () -> blobContainer.listBlobs(randomPurpose()));
233233
assertEquals(maxRetries + 1, requestCounters.get("/storage/v1/b/bucket/o").get());
234234
}
@@ -237,7 +237,7 @@ public void testReadLargeBlobWithRetries() throws Exception {
237237
final int maxRetries = randomIntBetween(2, 10);
238238
final AtomicInteger countDown = new AtomicInteger(maxRetries);
239239

240-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null, null);
240+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(maxRetries).build();
241241

242242
// SDK reads in 2 MB chunks so we use twice that to simulate 2 chunks
243243
final byte[] bytes = randomBytes(1 << 22);
@@ -266,7 +266,7 @@ public void testWriteBlobWithRetries() throws Exception {
266266
final int maxRetries = randomIntBetween(2, 10);
267267
final CountDown countDown = new CountDown(maxRetries);
268268

269-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null, null);
269+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(maxRetries).build();
270270
final byte[] bytes = randomBlobContent();
271271
httpServer.createContext("/upload/storage/v1/b/bucket/o", safeHandler(exchange -> {
272272
assertThat(exchange.getRequestURI().getQuery(), containsString("uploadType=multipart"));
@@ -308,7 +308,7 @@ public void testWriteBlobWithRetries() throws Exception {
308308
public void testWriteBlobWithReadTimeouts() {
309309
final byte[] bytes = randomByteArrayOfLength(randomIntBetween(10, 128));
310310
final TimeValue readTimeout = TimeValue.timeValueMillis(randomIntBetween(100, 500));
311-
final BlobContainer blobContainer = createBlobContainer(1, readTimeout, null, null, null, null);
311+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(1).readTimeout(readTimeout).build();
312312

313313
// HTTP server does not send a response
314314
httpServer.createContext("/upload/storage/v1/b/bucket/o", exchange -> {
@@ -360,7 +360,7 @@ public void testWriteLargeBlob() throws IOException {
360360
logger.debug("starting with resumable upload id [{}]", sessionUploadId.get());
361361

362362
final TimeValue readTimeout = allowReadTimeout.get() ? TimeValue.timeValueSeconds(3) : null;
363-
final BlobContainer blobContainer = createBlobContainer(nbErrors + 1, readTimeout, null, null, null, null);
363+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(nbErrors + 1).readTimeout(readTimeout).build();
364364

365365
httpServer.createContext("/upload/storage/v1/b/bucket/o", safeHandler(exchange -> {
366366
final BytesReference requestBody = Streams.readFully(exchange.getRequestBody());
@@ -507,7 +507,7 @@ public String next() {
507507
return Integer.toString(totalDeletesSent++);
508508
}
509509
};
510-
final BlobContainer blobContainer = createBlobContainer(1, null, null, null, null, null);
510+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(1).build();
511511
httpServer.createContext("/batch/storage/v1", safeHandler(exchange -> {
512512
assert pendingDeletes.get() <= MAX_DELETES_PER_BATCH;
513513

@@ -543,7 +543,7 @@ public void testCompareAndExchangeWhenThrottled() throws IOException {
543543
httpServer.createContext("/", new ResponseInjectingHttpHandler(requestHandlers, new GoogleCloudStorageHttpHandler("bucket")));
544544

545545
final int maxRetries = randomIntBetween(1, 3);
546-
final BlobContainer container = createBlobContainer(maxRetries, null, null, null, null, null);
546+
final BlobContainer container = blobContainerBuilder().maxRetries(maxRetries).build();
547547
final byte[] data = randomBytes(randomIntBetween(1, BlobContainerUtils.MAX_REGISTER_CONTENT_LENGTH));
548548
final String key = randomIdentifier();
549549

‎modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java‎

Lines changed: 59 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -265,7 +265,7 @@ public void testWriteBlobWithRetries() throws Exception {
265265
final int maxRetries = randomInt(5);
266266
final CountDown countDown = new CountDown(maxRetries + 1);
267267

268-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, null, null, null);
268+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(maxRetries).disableChunkedEncoding(true).build();
269269

270270
final byte[] bytes = randomBlobContent();
271271
httpServer.createContext(downloadStorageEndpoint(blobContainer, "write_blob_max_retries"), exchange -> {
@@ -313,7 +313,10 @@ public void testWriteBlobWithRetries() throws Exception {
313313
public void testWriteBlobWithReadTimeouts() {
314314
final byte[] bytes = randomByteArrayOfLength(randomIntBetween(10, 128));
315315
final TimeValue readTimeout = TimeValue.timeValueMillis(randomIntBetween(100, 500));
316-
final BlobContainer blobContainer = createBlobContainer(1, readTimeout, true, null, null, null);
316+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(1)
317+
.readTimeout(readTimeout)
318+
.disableChunkedEncoding(true)
319+
.build();
317320

318321
// HTTP server does not send a response
319322
httpServer.createContext(downloadStorageEndpoint(blobContainer, "write_blob_timeout"), exchange -> {
@@ -347,7 +350,11 @@ public void testWriteLargeBlob() throws Exception {
347350
final boolean useTimeout = rarely();
348351
final TimeValue readTimeout = useTimeout ? TimeValue.timeValueMillis(randomIntBetween(100, 500)) : null;
349352
final ByteSizeValue bufferSize = ByteSizeValue.of(5, ByteSizeUnit.MB);
350-
final BlobContainer blobContainer = createBlobContainer(null, readTimeout, true, bufferSize, null, null);
353+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(null)
354+
.readTimeout(readTimeout)
355+
.disableChunkedEncoding(true)
356+
.bufferSize(bufferSize)
357+
.build();
351358

352359
final int parts = randomIntBetween(1, 5);
353360
final long lastPartSize = randomLongBetween(10, 512);
@@ -443,7 +450,11 @@ public void testWriteLargeBlobStreaming() throws Exception {
443450
final boolean useTimeout = rarely();
444451
final TimeValue readTimeout = useTimeout ? TimeValue.timeValueMillis(randomIntBetween(100, 500)) : null;
445452
final ByteSizeValue bufferSize = ByteSizeValue.of(5, ByteSizeUnit.MB);
446-
final BlobContainer blobContainer = createBlobContainer(null, readTimeout, true, bufferSize, null, null);
453+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(null)
454+
.readTimeout(readTimeout)
455+
.disableChunkedEncoding(true)
456+
.bufferSize(bufferSize)
457+
.build();
447458

448459
final int parts = randomIntBetween(1, 5);
449460
final long lastPartSize = randomLongBetween(10, 512);
@@ -552,7 +563,10 @@ public void testReadRetriesAfterMeaningfulProgress() throws Exception {
552563
0,
553564
randomFrom(1000, Math.toIntExact(S3Repository.BUFFER_SIZE_SETTING.get(Settings.EMPTY).getBytes()))
554565
);
555-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, ByteSizeValue.ofBytes(bufferSizeBytes), null, null);
566+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(maxRetries)
567+
.disableChunkedEncoding(true)
568+
.bufferSize(ByteSizeValue.ofBytes(bufferSizeBytes))
569+
.build();
556570
final int meaningfulProgressBytes = Math.max(1, bufferSizeBytes / 100);
557571

558572
final byte[] bytes = randomBlobContent();
@@ -625,7 +639,10 @@ public void testReadDoesNotRetryForRepositoryAnalysis() {
625639
0,
626640
randomFrom(1000, Math.toIntExact(S3Repository.BUFFER_SIZE_SETTING.get(Settings.EMPTY).getBytes()))
627641
);
628-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, ByteSizeValue.ofBytes(bufferSizeBytes), null, null);
642+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(maxRetries)
643+
.disableChunkedEncoding(true)
644+
.bufferSize(ByteSizeValue.ofBytes(bufferSizeBytes))
645+
.build();
629646

630647
final byte[] bytes = randomBlobContent();
631648

@@ -663,7 +680,10 @@ public void testReadWithIndicesPurposeRetriesForever() throws IOException {
663680
0,
664681
randomFrom(1000, Math.toIntExact(S3Repository.BUFFER_SIZE_SETTING.get(Settings.EMPTY).getBytes()))
665682
);
666-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, ByteSizeValue.ofBytes(bufferSizeBytes), null, null);
683+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(maxRetries)
684+
.disableChunkedEncoding(true)
685+
.bufferSize(ByteSizeValue.ofBytes(bufferSizeBytes))
686+
.build();
667687
final int meaningfulProgressBytes = Math.max(1, bufferSizeBytes / 100);
668688

669689
final byte[] bytes = randomBlobContent(512);
@@ -756,7 +776,7 @@ public void handle(HttpExchange exchange) throws IOException {
756776

757777
public void testDoesNotRetryOnNotFound() {
758778
final int maxRetries = between(3, 5);
759-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, null, null, null);
779+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(maxRetries).disableChunkedEncoding(true).build();
760780

761781
final AtomicInteger numberOfReads = new AtomicInteger(0);
762782
@SuppressForbidden(reason = "use a http server")
@@ -787,8 +807,11 @@ public void handle(HttpExchange exchange) throws IOException {
787807
}
788808

789809
public void testSnapshotDeletesRetryOnThrottlingError() throws IOException {
790-
// disable AWS-client retries
791-
final BlobContainer blobContainer = createBlobContainer(0, null, true, null, null, null);
810+
final BlobContainer blobContainer = blobContainerBuilder()
811+
// disable AWS-client retries
812+
.maxRetries(0)
813+
.disableChunkedEncoding(true)
814+
.build();
792815

793816
int numBlobsToDelete = randomIntBetween(500, 3000);
794817
List<String> blobsToDelete = new ArrayList<>();
@@ -807,8 +830,11 @@ public void testSnapshotDeletesRetryOnThrottlingError() throws IOException {
807830
}
808831

809832
public void testSnapshotDeletesAbortRetriesWhenThreadIsInterrupted() {
810-
// disable AWS-client retries
811-
final BlobContainer blobContainer = createBlobContainer(0, null, true, null, null, null);
833+
final BlobContainer blobContainer = blobContainerBuilder()
834+
// disable AWS-client retries
835+
.maxRetries(0)
836+
.disableChunkedEncoding(true)
837+
.build();
812838

813839
int numBlobsToDelete = randomIntBetween(500, 3000);
814840
List<String> blobsToDelete = new ArrayList<>();
@@ -844,8 +870,11 @@ public void testSnapshotDeletesAbortRetriesWhenThreadIsInterrupted() {
844870
}
845871

846872
public void testNonSnapshotDeletesAreNotRetried() {
847-
// disable AWS-client retries
848-
final BlobContainer blobContainer = createBlobContainer(0, null, true, null, null, null);
873+
final BlobContainer blobContainer = blobContainerBuilder()
874+
// disable AWS-client retries
875+
.maxRetries(0)
876+
.disableChunkedEncoding(true)
877+
.build();
849878

850879
int numBlobsToDelete = randomIntBetween(500, 3000);
851880
List<String> blobsToDelete = new ArrayList<>();
@@ -873,8 +902,11 @@ public void testNonSnapshotDeletesAreNotRetried() {
873902
}
874903

875904
public void testNonThrottlingErrorsAreNotRetried() {
876-
// disable AWS-client retries
877-
final BlobContainer blobContainer = createBlobContainer(0, null, true, null, null, null);
905+
final BlobContainer blobContainer = blobContainerBuilder()
906+
// disable AWS-client retries
907+
.maxRetries(0)
908+
.disableChunkedEncoding(true)
909+
.build();
878910

879911
int numBlobsToDelete = randomIntBetween(500, 3000);
880912
List<String> blobsToDelete = new ArrayList<>();
@@ -953,7 +985,7 @@ private Set<OperationPurpose> operationPurposesThatRetryOnDelete() {
953985

954986
public void testGetRegisterRetries() {
955987
final var maxRetries = between(0, 3);
956-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null, null);
988+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(maxRetries).build();
957989

958990
interface FailingHandlerFactory {
959991
void addHandler(String blobName, Integer... responseCodes);
@@ -1023,7 +1055,11 @@ interface FailingHandlerFactory {
10231055
public void testSuppressedDeletionErrorsAreCapped() {
10241056
final TimeValue readTimeout = TimeValue.timeValueMillis(randomIntBetween(100, 500));
10251057
int maxBulkDeleteSize = randomIntBetween(1, 10);
1026-
final BlobContainer blobContainer = createBlobContainer(1, readTimeout, true, null, maxBulkDeleteSize, null);
1058+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(1)
1059+
.readTimeout(readTimeout)
1060+
.disableChunkedEncoding(true)
1061+
.maxBulkDeletes(maxBulkDeleteSize)
1062+
.build();
10271063
httpServer.createContext("/", exchange -> {
10281064
if (isMultiDeleteRequest(exchange)) {
10291065
exchange.sendResponseHeaders(
@@ -1055,7 +1091,11 @@ public void testSuppressedDeletionErrorsAreCapped() {
10551091
public void testTrimmedLogAndCappedSuppressedErrorOnMultiObjectDeletionException() {
10561092
final TimeValue readTimeout = TimeValue.timeValueMillis(randomIntBetween(100, 500));
10571093
int maxBulkDeleteSize = randomIntBetween(10, 30);
1058-
final BlobContainer blobContainer = createBlobContainer(1, readTimeout, true, null, maxBulkDeleteSize, null);
1094+
final BlobContainer blobContainer = blobContainerBuilder().maxRetries(1)
1095+
.readTimeout(readTimeout)
1096+
.disableChunkedEncoding(true)
1097+
.maxBulkDeletes(maxBulkDeleteSize)
1098+
.build();
10591099

10601100
final Pattern pattern = Pattern.compile("<Key>(.+?)</Key>");
10611101
httpServer.createContext("/", exchange -> {

0 commit comments

Comments
 (0)