Skip to content

Commit fbbbdd7

Browse files
DaveCTurnerDaveCTurner
authored
Allow overriding blob container path in tests (#126391)
Some `AbstractBlobContainerRetriesTestCase#createBlobContainer` implementations choose a path for the container randomly, but we have a need for a test which re-creates the same container against a different `S3Service` and `BlobStore` and must therefore specify the same path each time. This commit exposes a parameter that lets callers specify a container path.
1 parent 5dc7ab7 commit fbbbdd7

File tree

4 files changed

+47
-35
lines changed

4 files changed

+47
-35
lines changed

‎modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,8 @@ protected BlobContainer createBlobContainer(
127127
final @Nullable TimeValue readTimeout,
128128
final @Nullable Boolean disableChunkedEncoding,
129129
final @Nullable ByteSizeValue bufferSize,
130-
final @Nullable Integer maxBulkDeletes
130+
final @Nullable Integer maxBulkDeletes,
131+
final @Nullable BlobPath blobContainerPath
131132
) {
132133
final Settings.Builder clientSettings = Settings.builder();
133134
final String client = randomAlphaOfLength(5).toLowerCase(Locale.ROOT);
@@ -207,7 +208,10 @@ public HttpRequestInitializer getHttpRequestInitializer(ServiceOptions<?, ?> ser
207208
new GcsRepositoryStatsCollector()
208209
);
209210

210-
return new GoogleCloudStorageBlobContainer(randomBoolean() ? BlobPath.EMPTY : BlobPath.EMPTY.add("foo"), blobStore);
211+
return new GoogleCloudStorageBlobContainer(
212+
Objects.requireNonNullElse(blobContainerPath, randomBoolean() ? BlobPath.EMPTY : BlobPath.EMPTY.add("foo")),
213+
blobStore
214+
);
211215
}
212216

213217
public void testShouldRetryOnConnectionRefused() {
@@ -224,7 +228,7 @@ public void testShouldRetryOnUnresolvableHost() {
224228

225229
private void executeListBlobsAndAssertRetries() {
226230
final int maxRetries = randomIntBetween(3, 5);
227-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null);
231+
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null, null);
228232
expectThrows(StorageException.class, () -> blobContainer.listBlobs(randomPurpose()));
229233
assertEquals(maxRetries + 1, requestCounters.get("/storage/v1/b/bucket/o").get());
230234
}
@@ -233,7 +237,7 @@ public void testReadLargeBlobWithRetries() throws Exception {
233237
final int maxRetries = randomIntBetween(2, 10);
234238
final AtomicInteger countDown = new AtomicInteger(maxRetries);
235239

236-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null);
240+
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null, null);
237241

238242
// SDK reads in 2 MB chunks so we use twice that to simulate 2 chunks
239243
final byte[] bytes = randomBytes(1 << 22);
@@ -262,7 +266,7 @@ public void testWriteBlobWithRetries() throws Exception {
262266
final int maxRetries = randomIntBetween(2, 10);
263267
final CountDown countDown = new CountDown(maxRetries);
264268

265-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null);
269+
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null, null);
266270
final byte[] bytes = randomBlobContent();
267271
httpServer.createContext("/upload/storage/v1/b/bucket/o", safeHandler(exchange -> {
268272
assertThat(exchange.getRequestURI().getQuery(), containsString("uploadType=multipart"));
@@ -304,7 +308,7 @@ public void testWriteBlobWithRetries() throws Exception {
304308
public void testWriteBlobWithReadTimeouts() {
305309
final byte[] bytes = randomByteArrayOfLength(randomIntBetween(10, 128));
306310
final TimeValue readTimeout = TimeValue.timeValueMillis(randomIntBetween(100, 500));
307-
final BlobContainer blobContainer = createBlobContainer(1, readTimeout, null, null, null);
311+
final BlobContainer blobContainer = createBlobContainer(1, readTimeout, null, null, null, null);
308312

309313
// HTTP server does not send a response
310314
httpServer.createContext("/upload/storage/v1/b/bucket/o", exchange -> {
@@ -356,7 +360,7 @@ public void testWriteLargeBlob() throws IOException {
356360
logger.debug("starting with resumable upload id [{}]", sessionUploadId.get());
357361

358362
final TimeValue readTimeout = allowReadTimeout.get() ? TimeValue.timeValueSeconds(3) : null;
359-
final BlobContainer blobContainer = createBlobContainer(nbErrors + 1, readTimeout, null, null, null);
363+
final BlobContainer blobContainer = createBlobContainer(nbErrors + 1, readTimeout, null, null, null, null);
360364

361365
httpServer.createContext("/upload/storage/v1/b/bucket/o", safeHandler(exchange -> {
362366
final BytesReference requestBody = Streams.readFully(exchange.getRequestBody());
@@ -503,7 +507,7 @@ public String next() {
503507
return Integer.toString(totalDeletesSent++);
504508
}
505509
};
506-
final BlobContainer blobContainer = createBlobContainer(1, null, null, null, null);
510+
final BlobContainer blobContainer = createBlobContainer(1, null, null, null, null, null);
507511
httpServer.createContext("/batch/storage/v1", safeHandler(exchange -> {
508512
assert pendingDeletes.get() <= MAX_DELETES_PER_BATCH;
509513

@@ -539,7 +543,7 @@ public void testCompareAndExchangeWhenThrottled() throws IOException {
539543
httpServer.createContext("/", new ResponseInjectingHttpHandler(requestHandlers, new GoogleCloudStorageHttpHandler("bucket")));
540544

541545
final int maxRetries = randomIntBetween(1, 3);
542-
final BlobContainer container = createBlobContainer(maxRetries, null, null, null, null);
546+
final BlobContainer container = createBlobContainer(maxRetries, null, null, null, null, null);
543547
final byte[] data = randomBytes(randomIntBetween(1, BlobContainerUtils.MAX_REGISTER_CONTENT_LENGTH));
544548
final String key = randomIdentifier();
545549

‎modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java‎

Lines changed: 21 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,8 @@ protected BlobContainer createBlobContainer(
164164
final @Nullable TimeValue readTimeout,
165165
final @Nullable Boolean disableChunkedEncoding,
166166
final @Nullable ByteSizeValue bufferSize,
167-
final @Nullable Integer maxBulkDeletes
167+
final @Nullable Integer maxBulkDeletes,
168+
final @Nullable BlobPath blobContainerPath
168169
) {
169170
final Settings.Builder clientSettings = Settings.builder();
170171
final String clientName = randomAlphaOfLength(5).toLowerCase(Locale.ROOT);
@@ -216,7 +217,10 @@ protected BlobContainer createBlobContainer(
216217
new S3RepositoriesMetrics(new RepositoriesMetrics(recordingMeterRegistry)),
217218
BackoffPolicy.constantBackoff(TimeValue.timeValueMillis(1), MAX_NUMBER_SNAPSHOT_DELETE_RETRIES)
218219
);
219-
return new S3BlobContainer(randomBoolean() ? BlobPath.EMPTY : BlobPath.EMPTY.add("foo"), s3BlobStore) {
220+
return new S3BlobContainer(
221+
Objects.requireNonNullElse(blobContainerPath, randomBoolean() ? BlobPath.EMPTY : BlobPath.EMPTY.add("foo")),
222+
s3BlobStore
223+
) {
220224
@Override
221225
public InputStream readBlob(OperationPurpose purpose, String blobName) throws IOException {
222226
return new AssertingInputStream(new S3RetryingInputStream(purpose, s3BlobStore, buildKey(blobName)) {
@@ -261,7 +265,7 @@ public void testWriteBlobWithRetries() throws Exception {
261265
final int maxRetries = randomInt(5);
262266
final CountDown countDown = new CountDown(maxRetries + 1);
263267

264-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, null, null);
268+
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, null, null, null);
265269

266270
final byte[] bytes = randomBlobContent();
267271
httpServer.createContext(downloadStorageEndpoint(blobContainer, "write_blob_max_retries"), exchange -> {
@@ -309,7 +313,7 @@ public void testWriteBlobWithRetries() throws Exception {
309313
public void testWriteBlobWithReadTimeouts() {
310314
final byte[] bytes = randomByteArrayOfLength(randomIntBetween(10, 128));
311315
final TimeValue readTimeout = TimeValue.timeValueMillis(randomIntBetween(100, 500));
312-
final BlobContainer blobContainer = createBlobContainer(1, readTimeout, true, null, null);
316+
final BlobContainer blobContainer = createBlobContainer(1, readTimeout, true, null, null, null);
313317

314318
// HTTP server does not send a response
315319
httpServer.createContext(downloadStorageEndpoint(blobContainer, "write_blob_timeout"), exchange -> {
@@ -343,7 +347,7 @@ public void testWriteLargeBlob() throws Exception {
343347
final boolean useTimeout = rarely();
344348
final TimeValue readTimeout = useTimeout ? TimeValue.timeValueMillis(randomIntBetween(100, 500)) : null;
345349
final ByteSizeValue bufferSize = ByteSizeValue.of(5, ByteSizeUnit.MB);
346-
final BlobContainer blobContainer = createBlobContainer(null, readTimeout, true, bufferSize, null);
350+
final BlobContainer blobContainer = createBlobContainer(null, readTimeout, true, bufferSize, null, null);
347351

348352
final int parts = randomIntBetween(1, 5);
349353
final long lastPartSize = randomLongBetween(10, 512);
@@ -439,7 +443,7 @@ public void testWriteLargeBlobStreaming() throws Exception {
439443
final boolean useTimeout = rarely();
440444
final TimeValue readTimeout = useTimeout ? TimeValue.timeValueMillis(randomIntBetween(100, 500)) : null;
441445
final ByteSizeValue bufferSize = ByteSizeValue.of(5, ByteSizeUnit.MB);
442-
final BlobContainer blobContainer = createBlobContainer(null, readTimeout, true, bufferSize, null);
446+
final BlobContainer blobContainer = createBlobContainer(null, readTimeout, true, bufferSize, null, null);
443447

444448
final int parts = randomIntBetween(1, 5);
445449
final long lastPartSize = randomLongBetween(10, 512);
@@ -548,7 +552,7 @@ public void testReadRetriesAfterMeaningfulProgress() throws Exception {
548552
0,
549553
randomFrom(1000, Math.toIntExact(S3Repository.BUFFER_SIZE_SETTING.get(Settings.EMPTY).getBytes()))
550554
);
551-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, ByteSizeValue.ofBytes(bufferSizeBytes), null);
555+
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, ByteSizeValue.ofBytes(bufferSizeBytes), null, null);
552556
final int meaningfulProgressBytes = Math.max(1, bufferSizeBytes / 100);
553557

554558
final byte[] bytes = randomBlobContent();
@@ -621,7 +625,7 @@ public void testReadDoesNotRetryForRepositoryAnalysis() {
621625
0,
622626
randomFrom(1000, Math.toIntExact(S3Repository.BUFFER_SIZE_SETTING.get(Settings.EMPTY).getBytes()))
623627
);
624-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, ByteSizeValue.ofBytes(bufferSizeBytes), null);
628+
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, ByteSizeValue.ofBytes(bufferSizeBytes), null, null);
625629

626630
final byte[] bytes = randomBlobContent();
627631

@@ -659,7 +663,7 @@ public void testReadWithIndicesPurposeRetriesForever() throws IOException {
659663
0,
660664
randomFrom(1000, Math.toIntExact(S3Repository.BUFFER_SIZE_SETTING.get(Settings.EMPTY).getBytes()))
661665
);
662-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, ByteSizeValue.ofBytes(bufferSizeBytes), null);
666+
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, ByteSizeValue.ofBytes(bufferSizeBytes), null, null);
663667
final int meaningfulProgressBytes = Math.max(1, bufferSizeBytes / 100);
664668

665669
final byte[] bytes = randomBlobContent(512);
@@ -752,7 +756,7 @@ public void handle(HttpExchange exchange) throws IOException {
752756

753757
public void testDoesNotRetryOnNotFound() {
754758
final int maxRetries = between(3, 5);
755-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, null, null);
759+
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, null, null, null);
756760

757761
final AtomicInteger numberOfReads = new AtomicInteger(0);
758762
@SuppressForbidden(reason = "use a http server")
@@ -784,7 +788,7 @@ public void handle(HttpExchange exchange) throws IOException {
784788

785789
public void testSnapshotDeletesRetryOnThrottlingError() throws IOException {
786790
// disable AWS-client retries
787-
final BlobContainer blobContainer = createBlobContainer(0, null, true, null, null);
791+
final BlobContainer blobContainer = createBlobContainer(0, null, true, null, null, null);
788792

789793
int numBlobsToDelete = randomIntBetween(500, 3000);
790794
List<String> blobsToDelete = new ArrayList<>();
@@ -804,7 +808,7 @@ public void testSnapshotDeletesRetryOnThrottlingError() throws IOException {
804808

805809
public void testSnapshotDeletesAbortRetriesWhenThreadIsInterrupted() {
806810
// disable AWS-client retries
807-
final BlobContainer blobContainer = createBlobContainer(0, null, true, null, null);
811+
final BlobContainer blobContainer = createBlobContainer(0, null, true, null, null, null);
808812

809813
int numBlobsToDelete = randomIntBetween(500, 3000);
810814
List<String> blobsToDelete = new ArrayList<>();
@@ -841,7 +845,7 @@ public void testSnapshotDeletesAbortRetriesWhenThreadIsInterrupted() {
841845

842846
public void testNonSnapshotDeletesAreNotRetried() {
843847
// disable AWS-client retries
844-
final BlobContainer blobContainer = createBlobContainer(0, null, true, null, null);
848+
final BlobContainer blobContainer = createBlobContainer(0, null, true, null, null, null);
845849

846850
int numBlobsToDelete = randomIntBetween(500, 3000);
847851
List<String> blobsToDelete = new ArrayList<>();
@@ -870,7 +874,7 @@ public void testNonSnapshotDeletesAreNotRetried() {
870874

871875
public void testNonThrottlingErrorsAreNotRetried() {
872876
// disable AWS-client retries
873-
final BlobContainer blobContainer = createBlobContainer(0, null, true, null, null);
877+
final BlobContainer blobContainer = createBlobContainer(0, null, true, null, null, null);
874878

875879
int numBlobsToDelete = randomIntBetween(500, 3000);
876880
List<String> blobsToDelete = new ArrayList<>();
@@ -949,7 +953,7 @@ private Set<OperationPurpose> operationPurposesThatRetryOnDelete() {
949953

950954
public void testGetRegisterRetries() {
951955
final var maxRetries = between(0, 3);
952-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null);
956+
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null, null);
953957

954958
interface FailingHandlerFactory {
955959
void addHandler(String blobName, Integer... responseCodes);
@@ -1019,7 +1023,7 @@ interface FailingHandlerFactory {
10191023
public void testSuppressedDeletionErrorsAreCapped() {
10201024
final TimeValue readTimeout = TimeValue.timeValueMillis(randomIntBetween(100, 500));
10211025
int maxBulkDeleteSize = randomIntBetween(1, 10);
1022-
final BlobContainer blobContainer = createBlobContainer(1, readTimeout, true, null, maxBulkDeleteSize);
1026+
final BlobContainer blobContainer = createBlobContainer(1, readTimeout, true, null, maxBulkDeleteSize, null);
10231027
httpServer.createContext("/", exchange -> {
10241028
if (isMultiDeleteRequest(exchange)) {
10251029
exchange.sendResponseHeaders(
@@ -1051,7 +1055,7 @@ public void testSuppressedDeletionErrorsAreCapped() {
10511055
public void testTrimmedLogAndCappedSuppressedErrorOnMultiObjectDeletionException() {
10521056
final TimeValue readTimeout = TimeValue.timeValueMillis(randomIntBetween(100, 500));
10531057
int maxBulkDeleteSize = randomIntBetween(10, 30);
1054-
final BlobContainer blobContainer = createBlobContainer(1, readTimeout, true, null, maxBulkDeleteSize);
1058+
final BlobContainer blobContainer = createBlobContainer(1, readTimeout, true, null, maxBulkDeleteSize, null);
10551059

10561060
final Pattern pattern = Pattern.compile("<Key>(.+?)</Key>");
10571061
httpServer.createContext("/", exchange -> {

‎modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobContainerRetriesTests.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import java.net.MalformedURLException;
3030
import java.net.SocketTimeoutException;
3131
import java.net.URL;
32+
import java.util.Objects;
3233

3334
import static org.hamcrest.Matchers.either;
3435
import static org.hamcrest.Matchers.instanceOf;
@@ -77,7 +78,8 @@ protected BlobContainer createBlobContainer(
7778
TimeValue readTimeout,
7879
Boolean disableChunkedEncoding,
7980
ByteSizeValue bufferSize,
80-
Integer maxBulkDeletes
81+
Integer maxBulkDeletes,
82+
BlobPath blobContainerPath
8183
) {
8284
Settings.Builder settingsBuilder = Settings.builder();
8385

@@ -98,7 +100,7 @@ protected BlobContainer createBlobContainer(
98100
factory.create(httpClientSettings),
99101
httpClientSettings
100102
);
101-
return urlBlobStore.blobContainer(BlobPath.EMPTY);
103+
return urlBlobStore.blobContainer(Objects.requireNonNullElse(blobContainerPath, BlobPath.EMPTY));
102104
} catch (MalformedURLException e) {
103105
throw new RuntimeException("Unable to create URLBlobStore", e);
104106
}

0 commit comments

Comments
 (0)