Skip to content

Commit e00de75

Browse files
DaveCTurnerDaveCTurner
authored
Allow overriding blob container path in tests (elastic#126428)
Some `AbstractBlobContainerRetriesTestCase#createBlobContainer` implementations choose a path for the container randomly, but we have a need for a test which re-creates the same container against a different `S3Service` and `BlobStore` and must therefore specify the same path each time. This commit exposes a parameter that lets callers specify a container path. Backport of elastic#126391 to `8.x`
1 parent 2778f2a commit e00de75

File tree

4 files changed

+40
-28
lines changed

4 files changed

+40
-28
lines changed

‎modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,8 @@ protected BlobContainer createBlobContainer(
112112
final @Nullable TimeValue readTimeout,
113113
final @Nullable Boolean disableChunkedEncoding,
114114
final @Nullable ByteSizeValue bufferSize,
115-
final @Nullable Integer maxBulkDeletes
115+
final @Nullable Integer maxBulkDeletes,
116+
final @Nullable BlobPath blobContainerPath
116117
) {
117118
final Settings.Builder clientSettings = Settings.builder();
118119
final String client = randomAlphaOfLength(5).toLowerCase(Locale.ROOT);
@@ -165,14 +166,17 @@ StorageOptions createStorageOptions(
165166
randomIntBetween(1, 8) * 1024
166167
);
167168

168-
return new GoogleCloudStorageBlobContainer(randomBoolean() ? BlobPath.EMPTY : BlobPath.EMPTY.add("foo"), blobStore);
169+
return new GoogleCloudStorageBlobContainer(
170+
Objects.requireNonNullElse(blobContainerPath, randomBoolean() ? BlobPath.EMPTY : BlobPath.EMPTY.add("foo")),
171+
blobStore
172+
);
169173
}
170174

171175
public void testReadLargeBlobWithRetries() throws Exception {
172176
final int maxRetries = randomIntBetween(2, 10);
173177
final AtomicInteger countDown = new AtomicInteger(maxRetries);
174178

175-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null);
179+
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null, null);
176180

177181
// SDK reads in 2 MB chunks so we use twice that to simulate 2 chunks
178182
final byte[] bytes = randomBytes(1 << 22);
@@ -201,7 +205,7 @@ public void testWriteBlobWithRetries() throws Exception {
201205
final int maxRetries = randomIntBetween(2, 10);
202206
final CountDown countDown = new CountDown(maxRetries);
203207

204-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null);
208+
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null, null);
205209
final byte[] bytes = randomBlobContent();
206210
httpServer.createContext("/upload/storage/v1/b/bucket/o", safeHandler(exchange -> {
207211
assertThat(exchange.getRequestURI().getQuery(), containsString("uploadType=multipart"));
@@ -243,7 +247,7 @@ public void testWriteBlobWithRetries() throws Exception {
243247
public void testWriteBlobWithReadTimeouts() {
244248
final byte[] bytes = randomByteArrayOfLength(randomIntBetween(10, 128));
245249
final TimeValue readTimeout = TimeValue.timeValueMillis(randomIntBetween(100, 500));
246-
final BlobContainer blobContainer = createBlobContainer(1, readTimeout, null, null, null);
250+
final BlobContainer blobContainer = createBlobContainer(1, readTimeout, null, null, null, null);
247251

248252
// HTTP server does not send a response
249253
httpServer.createContext("/upload/storage/v1/b/bucket/o", exchange -> {
@@ -296,7 +300,7 @@ public void testWriteLargeBlob() throws IOException {
296300
logger.debug("starting with resumable upload id [{}]", sessionUploadId.get());
297301

298302
final TimeValue readTimeout = allowReadTimeout.get() ? TimeValue.timeValueSeconds(3) : null;
299-
final BlobContainer blobContainer = createBlobContainer(nbErrors + 1, readTimeout, null, null, null);
303+
final BlobContainer blobContainer = createBlobContainer(nbErrors + 1, readTimeout, null, null, null, null);
300304

301305
httpServer.createContext("/upload/storage/v1/b/bucket/o", safeHandler(exchange -> {
302306
final BytesReference requestBody = Streams.readFully(exchange.getRequestBody());
@@ -436,7 +440,7 @@ public String next() {
436440
return Integer.toString(totalDeletesSent++);
437441
}
438442
};
439-
final BlobContainer blobContainer = createBlobContainer(1, null, null, null, null);
443+
final BlobContainer blobContainer = createBlobContainer(1, null, null, null, null, null);
440444
httpServer.createContext("/batch/storage/v1", safeHandler(exchange -> {
441445
assert pendingDeletes.get() <= MAX_DELETES_PER_BATCH;
442446

‎modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java‎

Lines changed: 16 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,8 @@ protected BlobContainer createBlobContainer(
155155
final @Nullable TimeValue readTimeout,
156156
final @Nullable Boolean disableChunkedEncoding,
157157
final @Nullable ByteSizeValue bufferSize,
158-
final @Nullable Integer maxBulkDeletes
158+
final @Nullable Integer maxBulkDeletes,
159+
final @Nullable BlobPath blobContainerPath
159160
) {
160161
final Settings.Builder clientSettings = Settings.builder();
161162
final String clientName = randomAlphaOfLength(5).toLowerCase(Locale.ROOT);
@@ -204,7 +205,10 @@ protected BlobContainer createBlobContainer(
204205
new DeterministicTaskQueue().getThreadPool(),
205206
new S3RepositoriesMetrics(new RepositoriesMetrics(recordingMeterRegistry))
206207
);
207-
return new S3BlobContainer(randomBoolean() ? BlobPath.EMPTY : BlobPath.EMPTY.add("foo"), s3BlobStore) {
208+
return new S3BlobContainer(
209+
Objects.requireNonNullElse(blobContainerPath, randomBoolean() ? BlobPath.EMPTY : BlobPath.EMPTY.add("foo")),
210+
s3BlobStore
211+
) {
208212
@Override
209213
public InputStream readBlob(OperationPurpose purpose, String blobName) throws IOException {
210214
return new AssertingInputStream(new S3RetryingInputStream(purpose, s3BlobStore, buildKey(blobName)) {
@@ -249,7 +253,7 @@ public void testWriteBlobWithRetries() throws Exception {
249253
final int maxRetries = randomInt(5);
250254
final CountDown countDown = new CountDown(maxRetries + 1);
251255

252-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, null, null);
256+
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, null, null, null);
253257

254258
final byte[] bytes = randomBlobContent();
255259
httpServer.createContext(downloadStorageEndpoint(blobContainer, "write_blob_max_retries"), exchange -> {
@@ -297,7 +301,7 @@ public void testWriteBlobWithRetries() throws Exception {
297301
public void testWriteBlobWithReadTimeouts() {
298302
final byte[] bytes = randomByteArrayOfLength(randomIntBetween(10, 128));
299303
final TimeValue readTimeout = TimeValue.timeValueMillis(randomIntBetween(100, 500));
300-
final BlobContainer blobContainer = createBlobContainer(1, readTimeout, true, null, null);
304+
final BlobContainer blobContainer = createBlobContainer(1, readTimeout, true, null, null, null);
301305

302306
// HTTP server does not send a response
303307
httpServer.createContext(downloadStorageEndpoint(blobContainer, "write_blob_timeout"), exchange -> {
@@ -331,7 +335,7 @@ public void testWriteLargeBlob() throws Exception {
331335
final boolean useTimeout = rarely();
332336
final TimeValue readTimeout = useTimeout ? TimeValue.timeValueMillis(randomIntBetween(100, 500)) : null;
333337
final ByteSizeValue bufferSize = new ByteSizeValue(5, ByteSizeUnit.MB);
334-
final BlobContainer blobContainer = createBlobContainer(null, readTimeout, true, bufferSize, null);
338+
final BlobContainer blobContainer = createBlobContainer(null, readTimeout, true, bufferSize, null, null);
335339

336340
final int parts = randomIntBetween(1, 5);
337341
final long lastPartSize = randomLongBetween(10, 512);
@@ -427,7 +431,7 @@ public void testWriteLargeBlobStreaming() throws Exception {
427431
final boolean useTimeout = rarely();
428432
final TimeValue readTimeout = useTimeout ? TimeValue.timeValueMillis(randomIntBetween(100, 500)) : null;
429433
final ByteSizeValue bufferSize = new ByteSizeValue(5, ByteSizeUnit.MB);
430-
final BlobContainer blobContainer = createBlobContainer(null, readTimeout, true, bufferSize, null);
434+
final BlobContainer blobContainer = createBlobContainer(null, readTimeout, true, bufferSize, null, null);
431435

432436
final int parts = randomIntBetween(1, 5);
433437
final long lastPartSize = randomLongBetween(10, 512);
@@ -536,7 +540,7 @@ public void testReadRetriesAfterMeaningfulProgress() throws Exception {
536540
0,
537541
randomFrom(1000, Math.toIntExact(S3Repository.BUFFER_SIZE_SETTING.get(Settings.EMPTY).getBytes()))
538542
);
539-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, ByteSizeValue.ofBytes(bufferSizeBytes), null);
543+
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, ByteSizeValue.ofBytes(bufferSizeBytes), null, null);
540544
final int meaningfulProgressBytes = Math.max(1, bufferSizeBytes / 100);
541545

542546
final byte[] bytes = randomBlobContent();
@@ -609,7 +613,7 @@ public void testReadDoesNotRetryForRepositoryAnalysis() {
609613
0,
610614
randomFrom(1000, Math.toIntExact(S3Repository.BUFFER_SIZE_SETTING.get(Settings.EMPTY).getBytes()))
611615
);
612-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, ByteSizeValue.ofBytes(bufferSizeBytes), null);
616+
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, ByteSizeValue.ofBytes(bufferSizeBytes), null, null);
613617

614618
final byte[] bytes = randomBlobContent();
615619

@@ -647,7 +651,7 @@ public void testReadWithIndicesPurposeRetriesForever() throws IOException {
647651
0,
648652
randomFrom(1000, Math.toIntExact(S3Repository.BUFFER_SIZE_SETTING.get(Settings.EMPTY).getBytes()))
649653
);
650-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, ByteSizeValue.ofBytes(bufferSizeBytes), null);
654+
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, ByteSizeValue.ofBytes(bufferSizeBytes), null, null);
651655
final int meaningfulProgressBytes = Math.max(1, bufferSizeBytes / 100);
652656

653657
final byte[] bytes = randomBlobContent(512);
@@ -740,7 +744,7 @@ public void handle(HttpExchange exchange) throws IOException {
740744

741745
public void testDoesNotRetryOnNotFound() {
742746
final int maxRetries = between(3, 5);
743-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, null, null);
747+
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, null, null, null);
744748

745749
final AtomicInteger numberOfReads = new AtomicInteger(0);
746750
@SuppressForbidden(reason = "use a http server")
@@ -773,7 +777,7 @@ public void handle(HttpExchange exchange) throws IOException {
773777
public void testSuppressedDeletionErrorsAreCapped() {
774778
final TimeValue readTimeout = TimeValue.timeValueMillis(randomIntBetween(100, 500));
775779
int maxBulkDeleteSize = randomIntBetween(1, 10);
776-
final BlobContainer blobContainer = createBlobContainer(1, readTimeout, true, null, maxBulkDeleteSize);
780+
final BlobContainer blobContainer = createBlobContainer(1, readTimeout, true, null, maxBulkDeleteSize, null);
777781
httpServer.createContext("/", exchange -> {
778782
if (isMultiDeleteRequest(exchange)) {
779783
exchange.sendResponseHeaders(
@@ -805,7 +809,7 @@ public void testSuppressedDeletionErrorsAreCapped() {
805809
public void testTrimmedLogAndCappedSuppressedErrorOnMultiObjectDeletionException() {
806810
final TimeValue readTimeout = TimeValue.timeValueMillis(randomIntBetween(100, 500));
807811
int maxBulkDeleteSize = randomIntBetween(10, 30);
808-
final BlobContainer blobContainer = createBlobContainer(1, readTimeout, true, null, maxBulkDeleteSize);
812+
final BlobContainer blobContainer = createBlobContainer(1, readTimeout, true, null, maxBulkDeleteSize, null);
809813

810814
final Pattern pattern = Pattern.compile("<Key>(.+?)</Key>");
811815
httpServer.createContext("/", exchange -> {

‎modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobContainerRetriesTests.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import java.net.MalformedURLException;
3030
import java.net.SocketTimeoutException;
3131
import java.net.URL;
32+
import java.util.Objects;
3233

3334
import static org.hamcrest.Matchers.either;
3435
import static org.hamcrest.Matchers.instanceOf;
@@ -77,7 +78,8 @@ protected BlobContainer createBlobContainer(
7778
TimeValue readTimeout,
7879
Boolean disableChunkedEncoding,
7980
ByteSizeValue bufferSize,
80-
Integer maxBulkDeletes
81+
Integer maxBulkDeletes,
82+
BlobPath blobContainerPath
8183
) {
8284
Settings.Builder settingsBuilder = Settings.builder();
8385

@@ -98,7 +100,7 @@ protected BlobContainer createBlobContainer(
98100
factory.create(httpClientSettings),
99101
httpClientSettings
100102
);
101-
return urlBlobStore.blobContainer(BlobPath.EMPTY);
103+
return urlBlobStore.blobContainer(Objects.requireNonNullElse(blobContainerPath, BlobPath.EMPTY));
102104
} catch (MalformedURLException e) {
103105
throw new RuntimeException("Unable to create URLBlobStore", e);
104106
}

‎test/framework/src/main/java/org/elasticsearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import org.apache.http.ConnectionClosedException;
1616
import org.apache.http.HttpStatus;
1717
import org.elasticsearch.common.blobstore.BlobContainer;
18+
import org.elasticsearch.common.blobstore.BlobPath;
1819
import org.elasticsearch.common.blobstore.OperationPurpose;
1920
import org.elasticsearch.common.bytes.BytesReference;
2021
import org.elasticsearch.common.io.Streams;
@@ -83,7 +84,8 @@ protected abstract BlobContainer createBlobContainer(
8384
@Nullable TimeValue readTimeout,
8485
@Nullable Boolean disableChunkedEncoding,
8586
@Nullable ByteSizeValue bufferSize,
86-
@Nullable Integer maxBulkDeletes
87+
@Nullable Integer maxBulkDeletes,
88+
@Nullable BlobPath blobContainerPath
8789
);
8890

8991
protected org.hamcrest.Matcher<Object> readTimeoutExceptionMatcher() {
@@ -92,7 +94,7 @@ protected org.hamcrest.Matcher<Object> readTimeoutExceptionMatcher() {
9294
}
9395

9496
public void testReadNonexistentBlobThrowsNoSuchFileException() {
95-
final BlobContainer blobContainer = createBlobContainer(between(1, 5), null, null, null, null);
97+
final BlobContainer blobContainer = createBlobContainer(between(1, 5), null, null, null, null, null);
9698
final long position = randomLongBetween(0, MAX_RANGE_VAL);
9799
final int length = randomIntBetween(1, Math.toIntExact(Math.min(Integer.MAX_VALUE, MAX_RANGE_VAL - position)));
98100
final Exception exception = expectThrows(NoSuchFileException.class, () -> {
@@ -119,7 +121,7 @@ public void testReadBlobWithRetries() throws Exception {
119121

120122
final byte[] bytes = randomBlobContent();
121123
final TimeValue readTimeout = TimeValue.timeValueSeconds(between(1, 3));
122-
final BlobContainer blobContainer = createBlobContainer(maxRetries, readTimeout, null, null, null);
124+
final BlobContainer blobContainer = createBlobContainer(maxRetries, readTimeout, null, null, null, null);
123125
httpServer.createContext(downloadStorageEndpoint(blobContainer, "read_blob_max_retries"), exchange -> {
124126
Streams.readFully(exchange.getRequestBody());
125127
if (countDown.countDown()) {
@@ -176,7 +178,7 @@ public void testReadRangeBlobWithRetries() throws Exception {
176178
final CountDown countDown = new CountDown(maxRetries + 1);
177179

178180
final TimeValue readTimeout = TimeValue.timeValueSeconds(between(5, 10));
179-
final BlobContainer blobContainer = createBlobContainer(maxRetries, readTimeout, null, null, null);
181+
final BlobContainer blobContainer = createBlobContainer(maxRetries, readTimeout, null, null, null, null);
180182
final byte[] bytes = randomBlobContent();
181183
httpServer.createContext(downloadStorageEndpoint(blobContainer, "read_range_blob_max_retries"), exchange -> {
182184
Streams.readFully(exchange.getRequestBody());
@@ -248,7 +250,7 @@ public void testReadRangeBlobWithRetries() throws Exception {
248250
public void testReadBlobWithReadTimeouts() {
249251
final int maxRetries = randomInt(5);
250252
final TimeValue readTimeout = TimeValue.timeValueMillis(between(100, 200));
251-
final BlobContainer blobContainer = createBlobContainer(maxRetries, readTimeout, null, null, null);
253+
final BlobContainer blobContainer = createBlobContainer(maxRetries, readTimeout, null, null, null, null);
252254

253255
// HTTP server does not send a response
254256
httpServer.createContext(downloadStorageEndpoint(blobContainer, "read_blob_unresponsive"), exchange -> {});
@@ -305,7 +307,7 @@ protected OperationPurpose randomFiniteRetryingPurpose() {
305307

306308
public void testReadBlobWithNoHttpResponse() {
307309
final TimeValue readTimeout = TimeValue.timeValueMillis(between(100, 200));
308-
final BlobContainer blobContainer = createBlobContainer(randomInt(5), readTimeout, null, null, null);
310+
final BlobContainer blobContainer = createBlobContainer(randomInt(5), readTimeout, null, null, null, null);
309311

310312
// HTTP server closes connection immediately
311313
httpServer.createContext(downloadStorageEndpoint(blobContainer, "read_blob_no_response"), HttpExchange::close);
@@ -325,7 +327,7 @@ public void testReadBlobWithNoHttpResponse() {
325327

326328
public void testReadBlobWithPrematureConnectionClose() {
327329
final int maxRetries = randomInt(20);
328-
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null);
330+
final BlobContainer blobContainer = createBlobContainer(maxRetries, null, null, null, null, null);
329331

330332
final boolean alwaysFlushBody = randomBoolean();
331333

0 commit comments

Comments
 (0)