diff --git a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerRetriesTests.java b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerRetriesTests.java index 83906f6a36387..6c03ed45eaa39 100644 --- a/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerRetriesTests.java +++ b/modules/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerRetriesTests.java @@ -352,7 +352,7 @@ public void testWriteBlobWithRetries() throws Exception { } public void testWriteLargeBlob() throws Exception { - final int maxRetries = randomIntBetween(2, 5); + final int maxRetries = randomIntBetween(4, 8); logger.info("--> max retries: {}", maxRetries); final byte[] data = randomBytes(ByteSizeUnit.MB.toIntBytes(10) + randomIntBetween(0, ByteSizeUnit.MB.toIntBytes(1))); @@ -427,7 +427,6 @@ public void testWriteLargeBlob() throws Exception { try (InputStream stream = new InputStreamIndexInput(new ByteArrayIndexInput("desc", data), data.length)) { blobContainer.writeBlob(randomPurpose(), "write_large_blob", stream, data.length, false); } - assertThat(countDownUploads.get(), equalTo(0)); assertThat(countDownComplete.isCountedDown(), is(true)); assertThat(blocks.isEmpty(), is(true)); @@ -669,11 +668,11 @@ public void testRetriesAreTerminatedWhenClientProviderIsClosed() { } private BlobContainer createBlobContainer(int maxRetries, String secondaryHost, LocationMode locationMode) { - return createBlobContainer(maxRetries, null, null, null, null, null, BlobPath.EMPTY, secondaryHost, locationMode); + return createBlobContainer(maxRetries, null, null, null, null, null, null, BlobPath.EMPTY, secondaryHost, locationMode); } private BlobContainer createBlobContainer(int maxRetries) { - return createBlobContainer(maxRetries, null, null, null, null, null, null); + return createBlobContainer(maxRetries, null, null, null, null, null, null, null); } @Override @@ -694,17 +693,19 @@ protected Class unresponsiveExceptionType() { @Override protected BlobContainer createBlobContainer( - @Nullable Integer maxRetries, - @Nullable TimeValue readTimeout, - @Nullable Boolean disableChunkedEncoding, - @Nullable Integer maxConnections, - @Nullable ByteSizeValue bufferSize, - @Nullable Integer maxBulkDeletes, - @Nullable BlobPath blobContainerPath + final @Nullable Integer maxRetries, + final @Nullable TimeValue readTimeout, + final @Nullable TimeValue requestTimeout, + final @Nullable Boolean disableChunkedEncoding, + final @Nullable Integer maxConnections, + final @Nullable ByteSizeValue bufferSize, + final @Nullable Integer maxBulkDeletes, + final @Nullable BlobPath blobContainerPath ) { return createBlobContainer( maxRetries, readTimeout, + requestTimeout, disableChunkedEncoding, maxConnections, bufferSize, @@ -718,6 +719,7 @@ protected BlobContainer createBlobContainer( private BlobContainer createBlobContainer( @Nullable Integer maxRetries, @Nullable TimeValue readTimeout, + @Nullable TimeValue timeout, @Nullable Boolean disableChunkedEncoding, @Nullable Integer maxConnections, @Nullable ByteSizeValue bufferSize, @@ -742,7 +744,11 @@ private BlobContainer createBlobContainer( if (maxRetries != null) { clientSettings.put(MAX_RETRIES_SETTING.getConcreteSettingForNamespace(clientName).getKey(), maxRetries); } - clientSettings.put(TIMEOUT_SETTING.getConcreteSettingForNamespace(clientName).getKey(), TimeValue.timeValueSeconds(1)); + if (timeout != null) { + clientSettings.put(TIMEOUT_SETTING.getConcreteSettingForNamespace(clientName).getKey(), timeout); + } else { + clientSettings.put(TIMEOUT_SETTING.getConcreteSettingForNamespace(clientName).getKey(), SAFE_AWAIT_TIMEOUT); + } if (readTimeout != null) { clientSettings.put(READ_TIMEOUT_SETTING.getConcreteSettingForNamespace(clientName).getKey(), readTimeout); } diff --git a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java index a4b99d9740487..ef856f9d0f6ef 100644 --- a/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java +++ b/modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java @@ -141,6 +141,7 @@ protected Class unresponsiveExceptionType() { protected BlobContainer createBlobContainer( final @Nullable Integer maxRetries, final @Nullable TimeValue readTimeout, + final @Nullable TimeValue requestTimeout, final @Nullable Boolean disableChunkedEncoding, final @Nullable Integer maxConnections, final @Nullable ByteSizeValue bufferSize, @@ -633,7 +634,7 @@ public void testContentsChangeWhileStreaming() throws IOException { // The blob needs to be large enough that it won't be entirely buffered on the first request final int enoughBytesToNotBeEntirelyBuffered = Math.toIntExact(ByteSizeValue.ofMb(30).getBytes()); - final BlobContainer container = createBlobContainer(1, null, null, null, null, null, null); + final BlobContainer container = createBlobContainer(1, null, null, null, null, null, null, null); final String key = randomIdentifier(); byte[] initialValue = randomByteArrayOfLength(enoughBytesToNotBeEntirelyBuffered); diff --git a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java index e57eff50bee15..3962ae104b1e3 100644 --- a/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java +++ b/modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java @@ -194,6 +194,7 @@ protected Class unresponsiveExceptionType() { protected BlobContainer createBlobContainer( final @Nullable Integer maxRetries, final @Nullable TimeValue readTimeout, + final @Nullable TimeValue requestTimeout, final @Nullable Boolean disableChunkedEncoding, final @Nullable Integer maxConnections, final @Nullable ByteSizeValue bufferSize, @@ -599,7 +600,7 @@ public void testMaxConnections() throws InterruptedException, IOException { final CountDownLatch requestReceived = new CountDownLatch(1); final CountDownLatch releaseRequest = new CountDownLatch(1); int maxConnections = 1; - final BlobContainer blobContainer = createBlobContainer(null, null, null, maxConnections, null, null, null); + final BlobContainer blobContainer = createBlobContainer(null, null, null, null, maxConnections, null, null, null); // Setting up a simple request handler that returns NOT_FOUND, so as to avoid setting up a response. @SuppressForbidden(reason = "use a http server") @@ -1267,7 +1268,7 @@ public void testRetryOn403InStateless() { final var denyAccessAfterAttempt = between(1, 5); logger.info("--> maxRetries = {}, denyAccessAfterAttempt = {}", maxRetries, denyAccessAfterAttempt); final var blobContainerPath = BlobPath.EMPTY.add(getTestName()); - final var statefulBlobContainer = createBlobContainer(maxRetries, null, null, null, null, null, blobContainerPath); + final var statefulBlobContainer = createBlobContainer(maxRetries, null, null, null, null, null, null, blobContainerPath); final var requestCount = new AtomicInteger(); final var invalidAccessKeyIdResponseCount = new AtomicInteger(); final var accessDeniedResponseCount = new AtomicInteger(); @@ -1330,7 +1331,7 @@ public void testRetryOn403InStateless() { ); service.start(); recordingMeterRegistry = new RecordingMeterRegistry(); - final var statelessBlobContainer = createBlobContainer(maxRetries, null, null, null, null, null, blobContainerPath); + final var statelessBlobContainer = createBlobContainer(maxRetries, null, null, null, null, null, null, blobContainerPath); assertThat( ExceptionsHelper.unwrap( @@ -1349,7 +1350,7 @@ public void testRetryOn403InStateless() { public void testUploadNotFoundInCompareAndExchange() { final var blobContainerPath = BlobPath.EMPTY.add(getTestName()); - final var statefulBlobContainer = createBlobContainer(1, null, null, null, null, null, blobContainerPath); + final var statefulBlobContainer = createBlobContainer(1, null, null, null, null, null, null, blobContainerPath); @SuppressForbidden(reason = "use a http server") class RejectsUploadPartRequests extends S3HttpHandler { @@ -1386,7 +1387,7 @@ public void handle(HttpExchange exchange) throws IOException { public void testCompareAndExchangeWithConcurrentPutObject() throws Exception { final var blobContainerPath = BlobPath.EMPTY.add(getTestName()); - final var statefulBlobContainer = createBlobContainer(1, null, null, null, null, null, blobContainerPath); + final var statefulBlobContainer = createBlobContainer(1, null, null, null, null, null, null, blobContainerPath); final var objectContentsRequestedLatch = new CountDownLatch(1); @@ -1460,7 +1461,7 @@ public void testContentsChangeWhileStreaming() throws IOException { // The blob needs to be large enough that it won't be entirely buffered on the first request final int enoughBytesToNotBeEntirelyBuffered = Math.toIntExact(ByteSizeValue.ofMb(30).getBytes()); - final BlobContainer container = createBlobContainer(1, null, null, null, null, null, null); + final BlobContainer container = createBlobContainer(1, null, null, null, null, null, null, null); final String key = randomIdentifier(); byte[] initialValue = randomByteArrayOfLength(enoughBytesToNotBeEntirelyBuffered); diff --git a/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobContainerRetriesTests.java b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobContainerRetriesTests.java index e0df31e0d5297..0904947c8d8c5 100644 --- a/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobContainerRetriesTests.java +++ b/modules/repository-url/src/test/java/org/elasticsearch/common/blobstore/url/URLBlobContainerRetriesTests.java @@ -76,6 +76,7 @@ protected Matcher readTimeoutExceptionMatcher() { protected BlobContainer createBlobContainer( Integer maxRetries, TimeValue readTimeout, + TimeValue requestTimeout, Boolean disableChunkedEncoding, Integer maxConnections, ByteSizeValue bufferSize, diff --git a/muted-tests.yml b/muted-tests.yml index 72214ac4e4953..8ecc5a81bb3ab 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -228,9 +228,6 @@ tests: - class: org.elasticsearch.xpack.esql.heap_attack.HeapAttackLimitByIT method: testTopNByKeyEncoderTooMuchMemory issue: https://github.com/elastic/elasticsearch/issues/145627 -- class: org.elasticsearch.repositories.azure.AzureBlobContainerRetriesTests - method: testWriteLargeBlob - issue: https://github.com/elastic/elasticsearch/issues/145654 - class: org.elasticsearch.xpack.esql.CsvIT method: test {csv-spec:spatial_shapes.convertFromStringParseError} issue: https://github.com/elastic/elasticsearch/issues/145655 diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java index b8b9f1894ff21..bdd563d1526a0 100644 --- a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/AbstractBlobContainerRetriesTestCase.java @@ -274,6 +274,7 @@ public void testReadBlobWithReadTimeouts() { final BlobContainer blobContainer = blobContainerBuilder().maxRetries(maxRetries) .bufferSize(bufferSize) .readTimeout(readTimeout) + .requestTimeout(TimeValue.timeValueSeconds(1)) .build(); // HTTP server does not send a response @@ -585,6 +586,7 @@ private void ensureOpen() throws IOException { protected abstract BlobContainer createBlobContainer( @Nullable Integer maxRetries, @Nullable TimeValue readTimeout, + @Nullable TimeValue requestTimeout, @Nullable Boolean disableChunkedEncoding, @Nullable Integer maxConnections, @Nullable ByteSizeValue bufferSize, @@ -598,6 +600,8 @@ protected final class TestBlobContainerBuilder { @Nullable private TimeValue readTimeout; @Nullable + private TimeValue requestTimeout; + @Nullable private Boolean disableChunkedEncoding; @Nullable private Integer maxConnections; @@ -618,6 +622,11 @@ public TestBlobContainerBuilder readTimeout(@Nullable TimeValue readTimeout) { return this; } + public TestBlobContainerBuilder requestTimeout(@Nullable TimeValue requestTimeout) { + this.requestTimeout = requestTimeout; + return this; + } + public TestBlobContainerBuilder disableChunkedEncoding(@Nullable Boolean disableChunkedEncoding) { this.disableChunkedEncoding = disableChunkedEncoding; return this; @@ -647,6 +656,7 @@ public BlobContainer build() { return createBlobContainer( maxRetries, readTimeout, + requestTimeout, disableChunkedEncoding, maxConnections, bufferSize,