diff --git a/api/src/main/java/io/minio/BaseS3Client.java b/api/src/main/java/io/minio/BaseS3Client.java index aaf7c18a8..f8a5b404e 100644 --- a/api/src/main/java/io/minio/BaseS3Client.java +++ b/api/src/main/java/io/minio/BaseS3Client.java @@ -1,6 +1,6 @@ /* * MinIO Java SDK for Amazon S3 Compatible Cloud Storage, - * (C) 2025 MinIO, Inc. + * (C) 2025-2026 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -104,28 +104,54 @@ public abstract class BaseS3Client implements AutoCloseable { private static final String UPLOAD_ID = "uploadId"; private static final Set TRACE_QUERY_PARAMS = ImmutableSet.of("retention", "legal-hold", "tagging", UPLOAD_ID, "acl", "attributes"); + private PrintWriter traceStream; protected final Map regionCache = new ConcurrentHashMap<>(); protected String userAgent = Utils.getDefaultUserAgent(); protected Http.BaseUrl baseUrl; protected Provider provider; - protected OkHttpClient httpClient; + protected volatile OkHttpClient httpClient; protected boolean closeHttpClient; + /** + * Maximum attempts per S3 request. Effective only on the SDK-default {@link OkHttpClient}; + * caller-supplied clients are used verbatim and the SDK does not modify their retry policy. + */ + volatile int maxRetries = Http.RetryInterceptor.MAX_RETRY; + protected BaseS3Client( Http.BaseUrl baseUrl, Provider provider, OkHttpClient httpClient, boolean closeHttpClient) { this.baseUrl = baseUrl; this.provider = provider; - this.httpClient = httpClient; this.closeHttpClient = closeHttpClient; + this.httpClient = httpClient; } + /** + * Copies share the source's {@link OkHttpClient} (and its retry interceptor binding), so {@link + * #setMaxRetries(int)} on a copy does not affect the shared retry budget. + */ protected BaseS3Client(BaseS3Client client) { this.baseUrl = client.baseUrl; this.provider = client.provider; - this.httpClient = client.httpClient; this.closeHttpClient = client.closeHttpClient; + this.maxRetries = client.maxRetries; + this.httpClient = client.httpClient; + } + + /** + * Sets the maximum number of attempts for transient HTTP failures. Pass {@code 1} to disable + * automatic retries. Default {@code 10}. + * + *

Effective only on the SDK-default {@link OkHttpClient}; caller-supplied clients are used + * verbatim. + * + * @param maxRetries maximum attempts (must be {@code >= 1}). + */ + public void setMaxRetries(int maxRetries) { + if (maxRetries < 1) throw new IllegalArgumentException("maxRetries must be >= 1"); + this.maxRetries = maxRetries; } /** Closes underneath HTTP client. */ @@ -182,6 +208,11 @@ public void setAppInfo(String name, String version) { /** * Enables HTTP call tracing and written to traceStream. * + *

Retry caveat. Tracing happens at the SDK callback level, so only the final response + * of a retry sequence is recorded. Per-attempt traces (which the {@link Http.RetryInterceptor} + * sees and discards) are not surfaced here. To inspect individual retry attempts, register an + * OkHttp {@code HttpLoggingInterceptor} on a custom client. + * * @param traceStream {@link OutputStream} for writing HTTP call tracing. * @see #traceOff */ @@ -268,7 +299,10 @@ private String[] handleRedirectResponse( return new String[] {code, message}; } - /** Execute HTTP request asynchronously for given parameters. */ + /** + * Execute HTTP request asynchronously. Retry handling is delegated to {@link + * Http.RetryInterceptor} on the {@link OkHttpClient}, so this method itself never loops. + */ protected CompletableFuture executeAsync(Http.S3Request s3request, String region) { Credentials credentials = (provider == null) ? null : provider.fetch(); Http.Request request = null; @@ -282,15 +316,9 @@ protected CompletableFuture executeAsync(Http.S3Request s3request, Str PrintWriter traceStream = this.traceStream; if (traceStream != null) traceStream.print(request.httpTraces()); - OkHttpClient httpClient = this.httpClient; - // FIXME: enable retry for all request. - // if (!s3request.retryFailure()) { - // httpClient = httpClient.newBuilder().retryOnConnectionFailure(false).build(); - // } - okhttp3.Request httpRequest = request.httpRequest(); CompletableFuture completableFuture = newCompleteableFuture(); - httpClient + this.httpClient .newCall(httpRequest) .enqueue( new Callback() { diff --git a/api/src/main/java/io/minio/Http.java b/api/src/main/java/io/minio/Http.java index 846bfc3b9..021c755ed 100644 --- a/api/src/main/java/io/minio/Http.java +++ b/api/src/main/java/io/minio/Http.java @@ -1,5 +1,5 @@ /* - * MinIO Java SDK for Amazon S3 Compatible Cloud Storage, (C) 2025 MinIO, Inc. + * MinIO Java SDK for Amazon S3 Compatible Cloud Storage, (C) 2025-2026 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,6 +35,8 @@ import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; import java.security.SecureRandom; +import java.security.cert.CertPathBuilderException; +import java.security.cert.CertPathValidatorException; import java.security.cert.CertificateException; import java.security.cert.CertificateFactory; import java.security.cert.X509Certificate; @@ -52,7 +54,9 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; +import java.util.function.IntSupplier; import java.util.regex.Matcher; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -60,11 +64,15 @@ import javax.net.ssl.HostnameVerifier; import javax.net.ssl.KeyManagerFactory; import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLException; +import javax.net.ssl.SSLHandshakeException; +import javax.net.ssl.SSLPeerUnverifiedException; import javax.net.ssl.SSLSession; import javax.net.ssl.SSLSocketFactory; import javax.net.ssl.TrustManager; import javax.net.ssl.TrustManagerFactory; import javax.net.ssl.X509TrustManager; +import okhttp3.Interceptor; import okhttp3.MediaType; import okhttp3.OkHttpClient; import okhttp3.Protocol; @@ -609,10 +617,19 @@ public static OkHttpClient enableExternalCertificatesFromEnv(OkHttpClient client } /** - * Creates new HTTP client with default timeout with additional TLS certificates from - * SSL_CERT_FILE and SSL_CERT_DIR environment variables if present. + * Creates the SDK's default HTTP client with a fixed retry budget of {@link + * RetryInterceptor#MAX_RETRY}. For a runtime-tunable budget, use {@link + * #newDefaultClient(IntSupplier)}. */ public static OkHttpClient newDefaultClient() { + return newDefaultClient(() -> RetryInterceptor.MAX_RETRY); + } + + /** + * Creates the SDK's default HTTP client; the {@link RetryInterceptor}'s attempt budget is read + * from {@code maxAttemptsSupplier} on each call so it can track a runtime-tunable value. + */ + public static OkHttpClient newDefaultClient(IntSupplier maxAttemptsSupplier) { OkHttpClient client = new OkHttpClient() .newBuilder() @@ -620,6 +637,8 @@ public static OkHttpClient newDefaultClient() { .writeTimeout(DEFAULT_TIMEOUT, TimeUnit.MILLISECONDS) .readTimeout(DEFAULT_TIMEOUT, TimeUnit.MILLISECONDS) .protocols(Arrays.asList(Protocol.HTTP_1_1)) + .retryOnConnectionFailure(false) + .addInterceptor(new RetryInterceptor(maxAttemptsSupplier)) .build(); try { return enableExternalCertificatesFromEnv(client); @@ -683,6 +702,194 @@ public static OkHttpClient setTimeout( .build(); } + /** + * OkHttp interceptor that retries transient HTTP failures with full-jitter exponential backoff. + * + *

Installed automatically by {@link Http#newDefaultClient(IntSupplier)} on the SDK-default + * client. Not installed on caller-supplied clients passed via {@code + * MinioClient.Builder.httpClient(...)} — those are used verbatim. To opt into the SDK retry + * policy on a custom client, register this interceptor and pair it with {@code + * retryOnConnectionFailure(false)}. + * + *

Retries on: + * + *

+ * + *

Backoff is full-jitter exponential, with a 200 ms unit and a 1 s per-attempt cap. + * Attempt budget is read from an {@link IntSupplier} on every call so SDK clients can expose + * runtime tuning while the interceptor itself stays stateless. + * + *

Threading. Backoff sleeps on the OkHttp dispatcher thread that owns the call; under + * sustained 5xx/429 storms this can hold dispatcher slots idle. Size {@link + * okhttp3.Dispatcher#setMaxRequests} / {@code setMaxRequestsPerHost} accordingly. + * + *

Cancellation. {@code Call.cancel()} short-circuits the retry loop instead of being + * mistaken for a retryable transport error. + * + *

Replayability. SDK-owned body types ({@link Body} over {@code byte[]}, {@link + * ByteBuffer}, {@link RandomAccessFile}) are retry-safe. A caller-supplied {@link + * okhttp3.RequestBody} that overrides {@code isOneShot()} to {@code true} MUST NOT be retried; + * either disable retries for those calls via {@link BaseS3Client#setMaxRetries(int)} {@code (1)} + * or wrap the body in a replayable form. + * + *

Request reuse. Each attempt sends the same signed {@link okhttp3.Request}, so {@code + * X-Amz-Date}/{@code Authorization} are not refreshed. Harmless at the default budget; an extreme + * {@code maxRetries} combined with high backoff could outlast the 15-minute signing window or a + * short-lived STS credential. (minio-go re-signs per attempt; this Java implementation + * deliberately does not, in exchange for a simpler interceptor model.) + */ + public static class RetryInterceptor implements Interceptor { + /** Default maximum number of attempts per request. */ + static final int MAX_RETRY = 10; + + /** Base unit per retry attempt, in milliseconds. */ + static final long DEFAULT_RETRY_UNIT_MS = 200L; + + /** Per-attempt sleep cap, in milliseconds. */ + static final long DEFAULT_RETRY_CAP_MS = 1_000L; + + /** Maximum jitter fraction in {@code [0.0, 1.0]}. {@code 1.0} = full jitter. */ + static final double MAX_JITTER = 1.0; + + /** Retryable HTTP status codes. */ + static final Set RETRYABLE_HTTP_STATUS_CODES = + ImmutableSet.of( + 408, // Request Timeout + 429, // Too Many Requests + 499, // Client Closed Request (nginx) + 500, // Internal Server Error + 502, // Bad Gateway + 503, // Service Unavailable + 504, // Gateway Timeout + 520); // Cloudflare unknown error + + static boolean isHttpStatusRetryable(int code) { + return RETRYABLE_HTTP_STATUS_CODES.contains(code); + } + + /** + * Returns true if {@code e} represents a transient transport failure that should be retried. + * TLS handshake failure, unknown-CA / cert-path errors, and the "server gave HTTP response to + * HTTPS client" protocol mismatch are NOT retryable; everything else (connection reset, EOF, + * server closed idle connection, socket timeout, …) is. + */ + static boolean isRequestErrorRetryable(IOException e) { + if (e instanceof SSLHandshakeException) return false; + if (e instanceof SSLPeerUnverifiedException) return false; + if (e instanceof SSLException) { + Throwable cause = e.getCause(); + if (cause instanceof CertPathBuilderException + || cause instanceof CertPathValidatorException + || cause instanceof CertificateException) { + return false; + } + } + String msg = e.getMessage(); + if (msg != null && msg.contains("server gave HTTP response to HTTPS client")) return false; + return true; + } + + /** + * Computes the exponential-backoff-with-full-jitter delay for retry {@code attempt} (0-indexed: + * {@code 0} = before the second attempt, {@code 1} = before the third, …): + * + * 

+     *   sleep = min(DEFAULT_RETRY_CAP_MS, DEFAULT_RETRY_UNIT_MS * 2^attempt)
+     *   sleep -= (long)(random.nextDouble() * sleep * MAX_JITTER)   // full jitter when MAX_JITTER == 1.0
+     *
+ * + *

With {@code MAX_JITTER == 1.0}, returns a value in {@code [1, min(cap, base * + * 2^attempt)]}. The lower bound is {@code 1} rather than {@code 0} because {@link + * java.util.concurrent.ThreadLocalRandom#nextDouble()} is in {@code [0.0, 1.0)} and the {@code + * (long)} cast truncates {@code rand * sleep} to at most {@code sleep - 1}. This matches the + * behaviour of minio-go's {@code exponentialBackoffWait}, which uses the same formula and + * therefore the same bounds. + */ + static long exponentialBackoffMs(int attempt) { + int exp = Math.min(Math.max(attempt, 0), 30); + long sleep = DEFAULT_RETRY_UNIT_MS * (1L << exp); + if (sleep > DEFAULT_RETRY_CAP_MS) sleep = DEFAULT_RETRY_CAP_MS; + sleep -= (long) (ThreadLocalRandom.current().nextDouble() * (double) sleep * MAX_JITTER); + return Math.max(0L, sleep); + } + + private final IntSupplier maxAttemptsSupplier; + + /** Uses a fixed budget of {@link #MAX_RETRY} attempts. */ + public RetryInterceptor() { + this(() -> MAX_RETRY); + } + + /** + * Reads the attempt budget from {@code maxAttemptsSupplier} on every call so it can track a + * runtime-tunable value. Values below 1 are clamped to 1 (= retry off). + */ + public RetryInterceptor(IntSupplier maxAttemptsSupplier) { + this.maxAttemptsSupplier = Objects.requireNonNull(maxAttemptsSupplier, "maxAttemptsSupplier"); + } + + @Override + public okhttp3.Response intercept(Chain chain) throws IOException { + okhttp3.Request request = chain.request(); + int maxAttempts = Math.max(1, maxAttemptsSupplier.getAsInt()); + + okhttp3.Response response = null; + IOException lastException = null; + + for (int attempt = 0; attempt < maxAttempts; attempt++) { + // Honour caller cancellation: mirrors minio-go's + // `errors.Is(err, context.Canceled)` short-circuit so the loop does not + // burn attempts re-issuing a call the caller has already abandoned. + if (chain.call().isCanceled()) { + if (response != null) response.close(); + throw new IOException("Canceled"); + } + + if (attempt > 0) { + long delayMs = exponentialBackoffMs(attempt - 1); + if (delayMs > 0L) { + try { + Thread.sleep(delayMs); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new IOException("Retry interrupted", ie); + } + } + } + + if (response != null) { + response.close(); + response = null; + } + + try { + response = chain.proceed(request); + } catch (IOException e) { + // A cancelled call surfaces as IOException with a message that would + // otherwise pass `isRequestErrorRetryable`. Bail out instead. + if (chain.call().isCanceled()) throw e; + if (!isRequestErrorRetryable(e)) throw e; + lastException = e; + continue; + } + + if (isHttpStatusRetryable(response.code())) { + lastException = null; + continue; + } + + return response; + } + + if (lastException != null) throw lastException; + return response; + } + } + /** HTTP body of {@link RandomAccessFile}, {@link ByteBuffer} or {@link byte} array. */ public static class Body { private okhttp3.RequestBody requestBody; @@ -695,7 +902,14 @@ public static class Body { private String md5Hash; private boolean bodyString; - /** Creates Body for okhttp3 RequestBody. */ + /** + * Creates Body for okhttp3 RequestBody. + * + *

Retry caveat. {@link Http.RetryInterceptor} re-invokes {@code writeTo} on each + * retry. Pass only replayable bodies; do not pass a body that overrides {@code isOneShot()} to + * return {@code true} (or otherwise consumes its source on first write) unless retries are + * disabled for that call. + */ public Body(okhttp3.RequestBody requestBody) { this.requestBody = requestBody; this.contentType = requestBody.contentType(); diff --git a/api/src/main/java/io/minio/MinioAsyncClient.java b/api/src/main/java/io/minio/MinioAsyncClient.java index d0e21a4f2..8837f6abd 100644 --- a/api/src/main/java/io/minio/MinioAsyncClient.java +++ b/api/src/main/java/io/minio/MinioAsyncClient.java @@ -1,6 +1,6 @@ /* * MinIO Java SDK for Amazon S3 Compatible Cloud Storage, - * (C) 2022 MinIO, Inc. + * (C) 2022-2026 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -156,6 +156,7 @@ public static final class Builder { private Provider provider; private OkHttpClient httpClient; private boolean closeHttpClient; + private int maxRetries = Http.RetryInterceptor.MAX_RETRY; public Builder baseUrl(Http.BaseUrl baseUrl) { if (baseUrl.region() == null) { @@ -204,12 +205,22 @@ public Builder credentialsProvider(Provider provider) { return this; } + /** + * Supplies a custom {@link OkHttpClient}; the SDK uses it verbatim. Retry policy on this client + * is the caller's responsibility — neither {@link #maxRetries(int)} nor {@link + * MinioAsyncClient#setMaxRetries(int)} apply unless the caller has wired {@link + * Http.RetryInterceptor} to read from them. + */ public Builder httpClient(OkHttpClient httpClient) { Utils.validateNotNull(httpClient, "http client"); this.httpClient = httpClient; return this; } + /** + * Same as {@link #httpClient(OkHttpClient)} but additionally controls whether the SDK shuts + * down the underlying dispatcher on {@link MinioAsyncClient#close()}. + */ public Builder httpClient(OkHttpClient httpClient, boolean close) { Utils.validateNotNull(httpClient, "http client"); this.httpClient = httpClient; @@ -217,6 +228,17 @@ public Builder httpClient(OkHttpClient httpClient, boolean close) { return this; } + /** + * Sets the maximum number of attempts per request. Pass {@code 1} to disable automatic retries. + * Default {@code 10}. Effective only on the SDK-default client; see {@link + * #httpClient(OkHttpClient)}. + */ + public Builder maxRetries(int maxRetries) { + if (maxRetries < 1) throw new IllegalArgumentException("maxRetries must be >= 1"); + this.maxRetries = maxRetries; + return this; + } + public MinioAsyncClient build() { Utils.validateNotNull(baseUrl, "endpoint"); @@ -227,12 +249,16 @@ public MinioAsyncClient build() { throw new IllegalArgumentException("Region missing in Amazon S3 China endpoint " + baseUrl); } + // Construct the client before the default httpClient so the RetryInterceptor supplier can + // capture client.maxRetries. + MinioAsyncClient client = + new MinioAsyncClient(baseUrl, provider, httpClient, closeHttpClient); + client.setMaxRetries(maxRetries); if (httpClient == null) { - closeHttpClient = true; - httpClient = Http.newDefaultClient(); + client.httpClient = Http.newDefaultClient(() -> client.maxRetries); + client.closeHttpClient = true; } - - return new MinioAsyncClient(baseUrl, provider, httpClient, closeHttpClient); + return client; } } diff --git a/api/src/main/java/io/minio/MinioClient.java b/api/src/main/java/io/minio/MinioClient.java index 03dd587a9..f37988225 100644 --- a/api/src/main/java/io/minio/MinioClient.java +++ b/api/src/main/java/io/minio/MinioClient.java @@ -1,6 +1,6 @@ /* * MinIO Java SDK for Amazon S3 Compatible Cloud Storage, - * (C) 2015-2021 MinIO, Inc. + * (C) 2015-2026 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -1921,6 +1921,16 @@ public void ignoreCertCheck() throws MinioException { asyncClient.ignoreCertCheck(); } + /** + * Sets the maximum number of attempts for transient HTTP failures. Pass {@code 1} to disable + * automatic retries. Defaults to 10. + * + * @param maxRetries maximum attempts (must be {@code >= 1}). + */ + public void setMaxRetries(int maxRetries) { + asyncClient.setMaxRetries(maxRetries); + } + /** * Sets application's name/version to user agent. For more information about user agent refer #rfc2616. @@ -2040,6 +2050,15 @@ public Builder httpClient(OkHttpClient httpClient, boolean close) { return this; } + /** + * Sets the maximum number of attempts per request. Pass {@code 1} to disable automatic retries. + * Defaults to 10. + */ + public Builder maxRetries(int maxRetries) { + asyncClientBuilder.maxRetries(maxRetries); + return this; + } + public MinioClient build() { MinioAsyncClient asyncClient = asyncClientBuilder.build(); return new MinioClient(asyncClient); diff --git a/api/src/test/java/io/minio/RetryTest.java b/api/src/test/java/io/minio/RetryTest.java new file mode 100644 index 000000000..36cc94d68 --- /dev/null +++ b/api/src/test/java/io/minio/RetryTest.java @@ -0,0 +1,722 @@ +/* + * MinIO Java SDK for Amazon S3 Compatible Cloud Storage, (C) 2026 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.minio; + +import io.minio.errors.ErrorResponseException; +import io.minio.errors.InvalidResponseException; +import io.minio.errors.MinioException; +import java.io.IOException; +import java.net.SocketException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.cert.CertPathBuilderException; +import java.security.cert.CertPathValidatorException; +import java.security.cert.CertificateException; +import java.util.concurrent.ThreadLocalRandom; +import javax.net.ssl.SSLException; +import javax.net.ssl.SSLHandshakeException; +import javax.net.ssl.SSLPeerUnverifiedException; +import okhttp3.Call; +import okhttp3.OkHttpClient; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import okhttp3.mockwebserver.RecordedRequest; +import okhttp3.mockwebserver.SocketPolicy; +import okio.Buffer; +import org.junit.Assert; +import org.junit.Test; + +/** Unit + integration tests for {@link Http.RetryInterceptor}. */ +public class RetryTest { + + private static void closeQuietly(MinioClient client) { + if (client == null) return; + try { + client.close(); + } catch (Exception ignored) { + // Tests do not rely on close(); swallowing keeps the throws clauses narrow. + } + } + + // --------------------------------------------------------------------------- + // Http.RetryInterceptor.isHttpStatusRetryable + // --------------------------------------------------------------------------- + + @Test + public void testIsHttpStatusRetryable_retryable() { + Assert.assertTrue(Http.RetryInterceptor.isHttpStatusRetryable(408)); + Assert.assertTrue(Http.RetryInterceptor.isHttpStatusRetryable(429)); + Assert.assertTrue(Http.RetryInterceptor.isHttpStatusRetryable(499)); + Assert.assertTrue(Http.RetryInterceptor.isHttpStatusRetryable(500)); + Assert.assertTrue(Http.RetryInterceptor.isHttpStatusRetryable(502)); + Assert.assertTrue(Http.RetryInterceptor.isHttpStatusRetryable(503)); + Assert.assertTrue(Http.RetryInterceptor.isHttpStatusRetryable(504)); + Assert.assertTrue(Http.RetryInterceptor.isHttpStatusRetryable(520)); + } + + @Test + public void testIsHttpStatusRetryable_notRetryable() { + for (int code : new int[] {200, 201, 204, 301, 304, 400, 401, 403, 404, 409, 412, 416, 501}) { + Assert.assertFalse( + "status " + code + " must not be retryable", + Http.RetryInterceptor.isHttpStatusRetryable(code)); + } + } + + // --------------------------------------------------------------------------- + // Http.RetryInterceptor.isRequestErrorRetryable + // --------------------------------------------------------------------------- + + @Test + public void testIsRequestErrorRetryable_retryable() { + Assert.assertTrue( + Http.RetryInterceptor.isRequestErrorRetryable(new IOException("connection reset"))); + Assert.assertTrue(Http.RetryInterceptor.isRequestErrorRetryable(new IOException("EOF"))); + Assert.assertTrue( + Http.RetryInterceptor.isRequestErrorRetryable( + new IOException("http: server closed idle connection"))); + Assert.assertTrue( + Http.RetryInterceptor.isRequestErrorRetryable(new SocketException("Connection timed out"))); + Assert.assertTrue( + Http.RetryInterceptor.isRequestErrorRetryable(new java.net.SocketTimeoutException("read"))); + } + + @Test + public void testIsRequestErrorRetryable_sslHandshakeNotRetryable() { + Assert.assertFalse( + Http.RetryInterceptor.isRequestErrorRetryable( + new SSLHandshakeException("cert not trusted"))); + } + + @Test + public void testIsRequestErrorRetryable_protocolMismatchNotRetryable() { + Assert.assertFalse( + Http.RetryInterceptor.isRequestErrorRetryable( + new IOException("server gave HTTP response to HTTPS client"))); + } + + // --------------------------------------------------------------------------- + // Http.RetryInterceptor.exponentialBackoffMs + // --------------------------------------------------------------------------- + + @Test + public void testExponentialBackoffMs_attempt0WithinFirstUnit() { + // attempt=0: cap = min(1000, 200*2^0) = 200ms; with full jitter, result in [0, 200]. + for (int i = 0; i < 100; i++) { + long delay = Http.RetryInterceptor.exponentialBackoffMs(0); + Assert.assertTrue( + "attempt 0 delay must be in [0, 200ms], got " + delay, delay >= 0 && delay <= 200); + } + } + + @Test + public void testExponentialBackoffMs_attempt2WithinSecondCap() { + // attempt=2: cap = min(1000, 200*2^2) = 800ms. + for (int i = 0; i < 100; i++) { + long delay = Http.RetryInterceptor.exponentialBackoffMs(2); + Assert.assertTrue( + "attempt 2 delay must be in [0, 800ms], got " + delay, delay >= 0 && delay <= 800); + } + } + + @Test + public void testExponentialBackoffMs_cappedAtRetryCap() { + // attempt=10: uncapped 200*2^10 = 204800ms; capped at 1000ms. + for (int i = 0; i < 100; i++) { + long delay = Http.RetryInterceptor.exponentialBackoffMs(10); + Assert.assertTrue( + "delay must be <= cap, got " + delay, + delay <= Http.RetryInterceptor.DEFAULT_RETRY_CAP_MS); + Assert.assertTrue(delay >= 0); + } + } + + @Test + public void testExponentialBackoffMs_negativeAttemptIsClamped() { + // Negative attempt clamps to 0; cap = 200ms. + long delay = Http.RetryInterceptor.exponentialBackoffMs(-5); + Assert.assertTrue(delay >= 0 && delay <= 200); + } + + @Test + public void testExponentialBackoffMs_highAttemptDoesNotOverflow() { + // High attempts must not bit-shift overflow; cap saturates. + for (int attempt : new int[] {30, 31, 60, 100, 1000}) { + long delay = Http.RetryInterceptor.exponentialBackoffMs(attempt); + Assert.assertTrue( + "attempt=" + attempt + " delay must be <= cap, got " + delay, + delay <= Http.RetryInterceptor.DEFAULT_RETRY_CAP_MS); + Assert.assertTrue("attempt=" + attempt + " delay must be >= 0", delay >= 0); + } + } + + // --------------------------------------------------------------------------- + // Integration tests via MockWebServer + // --------------------------------------------------------------------------- + + private static final String LIST_BUCKETS_OK = + "" + + "" + + "testtest" + + ""; + + private MockResponse successResponse() { + return new MockResponse() + .setResponseCode(200) + .setHeader("Content-Type", "application/xml") + .setBody(new Buffer().writeUtf8(LIST_BUCKETS_OK)); + } + + private MockResponse htmlServerError(int code) { + return new MockResponse() + .setResponseCode(code) + .setHeader("Content-Type", "text/html") + .setBody(new Buffer().writeUtf8("" + code + "")); + } + + private MockResponse xmlError(int code, String s3Code) { + String xml = + "" + + "" + + s3Code + + "m/id"; + return new MockResponse() + .setResponseCode(code) + .setHeader("Content-Type", "application/xml") + .setBody(new Buffer().writeUtf8(xml)); + } + + @Test + public void testRetryOn503ThenSuccess() throws IOException, MinioException { + try (MockWebServer server = new MockWebServer()) { + server.enqueue(htmlServerError(503)); + server.enqueue(successResponse()); + server.start(); + + MinioClient client = + MinioClient.builder().endpoint(server.url("").toString()).maxRetries(2).build(); + try { + client.listBuckets(); + Assert.assertEquals(2, server.getRequestCount()); + } finally { + closeQuietly(client); + } + } + } + + @Test + public void testRetryOnEachRetryableHttpCode() throws IOException, MinioException { + for (int code : new int[] {408, 429, 499, 500, 502, 503, 504, 520}) { + try (MockWebServer server = new MockWebServer()) { + server.enqueue(htmlServerError(code)); + server.enqueue(successResponse()); + server.start(); + + MinioClient client = + MinioClient.builder().endpoint(server.url("").toString()).maxRetries(2).build(); + try { + client.listBuckets(); + Assert.assertEquals("status " + code + " expected to retry", 2, server.getRequestCount()); + } finally { + closeQuietly(client); + } + } + } + } + + @Test + public void testNoRetryOn404() throws IOException, MinioException { + // Bucket-scoped operation returning 404 NoSuchBucket — a wire-realistic shape + // (NoSuchBucket only ever applies to bucket-scoped calls). Verifies that 404 + // does not trigger retry and that the parsed error surfaces both code and status. + try (MockWebServer server = new MockWebServer()) { + server.enqueue(xmlError(404, "NoSuchBucket")); + server.start(); + + MinioClient client = + MinioClient.builder().endpoint(server.url("").toString()).maxRetries(3).build(); + try { + try { + client.removeBucket( + RemoveBucketArgs.builder().bucket("missing-bucket-for-no-retry").build()); + Assert.fail("expected ErrorResponseException"); + } catch (ErrorResponseException e) { + Assert.assertEquals("NoSuchBucket", e.errorResponse().code()); + Assert.assertEquals(404, e.response().code()); + } + Assert.assertEquals(1, server.getRequestCount()); + } finally { + closeQuietly(client); + } + } + } + + @Test + public void testNoRetryOn403() throws IOException, MinioException { + try (MockWebServer server = new MockWebServer()) { + server.enqueue(xmlError(403, "AccessDenied")); + server.start(); + + MinioClient client = + MinioClient.builder().endpoint(server.url("").toString()).maxRetries(3).build(); + try { + try { + client.listBuckets(); + Assert.fail("expected ErrorResponseException"); + } catch (ErrorResponseException e) { + Assert.assertEquals("AccessDenied", e.errorResponse().code()); + } + Assert.assertEquals(1, server.getRequestCount()); + } finally { + closeQuietly(client); + } + } + } + + @Test + public void testRetryExhaustedReturnsLastResponse() throws IOException, MinioException { + // 3 attempts, all 500 (HTML so server-side response dispatches to InvalidResponseException). + try (MockWebServer server = new MockWebServer()) { + server.enqueue(htmlServerError(500)); + server.enqueue(htmlServerError(500)); + server.enqueue(htmlServerError(500)); + server.start(); + + MinioClient client = + MinioClient.builder().endpoint(server.url("").toString()).maxRetries(3).build(); + try { + try { + client.listBuckets(); + Assert.fail("expected exception after exhausted retries"); + } catch (InvalidResponseException e) { + // Terminal exception must surface the underlying 500 status so callers can distinguish + // exhausted retries from unrelated failures (e.g. NPE, parser bugs). + Assert.assertTrue( + "exhausted-retries exception must reflect HTTP 500, got: " + e.getMessage(), + e.getMessage().contains("Response code: 500")); + } + Assert.assertEquals(3, server.getRequestCount()); + } finally { + closeQuietly(client); + } + } + } + + @Test + public void testRetryExhaustedSurfacesXmlErrorResponse() throws IOException, MinioException { + // 3 attempts, all 503 with XML InternalError — confirms the terminal + // ErrorResponseException carries both the 503 status and the parsed S3 code after the retry + // loop gives up. + try (MockWebServer server = new MockWebServer()) { + server.enqueue(xmlError(503, "InternalError")); + server.enqueue(xmlError(503, "InternalError")); + server.enqueue(xmlError(503, "InternalError")); + server.start(); + + MinioClient client = + MinioClient.builder().endpoint(server.url("").toString()).maxRetries(3).build(); + try { + try { + client.listBuckets(); + Assert.fail("expected ErrorResponseException after exhausted retries"); + } catch (ErrorResponseException e) { + Assert.assertEquals(503, e.response().code()); + Assert.assertEquals("InternalError", e.errorResponse().code()); + } + Assert.assertEquals(3, server.getRequestCount()); + } finally { + closeQuietly(client); + } + } + } + + @Test + public void testMaxRetriesOneDisablesRetry() throws IOException, MinioException { + try (MockWebServer server = new MockWebServer()) { + server.enqueue(htmlServerError(503)); + // Second response should never be reached. + server.enqueue(successResponse()); + server.start(); + + MinioClient client = + MinioClient.builder().endpoint(server.url("").toString()).maxRetries(1).build(); + try { + try { + client.listBuckets(); + Assert.fail("expected exception"); + } catch (InvalidResponseException e) { + // expected + } + Assert.assertEquals(1, server.getRequestCount()); + } finally { + closeQuietly(client); + } + } + } + + @Test + public void testSetMaxRetriesPostConstruction() throws IOException, MinioException { + try (MockWebServer server = new MockWebServer()) { + server.enqueue(htmlServerError(503)); + server.enqueue(successResponse()); + server.start(); + + MinioClient client = + MinioClient.builder().endpoint(server.url("").toString()).maxRetries(2).build(); + client.setMaxRetries(1); // disable + try { + try { + client.listBuckets(); + Assert.fail("expected exception"); + } catch (InvalidResponseException e) { + // expected + } + Assert.assertEquals(1, server.getRequestCount()); + } finally { + closeQuietly(client); + } + } + } + + @Test(expected = IllegalArgumentException.class) + public void testMaxRetriesBuilderValidation() { + MinioClient.builder().endpoint("http://localhost:9000").maxRetries(0).build(); + } + + @Test(expected = IllegalArgumentException.class) + public void testSetMaxRetriesValidation() { + MinioClient client = MinioClient.builder().endpoint("http://localhost:9000").build(); + client.setMaxRetries(0); + } + + @Test + public void testMultipleRetrySucceedsOnThirdAttempt() throws IOException, MinioException { + try (MockWebServer server = new MockWebServer()) { + server.enqueue(htmlServerError(500)); + server.enqueue(htmlServerError(503)); + server.enqueue(successResponse()); + server.start(); + + MinioClient client = + MinioClient.builder().endpoint(server.url("").toString()).maxRetries(3).build(); + try { + client.listBuckets(); + Assert.assertEquals(3, server.getRequestCount()); + } finally { + closeQuietly(client); + } + } + } + + // --------------------------------------------------------------------------- + // Additional coverage: SSL cert-path classifier, negative-validation, IOException + // path, user-supplied OkHttpClient, and cancellation short-circuit. + // --------------------------------------------------------------------------- + + @Test + public void testIsRequestErrorRetryable_certPathErrorsNotRetryable() { + // SSLException whose cause is one of the cert-path error types must NOT retry. + SSLException certPathBuilder = new SSLException("x", new CertPathBuilderException("bad")); + SSLException certPathValidator = new SSLException("x", new CertPathValidatorException("bad")); + SSLException certError = new SSLException("x", new CertificateException("bad")); + Assert.assertFalse(Http.RetryInterceptor.isRequestErrorRetryable(certPathBuilder)); + Assert.assertFalse(Http.RetryInterceptor.isRequestErrorRetryable(certPathValidator)); + Assert.assertFalse(Http.RetryInterceptor.isRequestErrorRetryable(certError)); + + // SSLPeerUnverifiedException must NOT retry. + Assert.assertFalse( + Http.RetryInterceptor.isRequestErrorRetryable(new SSLPeerUnverifiedException("untrusted"))); + + // SSLException with an unrelated cause is still retryable (transient TLS hiccup). + Assert.assertTrue( + Http.RetryInterceptor.isRequestErrorRetryable( + new SSLException("read", new IOException("boom")))); + } + + @Test(expected = IllegalArgumentException.class) + public void testMaxRetriesBuilderValidation_negative() { + MinioClient.builder().endpoint("http://localhost:9000").maxRetries(-1).build(); + } + + @Test(expected = IllegalArgumentException.class) + public void testSetMaxRetriesValidation_negative() { + MinioClient client = MinioClient.builder().endpoint("http://localhost:9000").build(); + try { + client.setMaxRetries(-1); + } finally { + closeQuietly(client); + } + } + + @Test + public void testRetryOnConnectionDropThenSuccess() throws IOException, MinioException { + // Simulate a transient transport failure: server drops the socket on first attempt, returns + // the success body on the second. Verifies the IOException retry path end-to-end (only the + // predicate was previously tested in isolation). + try (MockWebServer server = new MockWebServer()) { + server.enqueue(new MockResponse().setSocketPolicy(SocketPolicy.DISCONNECT_AT_START)); + server.enqueue(successResponse()); + server.start(); + + MinioClient client = + MinioClient.builder().endpoint(server.url("").toString()).maxRetries(3).build(); + try { + client.listBuckets(); + Assert.assertEquals(2, server.getRequestCount()); + } finally { + closeQuietly(client); + } + } + } + + @Test + public void testUserSuppliedClientWithoutInterceptorDoesNotRetry() + throws IOException, MinioException { + // Locks in the contract that caller-supplied clients are not modified by the SDK. + try (MockWebServer server = new MockWebServer()) { + server.enqueue(htmlServerError(503)); + server.enqueue(successResponse()); + server.start(); + + OkHttpClient custom = new OkHttpClient.Builder().retryOnConnectionFailure(false).build(); + MinioClient client = + MinioClient.builder() + .endpoint(server.url("").toString()) + .httpClient(custom, false) + .maxRetries(3) + .build(); + try { + try { + client.listBuckets(); + Assert.fail( + "expected exception — user-supplied client without interceptor must not retry"); + } catch (InvalidResponseException e) { + // expected + } + Assert.assertEquals(1, server.getRequestCount()); + } finally { + closeQuietly(client); + } + } + } + + @Test + public void testUserSuppliedClientWithInterceptorRetries() throws IOException, MinioException { + // Proves user-installed retry is the caller's own configuration, not an SDK side-effect. + try (MockWebServer server = new MockWebServer()) { + server.enqueue(htmlServerError(503)); + server.enqueue(htmlServerError(503)); + server.enqueue(successResponse()); + server.start(); + + OkHttpClient custom = + new OkHttpClient.Builder() + .retryOnConnectionFailure(false) + .addInterceptor(new Http.RetryInterceptor()) + .build(); + MinioClient client = + MinioClient.builder() + .endpoint(server.url("").toString()) + .httpClient(custom, false) + .build(); + try { + client.listBuckets(); + Assert.assertEquals(3, server.getRequestCount()); + } finally { + closeQuietly(client); + } + } + } + + @Test(timeout = 10_000) + public void testCancelDuringRetryStopsLoop() throws IOException, InterruptedException { + // Without cancellation handling the interceptor would loop maxRetries=10 times with backoff + // (~6s of sleep). With the chain.call().isCanceled() check, cancel mid-loop must short-circuit + // and surface IOException quickly — the timeout guards against regression. + try (MockWebServer server = new MockWebServer()) { + for (int i = 0; i < 20; i++) { + server.enqueue(htmlServerError(503)); + } + server.start(); + + OkHttpClient client = + new OkHttpClient.Builder() + .retryOnConnectionFailure(false) + .addInterceptor(new Http.RetryInterceptor()) + .build(); + try { + okhttp3.Request request = new okhttp3.Request.Builder().url(server.url("/x")).get().build(); + Call call = client.newCall(request); + + Thread canceler = + new Thread( + () -> { + try { + Thread.sleep(150); + } catch (InterruptedException ignored) { + Thread.currentThread().interrupt(); + } + call.cancel(); + }); + canceler.start(); + + long start = System.currentTimeMillis(); + try { + call.execute().close(); + Assert.fail("expected IOException after cancel"); + } catch (IOException e) { + // expected: cancel surfaces as IOException + } + long elapsed = System.currentTimeMillis() - start; + canceler.join(); + + Assert.assertTrue( + "cancel must short-circuit the retry loop, took " + elapsed + "ms", elapsed < 5_000); + } finally { + client.dispatcher().executorService().shutdown(); + client.connectionPool().evictAll(); + } + } + } + + // --------------------------------------------------------------------------- + // File-body retry validation + // + // Validates the file-PUT retry path end-to-end after BaseS3Client.createBody + // was reverted to master form (no in-method getFilePointer/seek bracket). + // Retry-safety relies on: + // - MinioAsyncClient pre-computing checksums and bracketing the file + // pointer so it lands at start-of-payload before putObject is called; + // - those pre-computed headers short-circuiting createBody's own + // Checksum.update guards, so the pointer is never touched here; + // - Http.RequestBody.writeTo seeking to the captured start position on + // every attempt, so the retry interceptor's repeated writeTo calls all + // replay the same payload bytes. + // --------------------------------------------------------------------------- + + private MockResponse putSuccessResponse() { + return new MockResponse() + .setResponseCode(200) + .setHeader("ETag", "\"deadbeef\"") + .setHeader("x-amz-version-id", "v0"); + } + + @Test + public void testRetryWithFileBody_replaysSameBytesOnEveryAttempt() + throws IOException, InterruptedException, MinioException { + byte[] payload = new byte[8 * 1024]; + ThreadLocalRandom.current().nextBytes(payload); + Path tempFile = Files.createTempFile("minio-retry-file-put-", ".dat"); + try { + Files.write(tempFile, payload); + + try (MockWebServer server = new MockWebServer()) { + server.enqueue(htmlServerError(503)); + server.enqueue(htmlServerError(503)); + server.enqueue(htmlServerError(503)); + server.enqueue(putSuccessResponse()); + server.start(); + + MinioClient client = + MinioClient.builder() + .endpoint(server.url("").toString()) + .region("us-east-1") + .credentials("ACCESS", "SECRET") + .maxRetries(4) + .build(); + try { + client.uploadObject( + UploadObjectArgs.builder() + .bucket("retry-file-bucket") + .object("retry-file-object") + .filename(tempFile.toString()) + .build()); + + Assert.assertEquals( + "expected 3 retryable failures + 1 success", 4, server.getRequestCount()); + + for (int i = 0; i < 4; i++) { + RecordedRequest req = server.takeRequest(); + Assert.assertEquals("attempt " + (i + 1) + " must be PUT", "PUT", req.getMethod()); + byte[] received = req.getBody().readByteArray(); + Assert.assertArrayEquals( + "attempt " + (i + 1) + " body must match the original file payload", + payload, + received); + } + } finally { + closeQuietly(client); + } + } + } finally { + Files.deleteIfExists(tempFile); + } + } + + @Test + public void testRetryWithFileBody_transportFailureReplaysFromStart() + throws IOException, InterruptedException, MinioException { + // Locks in that the body source is replayable across a transport-level failure: the first + // attempt is dropped before the server sends a response, the second attempt must seek back + // to start-of-payload and resend the full file. + byte[] payload = new byte[4 * 1024]; + ThreadLocalRandom.current().nextBytes(payload); + Path tempFile = Files.createTempFile("minio-retry-file-put-drop-", ".dat"); + try { + Files.write(tempFile, payload); + + try (MockWebServer server = new MockWebServer()) { + server.enqueue(new MockResponse().setSocketPolicy(SocketPolicy.DISCONNECT_AT_START)); + server.enqueue(putSuccessResponse()); + server.start(); + + MinioClient client = + MinioClient.builder() + .endpoint(server.url("").toString()) + .region("us-east-1") + .credentials("ACCESS", "SECRET") + .maxRetries(3) + .build(); + try { + client.uploadObject( + UploadObjectArgs.builder() + .bucket("retry-file-bucket-drop") + .object("retry-file-object-drop") + .filename(tempFile.toString()) + .build()); + + Assert.assertEquals( + "expected 1 disconnected attempt + 1 successful retry", 2, server.getRequestCount()); + + // First request was disconnected at start; MockWebServer may or may not have a body for + // it. The second attempt — the actual retry — MUST carry the full original payload. + server.takeRequest(); + RecordedRequest retry = server.takeRequest(); + Assert.assertEquals("retry must be PUT", "PUT", retry.getMethod()); + Assert.assertArrayEquals( + "retry body must match the original file payload after seek-back", + payload, + retry.getBody().readByteArray()); + } finally { + closeQuietly(client); + } + } + } finally { + Files.deleteIfExists(tempFile); + } + } +} diff --git a/docs/API.md b/docs/API.md index ee72ac8f1..25101d911 100644 --- a/docs/API.md +++ b/docs/API.md @@ -71,6 +71,7 @@ MinIO Client Builder is used to create MinIO client. Builder has below methods t | `credentials()` | Accepts access key (aka user ID) and secret key (aka password) of an account in S3 service. | | `region()` | Accepts region name of S3 service. If specified, all operations use this region otherwise region is probed per bucket. | | `httpClient()` | Custom HTTP client to override default. | +| `maxRetries()` | Maximum number of attempts per request for transient HTTP failures (retryable status codes 408/429/499/500/502/503/504/520, retryable S3 codes such as `SlowDown`/`InternalError`/`ExpiredToken`, and retryable IOExceptions). Pass `1` to disable automatic retries. Defaults to `10`. | __Examples__