Fix remaining S3BlobStoreRepositoryTests

Author: DaveCTurner

modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java

@@ -81,6 +81,7 @@
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Semaphore;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Collectors;
@@ -675,10 +676,24 @@ private void validateAuthHeader(HttpExchange exchange) {
     @SuppressForbidden(reason = "this test uses a HttpServer to emulate an S3 endpoint")
     protected static class S3ErroneousHttpHandler extends ErroneousHttpHandler {
 
+        // S3 SDK stops retrying after TOKEN_BUCKET_SIZE/DEFAULT_EXCEPTION_TOKEN_COST == 500/5 == 100 failures in quick succession
+        // see software.amazon.awssdk.retries.DefaultRetryStrategy.Legacy.TOKEN_BUCKET_SIZE
+        // see software.amazon.awssdk.retries.DefaultRetryStrategy.Legacy.DEFAULT_EXCEPTION_TOKEN_COST
+        private final Semaphore failurePermits = new Semaphore(100);
+
         S3ErroneousHttpHandler(final HttpHandler delegate, final int maxErrorsPerRequest) {
             super(delegate, maxErrorsPerRequest);
         }
 
+        @Override
+        public void handle(HttpExchange exchange) throws IOException {
+            if (failurePermits.tryAcquire()) {
+                super.handle(exchange);
+            } else {
+                delegate.handle(exchange);
+            }
+        }
+
         @Override
         protected String requestUniqueId(final HttpExchange exchange) {
             // Amazon SDK client provides a unique ID per request

test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESBlobStoreRepositoryIntegTestCase.java

@@ -396,7 +396,6 @@ protected void testSnapshotAndRestore(boolean recreateRepositoryBeforeRestore) t
         );
     }
 
-    @AwaitsFix(bugUrl = "TODO NOMERGE")
     public void testMultipleSnapshotAndRollback() throws Exception {
         final String repoName = createRepository(randomRepositoryName());
         int iterationCount = randomIntBetween(2, 5);
@@ -461,7 +460,6 @@ private void deleteRandomDocs(String indexName, int existingDocCount) {
         client().admin().indices().prepareRefresh(indexName).get();
     }
 
-    @AwaitsFix(bugUrl = "TODO NOMERGE")
     public void testIndicesDeletedFromRepository() throws Exception {
         final String repoName = createRepository(randomRepositoryName());
         Client client = client();

test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESMockAPIBasedRepositoryIntegTestCase.java

@@ -282,7 +282,7 @@ protected abstract static class ErroneousHttpHandler implements DelegatingHttpHa
         // value is the number of times the request has been seen
         private final Map<String, AtomicInteger> requests;
 
-        private final HttpHandler delegate;
+        protected final HttpHandler delegate;
         private final int maxErrorsPerRequest;
 
         @SuppressForbidden(reason = "this test uses a HttpServer to emulate a cloud-based storage service")