Reinstate DnsResolver failure testing

Author: DaveCTurner

modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java

@@ -33,6 +33,7 @@
 import software.amazon.awssdk.utils.SdkAutoCloseable;
 
 import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.conn.DnsResolver;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.ElasticsearchException;
@@ -43,6 +44,7 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.Maps;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Releasables;
 import org.elasticsearch.core.TimeValue;
@@ -157,7 +159,7 @@ public AmazonS3Reference client(RepositoryMetadata repositoryMetadata) {
                 return existing;
             }
             // TODO NOMERGE: consider alternative methods of retaining an httpClient reference for an explicit close() call.
-            SdkHttpClient httpClient = buildHttpClient(clientSettings);
+            SdkHttpClient httpClient = buildHttpClient(clientSettings, getCustomDnsResolver());
             Releasable toRelease = httpClient::close;
             try {
                 final AmazonS3Reference clientReference = new AmazonS3Reference(buildClient(clientSettings, httpClient), httpClient);
@@ -240,13 +242,23 @@ protected S3ClientBuilder buildClientBuilder(S3ClientSettings clientSettings, Sd
         return s3clientBuilder;
     }
 
-    static SdkHttpClient buildHttpClient(S3ClientSettings clientSettings) {
+    @Nullable // in production, but exposed for tests to override
+    DnsResolver getCustomDnsResolver() {
+        return null;
+    }
+
+    static SdkHttpClient buildHttpClient(S3ClientSettings clientSettings, @Nullable /* to use default resolver */ DnsResolver dnsResolver) {
         ApacheHttpClient.Builder httpClientBuilder = ApacheHttpClient.builder();
 
         httpClientBuilder.maxConnections(clientSettings.maxConnections);
         httpClientBuilder.socketTimeout(Duration.ofMillis(clientSettings.readTimeoutMillis));
 
         applyProxyConfiguration(clientSettings, httpClientBuilder);
+
+        if (dnsResolver != null) {
+            httpClientBuilder.dnsResolver(dnsResolver);
+        }
+
         return httpClientBuilder.build();
     }
 

modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java

@@ -209,7 +209,7 @@ private void launchAWSConfigurationTest(
     ) {
 
         final S3ClientSettings clientSettings = S3ClientSettings.getClientSettings(settings, "default");
-        final var httpClient = S3Service.buildHttpClient(clientSettings);
+        final var httpClient = S3Service.buildHttpClient(clientSettings, null);
         final ClientOverrideConfiguration configuration = S3Service.buildConfiguration(clientSettings, false);
 
         // TODO NOMERGE

modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobContainerRetriesTests.java

@@ -17,6 +17,7 @@
 import com.sun.net.httpserver.HttpHandler;
 
 import org.apache.http.HttpStatus;
+import org.apache.http.conn.DnsResolver;
 import org.apache.logging.log4j.Level;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.cluster.metadata.RepositoryMetadata;
@@ -66,8 +67,10 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.SocketTimeoutException;
+import java.net.UnknownHostException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.NoSuchFileException;
 import java.security.MessageDigest;
@@ -112,12 +115,26 @@ public class S3BlobContainerRetriesTests extends AbstractBlobContainerRetriesTes
 
     private static final int MAX_NUMBER_SNAPSHOT_DELETE_RETRIES = 10;
     private S3Service service;
-    // private AtomicBoolean shouldErrorOnDns; // TODO NOMERGE do we need to cover this too?
+    private volatile boolean shouldErrorOnDns;
     private RecordingMeterRegistry recordingMeterRegistry;
 
     @Before
     public void setUp() throws Exception {
-        service = new S3Service(Mockito.mock(Environment.class), Settings.EMPTY, Mockito.mock(ResourceWatcherService.class));
+        shouldErrorOnDns = false;
+        service = new S3Service(Mockito.mock(Environment.class), Settings.EMPTY, Mockito.mock(ResourceWatcherService.class)) {
+            private InetAddress[] resolveHost(String host) throws UnknownHostException {
+                assertEquals("127.0.0.1", host);
+                if (shouldErrorOnDns && randomBoolean() && randomBoolean()) {
+                    throw new UnknownHostException(host);
+                }
+                return new InetAddress[] { InetAddress.getLoopbackAddress() };
+            }
+
+            @Override
+            DnsResolver getCustomDnsResolver() {
+                return this::resolveHost;
+            }
+        };
         recordingMeterRegistry = new RecordingMeterRegistry();
         super.setUp();
     }
@@ -658,7 +675,7 @@ public void testReadWithIndicesPurposeRetriesForever() throws IOException {
 
         final byte[] bytes = randomBlobContent(512);
 
-        // shouldErrorOnDns.set(true); // TODO NOMERGE do we need to cover this too?
+        shouldErrorOnDns = true;
         final AtomicInteger failures = new AtomicInteger();
         @SuppressForbidden(reason = "use a http server")
         class FlakyReadHandler implements HttpHandler {