Default S3 endpoint scheme to HTTPS when not specified (elastic#127489)

Author: nicktindall

modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java

@@ -254,7 +254,21 @@ protected S3ClientBuilder buildClientBuilder(S3ClientSettings clientSettings, Sd
         }
 
         if (Strings.hasLength(clientSettings.endpoint)) {
-            s3clientBuilder.endpointOverride(URI.create(clientSettings.endpoint));
+            String endpoint = clientSettings.endpoint;
+            if ((endpoint.startsWith("http://") || endpoint.startsWith("https://")) == false) {
+                // The SDK does not know how to interpret endpoints without a scheme prefix and will error. Therefore, when the scheme is
+                // absent, we'll supply HTTPS as a default to avoid errors.
+                // See https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/client-configuration.html#client-config-other-diffs
+                endpoint = "https://" + endpoint;
+                LOGGER.warn(
+                    """
+                        found S3 client with endpoint [{}] that is missing a scheme, guessing it should use 'https://'; \
+                        to suppress this warning, add a scheme prefix to the [{}] setting on this node""",
+                    clientSettings.endpoint,
+                    S3ClientSettings.ENDPOINT_SETTING.getConcreteSettingForNamespace("CLIENT_NAME").getKey()
+                );
+            }
+            s3clientBuilder.endpointOverride(URI.create(endpoint));
         }
 
         return s3clientBuilder;

modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ServiceTests.java

@@ -12,7 +12,9 @@
 import software.amazon.awssdk.awscore.exception.AwsServiceException;
 import software.amazon.awssdk.core.retry.RetryPolicyContext;
 import software.amazon.awssdk.core.retry.conditions.RetryCondition;
+import software.amazon.awssdk.http.SdkHttpClient;
 import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
 import software.amazon.awssdk.services.s3.endpoints.S3EndpointParams;
 import software.amazon.awssdk.services.s3.endpoints.internal.DefaultS3EndpointProvider;
 import software.amazon.awssdk.services.s3.model.S3Exception;
@@ -29,8 +31,10 @@
 import org.elasticsearch.watcher.ResourceWatcherService;
 
 import java.io.IOException;
+import java.net.URI;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.Mockito.mock;
 
 public class S3ServiceTests extends ESTestCase {
@@ -217,4 +221,22 @@ public void testGetClientRegionFallbackToUsEast1() {
             );
         }
     }
+
+    public void testEndpointOverrideSchemeDefaultsToHttpsWhenNotSpecified() {
+        final S3Service s3Service = new S3Service(
+            mock(Environment.class),
+            Settings.EMPTY,
+            mock(ResourceWatcherService.class),
+            () -> Region.of("es-test-region")
+        );
+        final String endpointWithoutScheme = randomIdentifier() + ".ignore";
+        S3Client s3Client = s3Service.buildClient(
+            S3ClientSettings.getClientSettings(
+                Settings.builder().put("s3.client.test-client.endpoint", endpointWithoutScheme).build(),
+                "test-client"
+            ),
+            mock(SdkHttpClient.class)
+        );
+        assertThat(s3Client.serviceClientConfiguration().endpointOverride().get(), equalTo(URI.create("https://" + endpointWithoutScheme)));
+    }
 }