Add end-to-end test for reloading S3 credentials (#116994)

We don't seem to have a test that completely verifies that a S3
repository can reload credentials from an updated keystore. This commit
adds such a test.

Backport of #116762 to 8.16.

Author: DaveCTurner

modules/repository-s3/build.gradle

@@ -12,6 +12,7 @@ import org.elasticsearch.gradle.internal.test.InternalClusterTestPlugin
  */
 apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
+apply plugin: 'elasticsearch.internal-java-rest-test'
 
 esplugin {
   description 'The S3 repository plugin adds S3 repositories'
@@ -48,6 +49,10 @@ dependencies {
   yamlRestTestImplementation project(':test:fixtures:minio-fixture')
   internalClusterTestImplementation project(':test:fixtures:minio-fixture')
 
+  javaRestTestImplementation project(":test:framework")
+  javaRestTestImplementation project(':test:fixtures:s3-fixture')
+  javaRestTestImplementation project(':modules:repository-s3')
+
   yamlRestTestRuntimeOnly "org.slf4j:slf4j-simple:${versions.slf4j}"
   internalClusterTestRuntimeOnly "org.slf4j:slf4j-simple:${versions.slf4j}"
 }

modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3RestIT.java

@@ -0,0 +1,97 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.repositories.s3;
+
+import fixture.s3.S3HttpFixture;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.MutableSettingsProvider;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.junit.ClassRule;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
+
+import java.io.IOException;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.equalTo;
+
+public class RepositoryS3RestIT extends ESRestTestCase {
+
+    private static final String BUCKET = "RepositoryS3JavaRestTest-bucket";
+    private static final String BASE_PATH = "RepositoryS3JavaRestTest-base-path";
+
+    public static final S3HttpFixture s3Fixture = new S3HttpFixture(true, BUCKET, BASE_PATH, "ignored");
+
+    private static final MutableSettingsProvider keystoreSettings = new MutableSettingsProvider();
+
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .module("repository-s3")
+        .keystore(keystoreSettings)
+        .setting("s3.client.default.endpoint", s3Fixture::getAddress)
+        .build();
+
+    @ClassRule
+    public static TestRule ruleChain = RuleChain.outerRule(s3Fixture).around(cluster);
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+    public void testReloadCredentialsFromKeystore() throws IOException {
+        // Register repository (?verify=false because we don't have access to the blob store yet)
+        final var repositoryName = randomIdentifier();
+        registerRepository(
+            repositoryName,
+            S3Repository.TYPE,
+            false,
+            Settings.builder().put("bucket", BUCKET).put("base_path", BASE_PATH).build()
+        );
+        final var verifyRequest = new Request("POST", "/_snapshot/" + repositoryName + "/_verify");
+
+        // Set up initial credentials
+        final var accessKey1 = randomIdentifier();
+        s3Fixture.setAccessKey(accessKey1);
+        keystoreSettings.put("s3.client.default.access_key", accessKey1);
+        keystoreSettings.put("s3.client.default.secret_key", randomIdentifier());
+        cluster.updateStoredSecureSettings();
+        assertOK(client().performRequest(new Request("POST", "/_nodes/reload_secure_settings")));
+
+        // Check access using initial credentials
+        assertOK(client().performRequest(verifyRequest));
+
+        // Rotate credentials in blob store
+        final var accessKey2 = randomValueOtherThan(accessKey1, ESTestCase::randomIdentifier);
+        s3Fixture.setAccessKey(accessKey2);
+
+        // Ensure that initial credentials now invalid
+        final var accessDeniedException2 = expectThrows(ResponseException.class, () -> client().performRequest(verifyRequest));
+        assertThat(accessDeniedException2.getResponse().getStatusLine().getStatusCode(), equalTo(500));
+        assertThat(
+            accessDeniedException2.getMessage(),
+            allOf(containsString("Bad access key"), containsString("Status Code: 403"), containsString("Error Code: AccessDenied"))
+        );
+
+        // Set up refreshed credentials
+        keystoreSettings.put("s3.client.default.access_key", accessKey2);
+        cluster.updateStoredSecureSettings();
+        assertOK(client().performRequest(new Request("POST", "/_nodes/reload_secure_settings")));
+
+        // Check access using refreshed credentials
+        assertOK(client().performRequest(verifyRequest));
+    }
+
+}

test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpFixture.java

@@ -26,10 +26,10 @@ public class S3HttpFixture extends ExternalResource {
 
     private HttpServer server;
 
-    private boolean enabled;
+    private final boolean enabled;
     private final String bucket;
     private final String basePath;
-    protected final String accessKey;
+    protected volatile String accessKey;
 
     public S3HttpFixture() {
         this(true);
@@ -98,4 +98,8 @@ private static InetSocketAddress resolveAddress(String address, int port) {
             throw new RuntimeException(e);
         }
     }
+
+    public void setAccessKey(String accessKey) {
+        this.accessKey = accessKey;
+    }
 }