Support dynamic credentials in S3HttpFixture (#117749)

Rephrase the authorization check in `S3HttpFixture` in terms of a
predicate provided by the caller so that there's no need for a separate
subclass that handles session tokens, and so that it can support
auto-generated credentials more naturally.

Also adapts `Ec2ImdsHttpFixture` to dynamically generate credentials
this way.

Also extracts the STS fixture in `S3HttpFixtureWithSTS` into a separate
service, similarly to #117324, and adapts this new fixture to
dynamically generate credentials too.

Relates ES-9984

Backport of #117458 to 8.x

Author: DaveCTurner

modules/repository-s3/build.gradle

@@ -47,6 +47,7 @@ dependencies {
   yamlRestTestImplementation project(":test:framework")
   yamlRestTestImplementation project(':test:fixtures:s3-fixture')
   yamlRestTestImplementation project(':test:fixtures:ec2-imds-fixture')
+  yamlRestTestImplementation project(':test:fixtures:aws-sts-fixture')
   yamlRestTestImplementation project(':test:fixtures:minio-fixture')
   internalClusterTestImplementation project(':test:fixtures:minio-fixture')
 

modules/repository-s3/src/javaRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3RestReloadCredentialsIT.java

@@ -36,7 +36,14 @@ public class RepositoryS3RestReloadCredentialsIT extends ESRestTestCase {
     private static final String BUCKET = "RepositoryS3RestReloadCredentialsIT-bucket-" + HASHED_SEED;
     private static final String BASE_PATH = "RepositoryS3RestReloadCredentialsIT-base-path-" + HASHED_SEED;
 
-    public static final S3HttpFixture s3Fixture = new S3HttpFixture(true, BUCKET, BASE_PATH, "ignored");
+    private static volatile String repositoryAccessKey;
+
+    public static final S3HttpFixture s3Fixture = new S3HttpFixture(
+        true,
+        BUCKET,
+        BASE_PATH,
+        S3HttpFixture.mutableAccessKey(() -> repositoryAccessKey)
+    );
 
     private static final MutableSettingsProvider keystoreSettings = new MutableSettingsProvider();
 
@@ -67,7 +74,7 @@ public void testReloadCredentialsFromKeystore() throws IOException {
 
         // Set up initial credentials
         final var accessKey1 = randomIdentifier();
-        s3Fixture.setAccessKey(accessKey1);
+        repositoryAccessKey = accessKey1;
         keystoreSettings.put("s3.client.default.access_key", accessKey1);
         keystoreSettings.put("s3.client.default.secret_key", randomSecretKey());
         cluster.updateStoredSecureSettings();
@@ -79,14 +86,14 @@ public void testReloadCredentialsFromKeystore() throws IOException {
 
         // Rotate credentials in blob store
         final var accessKey2 = randomValueOtherThan(accessKey1, ESTestCase::randomSecretKey);
-        s3Fixture.setAccessKey(accessKey2);
+        repositoryAccessKey = accessKey2;
 
         // Ensure that initial credentials now invalid
         final var accessDeniedException2 = expectThrows(ResponseException.class, () -> client().performRequest(verifyRequest));
         assertThat(accessDeniedException2.getResponse().getStatusLine().getStatusCode(), equalTo(500));
         assertThat(
             accessDeniedException2.getMessage(),
-            allOf(containsString("Bad access key"), containsString("Status Code: 403"), containsString("Error Code: AccessDenied"))
+            allOf(containsString("Access denied"), containsString("Status Code: 403"), containsString("Error Code: AccessDenied"))
         );
 
         // Set up refreshed credentials

modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java

@@ -10,8 +10,8 @@
 package org.elasticsearch.repositories.s3;
 
 import fixture.aws.imds.Ec2ImdsHttpFixture;
+import fixture.s3.DynamicS3Credentials;
 import fixture.s3.S3HttpFixture;
-import fixture.s3.S3HttpFixtureWithSessionToken;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
@@ -34,27 +34,30 @@ public class RepositoryS3ClientYamlTestSuiteIT extends AbstractRepositoryS3Clien
 
     private static final String HASHED_SEED = Integer.toString(Murmur3HashFunction.hash(System.getProperty("tests.seed")));
     private static final String TEMPORARY_SESSION_TOKEN = "session_token-" + HASHED_SEED;
-    private static final String IMDS_ACCESS_KEY = "imds-access-key-" + HASHED_SEED;
-    private static final String IMDS_SESSION_TOKEN = "imds-session-token-" + HASHED_SEED;
 
     private static final S3HttpFixture s3Fixture = new S3HttpFixture();
 
-    private static final S3HttpFixtureWithSessionToken s3HttpFixtureWithSessionToken = new S3HttpFixtureWithSessionToken(
+    private static final S3HttpFixture s3HttpFixtureWithSessionToken = new S3HttpFixture(
+        true,
         "session_token_bucket",
         "session_token_base_path_integration_tests",
-        System.getProperty("s3TemporaryAccessKey"),
-        TEMPORARY_SESSION_TOKEN
+        S3HttpFixture.fixedAccessKeyAndToken(System.getProperty("s3TemporaryAccessKey"), TEMPORARY_SESSION_TOKEN)
     );
 
-    private static final S3HttpFixtureWithSessionToken s3HttpFixtureWithImdsSessionToken = new S3HttpFixtureWithSessionToken(
+    private static final DynamicS3Credentials dynamicS3Credentials = new DynamicS3Credentials();
+
+    private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture(
+        dynamicS3Credentials::addValidCredentials,
+        Set.of()
+    );
+
+    private static final S3HttpFixture s3HttpFixtureWithImdsSessionToken = new S3HttpFixture(
+        true,
         "ec2_bucket",
         "ec2_base_path",
-        IMDS_ACCESS_KEY,
-        IMDS_SESSION_TOKEN
+        dynamicS3Credentials::isAuthorized
     );
 
-    private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture(IMDS_ACCESS_KEY, IMDS_SESSION_TOKEN, Set.of());
-
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .module("repository-s3")
         .keystore("s3.client.integration_test_permanent.access_key", System.getProperty("s3PermanentAccessKey"))

modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3EcsClientYamlTestSuiteIT.java

@@ -10,12 +10,12 @@
 package org.elasticsearch.repositories.s3;
 
 import fixture.aws.imds.Ec2ImdsHttpFixture;
-import fixture.s3.S3HttpFixtureWithSessionToken;
+import fixture.s3.DynamicS3Credentials;
+import fixture.s3.S3HttpFixture;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
-import org.elasticsearch.cluster.routing.Murmur3HashFunction;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.junit.ClassRule;
@@ -26,23 +26,20 @@
 
 public class RepositoryS3EcsClientYamlTestSuiteIT extends AbstractRepositoryS3ClientYamlTestSuiteIT {
 
-    private static final String HASHED_SEED = Integer.toString(Murmur3HashFunction.hash(System.getProperty("tests.seed")));
-    private static final String ECS_ACCESS_KEY = "ecs-access-key-" + HASHED_SEED;
-    private static final String ECS_SESSION_TOKEN = "ecs-session-token-" + HASHED_SEED;
-
-    private static final S3HttpFixtureWithSessionToken s3Fixture = new S3HttpFixtureWithSessionToken(
-        "ecs_bucket",
-        "ecs_base_path",
-        ECS_ACCESS_KEY,
-        ECS_SESSION_TOKEN
-    );
+    private static final DynamicS3Credentials dynamicS3Credentials = new DynamicS3Credentials();
 
     private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture(
-        ECS_ACCESS_KEY,
-        ECS_SESSION_TOKEN,
+        dynamicS3Credentials::addValidCredentials,
         Set.of("/ecs_credentials_endpoint")
     );
 
+    private static final S3HttpFixture s3Fixture = new S3HttpFixture(
+        true,
+        "ecs_bucket",
+        "ecs_base_path",
+        dynamicS3Credentials::isAuthorized
+    );
+
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .module("repository-s3")
         .setting("s3.client.integration_test_ecs.endpoint", s3Fixture::getAddress)

modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3StsClientYamlTestSuiteIT.java

@@ -9,8 +9,9 @@
 
 package org.elasticsearch.repositories.s3;
 
+import fixture.aws.sts.AwsStsHttpFixture;
+import fixture.s3.DynamicS3Credentials;
 import fixture.s3.S3HttpFixture;
-import fixture.s3.S3HttpFixtureWithSTS;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
@@ -24,13 +25,27 @@
 
 public class RepositoryS3StsClientYamlTestSuiteIT extends AbstractRepositoryS3ClientYamlTestSuiteIT {
 
-    public static final S3HttpFixture s3Fixture = new S3HttpFixture();
-    private static final S3HttpFixtureWithSTS s3Sts = new S3HttpFixtureWithSTS();
+    private static final DynamicS3Credentials dynamicS3Credentials = new DynamicS3Credentials();
+
+    private static final S3HttpFixture s3HttpFixture = new S3HttpFixture(
+        true,
+        "sts_bucket",
+        "sts_base_path",
+        dynamicS3Credentials::isAuthorized
+    );
+
+    private static final AwsStsHttpFixture stsHttpFixture = new AwsStsHttpFixture(dynamicS3Credentials::addValidCredentials, """
+        Atza|IQEBLjAsAhRFiXuWpUXuRvQ9PZL3GMFcYevydwIUFAHZwXZXXXXXXXXJnrulxKDHwy87oGKPznh0D6bEQZTSCzyoCtL_8S07pLpr0zMbn6w1lfVZKNTBdDans\
+        FBmtGnIsIapjI6xKR02Yc_2bQ8LZbUXSGm6Ry6_BG7PrtLZtj_dfCTj92xNGed-CrKqjG7nPBjNIL016GGvuS5gSvPRUxWES3VYfm1wl7WTI7jn-Pcb6M-buCgHhFO\
+        zTQxod27L9CqnOLio7N3gZAGpsp6n1-AJBOCJckcyXe2c6uD0srOJeZlKUm2eTDVMf8IehDVI0r1QOnTV6KzzAI3OY87Vd_cVMQ""");
 
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .module("repository-s3")
-        .setting("s3.client.integration_test_sts.endpoint", s3Sts::getAddress)
-        .systemProperty("com.amazonaws.sdk.stsMetadataServiceEndpointOverride", () -> s3Sts.getAddress() + "/assume-role-with-web-identity")
+        .setting("s3.client.integration_test_sts.endpoint", s3HttpFixture::getAddress)
+        .systemProperty(
+            "com.amazonaws.sdk.stsMetadataServiceEndpointOverride",
+            () -> stsHttpFixture.getAddress() + "/assume-role-with-web-identity"
+        )
         .configFile("repository-s3/aws-web-identity-token-file", Resource.fromClasspath("aws-web-identity-token-file"))
         .environment("AWS_WEB_IDENTITY_TOKEN_FILE", System.getProperty("awsWebIdentityTokenExternalLocation"))
         // // The AWS STS SDK requires the role and session names to be set. We can verify that they are sent to S3S in the
@@ -40,7 +55,7 @@ public class RepositoryS3StsClientYamlTestSuiteIT extends AbstractRepositoryS3Cl
         .build();
 
     @ClassRule
-    public static TestRule ruleChain = RuleChain.outerRule(s3Fixture).around(s3Sts).around(cluster);
+    public static TestRule ruleChain = RuleChain.outerRule(s3HttpFixture).around(stsHttpFixture).around(cluster);
 
     @ParametersFactory
     public static Iterable<Object[]> parameters() throws Exception {

settings.gradle

@@ -88,6 +88,7 @@ List projects = [
   'distribution:tools:ansi-console',
   'server',
   'test:framework',
+  'test:fixtures:aws-sts-fixture',
   'test:fixtures:azure-fixture',
   'test:fixtures:ec2-imds-fixture',
   'test:fixtures:gcs-fixture',

test/fixtures/aws-sts-fixture/build.gradle

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+apply plugin: 'elasticsearch.java'
+
+description = 'Fixture for emulating the Security Token Service (STS) running in AWS'
+
+dependencies {
+  api project(':server')
+  api("junit:junit:${versions.junit}") {
+    transitive = false
+  }
+  api project(':test:framework')
+}

test/fixtures/aws-sts-fixture/src/main/java/fixture/aws/sts/AwsStsHttpFixture.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+package fixture.aws.sts;
+
+import com.sun.net.httpserver.HttpHandler;
+import com.sun.net.httpserver.HttpServer;
+
+import org.junit.rules.ExternalResource;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+import java.util.Objects;
+import java.util.function.BiConsumer;
+
+public class AwsStsHttpFixture extends ExternalResource {
+
+    private HttpServer server;
+
+    private final BiConsumer<String, String> newCredentialsConsumer;
+    private final String webIdentityToken;
+
+    public AwsStsHttpFixture(BiConsumer<String, String> newCredentialsConsumer, String webIdentityToken) {
+        this.newCredentialsConsumer = Objects.requireNonNull(newCredentialsConsumer);
+        this.webIdentityToken = Objects.requireNonNull(webIdentityToken);
+    }
+
+    protected HttpHandler createHandler() {
+        return new AwsStsHttpHandler(newCredentialsConsumer, webIdentityToken);
+    }
+
+    public String getAddress() {
+        return "http://" + server.getAddress().getHostString() + ":" + server.getAddress().getPort();
+    }
+
+    public void stop(int delay) {
+        server.stop(delay);
+    }
+
+    protected void before() throws Throwable {
+        server = HttpServer.create(resolveAddress(), 0);
+        server.createContext("/", Objects.requireNonNull(createHandler()));
+        server.start();
+    }
+
+    @Override
+    protected void after() {
+        stop(0);
+    }
+
+    private static InetSocketAddress resolveAddress() {
+        try {
+            return new InetSocketAddress(InetAddress.getByName("localhost"), 0);
+        } catch (UnknownHostException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}