Extract IMDS test fixture from S3 fixture (#117324)

The S3 and IMDS services are separate things in practice, we shouldn't
be conflating them as we do today. This commit introduces a new
independent test fixture just for the IMDS endpoint and migrates the
relevant tests to use it.

Relates ES-9984

Author: DaveCTurner

modules/repository-s3/build.gradle

@@ -45,6 +45,7 @@ dependencies {
   testImplementation project(':test:fixtures:s3-fixture')
   yamlRestTestImplementation project(":test:framework")
   yamlRestTestImplementation project(':test:fixtures:s3-fixture')
+  yamlRestTestImplementation project(':test:fixtures:ec2-imds-fixture')
   yamlRestTestImplementation project(':test:fixtures:minio-fixture')
   internalClusterTestImplementation project(':test:fixtures:minio-fixture')
 

modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3ClientYamlTestSuiteIT.java

@@ -9,47 +9,71 @@
 
 package org.elasticsearch.repositories.s3;
 
+import fixture.aws.imds.Ec2ImdsHttpFixture;
 import fixture.s3.S3HttpFixture;
-import fixture.s3.S3HttpFixtureWithEC2;
 import fixture.s3.S3HttpFixtureWithSessionToken;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
 
+import org.elasticsearch.cluster.routing.Murmur3HashFunction;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.fixtures.testcontainers.TestContainersThreadFilter;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.junit.ClassRule;
 import org.junit.rules.RuleChain;
 import org.junit.rules.TestRule;
 
+import java.util.Set;
+
 @ThreadLeakFilters(filters = { TestContainersThreadFilter.class })
 @ThreadLeakScope(ThreadLeakScope.Scope.NONE) // https://github.com/elastic/elasticsearch/issues/102482
 public class RepositoryS3ClientYamlTestSuiteIT extends AbstractRepositoryS3ClientYamlTestSuiteIT {
 
-    public static final S3HttpFixture s3Fixture = new S3HttpFixture();
-    public static final S3HttpFixtureWithSessionToken s3HttpFixtureWithSessionToken = new S3HttpFixtureWithSessionToken();
-    public static final S3HttpFixtureWithEC2 s3Ec2 = new S3HttpFixtureWithEC2();
+    private static final String HASHED_SEED = Integer.toString(Murmur3HashFunction.hash(System.getProperty("tests.seed")));
+    private static final String TEMPORARY_SESSION_TOKEN = "session_token-" + HASHED_SEED;
+    private static final String IMDS_ACCESS_KEY = "imds-access-key-" + HASHED_SEED;
+    private static final String IMDS_SESSION_TOKEN = "imds-session-token-" + HASHED_SEED;
+
+    private static final S3HttpFixture s3Fixture = new S3HttpFixture();
+
+    private static final S3HttpFixtureWithSessionToken s3HttpFixtureWithSessionToken = new S3HttpFixtureWithSessionToken(
+        "session_token_bucket",
+        "session_token_base_path_integration_tests",
+        System.getProperty("s3TemporaryAccessKey"),
+        TEMPORARY_SESSION_TOKEN
+    );
+
+    private static final S3HttpFixtureWithSessionToken s3HttpFixtureWithImdsSessionToken = new S3HttpFixtureWithSessionToken(
+        "ec2_bucket",
+        "ec2_base_path",
+        IMDS_ACCESS_KEY,
+        IMDS_SESSION_TOKEN
+    );
 
-    private static final String s3TemporarySessionToken = "session_token";
+    private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture(IMDS_ACCESS_KEY, IMDS_SESSION_TOKEN, Set.of());
 
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .module("repository-s3")
         .keystore("s3.client.integration_test_permanent.access_key", System.getProperty("s3PermanentAccessKey"))
         .keystore("s3.client.integration_test_permanent.secret_key", System.getProperty("s3PermanentSecretKey"))
         .keystore("s3.client.integration_test_temporary.access_key", System.getProperty("s3TemporaryAccessKey"))
         .keystore("s3.client.integration_test_temporary.secret_key", System.getProperty("s3TemporarySecretKey"))
-        .keystore("s3.client.integration_test_temporary.session_token", s3TemporarySessionToken)
+        .keystore("s3.client.integration_test_temporary.session_token", TEMPORARY_SESSION_TOKEN)
         .setting("s3.client.integration_test_permanent.endpoint", s3Fixture::getAddress)
         .setting("s3.client.integration_test_temporary.endpoint", s3HttpFixtureWithSessionToken::getAddress)
-        .setting("s3.client.integration_test_ec2.endpoint", s3Ec2::getAddress)
-        .systemProperty("com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", s3Ec2::getAddress)
+        .setting("s3.client.integration_test_ec2.endpoint", s3HttpFixtureWithImdsSessionToken::getAddress)
+        .systemProperty("com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", ec2ImdsHttpFixture::getAddress)
         .build();
 
     @ClassRule
-    public static TestRule ruleChain = RuleChain.outerRule(s3Fixture).around(s3Ec2).around(s3HttpFixtureWithSessionToken).around(cluster);
+    public static TestRule ruleChain = RuleChain.outerRule(s3Fixture)
+        .around(s3HttpFixtureWithSessionToken)
+        .around(s3HttpFixtureWithImdsSessionToken)
+        .around(ec2ImdsHttpFixture)
+        .around(cluster);
 
     @ParametersFactory
     public static Iterable<Object[]> parameters() throws Exception {

modules/repository-s3/src/yamlRestTest/java/org/elasticsearch/repositories/s3/RepositoryS3EcsClientYamlTestSuiteIT.java

@@ -9,28 +9,48 @@
 
 package org.elasticsearch.repositories.s3;
 
-import fixture.s3.S3HttpFixtureWithECS;
+import fixture.aws.imds.Ec2ImdsHttpFixture;
+import fixture.s3.S3HttpFixtureWithSessionToken;
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
+import org.elasticsearch.cluster.routing.Murmur3HashFunction;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.junit.ClassRule;
 import org.junit.rules.RuleChain;
 import org.junit.rules.TestRule;
 
+import java.util.Set;
+
 public class RepositoryS3EcsClientYamlTestSuiteIT extends AbstractRepositoryS3ClientYamlTestSuiteIT {
-    private static final S3HttpFixtureWithECS s3Ecs = new S3HttpFixtureWithECS();
+
+    private static final String HASHED_SEED = Integer.toString(Murmur3HashFunction.hash(System.getProperty("tests.seed")));
+    private static final String ECS_ACCESS_KEY = "ecs-access-key-" + HASHED_SEED;
+    private static final String ECS_SESSION_TOKEN = "ecs-session-token-" + HASHED_SEED;
+
+    private static final S3HttpFixtureWithSessionToken s3Fixture = new S3HttpFixtureWithSessionToken(
+        "ecs_bucket",
+        "ecs_base_path",
+        ECS_ACCESS_KEY,
+        ECS_SESSION_TOKEN
+    );
+
+    private static final Ec2ImdsHttpFixture ec2ImdsHttpFixture = new Ec2ImdsHttpFixture(
+        ECS_ACCESS_KEY,
+        ECS_SESSION_TOKEN,
+        Set.of("/ecs_credentials_endpoint")
+    );
 
     public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .module("repository-s3")
-        .setting("s3.client.integration_test_ecs.endpoint", s3Ecs::getAddress)
-        .environment("AWS_CONTAINER_CREDENTIALS_FULL_URI", () -> (s3Ecs.getAddress() + "/ecs_credentials_endpoint"))
+        .setting("s3.client.integration_test_ecs.endpoint", s3Fixture::getAddress)
+        .environment("AWS_CONTAINER_CREDENTIALS_FULL_URI", () -> ec2ImdsHttpFixture.getAddress() + "/ecs_credentials_endpoint")
         .build();
 
     @ClassRule
-    public static TestRule ruleChain = RuleChain.outerRule(s3Ecs).around(cluster);
+    public static TestRule ruleChain = RuleChain.outerRule(s3Fixture).around(ec2ImdsHttpFixture).around(cluster);
 
     @ParametersFactory
     public static Iterable<Object[]> parameters() throws Exception {

settings.gradle

@@ -87,6 +87,7 @@ List projects = [
   'server',
   'test:framework',
   'test:fixtures:azure-fixture',
+  'test:fixtures:ec2-imds-fixture',
   'test:fixtures:gcs-fixture',
   'test:fixtures:hdfs-fixture',
   'test:fixtures:krb5kdc-fixture',

test/fixtures/ec2-imds-fixture/build.gradle

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+apply plugin: 'elasticsearch.java'
+
+description = 'Fixture for emulating the Instance Metadata Service (IMDS) running in AWS EC2'
+
+dependencies {
+  api project(':server')
+  api("junit:junit:${versions.junit}") {
+    transitive = false
+  }
+  api project(':test:framework')
+}

test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsHttpFixture.java

@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+package fixture.aws.imds;
+
+import com.sun.net.httpserver.HttpHandler;
+import com.sun.net.httpserver.HttpServer;
+
+import org.junit.rules.ExternalResource;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+import java.util.Objects;
+import java.util.Set;
+
+public class Ec2ImdsHttpFixture extends ExternalResource {
+
+    private HttpServer server;
+
+    private final String accessKey;
+    private final String sessionToken;
+    private final Set<String> alternativeCredentialsEndpoints;
+
+    public Ec2ImdsHttpFixture(String accessKey, String sessionToken, Set<String> alternativeCredentialsEndpoints) {
+        this.accessKey = accessKey;
+        this.sessionToken = sessionToken;
+        this.alternativeCredentialsEndpoints = alternativeCredentialsEndpoints;
+    }
+
+    protected HttpHandler createHandler() {
+        return new Ec2ImdsHttpHandler(accessKey, sessionToken, alternativeCredentialsEndpoints);
+    }
+
+    public String getAddress() {
+        return "http://" + server.getAddress().getHostString() + ":" + server.getAddress().getPort();
+    }
+
+    public void stop(int delay) {
+        server.stop(delay);
+    }
+
+    protected void before() throws Throwable {
+        server = HttpServer.create(resolveAddress(), 0);
+        server.createContext("/", Objects.requireNonNull(createHandler()));
+        server.start();
+    }
+
+    @Override
+    protected void after() {
+        stop(0);
+    }
+
+    private static InetSocketAddress resolveAddress() {
+        try {
+            return new InetSocketAddress(InetAddress.getByName("localhost"), 0);
+        } catch (UnknownHostException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsHttpHandler.java

@@ -0,0 +1,98 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+package fixture.aws.imds;
+
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpHandler;
+
+import org.elasticsearch.ExceptionsHelper;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
+import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.rest.RestStatus;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.time.Clock;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.Collection;
+import java.util.Objects;
+import java.util.Set;
+
+import static org.elasticsearch.test.ESTestCase.randomIdentifier;
+
+/**
+ * Minimal HTTP handler that emulates the EC2 IMDS server
+ */
+@SuppressForbidden(reason = "this test uses a HttpServer to emulate the EC2 IMDS endpoint")
+public class Ec2ImdsHttpHandler implements HttpHandler {
+
+    private static final String IMDS_SECURITY_CREDENTIALS_PATH = "/latest/meta-data/iam/security-credentials/";
+
+    private final String accessKey;
+    private final String sessionToken;
+    private final Set<String> validCredentialsEndpoints = ConcurrentCollections.newConcurrentSet();
+
+    public Ec2ImdsHttpHandler(String accessKey, String sessionToken, Collection<String> alternativeCredentialsEndpoints) {
+        this.accessKey = Objects.requireNonNull(accessKey);
+        this.sessionToken = Objects.requireNonNull(sessionToken);
+        this.validCredentialsEndpoints.addAll(alternativeCredentialsEndpoints);
+    }
+
+    @Override
+    public void handle(final HttpExchange exchange) throws IOException {
+        // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html
+
+        try (exchange) {
+            final var path = exchange.getRequestURI().getPath();
+            final var requestMethod = exchange.getRequestMethod();
+
+            if ("PUT".equals(requestMethod) && "/latest/api/token".equals(path)) {
+                // Reject IMDSv2 probe
+                exchange.sendResponseHeaders(RestStatus.METHOD_NOT_ALLOWED.getStatus(), -1);
+                return;
+            }
+
+            if ("GET".equals(requestMethod)) {
+                if (path.equals(IMDS_SECURITY_CREDENTIALS_PATH)) {
+                    final var profileName = randomIdentifier();
+                    validCredentialsEndpoints.add(IMDS_SECURITY_CREDENTIALS_PATH + profileName);
+                    final byte[] response = profileName.getBytes(StandardCharsets.UTF_8);
+                    exchange.getResponseHeaders().add("Content-Type", "text/plain");
+                    exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length);
+                    exchange.getResponseBody().write(response);
+                    return;
+                } else if (validCredentialsEndpoints.contains(path)) {
+                    final byte[] response = Strings.format(
+                        """
+                            {
+                              "AccessKeyId": "%s",
+                              "Expiration": "%s",
+                              "RoleArn": "%s",
+                              "SecretAccessKey": "%s",
+                              "Token": "%s"
+                            }""",
+                        accessKey,
+                        ZonedDateTime.now(Clock.systemUTC()).plusDays(1L).format(DateTimeFormatter.ISO_DATE_TIME),
+                        randomIdentifier(),
+                        randomIdentifier(),
+                        sessionToken
+                    ).getBytes(StandardCharsets.UTF_8);
+                    exchange.getResponseHeaders().add("Content-Type", "application/json");
+                    exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length);
+                    exchange.getResponseBody().write(response);
+                    return;
+                }
+            }
+
+            ExceptionsHelper.maybeDieOnAnotherThread(new AssertionError("not supported: " + requestMethod + " " + path));
+        }
+    }
+}