> getSettings() {
@Override
public Settings additionalSettings() {
- final Settings.Builder builder = Settings.builder();
-
- // Adds a node attribute for the ec2 availability zone
- final String azMetadataUrl = EC2MetadataUtils.getHostAddressForEC2MetadataService()
- + "/latest/meta-data/placement/availability-zone";
- String azMetadataTokenUrl = EC2MetadataUtils.getHostAddressForEC2MetadataService() + "/latest/api/token";
- builder.put(getAvailabilityZoneNodeAttributes(settings, azMetadataUrl, azMetadataTokenUrl));
- return builder.build();
+ return getAvailabilityZoneNodeAttributes(settings);
}
- // pkg private for testing
- @SuppressForbidden(reason = "We call getInputStream in doPrivileged and provide SocketPermission")
- static Settings getAvailabilityZoneNodeAttributes(Settings settings, String azMetadataUrl, String azMetadataTokenUrl) {
- if (AwsEc2Service.AUTO_ATTRIBUTE_SETTING.get(settings) == false) {
- return Settings.EMPTY;
- }
- final Settings.Builder attrs = Settings.builder();
-
- final URL url;
- final URLConnection urlConnection;
- try {
- url = new URL(azMetadataUrl);
- logger.debug("obtaining ec2 [placement/availability-zone] from ec2 meta-data url {}", url);
- urlConnection = SocketAccess.doPrivilegedIOException(url::openConnection);
- urlConnection.setConnectTimeout(2000);
- AwsEc2Utils.getMetadataToken(azMetadataTokenUrl)
- .ifPresent(token -> urlConnection.setRequestProperty(X_AWS_EC_2_METADATA_TOKEN, token));
- } catch (final IOException e) {
- // should not happen, we know the url is not malformed, and openConnection does not actually hit network
- throw new UncheckedIOException(e);
- }
-
- try (
- InputStream in = SocketAccess.doPrivilegedIOException(urlConnection::getInputStream);
- BufferedReader urlReader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))
- ) {
+ private static final String IMDS_AVAILABILITY_ZONE_PATH = "/latest/meta-data/placement/availability-zone";
- final String metadataResult = urlReader.readLine();
- if ((metadataResult == null) || (metadataResult.length() == 0)) {
- throw new IllegalStateException("no ec2 metadata returned from " + url);
- } else {
- attrs.put(Node.NODE_ATTRIBUTES.getKey() + "aws_availability_zone", metadataResult);
+ // pkg private for testing
+ static Settings getAvailabilityZoneNodeAttributes(Settings settings) {
+ if (AwsEc2Service.AUTO_ATTRIBUTE_SETTING.get(settings)) {
+ try {
+ return Settings.builder()
+ .put(
+ Node.NODE_ATTRIBUTES.getKey() + "aws_availability_zone",
+ AwsEc2Utils.getInstanceMetadata(IMDS_AVAILABILITY_ZONE_PATH)
+ )
+ .build();
+ } catch (Exception e) {
+ // this is lenient so the plugin does not fail when installed outside of ec2
+ logger.error("failed to get metadata for [placement/availability-zone]", e);
}
- } catch (final IOException e) {
- // this is lenient so the plugin does not fail when installed outside of ec2
- logger.error("failed to get metadata for [placement/availability-zone]", e);
}
- return attrs.build();
+ return Settings.EMPTY;
}
@Override
@@ -179,8 +123,6 @@ public void close() throws IOException {
@Override
public void reload(Settings settingsToLoad) {
- // secure settings should be readable
- final Ec2ClientSettings clientSettings = Ec2ClientSettings.getClientSettings(settingsToLoad);
- ec2Service.refreshAndClearCache(clientSettings);
+ ec2Service.refreshAndClearCache(Ec2ClientSettings.getClientSettings(settingsToLoad));
}
}
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2NameResolver.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2NameResolver.java
index b4fbce7bb693a..b7deda32a9fc2 100644
--- a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2NameResolver.java
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/Ec2NameResolver.java
@@ -9,51 +9,28 @@
package org.elasticsearch.discovery.ec2;
-import com.amazonaws.util.EC2MetadataUtils;
-
import org.elasticsearch.common.network.NetworkService.CustomNameResolver;
-import org.elasticsearch.core.IOUtils;
-import org.elasticsearch.core.SuppressForbidden;
-import org.elasticsearch.logging.LogManager;
-import org.elasticsearch.logging.Logger;
-import java.io.BufferedReader;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
import java.net.InetAddress;
-import java.net.URL;
-import java.net.URLConnection;
-import java.nio.charset.StandardCharsets;
-
-import static org.elasticsearch.discovery.ec2.AwsEc2Utils.X_AWS_EC_2_METADATA_TOKEN;
/**
- * Resolves certain ec2 related 'meta' hostnames into an actual hostname
- * obtained from ec2 meta-data.
+ * Resolves certain EC2 related 'meta' hostnames into an actual hostname
+ * obtained from the EC2 instance metadata service
*
* Valid config values for {@link Ec2HostnameType}s are -
*
- * - _ec2_ - maps to privateIpv4
- * - _ec2:privateIp_ - maps to privateIpv4
- * - _ec2:privateIpv4_
- * - _ec2:privateDns_
- * - _ec2:publicIp_ - maps to publicIpv4
- * - _ec2:publicIpv4_
- * - _ec2:publicDns_
+ * - {@code _ec2_} (maps to privateIpv4)
+ * - {@code _ec2:privateIp_} (maps to privateIpv4)
+ * - {@code _ec2:privateIpv4_}
+ * - {@code _ec2:privateDns_}
+ * - {@code _ec2:publicIp_} (maps to publicIpv4)
+ * - {@code _ec2:publicIpv4_}
+ * - {@code _ec2:publicDns_}
*
- *
- * @author Paul_Loy (keteracel)
*/
class Ec2NameResolver implements CustomNameResolver {
- private static final Logger logger = LogManager.getLogger(Ec2NameResolver.class);
-
- /**
- * enum that can be added to over time with more meta-data types (such as ipv6 when this is available)
- *
- * @author Paul_Loy
- */
private enum Ec2HostnameType {
PRIVATE_IPv4("ec2:privateIpv4", "local-ipv4"),
@@ -75,51 +52,24 @@ private enum Ec2HostnameType {
}
}
- /**
- * @param type the ec2 hostname type to discover.
- * @return the appropriate host resolved from ec2 meta-data, or null if it cannot be obtained.
- * @see CustomNameResolver#resolveIfPossible(String)
- */
- @SuppressForbidden(reason = "We call getInputStream in doPrivileged and provide SocketPermission")
- public static InetAddress[] resolve(Ec2HostnameType type) throws IOException {
- InputStream in = null;
- String metadataUrl = EC2MetadataUtils.getHostAddressForEC2MetadataService() + "/latest/meta-data/" + type.ec2Name;
- String metadataTokenUrl = EC2MetadataUtils.getHostAddressForEC2MetadataService() + "/latest/api/token";
- try {
- URL url = new URL(metadataUrl);
- logger.debug("obtaining ec2 hostname from ec2 meta-data url {}", url);
- URLConnection urlConnection = SocketAccess.doPrivilegedIOException(url::openConnection);
- urlConnection.setConnectTimeout(2000);
- AwsEc2Utils.getMetadataToken(metadataTokenUrl)
- .ifPresent(token -> urlConnection.setRequestProperty(X_AWS_EC_2_METADATA_TOKEN, token));
-
- in = SocketAccess.doPrivilegedIOException(urlConnection::getInputStream);
- BufferedReader urlReader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8));
-
- String metadataResult = urlReader.readLine();
- if (metadataResult == null || metadataResult.length() == 0) {
- throw new IOException("no gce metadata returned from [" + url + "] for [" + type.configName + "]");
- }
- // only one address: because we explicitly ask for only one via the Ec2HostnameType
- return new InetAddress[] { InetAddress.getByName(metadataResult) };
- } catch (IOException e) {
- throw new IOException("IOException caught when fetching InetAddress from [" + metadataUrl + "]", e);
- } finally {
- IOUtils.closeWhileHandlingException(in);
- }
- }
-
@Override
public InetAddress[] resolveDefault() {
return null; // using this, one has to explicitly specify _ec2_ in network setting
- // return resolve(Ec2HostnameType.DEFAULT, false);
}
+ private static final String IMDS_ADDRESS_PATH_PREFIX = "/latest/meta-data/";
+
@Override
public InetAddress[] resolveIfPossible(String value) throws IOException {
for (Ec2HostnameType type : Ec2HostnameType.values()) {
if (type.configName.equals(value)) {
- return resolve(type);
+ final var metadataPath = IMDS_ADDRESS_PATH_PREFIX + type.ec2Name;
+ try {
+ // only one address: IMDS returns just one address/name, and if it's a name then it should resolve to one address
+ return new InetAddress[] { InetAddress.getByName(AwsEc2Utils.getInstanceMetadata(metadataPath)) };
+ } catch (Exception e) {
+ throw new IOException("Exception caught when resolving EC2 address from [" + metadataPath + "]", e);
+ }
}
}
return null;
diff --git a/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/HttpScheme.java b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/HttpScheme.java
new file mode 100644
index 0000000000000..34f883e461974
--- /dev/null
+++ b/plugins/discovery-ec2/src/main/java/org/elasticsearch/discovery/ec2/HttpScheme.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.discovery.ec2;
+
+public enum HttpScheme {
+ HTTP("http"),
+ HTTPS("https");
+
+ private final String schemeString;
+
+ HttpScheme(String schemeString) {
+ this.schemeString = schemeString;
+ }
+
+ public String getSchemeString() {
+ return schemeString;
+ }
+}
diff --git a/plugins/discovery-ec2/src/main/plugin-metadata/plugin-security.policy b/plugins/discovery-ec2/src/main/plugin-metadata/plugin-security.policy
index 4fb7eb7dda2e4..7827c4b9bb987 100644
--- a/plugins/discovery-ec2/src/main/plugin-metadata/plugin-security.policy
+++ b/plugins/discovery-ec2/src/main/plugin-metadata/plugin-security.policy
@@ -19,4 +19,5 @@ grant {
permission java.net.SocketPermission "*", "connect";
permission java.lang.reflect.ReflectPermission "suppressAccessChecks";
+ permission java.util.PropertyPermission "http.proxyHost", "read";
};
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AbstractEC2MockAPITestCase.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AbstractEC2MockAPITestCase.java
index 7e5ac435a412a..9f348abeee7a9 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AbstractEC2MockAPITestCase.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AbstractEC2MockAPITestCase.java
@@ -8,8 +8,9 @@
*/
package org.elasticsearch.discovery.ec2;
-import com.amazonaws.services.ec2.model.Instance;
-import com.amazonaws.services.ec2.model.Tag;
+import software.amazon.awssdk.services.ec2.model.Instance;
+import software.amazon.awssdk.services.ec2.model.Tag;
+
import com.sun.net.httpserver.HttpServer;
import org.elasticsearch.common.network.InetAddresses;
@@ -115,11 +116,11 @@ static byte[] generateDescribeInstancesResponse(List instances) {
sw.writeStartElement("item");
{
sw.writeStartElement("instanceId");
- sw.writeCharacters(instance.getInstanceId());
+ sw.writeCharacters(instance.instanceId());
sw.writeEndElement();
sw.writeStartElement("imageId");
- sw.writeCharacters(instance.getImageId());
+ sw.writeCharacters(instance.imageId());
sw.writeEndElement();
sw.writeStartElement("instanceState");
@@ -135,11 +136,11 @@ static byte[] generateDescribeInstancesResponse(List instances) {
sw.writeEndElement();
sw.writeStartElement("privateDnsName");
- sw.writeCharacters(instance.getPrivateDnsName());
+ sw.writeCharacters(instance.privateDnsName());
sw.writeEndElement();
sw.writeStartElement("dnsName");
- sw.writeCharacters(instance.getPublicDnsName());
+ sw.writeCharacters(instance.publicDnsName());
sw.writeEndElement();
sw.writeStartElement("instanceType");
@@ -161,23 +162,23 @@ static byte[] generateDescribeInstancesResponse(List instances) {
sw.writeEndElement();
sw.writeStartElement("privateIpAddress");
- sw.writeCharacters(instance.getPrivateIpAddress());
+ sw.writeCharacters(instance.privateIpAddress());
sw.writeEndElement();
sw.writeStartElement("ipAddress");
- sw.writeCharacters(instance.getPublicIpAddress());
+ sw.writeCharacters(instance.publicIpAddress());
sw.writeEndElement();
sw.writeStartElement("tagSet");
- for (Tag tag : instance.getTags()) {
+ for (Tag tag : instance.tags()) {
sw.writeStartElement("item");
{
sw.writeStartElement("key");
- sw.writeCharacters(tag.getKey());
+ sw.writeCharacters(tag.key());
sw.writeEndElement();
sw.writeStartElement("value");
- sw.writeCharacters(tag.getValue());
+ sw.writeCharacters(tag.value());
sw.writeEndElement();
}
sw.writeEndElement();
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImplTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImplTests.java
deleted file mode 100644
index cf12ebb302248..0000000000000
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/AwsEc2ServiceImplTests.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the "Elastic License
- * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
- * Public License v 1"; you may not use this file except in compliance with, at
- * your election, the "Elastic License 2.0", the "GNU Affero General Public
- * License v3.0 only", or the "Server Side Public License, v 1".
- */
-
-package org.elasticsearch.discovery.ec2;
-
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.Protocol;
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.BasicSessionCredentials;
-import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
-
-import org.apache.logging.log4j.Level;
-import org.elasticsearch.common.settings.MockSecureSettings;
-import org.elasticsearch.common.settings.Setting;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.settings.SettingsException;
-import org.elasticsearch.logging.LogManager;
-import org.elasticsearch.logging.Logger;
-import org.elasticsearch.test.ESTestCase;
-
-import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.is;
-
-public class AwsEc2ServiceImplTests extends ESTestCase {
-
- // we need our own ES logger, rather than log4j logging
- private static final Logger logger = LogManager.getLogger(AwsEc2ServiceImplTests.class);
-
- public void testAWSCredentialsWithSystemProviders() {
- final AWSCredentialsProvider credentialsProvider = AwsEc2ServiceImpl.buildCredentials(
- logger,
- Ec2ClientSettings.getClientSettings(Settings.EMPTY)
- );
- assertThat(credentialsProvider, instanceOf(DefaultAWSCredentialsProviderChain.class));
- }
-
- public void testAWSCredentialsWithElasticsearchAwsSettings() {
- final MockSecureSettings secureSettings = new MockSecureSettings();
- secureSettings.setString("discovery.ec2.access_key", "aws_key");
- secureSettings.setString("discovery.ec2.secret_key", "aws_secret");
- final AWSCredentials credentials = AwsEc2ServiceImpl.buildCredentials(
- logger,
- Ec2ClientSettings.getClientSettings(Settings.builder().setSecureSettings(secureSettings).build())
- ).getCredentials();
- assertThat(credentials.getAWSAccessKeyId(), is("aws_key"));
- assertThat(credentials.getAWSSecretKey(), is("aws_secret"));
- }
-
- public void testAWSSessionCredentialsWithElasticsearchAwsSettings() {
- final MockSecureSettings secureSettings = new MockSecureSettings();
- secureSettings.setString("discovery.ec2.access_key", "aws_key");
- secureSettings.setString("discovery.ec2.secret_key", "aws_secret");
- secureSettings.setString("discovery.ec2.session_token", "aws_session_token");
- final BasicSessionCredentials credentials = (BasicSessionCredentials) AwsEc2ServiceImpl.buildCredentials(
- logger,
- Ec2ClientSettings.getClientSettings(Settings.builder().setSecureSettings(secureSettings).build())
- ).getCredentials();
- assertThat(credentials.getAWSAccessKeyId(), is("aws_key"));
- assertThat(credentials.getAWSSecretKey(), is("aws_secret"));
- assertThat(credentials.getSessionToken(), is("aws_session_token"));
- }
-
- public void testDeprecationOfLoneAccessKey() {
- final MockSecureSettings secureSettings = new MockSecureSettings();
- secureSettings.setString("discovery.ec2.access_key", "aws_key");
- final AWSCredentials credentials = AwsEc2ServiceImpl.buildCredentials(
- logger,
- Ec2ClientSettings.getClientSettings(Settings.builder().setSecureSettings(secureSettings).build())
- ).getCredentials();
- assertThat(credentials.getAWSAccessKeyId(), is("aws_key"));
- assertThat(credentials.getAWSSecretKey(), is(""));
- assertSettingDeprecationsAndWarnings(
- new Setting[] {},
- new DeprecationWarning(
- Level.WARN,
- "Setting [discovery.ec2.access_key] is set but " + "[discovery.ec2.secret_key] is not, which will be unsupported in future"
- )
- );
- }
-
- public void testDeprecationOfLoneSecretKey() {
- final MockSecureSettings secureSettings = new MockSecureSettings();
- secureSettings.setString("discovery.ec2.secret_key", "aws_secret");
- final AWSCredentials credentials = AwsEc2ServiceImpl.buildCredentials(
- logger,
- Ec2ClientSettings.getClientSettings(Settings.builder().setSecureSettings(secureSettings).build())
- ).getCredentials();
- assertThat(credentials.getAWSAccessKeyId(), is(""));
- assertThat(credentials.getAWSSecretKey(), is("aws_secret"));
- assertSettingDeprecationsAndWarnings(
- new Setting[] {},
- new DeprecationWarning(
- Level.WARN,
- "Setting [discovery.ec2.secret_key] is set but " + "[discovery.ec2.access_key] is not, which will be unsupported in future"
- )
- );
- }
-
- public void testRejectionOfLoneSessionToken() {
- final MockSecureSettings secureSettings = new MockSecureSettings();
- secureSettings.setString("discovery.ec2.session_token", "aws_session_token");
- SettingsException e = expectThrows(
- SettingsException.class,
- () -> AwsEc2ServiceImpl.buildCredentials(
- logger,
- Ec2ClientSettings.getClientSettings(Settings.builder().setSecureSettings(secureSettings).build())
- )
- );
- assertThat(
- e.getMessage(),
- is("Setting [discovery.ec2.session_token] is set but [discovery.ec2.access_key] and [discovery.ec2.secret_key] are not")
- );
- }
-
- public void testAWSDefaultConfiguration() {
- launchAWSConfigurationTest(
- Settings.EMPTY,
- Protocol.HTTPS,
- null,
- -1,
- Protocol.HTTP,
- null,
- null,
- ClientConfiguration.DEFAULT_SOCKET_TIMEOUT
- );
- }
-
- public void testAWSConfigurationWithAwsSettings() {
- final MockSecureSettings secureSettings = new MockSecureSettings();
- secureSettings.setString("discovery.ec2.proxy.username", "aws_proxy_username");
- secureSettings.setString("discovery.ec2.proxy.password", "aws_proxy_password");
- final Settings settings = Settings.builder()
- .put("discovery.ec2.protocol", "http")
- .put("discovery.ec2.proxy.host", "aws_proxy_host")
- .put("discovery.ec2.proxy.port", 8080)
- .put("discovery.ec2.proxy.scheme", "http")
- .put("discovery.ec2.read_timeout", "10s")
- .setSecureSettings(secureSettings)
- .build();
- launchAWSConfigurationTest(
- settings,
- Protocol.HTTP,
- "aws_proxy_host",
- 8080,
- Protocol.HTTP,
- "aws_proxy_username",
- "aws_proxy_password",
- 10000
- );
- }
-
- protected void launchAWSConfigurationTest(
- Settings settings,
- Protocol expectedProtocol,
- String expectedProxyHost,
- int expectedProxyPort,
- Protocol expectedProxyScheme,
- String expectedProxyUsername,
- String expectedProxyPassword,
- int expectedReadTimeout
- ) {
- final ClientConfiguration configuration = AwsEc2ServiceImpl.buildConfiguration(Ec2ClientSettings.getClientSettings(settings));
-
- assertThat(configuration.getResponseMetadataCacheSize(), is(0));
- assertThat(configuration.getProtocol(), is(expectedProtocol));
- assertThat(configuration.getProxyHost(), is(expectedProxyHost));
- assertThat(configuration.getProxyPort(), is(expectedProxyPort));
- assertThat(configuration.getProxyProtocol(), is(expectedProxyScheme));
- assertThat(configuration.getProxyUsername(), is(expectedProxyUsername));
- assertThat(configuration.getProxyPassword(), is(expectedProxyPassword));
- assertThat(configuration.getSocketTimeout(), is(expectedReadTimeout));
- }
-
-}
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/EC2RetriesTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/EC2RetriesTests.java
index 78ea619d81f84..aa19c794dfb6a 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/EC2RetriesTests.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/EC2RetriesTests.java
@@ -9,8 +9,8 @@
package org.elasticsearch.discovery.ec2;
-import com.amazonaws.http.HttpMethodName;
-import com.amazonaws.services.ec2.model.Instance;
+import software.amazon.awssdk.http.SdkHttpMethod;
+import software.amazon.awssdk.services.ec2.model.Instance;
import org.apache.http.HttpStatus;
import org.apache.http.NameValuePair;
@@ -72,7 +72,7 @@ public void testEC2DiscoveryRetriesOnRateLimiting() throws IOException {
// retry the same request 5 times at most
final int maxRetries = randomIntBetween(1, 5);
httpServer.createContext("/", exchange -> {
- if (exchange.getRequestMethod().equals(HttpMethodName.POST.name())) {
+ if (SdkHttpMethod.POST.name().equals(exchange.getRequestMethod())) {
final String request = new String(exchange.getRequestBody().readAllBytes(), UTF_8);
final String userAgent = exchange.getRequestHeaders().getFirst("User-Agent");
if (userAgent != null && userAgent.startsWith("aws-sdk-java")) {
@@ -92,7 +92,9 @@ public void testEC2DiscoveryRetriesOnRateLimiting() throws IOException {
for (NameValuePair parse : URLEncodedUtils.parse(request, UTF_8)) {
if ("Action".equals(parse.getName())) {
responseBody = generateDescribeInstancesResponse(
- hosts.stream().map(address -> new Instance().withPublicIpAddress(address)).collect(Collectors.toList())
+ hosts.stream()
+ .map(address -> Instance.builder().publicIpAddress(address).build())
+ .collect(Collectors.toList())
);
break;
}
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java
index a5bcb10cdd468..76a2ca856b6bd 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryPluginTests.java
@@ -9,318 +9,341 @@
package org.elasticsearch.discovery.ec2;
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.Protocol;
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.BasicAWSCredentials;
-import com.amazonaws.auth.BasicSessionCredentials;
-import com.amazonaws.services.ec2.AbstractAmazonEC2;
-import com.amazonaws.services.ec2.AmazonEC2;
-import com.sun.net.httpserver.HttpExchange;
-import com.sun.net.httpserver.HttpHandler;
-import com.sun.net.httpserver.HttpServer;
+import fixture.aws.imds.Ec2ImdsHttpFixture;
+import fixture.aws.imds.Ec2ImdsServiceBuilder;
+import fixture.aws.imds.Ec2ImdsVersion;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
+import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
+import software.amazon.awssdk.http.apache.ApacheHttpClient;
+import software.amazon.awssdk.http.apache.ProxyConfiguration;
+import software.amazon.awssdk.services.ec2.Ec2Client;
+import software.amazon.awssdk.services.ec2.Ec2ClientBuilder;
+import software.amazon.awssdk.services.ec2.endpoints.Ec2EndpointParams;
+import software.amazon.awssdk.services.ec2.endpoints.Ec2EndpointProvider;
import org.elasticsearch.common.settings.MockSecureSettings;
import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.core.SuppressForbidden;
-import org.elasticsearch.mocksocket.MockHttpServer;
+import org.elasticsearch.common.settings.SettingsException;
+import org.elasticsearch.core.CheckedConsumer;
+import org.elasticsearch.core.TimeValue;
import org.elasticsearch.node.Node;
import org.elasticsearch.test.ESTestCase;
-
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.nio.charset.StandardCharsets;
-
+import org.mockito.ArgumentCaptor;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.function.Consumer;
+
+import static org.elasticsearch.discovery.ec2.Ec2ClientSettings.ACCESS_KEY_SETTING;
+import static org.elasticsearch.discovery.ec2.Ec2ClientSettings.PROXY_HOST_SETTING;
+import static org.elasticsearch.discovery.ec2.Ec2ClientSettings.PROXY_PASSWORD_SETTING;
+import static org.elasticsearch.discovery.ec2.Ec2ClientSettings.PROXY_PORT_SETTING;
+import static org.elasticsearch.discovery.ec2.Ec2ClientSettings.PROXY_SCHEME_SETTING;
+import static org.elasticsearch.discovery.ec2.Ec2ClientSettings.PROXY_USERNAME_SETTING;
+import static org.elasticsearch.discovery.ec2.Ec2ClientSettings.READ_TIMEOUT_SETTING;
+import static org.elasticsearch.discovery.ec2.Ec2ClientSettings.SECRET_KEY_SETTING;
+import static org.elasticsearch.discovery.ec2.Ec2ClientSettings.SESSION_TOKEN_SETTING;
import static org.hamcrest.Matchers.instanceOf;
-import static org.hamcrest.Matchers.is;
-
-@SuppressForbidden(reason = "Uses an HttpServer to emulate the Instance Metadata Service")
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Mostly just testing that the various plugin settings (see reference docs) result in appropriate calls on the client builder, using mocks.
+ */
public class Ec2DiscoveryPluginTests extends ESTestCase {
- private Settings getNodeAttributes(Settings settings, String url, String tokenUrl) {
- final Settings realSettings = Settings.builder().put(AwsEc2Service.AUTO_ATTRIBUTE_SETTING.getKey(), true).put(settings).build();
- return Ec2DiscoveryPlugin.getAvailabilityZoneNodeAttributes(realSettings, url, tokenUrl);
- }
-
- private void assertNodeAttributes(Settings settings, String url, String tokenUrl, String expected) {
- final Settings additional = getNodeAttributes(settings, url, tokenUrl);
- if (expected == null) {
- assertTrue(additional.isEmpty());
- } else {
- assertEquals(expected, additional.get(Node.NODE_ATTRIBUTES.getKey() + "aws_availability_zone"));
- }
+ public void testNodeAttributesDisabledByDefault() {
+ assertTrue(Ec2DiscoveryPlugin.getAvailabilityZoneNodeAttributes(Settings.EMPTY).isEmpty());
}
public void testNodeAttributesDisabled() {
- final Settings settings = Settings.builder().put(AwsEc2Service.AUTO_ATTRIBUTE_SETTING.getKey(), false).build();
- assertNodeAttributes(settings, "bogus", "", null);
+ assertTrue(
+ Ec2DiscoveryPlugin.getAvailabilityZoneNodeAttributes(
+ Settings.builder().put(AwsEc2Service.AUTO_ATTRIBUTE_SETTING.getKey(), false).build()
+ ).isEmpty()
+ );
}
- public void testNodeAttributes() throws Exception {
- try (var metadataServer = metadataServerWithoutToken()) {
- assertNodeAttributes(Settings.EMPTY, metadataServer.metadataUri(), "", "us-east-1c");
- }
+ public void testNodeAttributesEnabled() {
+ final var availabilityZone = randomIdentifier();
+ Ec2ImdsHttpFixture.runWithFixture(
+ new Ec2ImdsServiceBuilder(Ec2ImdsVersion.V2).availabilityZoneSupplier(() -> availabilityZone),
+ ec2ImdsHttpFixture -> {
+ try (var ignored = Ec2ImdsHttpFixture.withEc2MetadataServiceEndpointOverride(ec2ImdsHttpFixture.getAddress())) {
+ final var availabilityZoneNodeAttributeSettings = Ec2DiscoveryPlugin.getAvailabilityZoneNodeAttributes(
+ Settings.builder().put(AwsEc2Service.AUTO_ATTRIBUTE_SETTING.getKey(), true).build()
+ );
+ assertEquals(
+ availabilityZone,
+ availabilityZoneNodeAttributeSettings.get(Node.NODE_ATTRIBUTES.getKey() + "aws_availability_zone")
+ );
+ }
+ }
+ );
}
- public void testNodeAttributesBogusUrl() {
- final UncheckedIOException e = expectThrows(UncheckedIOException.class, () -> getNodeAttributes(Settings.EMPTY, "bogus", ""));
- assertNotNull(e.getCause());
- final String msg = e.getCause().getMessage();
- assertTrue(msg, msg.contains("no protocol: bogus"));
+ public void testDefaultEndpoint() {
+ // Ec2ClientSettings#ENDPOINT_SETTING is not set, so the builder method shouldn't be called
+ runPluginMockTest(Settings.builder(), plugin -> verify(plugin.ec2ClientBuilder, never()).endpointProvider(any()));
}
- public void testNodeAttributesEmpty() throws Exception {
- try (MetadataServer metadataServer = new MetadataServer("/metadata", exchange -> {
- exchange.sendResponseHeaders(200, -1);
- exchange.close();
- })) {
- final IllegalStateException e = expectThrows(
- IllegalStateException.class,
- () -> getNodeAttributes(Settings.EMPTY, metadataServer.metadataUri(), "")
- );
- assertTrue(e.getMessage(), e.getMessage().contains("no ec2 metadata returned"));
- }
+ public void testSpecificEndpoint() {
+ final var argumentCaptor = ArgumentCaptor.forClass(Ec2EndpointProvider.class);
+ final var endpoint = randomIdentifier() + ".local";
+ runPluginMockTest(
+ Settings.builder().put(Ec2ClientSettings.ENDPOINT_SETTING.getKey(), endpoint),
+ plugin -> verify(plugin.ec2ClientBuilder, times(1)).endpointProvider(argumentCaptor.capture())
+ );
+ assertEquals(endpoint, safeGet(argumentCaptor.getValue().resolveEndpoint(Ec2EndpointParams.builder().build())).url().toString());
}
- public void testNodeAttributesErrorLenient() throws Exception {
- try (var metadataServer = new MetadataServer("/metadata", exchange -> {
- exchange.sendResponseHeaders(404, -1);
- exchange.close();
- })) {
- assertNodeAttributes(Settings.EMPTY, metadataServer.metadataUri(), "", null);
- }
+ public void testDefaultHttpSocketTimeout() {
+ final var argumentCaptor = ArgumentCaptor.forClass(Duration.class);
+ runPluginMockTest(Settings.builder(), plugin -> verify(plugin.httpClientBuilder, times(1)).socketTimeout(argumentCaptor.capture()));
+ assertEquals(READ_TIMEOUT_SETTING.get(Settings.EMPTY).nanos(), argumentCaptor.getValue().toNanos());
}
- public void testNodeAttributesWithToken() throws Exception {
- try (var metadataServer = new MetadataServer("/metadata", exchange -> {
- assertEquals("imdsv2-token", exchange.getRequestHeaders().getFirst("X-aws-ec2-metadata-token"));
- exchange.sendResponseHeaders(200, 0);
- exchange.getResponseBody().write("us-east-1c".getBytes(StandardCharsets.UTF_8));
- exchange.close();
- }, "/latest/api/token", exchange -> {
- assertEquals("PUT", exchange.getRequestMethod());
- assertEquals("10", exchange.getRequestHeaders().getFirst("X-aws-ec2-metadata-token-ttl-seconds"));
- exchange.sendResponseHeaders(200, 0);
- exchange.getResponseBody().write("imdsv2-token".getBytes(StandardCharsets.UTF_8));
- exchange.close();
- })) {
- assertNodeAttributes(Settings.EMPTY, metadataServer.metadataUri(), metadataServer.tokenUri(), "us-east-1c");
- }
+ public void testSpecificHttpSocketTimeout() {
+ final var argumentCaptor = ArgumentCaptor.forClass(Duration.class);
+ final var timeoutValue = TimeValue.timeValueMillis(between(0, 100000));
+ runPluginMockTest(
+ Settings.builder().put(READ_TIMEOUT_SETTING.getKey(), timeoutValue),
+ plugin -> verify(plugin.httpClientBuilder, times(1)).socketTimeout(argumentCaptor.capture())
+ );
+ assertEquals(timeoutValue.nanos(), argumentCaptor.getValue().toNanos());
}
- public void testTokenMetadataApiIsMisbehaving() throws Exception {
- try (var metadataServer = new MetadataServer("/metadata", exchange -> {
- assertNull(exchange.getRequestHeaders().getFirst("X-aws-ec2-metadata-token"));
- exchange.sendResponseHeaders(200, 0);
- exchange.getResponseBody().write("us-east-1c".getBytes(StandardCharsets.UTF_8));
- exchange.close();
- }, "/latest/api/token", HttpExchange::close)) {
- assertNodeAttributes(Settings.EMPTY, metadataServer.metadataUri(), metadataServer.tokenUri(), "us-east-1c");
- }
+ public void testDefaultProxyConfiguration() {
+ runPluginMockTest(Settings.builder(), plugin -> verify(plugin.httpClientBuilder, never()).proxyConfiguration(any()));
}
- public void testTokenMetadataApiDoesNotRespond() throws Exception {
- try (var metadataServer = new MetadataServer("/metadata", exchange -> {
- assertNull(exchange.getRequestHeaders().getFirst("X-aws-ec2-metadata-token"));
- exchange.sendResponseHeaders(200, 0);
- exchange.getResponseBody().write("us-east-1c".getBytes(StandardCharsets.UTF_8));
- exchange.close();
- }, "/latest/api/token", ex -> {
- // Intentionally don't close the connection, so the client has to time out
- })) {
- assertNodeAttributes(Settings.EMPTY, metadataServer.metadataUri(), metadataServer.tokenUri(), "us-east-1c");
- }
- }
+ public void testSpecificProxyConfiguration() {
+ // generates a random proxy configuration (i.e. randomly setting/omitting all the settings) and verifies that the resulting
+ // ProxyConfiguration is as expected with a sequence of assertions that match the configuration we generated
- public void testTokenMetadataApiIsNotAvailable() throws Exception {
- try (var metadataServer = metadataServerWithoutToken()) {
- assertNodeAttributes(Settings.EMPTY, metadataServer.metadataUri(), metadataServer.tokenUri(), "us-east-1c");
- }
- }
+ final var argumentCaptor = ArgumentCaptor.forClass(ProxyConfiguration.class);
- public void testBogusTokenMetadataUrl() throws Exception {
- try (var metadataServer = metadataServerWithoutToken();) {
- assertNodeAttributes(Settings.EMPTY, metadataServer.metadataUri(), "bogus", "us-east-1c");
- }
- }
+ final var proxySettings = Settings.builder();
+ final var assertions = new ArrayList>();
- public void testDefaultEndpoint() throws IOException {
- try (Ec2DiscoveryPluginMock plugin = new Ec2DiscoveryPluginMock(Settings.EMPTY)) {
- final String endpoint = ((AmazonEC2Mock) plugin.ec2Service.client().client()).endpoint;
- assertThat(endpoint, is(""));
- }
- }
+ final var proxyHost = "proxy." + randomIdentifier() + ".host";
+ proxySettings.put(PROXY_HOST_SETTING.getKey(), proxyHost);
+ assertions.add(proxyConfiguration -> assertEquals(proxyHost, proxyConfiguration.host()));
- public void testSpecificEndpoint() throws IOException {
- final Settings settings = Settings.builder().put(Ec2ClientSettings.ENDPOINT_SETTING.getKey(), "ec2.endpoint").build();
- try (Ec2DiscoveryPluginMock plugin = new Ec2DiscoveryPluginMock(settings)) {
- final String endpoint = ((AmazonEC2Mock) plugin.ec2Service.client().client()).endpoint;
- assertThat(endpoint, is("ec2.endpoint"));
+ // randomly set, or not, the port
+ if (randomBoolean()) {
+ final var proxyPort = between(1, 65535);
+ proxySettings.put(PROXY_PORT_SETTING.getKey(), proxyPort);
+ assertions.add(proxyConfiguration -> assertEquals(proxyPort, proxyConfiguration.port()));
+ } else {
+ assertions.add(proxyConfiguration -> assertEquals((int) PROXY_PORT_SETTING.get(Settings.EMPTY), proxyConfiguration.port()));
}
- }
- public void testClientSettingsReInit() throws IOException {
- final MockSecureSettings mockSecure1 = new MockSecureSettings();
- mockSecure1.setString(Ec2ClientSettings.ACCESS_KEY_SETTING.getKey(), "ec2_access_1");
- mockSecure1.setString(Ec2ClientSettings.SECRET_KEY_SETTING.getKey(), "ec2_secret_key_1");
- final boolean mockSecure1HasSessionToken = randomBoolean();
- if (mockSecure1HasSessionToken) {
- mockSecure1.setString(Ec2ClientSettings.SESSION_TOKEN_SETTING.getKey(), "ec2_session_token_1");
- }
- mockSecure1.setString(Ec2ClientSettings.PROXY_USERNAME_SETTING.getKey(), "proxy_username_1");
- mockSecure1.setString(Ec2ClientSettings.PROXY_PASSWORD_SETTING.getKey(), "proxy_password_1");
- final Settings settings1 = Settings.builder()
- .put(Ec2ClientSettings.PROXY_HOST_SETTING.getKey(), "proxy_host_1")
- .put(Ec2ClientSettings.PROXY_PORT_SETTING.getKey(), 881)
- .put(Ec2ClientSettings.PROXY_SCHEME_SETTING.getKey(), "http")
- .put(Ec2ClientSettings.ENDPOINT_SETTING.getKey(), "ec2_endpoint_1")
- .setSecureSettings(mockSecure1)
- .build();
- final MockSecureSettings mockSecure2 = new MockSecureSettings();
- mockSecure2.setString(Ec2ClientSettings.ACCESS_KEY_SETTING.getKey(), "ec2_access_2");
- mockSecure2.setString(Ec2ClientSettings.SECRET_KEY_SETTING.getKey(), "ec2_secret_key_2");
- final boolean mockSecure2HasSessionToken = randomBoolean();
- if (mockSecure2HasSessionToken) {
- mockSecure2.setString(Ec2ClientSettings.SESSION_TOKEN_SETTING.getKey(), "ec2_session_token_2");
+ // randomly set, or not, the scheme
+ if (randomBoolean()) {
+ final var proxyScheme = randomFrom("http", "https");
+ proxySettings.put(PROXY_SCHEME_SETTING.getKey(), proxyScheme);
+ assertions.add(proxyConfiguration -> assertEquals(proxyScheme, proxyConfiguration.scheme()));
+ } else {
+ assertions.add(
+ proxyConfiguration -> assertEquals(PROXY_SCHEME_SETTING.get(Settings.EMPTY).getSchemeString(), proxyConfiguration.scheme())
+ );
}
- mockSecure2.setString(Ec2ClientSettings.PROXY_USERNAME_SETTING.getKey(), "proxy_username_2");
- mockSecure2.setString(Ec2ClientSettings.PROXY_PASSWORD_SETTING.getKey(), "proxy_password_2");
- final Settings settings2 = Settings.builder()
- .put(Ec2ClientSettings.PROXY_HOST_SETTING.getKey(), "proxy_host_2")
- .put(Ec2ClientSettings.PROXY_PORT_SETTING.getKey(), 882)
- .put(Ec2ClientSettings.PROXY_SCHEME_SETTING.getKey(), "http")
- .put(Ec2ClientSettings.ENDPOINT_SETTING.getKey(), "ec2_endpoint_2")
- .setSecureSettings(mockSecure2)
- .build();
- try (Ec2DiscoveryPluginMock plugin = new Ec2DiscoveryPluginMock(settings1)) {
- try (AmazonEc2Reference clientReference = plugin.ec2Service.client()) {
- {
- final AWSCredentials credentials = ((AmazonEC2Mock) clientReference.client()).credentials.getCredentials();
- assertThat(credentials.getAWSAccessKeyId(), is("ec2_access_1"));
- assertThat(credentials.getAWSSecretKey(), is("ec2_secret_key_1"));
- if (mockSecure1HasSessionToken) {
- assertThat(credentials, instanceOf(BasicSessionCredentials.class));
- assertThat(((BasicSessionCredentials) credentials).getSessionToken(), is("ec2_session_token_1"));
- } else {
- assertThat(credentials, instanceOf(BasicAWSCredentials.class));
- }
- assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyUsername(), is("proxy_username_1"));
- assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPassword(), is("proxy_password_1"));
- assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyHost(), is("proxy_host_1"));
- assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPort(), is(881));
- assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyProtocol(), is(Protocol.HTTP));
- assertThat(((AmazonEC2Mock) clientReference.client()).endpoint, is("ec2_endpoint_1"));
- }
- // reload secure settings2
- plugin.reload(settings2);
- // client is not released, it is still using the old settings
- {
- final AWSCredentials credentials = ((AmazonEC2Mock) clientReference.client()).credentials.getCredentials();
- if (mockSecure1HasSessionToken) {
- assertThat(credentials, instanceOf(BasicSessionCredentials.class));
- assertThat(((BasicSessionCredentials) credentials).getSessionToken(), is("ec2_session_token_1"));
- } else {
- assertThat(credentials, instanceOf(BasicAWSCredentials.class));
- }
- assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyUsername(), is("proxy_username_1"));
- assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPassword(), is("proxy_password_1"));
- assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyHost(), is("proxy_host_1"));
- assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPort(), is(881));
- assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyProtocol(), is(Protocol.HTTP));
- assertThat(((AmazonEC2Mock) clientReference.client()).endpoint, is("ec2_endpoint_1"));
- }
- }
- try (AmazonEc2Reference clientReference = plugin.ec2Service.client()) {
- final AWSCredentials credentials = ((AmazonEC2Mock) clientReference.client()).credentials.getCredentials();
- assertThat(credentials.getAWSAccessKeyId(), is("ec2_access_2"));
- assertThat(credentials.getAWSSecretKey(), is("ec2_secret_key_2"));
- if (mockSecure2HasSessionToken) {
- assertThat(credentials, instanceOf(BasicSessionCredentials.class));
- assertThat(((BasicSessionCredentials) credentials).getSessionToken(), is("ec2_session_token_2"));
- } else {
- assertThat(credentials, instanceOf(BasicAWSCredentials.class));
- }
- assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyUsername(), is("proxy_username_2"));
- assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPassword(), is("proxy_password_2"));
- assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyHost(), is("proxy_host_2"));
- assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyPort(), is(882));
- assertThat(((AmazonEC2Mock) clientReference.client()).configuration.getProxyProtocol(), is(Protocol.HTTP));
- assertThat(((AmazonEC2Mock) clientReference.client()).endpoint, is("ec2_endpoint_2"));
- }
+
+ // randomly set, or not, the credentials
+ if (randomBoolean()) {
+ final var secureSettings = new MockSecureSettings();
+ final var proxyUsername = randomSecretKey();
+ final var proxyPassword = randomSecretKey();
+ secureSettings.setString(PROXY_USERNAME_SETTING.getKey(), proxyUsername);
+ secureSettings.setString(PROXY_PASSWORD_SETTING.getKey(), proxyPassword);
+ assertions.add(proxyConfiguration -> assertEquals(proxyUsername, proxyConfiguration.username()));
+ assertions.add(proxyConfiguration -> assertEquals(proxyPassword, proxyConfiguration.password()));
+ proxySettings.setSecureSettings(secureSettings);
+ } else {
+ assertions.add(proxyConfiguration -> assertEquals("", proxyConfiguration.username()));
+ assertions.add(proxyConfiguration -> assertEquals("", proxyConfiguration.password()));
}
- }
- private static class Ec2DiscoveryPluginMock extends Ec2DiscoveryPlugin {
+ // now verify
+ runPluginMockTest(proxySettings, plugin -> verify(plugin.httpClientBuilder, times(1)).proxyConfiguration(argumentCaptor.capture()));
+ final var proxyConfiguration = argumentCaptor.getValue();
+ assertions.forEach(a -> a.accept(proxyConfiguration));
+ }
- Ec2DiscoveryPluginMock(Settings settings) {
- super(settings, new AwsEc2ServiceImpl() {
- @Override
- AmazonEC2 buildClient(AWSCredentialsProvider credentials, ClientConfiguration configuration, String endpoint) {
- return new AmazonEC2Mock(credentials, configuration, endpoint);
- }
- });
- }
+ public void testCredentialsFromEnvironment() {
+ final var argumentCaptor = ArgumentCaptor.forClass(AwsCredentialsProvider.class);
+ runPluginMockTest(
+ Settings.builder(),
+ plugin -> verify(plugin.ec2ClientBuilder, times(1)).credentialsProvider(argumentCaptor.capture())
+ );
+ assertThat(argumentCaptor.getValue(), instanceOf(DefaultCredentialsProvider.class));
}
- private static class AmazonEC2Mock extends AbstractAmazonEC2 {
+ public void testPermanentCredentialsFromKeystore() {
+ final var accessKey = randomSecretKey();
+ final var secretKey = randomSecretKey();
- String endpoint;
- final AWSCredentialsProvider credentials;
- final ClientConfiguration configuration;
+ final var secureSettings = new MockSecureSettings();
+ secureSettings.setString(ACCESS_KEY_SETTING.getKey(), accessKey);
+ secureSettings.setString(SECRET_KEY_SETTING.getKey(), secretKey);
- AmazonEC2Mock(AWSCredentialsProvider credentials, ClientConfiguration configuration, String endpoint) {
- this.credentials = credentials;
- this.configuration = configuration;
- this.endpoint = endpoint;
- }
+ final var argumentCaptor = ArgumentCaptor.forClass(AwsCredentialsProvider.class);
+
+ runPluginMockTest(
+ Settings.builder().setSecureSettings(secureSettings),
+ plugin -> verify(plugin.ec2ClientBuilder, times(1)).credentialsProvider(argumentCaptor.capture())
+ );
+ final var awsCredentials = asInstanceOf(AwsBasicCredentials.class, argumentCaptor.getValue().resolveCredentials());
+ assertEquals(accessKey, awsCredentials.accessKeyId());
+ assertEquals(secretKey, awsCredentials.secretAccessKey());
+ }
- @Override
- public void shutdown() {}
+ public void testSessionCredentialsFromKeystore() {
+ final var accessKey = randomSecretKey();
+ final var secretKey = randomSecretKey();
+ final var sessionToken = randomSecretKey();
+
+ final var secureSettings = new MockSecureSettings();
+ secureSettings.setString(ACCESS_KEY_SETTING.getKey(), accessKey);
+ secureSettings.setString(SECRET_KEY_SETTING.getKey(), secretKey);
+ secureSettings.setString(SESSION_TOKEN_SETTING.getKey(), sessionToken);
+
+ final var argumentCaptor = ArgumentCaptor.forClass(AwsCredentialsProvider.class);
+
+ runPluginMockTest(
+ Settings.builder().setSecureSettings(secureSettings),
+ plugin -> verify(plugin.ec2ClientBuilder, times(1)).credentialsProvider(argumentCaptor.capture())
+ );
+ final var awsCredentials = asInstanceOf(AwsSessionCredentials.class, argumentCaptor.getValue().resolveCredentials());
+ assertEquals(accessKey, awsCredentials.accessKeyId());
+ assertEquals(secretKey, awsCredentials.secretAccessKey());
+ assertEquals(sessionToken, awsCredentials.sessionToken());
}
- @SuppressForbidden(reason = "Uses an HttpServer to emulate the Instance Metadata Service")
- private static MetadataServer metadataServerWithoutToken() throws IOException {
- return new MetadataServer("/metadata", exchange -> {
- assertNull(exchange.getRequestHeaders().getFirst("X-aws-ec2-metadata-token"));
- exchange.sendResponseHeaders(200, 0);
- exchange.getResponseBody().write("us-east-1c".getBytes(StandardCharsets.UTF_8));
- exchange.close();
- });
+ /**
+ * Sets up a plugin with the given {@code settings}, using mocks, and then calls the {@code pluginConsumer} on it.
+ */
+ private static void runPluginMockTest(Settings.Builder settings, CheckedConsumer pluginConsumer) {
+ final var httpClientBuilder = mock(ApacheHttpClient.Builder.class);
+ final var ec2ClientBuilder = mock(Ec2ClientBuilder.class);
+ when(ec2ClientBuilder.build()).thenReturn(mock(Ec2Client.class));
+
+ try (
+ var plugin = new Ec2DiscoveryPluginMock(settings.build(), httpClientBuilder, ec2ClientBuilder);
+ var ignored = plugin.ec2Service.client()
+ ) {
+ pluginConsumer.accept(plugin);
+ } catch (Exception e) {
+ throw new AssertionError("unexpected", e);
+ }
}
- @SuppressForbidden(reason = "Uses an HttpServer to emulate the Instance Metadata Service")
- private static class MetadataServer implements AutoCloseable {
+ public void testLoneAccessKeyError() {
+ final var secureSettings = new MockSecureSettings();
+ secureSettings.setString(ACCESS_KEY_SETTING.getKey(), randomSecretKey());
+ final var settings = Settings.builder().setSecureSettings(secureSettings).build();
+ assertEquals(
+ "Setting [discovery.ec2.access_key] is set but [discovery.ec2.secret_key] is not",
+ expectThrows(SettingsException.class, () -> new Ec2DiscoveryPlugin(settings)).getMessage()
+ );
+ }
- private final HttpServer httpServer;
+ public void testLoneSecretKeyError() {
+ final var secureSettings = new MockSecureSettings();
+ secureSettings.setString(SECRET_KEY_SETTING.getKey(), randomSecretKey());
+ final var settings = Settings.builder().setSecureSettings(secureSettings).build();
+ assertEquals(
+ "Setting [discovery.ec2.secret_key] is set but [discovery.ec2.access_key] is not",
+ expectThrows(SettingsException.class, () -> new Ec2DiscoveryPlugin(settings)).getMessage()
+ );
+ }
- private MetadataServer(String metadataPath, HttpHandler metadataHandler) throws IOException {
- this(metadataPath, metadataHandler, null, null);
- }
+ public void testLoneSessionTokenError() {
+ final var secureSettings = new MockSecureSettings();
+ secureSettings.setString(SESSION_TOKEN_SETTING.getKey(), randomSecretKey());
+ final var settings = Settings.builder().setSecureSettings(secureSettings).build();
+ assertEquals(
+ "Setting [discovery.ec2.session_token] is set but [discovery.ec2.access_key] and [discovery.ec2.secret_key] are not",
+ expectThrows(SettingsException.class, () -> new Ec2DiscoveryPlugin(settings)).getMessage()
+ );
+ }
- private MetadataServer(String metadataPath, HttpHandler metadataHandler, String tokenPath, HttpHandler tokenHandler)
- throws IOException {
- httpServer = MockHttpServer.createHttp(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0);
- httpServer.createContext(metadataPath, metadataHandler);
- if (tokenPath != null && tokenHandler != null) {
- httpServer.createContext(tokenPath, tokenHandler);
+ public void testReloadSettings() {
+ final var httpClientBuilder = mock(ApacheHttpClient.Builder.class);
+ final var ec2ClientBuilder = mock(Ec2ClientBuilder.class);
+
+ final var accessKey1 = randomSecretKey();
+ final var secretKey1 = randomSecretKey();
+ final var secureSettings1 = new MockSecureSettings();
+ secureSettings1.setString(ACCESS_KEY_SETTING.getKey(), accessKey1);
+ secureSettings1.setString(SECRET_KEY_SETTING.getKey(), secretKey1);
+ final var settings1 = Settings.builder().setSecureSettings(secureSettings1).build();
+
+ try (var plugin = new Ec2DiscoveryPluginMock(settings1, httpClientBuilder, ec2ClientBuilder)) {
+ final var client1 = mock(Ec2Client.class);
+ when(ec2ClientBuilder.build()).thenReturn(client1);
+
+ try (var clientReference = plugin.ec2Service.client()) {
+ assertSame(client1, clientReference.client());
+ final var argumentCaptor = ArgumentCaptor.forClass(AwsCredentialsProvider.class);
+ verify(plugin.ec2ClientBuilder, times(1)).credentialsProvider(argumentCaptor.capture());
+ final var awsCredentials = argumentCaptor.getValue().resolveCredentials();
+ assertEquals(accessKey1, awsCredentials.accessKeyId());
+ assertEquals(secretKey1, awsCredentials.secretAccessKey());
}
- httpServer.start();
+ verify(client1, never()).close(); // retaining client for future use
+
+ final var accessKey2 = randomSecretKey();
+ final var secretKey2 = randomSecretKey();
+ final var secureSettings2 = new MockSecureSettings();
+ secureSettings2.setString(ACCESS_KEY_SETTING.getKey(), accessKey2);
+ secureSettings2.setString(SECRET_KEY_SETTING.getKey(), secretKey2);
+ plugin.reload(Settings.builder().setSecureSettings(secureSettings2).build());
+
+ verify(client1, times(1)).close(); // client released on reload
+
+ final var client2 = mock(Ec2Client.class);
+ when(ec2ClientBuilder.build()).thenReturn(client2);
+
+ try (var clientReference = plugin.ec2Service.client()) {
+ assertSame(client2, clientReference.client());
+ final var argumentCaptor = ArgumentCaptor.forClass(AwsCredentialsProvider.class);
+ verify(plugin.ec2ClientBuilder, times(2)).credentialsProvider(argumentCaptor.capture());
+ final var awsCredentials = argumentCaptor.getAllValues().get(1).resolveCredentials();
+ assertEquals(accessKey2, awsCredentials.accessKeyId());
+ assertEquals(secretKey2, awsCredentials.secretAccessKey());
+ }
+ } catch (Exception e) {
+ throw new AssertionError("unexpected", e);
}
+ }
- @Override
- public void close() throws Exception {
- httpServer.stop(0);
- }
+ private static class Ec2DiscoveryPluginMock extends Ec2DiscoveryPlugin {
+ final ApacheHttpClient.Builder httpClientBuilder;
+ final Ec2ClientBuilder ec2ClientBuilder;
- private String metadataUri() {
- return "http://" + httpServer.getAddress().getHostString() + ":" + httpServer.getAddress().getPort() + "/metadata";
- }
+ Ec2DiscoveryPluginMock(Settings settings, ApacheHttpClient.Builder httpClientBuilder, Ec2ClientBuilder ec2ClientBuilder) {
+ super(settings, new AwsEc2ServiceImpl() {
+ @Override
+ ApacheHttpClient.Builder getHttpClientBuilder() {
+ return httpClientBuilder;
+ }
- private String tokenUri() {
- return "http://" + httpServer.getAddress().getHostString() + ":" + httpServer.getAddress().getPort() + "/latest/api/token";
+ @Override
+ Ec2ClientBuilder getEc2ClientBuilder() {
+ return ec2ClientBuilder;
+ }
+ });
+ this.httpClientBuilder = httpClientBuilder;
+ this.ec2ClientBuilder = ec2ClientBuilder;
}
}
+
}
diff --git a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java
index 135ddcee8da44..cbdbfab3ff41e 100644
--- a/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java
+++ b/plugins/discovery-ec2/src/test/java/org/elasticsearch/discovery/ec2/Ec2DiscoveryTests.java
@@ -9,11 +9,11 @@
package org.elasticsearch.discovery.ec2;
-import com.amazonaws.http.HttpMethodName;
-import com.amazonaws.services.ec2.model.Instance;
-import com.amazonaws.services.ec2.model.InstanceState;
-import com.amazonaws.services.ec2.model.InstanceStateName;
-import com.amazonaws.services.ec2.model.Tag;
+import software.amazon.awssdk.http.SdkHttpMethod;
+import software.amazon.awssdk.services.ec2.model.Instance;
+import software.amazon.awssdk.services.ec2.model.InstanceState;
+import software.amazon.awssdk.services.ec2.model.InstanceStateName;
+import software.amazon.awssdk.services.ec2.model.Tag;
import org.apache.http.HttpStatus;
import org.apache.http.NameValuePair;
@@ -90,7 +90,7 @@ protected List buildDynamicHosts(Settings nodeSettings, int no
try (Ec2DiscoveryPlugin plugin = new Ec2DiscoveryPlugin(buildSettings(accessKey))) {
AwsEc2SeedHostsProvider provider = new AwsEc2SeedHostsProvider(nodeSettings, transportService, plugin.ec2Service);
httpServer.createContext("/", exchange -> {
- if (exchange.getRequestMethod().equals(HttpMethodName.POST.name())) {
+ if (SdkHttpMethod.POST.name().equals(exchange.getRequestMethod())) {
final String request = new String(exchange.getRequestBody().readAllBytes(), UTF_8);
final String userAgent = exchange.getRequestHeaders().getFirst("User-Agent");
if (userAgent != null && userAgent.startsWith("aws-sdk-java")) {
@@ -114,25 +114,26 @@ protected List buildDynamicHosts(Settings nodeSettings, int no
});
final List instances = IntStream.range(1, nodes + 1).mapToObj(node -> {
final String instanceId = "node" + node;
- final Instance instance = new Instance().withInstanceId(instanceId)
- .withState(new InstanceState().withName(InstanceStateName.Running))
- .withPrivateDnsName(PREFIX_PRIVATE_DNS + instanceId + SUFFIX_PRIVATE_DNS)
- .withPublicDnsName(PREFIX_PUBLIC_DNS + instanceId + SUFFIX_PUBLIC_DNS)
- .withPrivateIpAddress(PREFIX_PRIVATE_IP + node)
- .withPublicIpAddress(PREFIX_PUBLIC_IP + node);
+ final Instance.Builder instanceBuilder = Instance.builder()
+ .instanceId(instanceId)
+ .state(InstanceState.builder().name(InstanceStateName.RUNNING).build())
+ .privateDnsName(PREFIX_PRIVATE_DNS + instanceId + SUFFIX_PRIVATE_DNS)
+ .publicDnsName(PREFIX_PUBLIC_DNS + instanceId + SUFFIX_PUBLIC_DNS)
+ .privateIpAddress(PREFIX_PRIVATE_IP + node)
+ .publicIpAddress(PREFIX_PUBLIC_IP + node);
if (tagsList != null) {
- instance.setTags(tagsList.get(node - 1));
+ instanceBuilder.tags(tagsList.get(node - 1));
}
- return instance;
+ return instanceBuilder.build();
})
.filter(
instance -> tagsIncluded.entrySet()
.stream()
.allMatch(
- entry -> instance.getTags()
+ entry -> instance.tags()
.stream()
- .filter(t -> t.getKey().equals(entry.getKey()))
- .map(Tag::getValue)
+ .filter(t -> t.key().equals(entry.getKey()))
+ .map(Tag::value)
.toList()
.containsAll(entry.getValue())
)
@@ -253,10 +254,10 @@ public void testFilterByTags() {
for (int node = 0; node < nodes; node++) {
List tags = new ArrayList<>();
if (randomBoolean()) {
- tags.add(new Tag("stage", "prod"));
+ tags.add(tag("stage", "prod"));
prodInstances++;
} else {
- tags.add(new Tag("stage", "dev"));
+ tags.add(tag("stage", "dev"));
}
tagsList.add(tags);
}
@@ -266,6 +267,10 @@ public void testFilterByTags() {
assertThat(dynamicHosts, hasSize(prodInstances));
}
+ private static Tag tag(String key, String value) {
+ return Tag.builder().key(key).value(value).build();
+ }
+
public void testFilterByMultipleTags() {
int nodes = randomIntBetween(5, 10);
Settings nodeSettings = Settings.builder().putList(AwsEc2Service.TAG_SETTING.getKey() + "stage", "prod", "preprod").build();
@@ -276,15 +281,15 @@ public void testFilterByMultipleTags() {
for (int node = 0; node < nodes; node++) {
List tags = new ArrayList<>();
if (randomBoolean()) {
- tags.add(new Tag("stage", "prod"));
+ tags.add(tag("stage", "prod"));
if (randomBoolean()) {
- tags.add(new Tag("stage", "preprod"));
+ tags.add(tag("stage", "preprod"));
prodInstances++;
}
} else {
- tags.add(new Tag("stage", "dev"));
+ tags.add(tag("stage", "dev"));
if (randomBoolean()) {
- tags.add(new Tag("stage", "preprod"));
+ tags.add(tag("stage", "preprod"));
}
}
tagsList.add(tags);
@@ -311,7 +316,7 @@ public void testReadHostFromTag() throws UnknownHostException {
for (int node = 0; node < nodes; node++) {
List tags = new ArrayList<>();
- tags.add(new Tag("foo", "node" + (node + 1)));
+ tags.add(tag("foo", "node" + (node + 1)));
tagsList.add(tags);
}
diff --git a/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsHttpFixture.java b/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsHttpFixture.java
index e232d10fdddbd..d2ab8d4f4c7b0 100644
--- a/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsHttpFixture.java
+++ b/test/fixtures/ec2-imds-fixture/src/main/java/fixture/aws/imds/Ec2ImdsHttpFixture.java
@@ -26,8 +26,17 @@
public class Ec2ImdsHttpFixture extends ExternalResource {
+ /**
+ * Name of the JVM system property that allows to override the IMDS endpoint address when using the AWS v1 SDK.
+ * Can be removed once we only use the v2 SDK.
+ */
public static final String ENDPOINT_OVERRIDE_SYSPROP_NAME = "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride";
+ /**
+ * Name of the JVM system property that allows to override the IMDS endpoint address when using the AWS v2 SDK.
+ */
+ public static final String ENDPOINT_OVERRIDE_SYSPROP_NAME_SDK2 = "aws.ec2MetadataServiceEndpoint";
+
private final Ec2ImdsServiceBuilder ec2ImdsServiceBuilder;
private HttpServer server;
@@ -62,12 +71,18 @@ private static InetSocketAddress resolveAddress() {
}
}
+ /**
+ * Overrides the EC2 service endpoint for the lifetime of the method response. Resets back to the original endpoint property when
+ * closed.
+ */
@SuppressForbidden(reason = "deliberately adjusting system property for endpoint override for use in internal-cluster tests")
public static Releasable withEc2MetadataServiceEndpointOverride(String endpointOverride) {
- final PrivilegedAction resetProperty = System.getProperty(ENDPOINT_OVERRIDE_SYSPROP_NAME) instanceof String originalValue
- ? () -> System.setProperty(ENDPOINT_OVERRIDE_SYSPROP_NAME, originalValue)
- : () -> System.clearProperty(ENDPOINT_OVERRIDE_SYSPROP_NAME);
- doPrivileged(() -> System.setProperty(ENDPOINT_OVERRIDE_SYSPROP_NAME, endpointOverride));
+ final PrivilegedAction resetProperty = System.getProperty(
+ ENDPOINT_OVERRIDE_SYSPROP_NAME_SDK2
+ ) instanceof String originalValue
+ ? () -> System.setProperty(ENDPOINT_OVERRIDE_SYSPROP_NAME_SDK2, originalValue)
+ : () -> System.clearProperty(ENDPOINT_OVERRIDE_SYSPROP_NAME_SDK2);
+ doPrivileged(() -> System.setProperty(ENDPOINT_OVERRIDE_SYSPROP_NAME_SDK2, endpointOverride));
return () -> doPrivileged(resetProperty);
}
@@ -75,6 +90,10 @@ private static void doPrivileged(PrivilegedAction privilegedAction) {
AccessController.doPrivileged(privilegedAction);
}
+ /**
+ * Adapter to allow running a {@link Ec2ImdsHttpFixture} directly rather than via a {@code @ClassRule}. Creates an HTTP handler (see
+ * {@link Ec2ImdsHttpHandler}) from the given builder, and provides the handler to the action, and then cleans up the handler.
+ */
public static void runWithFixture(Ec2ImdsServiceBuilder ec2ImdsServiceBuilder, CheckedConsumer action) {
final var imdsFixture = new Ec2ImdsHttpFixture(ec2ImdsServiceBuilder);
try {
@@ -86,6 +105,8 @@ public void evaluate() throws Exception {
}, Description.EMPTY).evaluate();
} catch (Throwable e) {
throw new AssertionError(e);
+ } finally {
+ imdsFixture.stop(0);
}
}