elastic · elasticsearchmachine · Mar 18, 2025 · Mar 18, 2025 · Mar 18, 2025
diff --git a/docs/changelog/122062.yaml b/docs/changelog/122062.yaml
@@ -0,0 +1,54 @@
+pr: 122062
+summary: Upgrade `discovery-ec2` to AWS SDK v2
+area: Discovery-Plugins
+type: breaking
+issues: []
+breaking:
+  title: Upgrade `discovery-ec2` to AWS SDK v2
+  area: Cluster and node setting
+  details: >-
+
+    In earlier versions of {es} the `discovery-ec2` plugin was based on the AWS
+    SDK v1. AWS will withdraw support for this SDK before the end of the life
+    of {es} {minor-version} so we must migrate to the newer AWS SDK v2.
+
+    Unfortunately there are several differences between the two AWS SDK
+    versions which may require you to adjust your system configuration when
+    upgrading to {es} {minor-version} or later. These differences include, but
+    may not be limited to, the following items.
+
+    * AWS SDK v2 does not support the EC2 IMDSv1 protocol.
+
+    * AWS SDK v2 does not support the `aws.secretKey` or
+      `com.amazonaws.sdk.ec2MetadataServiceEndpointOverride` system properties.
+
+    * AWS SDK v2 does not permit specifying a choice between HTTP and HTTPS so
+      the `discovery.ec2.protocol` setting is no longer effective.
+
+    * AWS SDK v2 does not accept an access key without a secret key or vice
+      versa.
+
+  impact: >-
+
+    If you use the `discovery-ec2` plugin, test your upgrade thoroughly before
+    upgrading any production workloads.
+
+    Adapt your configuration to the new SDK functionality. This includes, but
+    may not be limited to, the following items.
+
+    * If you use IMDS to determine the availability zone of a node or to obtain
+      credentials for accessing the EC2 API, ensure that it supports the IMDSv2
+      protocol.
+
+    * If applicable, discontinue use of the `aws.secretKey` and
+      `com.amazonaws.sdk.ec2MetadataServiceEndpointOverride` system properties.
+
+    * If applicable, specify that you wish to use the insecure HTTP protocol to
+      access the EC2 API by setting `discovery.ec2.endpoint` to a URL which
+      starts with `http://`.
+
+    * Either supply both an access key and a secret key using the keystore
+      settings `discovery.ec2.access_key` and `discovery.ec2.secret_key`, or
+      configure neither of these settings.
+
+  notable: true
diff --git a/docs/plugins/discovery-ec2.asciidoc b/docs/plugins/discovery-ec2.asciidoc
@@ -83,8 +83,8 @@ The available settings for the EC2 discovery plugin are as follows.
 
 `discovery.ec2.protocol`::
 
-    The protocol to use to connect to the EC2 service endpoint, which may be
-    either `http` or `https`. Defaults to `https`.
+    This setting has no effect. To use HTTP to connect to the EC2 service
+    endpoint, set `discovery.ec2.endpoint` to a URL that starts with `http://`.
 
 `discovery.ec2.proxy.host`::
 

diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml
@@ -4655,6 +4655,11 @@
             <sha256 value="64d8c2bcccd33c20ccdbafa101b01d8e0f750c4e4bd227b0b765046f601eb944" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="software.amazon.awssdk" name="apache-client" version="2.30.38">
+         <artifact name="apache-client-2.30.38.jar">
+            <sha256 value="ebb1d3d05711ccf2aa9bfc43fcc69fbe32e7be69e006e7952679c2f37d149f4d" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="software.amazon.awssdk" name="auth" version="2.30.38">
          <artifact name="auth-2.30.38.jar">
             <sha256 value="22d59f9af8111be5219eb33ef480d84c616565913da57cb4eac686076fea370e" origin="Generated by Gradle"/>
@@ -4670,6 +4675,11 @@
             <sha256 value="b62be02560a46135181342afc9fb2d99373a9f04f384caf30863e2e7fe5b3892" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="software.amazon.awssdk" name="aws-query-protocol" version="2.30.38">
+         <artifact name="aws-query-protocol-2.30.38.jar">
+            <sha256 value="bfd558e937de70c3260df2356b47a25b562c59b5ebeded6b199846cc9a354fe5" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="software.amazon.awssdk" name="bedrockruntime" version="2.30.38">
          <artifact name="bedrockruntime-2.30.38.jar">
             <sha256 value="4424437b49fdf263ea460f4da634d3279ada7f4763827d74fea48c0f8f2afea3" origin="Generated by Gradle"/>
@@ -4685,6 +4695,11 @@
             <sha256 value="82d97bcbb18d8f369b00c9971ca8c24ad94769d20836e0c4f86ebcdfea994cdb" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="software.amazon.awssdk" name="ec2" version="2.30.38">
+         <artifact name="ec2-2.30.38.jar">
+            <sha256 value="a2e52ca80aac79553f9da1463256db0177b12c5c24c1b5660a3bb7874b66222f" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="software.amazon.awssdk" name="endpoints-spi" version="2.30.38">
          <artifact name="endpoints-spi-2.30.38.jar">
             <sha256 value="80620e3020a29871073a8a4efbcaa4d546667eeb92dfd478de808dca7e0500aa" origin="Generated by Gradle"/>
@@ -4720,6 +4735,11 @@
             <sha256 value="e784929d8a51591b6ed51344f41b37f2a68582d2e912e8310ea3e57a56d4d6bf" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="software.amazon.awssdk" name="imds" version="2.30.38">
+         <artifact name="imds-2.30.38.jar">
+            <sha256 value="1586c80dfee0d09e26ad9cb313529530f1fb75d9360b05b1f1f1ca246f39d1b2" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="software.amazon.awssdk" name="json-utils" version="2.30.38">
          <artifact name="json-utils-2.30.38.jar">
             <sha256 value="823f565bc6d4031e4b3dada05c1e66c1344f34d498344b7186a2f2d048ba01d8" origin="Generated by Gradle"/>

diff --git a/plugins/discovery-ec2/build.gradle b/plugins/discovery-ec2/build.gradle
@@ -8,6 +8,7 @@ import org.elasticsearch.gradle.internal.info.BuildParams
  * your election, the "Elastic License 2.0", the "GNU Affero General Public
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
+apply plugin: 'elasticsearch.internal-cluster-test'
 apply plugin: 'elasticsearch.internal-java-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
@@ -17,30 +18,83 @@ esplugin {
 }
 
 dependencies {
-  api "com.amazonaws:aws-java-sdk-ec2:${versions.awsv1sdk}"
-  api "com.amazonaws:aws-java-sdk-core:${versions.awsv1sdk}"
-  api "org.apache.httpcomponents:httpclient:${versions.httpclient}"
-  api "org.apache.httpcomponents:httpcore:${versions.httpcore}"
-  api "commons-logging:commons-logging:${versions.commonslogging}"
-  api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
-  api "commons-codec:commons-codec:${versions.commonscodec}"
-  api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
-  api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
-  api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
-  api "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}"
-  api "joda-time:joda-time:2.10.10"
+
+  implementation "software.amazon.awssdk:annotations:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:apache-client:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:auth:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:aws-core:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:ec2:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:endpoints-spi:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:http-client-spi:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:identity-spi:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:imds:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:retries-spi:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:sdk-core:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:utils:${versions.awsv2sdk}"
+
+  runtimeOnly "software.amazon.awssdk:aws-query-protocol:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:checksums-spi:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:checksums:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:http-auth-aws:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:http-auth-spi:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:http-auth:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:json-utils:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:metrics-spi:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:profiles:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:protocol-core:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:regions:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:retries:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:third-party-jackson-core:${versions.awsv2sdk}"
+
+  implementation "org.apache.httpcomponents:httpclient:${versions.httpclient}"
+
+  runtimeOnly "commons-codec:commons-codec:${versions.commonscodec}"
+  runtimeOnly "commons-logging:commons-logging:${versions.commonslogging}"
+  runtimeOnly "joda-time:joda-time:2.10.10"
+  runtimeOnly "org.apache.httpcomponents:httpcore:${versions.httpcore}"
+  runtimeOnly "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
+  runtimeOnly "org.slf4j:slf4j-nop:${versions.slf4j}"
+  // runtimeOnly("org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}") https://github.com/elastic/elasticsearch/issues/93714
+  runtimeOnly "org.slf4j:slf4j-api:${versions.slf4j}"
+  runtimeOnly "org.reactivestreams:reactive-streams:${versions.reactive_streams}"
 
   javaRestTestImplementation project(':plugins:discovery-ec2')
   javaRestTestImplementation project(':test:fixtures:aws-fixture-utils')
   javaRestTestImplementation project(':test:fixtures:aws-ec2-fixture')
   javaRestTestImplementation project(':test:fixtures:ec2-imds-fixture')
 
+  testImplementation project(':test:fixtures:aws-fixture-utils')
+  testImplementation project(':test:fixtures:ec2-imds-fixture')
+
   internalClusterTestImplementation project(':test:fixtures:ec2-imds-fixture')
 }
 
 tasks.named("dependencyLicenses").configure {
-  mapping from: /aws-java-sdk-.*/, to: 'aws-java-sdk'
-  mapping from: /jackson-.*/, to: 'jackson'
+  mapping from: 'annotations',              to: 'aws-sdk-2'
+  mapping from: 'apache-client',            to: 'aws-sdk-2'
+  mapping from: 'auth',                     to: 'aws-sdk-2'
+  mapping from: 'aws-core',                 to: 'aws-sdk-2'
+  mapping from: 'aws-query-protocol',       to: 'aws-sdk-2'
+  mapping from: 'checksums',                to: 'aws-sdk-2'
+  mapping from: 'checksums-spi',            to: 'aws-sdk-2'
+  mapping from: 'ec2',                      to: 'aws-sdk-2'
+  mapping from: 'endpoints-spi',            to: 'aws-sdk-2'
+  mapping from: 'http-auth',                to: 'aws-sdk-2'
+  mapping from: 'http-auth-aws',            to: 'aws-sdk-2'
+  mapping from: 'http-auth-spi',            to: 'aws-sdk-2'
+  mapping from: 'http-client-spi',          to: 'aws-sdk-2'
+  mapping from: 'identity-spi',             to: 'aws-sdk-2'
+  mapping from: 'imds',                     to: 'aws-sdk-2'
+  mapping from: 'json-utils',               to: 'aws-sdk-2'
+  mapping from: 'metrics-spi',              to: 'aws-sdk-2'
+  mapping from: 'profiles',                 to: 'aws-sdk-2'
+  mapping from: 'protocol-core',            to: 'aws-sdk-2'
+  mapping from: 'regions',                  to: 'aws-sdk-2'
+  mapping from: 'retries',                  to: 'aws-sdk-2'
+  mapping from: 'retries-spi',              to: 'aws-sdk-2'
+  mapping from: 'sdk-core',                 to: 'aws-sdk-2'
+  mapping from: 'third-party-jackson-core', to: 'aws-sdk-2'
+  mapping from: 'utils',                    to: 'aws-sdk-2'
 }
 
 esplugin.bundleSpec.from('config/discovery-ec2') {
@@ -69,19 +123,23 @@ tasks.register("writeTestJavaPolicy") {
           "permission org.bouncycastle.crypto.CryptoServicesPermission \"exportSecretKey\";",
           "permission org.bouncycastle.crypto.CryptoServicesPermission \"exportPrivateKey\";",
           "permission java.io.FilePermission \"\${javax.net.ssl.trustStore}\", \"read\";",
-          "permission java.util.PropertyPermission \"com.amazonaws.sdk.ec2MetadataServiceEndpointOverride\", \"write\";",
           "permission java.security.SecurityPermission \"getProperty.jdk.tls.disabledAlgorithms\";",
           "permission java.security.SecurityPermission \"getProperty.jdk.certpath.disabledAlgorithms\";",
           "permission java.security.SecurityPermission \"getProperty.keystore.type.compat\";",
           "permission java.security.SecurityPermission \"getProperty.org.bouncycastle.ec.max_f2m_field_size\";",
+          "permission java.util.PropertyPermission \"aws.ec2MetadataServiceEndpoint\", \"write\";",
+          "permission java.util.PropertyPermission \"http.proxyHost\", \"read\";",
+          "permission java.util.PropertyPermission \"aws.region\", \"read\";",
           "};"
         ].join("\n")
       )
     } else {
       javaPolicy.write(
         [
           "grant {",
-          "  permission java.util.PropertyPermission \"com.amazonaws.sdk.ec2MetadataServiceEndpointOverride\", \"write\";",
+          "permission java.util.PropertyPermission \"aws.ec2MetadataServiceEndpoint\", \"write\";",
+          "permission java.util.PropertyPermission \"http.proxyHost\", \"read\";",
+          "permission java.util.PropertyPermission \"aws.region\", \"read\";",
           "};"
         ].join("\n"))
     }
@@ -93,27 +151,42 @@ tasks.withType(Test).configureEach {
   // this is needed for insecure plugins, remove if possible!
   systemProperty 'tests.artifact', project.name
 
-  // this is needed to manipulate com.amazonaws.sdk.ec2MetadataServiceEndpointOverride system property
+  // this is needed to manipulate aws.ec2MetadataServiceEndpoint system property
   // it is better rather disable security manager at all with `systemProperty 'tests.security.manager', 'false'`
   if (buildParams.inFipsJvm){
     nonInputProperties.systemProperty 'java.security.policy', "=file://${layout.buildDirectory.asFile.get()}/tmp/java.policy"
   } else {
     nonInputProperties.systemProperty 'java.security.policy', "file://${layout.buildDirectory.asFile.get()}/tmp/java.policy"
   }
+
+  systemProperty 'aws.region', 'es-test-region'
 }
 
 tasks.named("thirdPartyAudit").configure {
   ignoreMissingClasses(
           // classes are missing
-          'com.amazonaws.jmespath.JmesPathExpression',
-          'com.amazonaws.jmespath.ObjectMapperSingleton',
           'javax.servlet.ServletContextEvent',
           'javax.servlet.ServletContextListener',
           'org.apache.avalon.framework.logger.Logger',
           'org.apache.log.Hierarchy',
           'org.apache.log.Logger',
           'javax.jms.Message',
-          'javax.xml.bind.DatatypeConverter',
-          'javax.xml.bind.JAXBContext'
+
+          // eventstream not used by the sync client
+          'software.amazon.eventstream.HeaderValue',
+          'software.amazon.eventstream.Message',
+          'software.amazon.eventstream.MessageDecoder',
+
+          // crt?
+          'software.amazon.awssdk.crt.auth.credentials.Credentials',
+          'software.amazon.awssdk.crt.auth.signing.AwsSigner',
+          'software.amazon.awssdk.crt.auth.signing.AwsSigningConfig',
+          'software.amazon.awssdk.crt.auth.signing.AwsSigningConfig$AwsSignatureType',
+          'software.amazon.awssdk.crt.auth.signing.AwsSigningConfig$AwsSignedBodyHeaderType',
+          'software.amazon.awssdk.crt.auth.signing.AwsSigningConfig$AwsSigningAlgorithm',
+          'software.amazon.awssdk.crt.auth.signing.AwsSigningResult',
+          'software.amazon.awssdk.crt.http.HttpHeader',
+          'software.amazon.awssdk.crt.http.HttpRequest',
+          'software.amazon.awssdk.crt.http.HttpRequestBodyStream',
   )
 }