Upgrade discovery-ec2 to AWS SDK v2

docs/changelog/122062.yaml

@@ -0,0 +1,40 @@
+pr: 122062
+summary: Upgrade `discovery-ec2` to AWS SDK v2
+area: Discovery-Plugins
+type: breaking
+issues: []
+breaking:
+  title: Upgrade `discovery-ec2` to AWS SDK v2
+  area: Cluster and node setting
+  details: |2-
+
+    In earlier versions of {es} the `discovery-ec2` plugin was based on the AWS SDK v1. AWS will withdraw support for this SDK before the end of the life of {es} {minor-version} so we must migrate to the newer AWS SDK v2.
+    Unfortunately there are several differences between the two AWS SDK versions which may require you to adjust your system configuration when upgrading to {es} {minor-version} or later. These differences include, but may not be limited to, the following items.
+    * AWS SDK v2 does not support the EC2 IMDSv1 protocol.
+    * AWS SDK v2 does not support the `aws.secretKey` or
+      `com.amazonaws.sdk.ec2MetadataServiceEndpointOverride` system properties.
+
+    * AWS SDK v2 does not permit specifying a choice between HTTP and HTTPS so
+      the `discovery.ec2.protocol` setting is no longer effective.
+
+    * AWS SDK v2 does not accept an access key without a secret key or vice
+      versa.
+  impact: |2-
+
+    If you use the `discovery-ec2` plugin, test your upgrade thoroughly before upgrading any production workloads.
+    Adapt your configuration to the new SDK functionality. This includes, but may not be limited to, the following items.
+    * If you use IMDS to determine the availability zone of a node or to obtain
+      credentials for accessing the EC2 API, ensure that it supports the IMDSv2
+      protocol.
+
+    * If applicable, discontinue use of the `aws.secretKey` and
+      `com.amazonaws.sdk.ec2MetadataServiceEndpointOverride` system properties.
+
+    * If applicable, specify that you wish to use the insecure HTTP protocol to
+      access the EC2 API by setting `discovery.ec2.endpoint` to a URL which
+      starts with `http://`.
+
+    * Either supply both an access key and a secret key using the keystore
+      settings `discovery.ec2.access_key` and `discovery.ec2.secret_key`, or
+      configure neither of these settings.
+  notable: true

gradle/verification-metadata.xml

@@ -4719,6 +4719,11 @@
             <sha256 value="f8f0df5ee1fcfef0381d167ae50d85ce635b7e5b32d5d620bbb8019f183c6b41" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="software.amazon.awssdk" name="apache-client" version="2.28.13">
+         <artifact name="apache-client-2.28.13.jar">
+            <sha256 value="5099b4417adb661410b3213426319c8b0e87d7216d8f271b2e43533672122e26" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="software.amazon.awssdk" name="auth" version="2.28.13">
          <artifact name="auth-2.28.13.jar">
             <sha256 value="494db83a2a06f09ba6717bb7fff07d50eb85b0b0d51904bf76601ee48e728741" origin="Generated by Gradle"/>
@@ -4734,6 +4739,11 @@
             <sha256 value="63adac3a637c67f779cc56099e264f1cdd2fc4ac85c27e281b2cad53a693f7d2" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="software.amazon.awssdk" name="aws-query-protocol" version="2.28.13">
+         <artifact name="aws-query-protocol-2.28.13.jar">
+            <sha256 value="e967617a6a8b0b76187218d144a4058462a13a993aa18400cb6f783d65e5b947" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="software.amazon.awssdk" name="bedrockruntime" version="2.28.13">
          <artifact name="bedrockruntime-2.28.13.jar">
             <sha256 value="9ff1571e87a11114407eade316e4439b63275283ff49b6aaf52549c37d8e6a92" origin="Generated by Gradle"/>
@@ -4749,6 +4759,11 @@
             <sha256 value="20dfb45d582c175e48aa50237fd44704e31e91418b5d3da1092508dbcb9a4d11" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="software.amazon.awssdk" name="ec2" version="2.28.13">
+         <artifact name="ec2-2.28.13.jar">
+            <sha256 value="667a1f24610fd9b5d68db7dc304bfc5d9df9f294d9a3e320e96ad415265b112d" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="software.amazon.awssdk" name="endpoints-spi" version="2.28.13">
          <artifact name="endpoints-spi-2.28.13.jar">
             <sha256 value="b18dd1d66f03bf5e192ab51d7f3a8139e5bf1e7bab27501b00338f1d8e260f61" origin="Generated by Gradle"/>
@@ -4784,6 +4799,11 @@
             <sha256 value="8baf158caf32cbab7cfdc2fabf48bac90e737917703c2a6e0502f46c46e3ef71" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="software.amazon.awssdk" name="imds" version="2.28.13">
+         <artifact name="imds-2.28.13.jar">
+            <sha256 value="b7ff330aae712ed5dfc35b7d4612bddbfb483cbc945977420830044f6a609eed" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="software.amazon.awssdk" name="json-utils" version="2.28.13">
          <artifact name="json-utils-2.28.13.jar">
             <sha256 value="369ed42586213a33bc7f94e9d21594ee64fec1152819476c24c82b312b27b170" origin="Generated by Gradle"/>

plugins/discovery-ec2/build.gradle

@@ -6,6 +6,7 @@
  * your election, the "Elastic License 2.0", the "GNU Affero General Public
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
+apply plugin: 'elasticsearch.internal-cluster-test'
 apply plugin: 'elasticsearch.internal-java-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
 
@@ -15,30 +16,83 @@ esplugin {
 }
 
 dependencies {
-  api "com.amazonaws:aws-java-sdk-ec2:${versions.awsv1sdk}"
-  api "com.amazonaws:aws-java-sdk-core:${versions.awsv1sdk}"
-  api "org.apache.httpcomponents:httpclient:${versions.httpclient}"
-  api "org.apache.httpcomponents:httpcore:${versions.httpcore}"
-  api "commons-logging:commons-logging:${versions.commonslogging}"
-  api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
-  api "commons-codec:commons-codec:${versions.commonscodec}"
-  api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
-  api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
-  api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
-  api "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}"
-  api "joda-time:joda-time:2.10.10"
+
+  implementation "software.amazon.awssdk:annotations:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:apache-client:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:auth:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:aws-core:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:ec2:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:endpoints-spi:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:http-client-spi:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:identity-spi:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:imds:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:retries-spi:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:sdk-core:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:utils:${versions.awsv2sdk}"
+
+  runtimeOnly "software.amazon.awssdk:aws-query-protocol:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:checksums-spi:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:checksums:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:http-auth-aws:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:http-auth-spi:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:http-auth:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:json-utils:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:metrics-spi:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:profiles:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:protocol-core:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:regions:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:retries:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:third-party-jackson-core:${versions.awsv2sdk}"
+
+  implementation "org.apache.httpcomponents:httpclient:${versions.httpclient}"
+
+  runtimeOnly "commons-codec:commons-codec:${versions.commonscodec}"
+  runtimeOnly "commons-logging:commons-logging:${versions.commonslogging}"
+  runtimeOnly "joda-time:joda-time:2.10.10"
+  runtimeOnly "org.apache.httpcomponents:httpcore:${versions.httpcore}"
+  runtimeOnly "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
+  runtimeOnly "org.slf4j:slf4j-nop:${versions.slf4j}"
+  // runtimeOnly("org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}") https://github.com/elastic/elasticsearch/issues/93714
+  runtimeOnly "org.slf4j:slf4j-api:${versions.slf4j}"
+  runtimeOnly "org.reactivestreams:reactive-streams:${versions.reactive_streams}"
 
   javaRestTestImplementation project(':plugins:discovery-ec2')
   javaRestTestImplementation project(':test:fixtures:aws-fixture-utils')
   javaRestTestImplementation project(':test:fixtures:aws-ec2-fixture')
   javaRestTestImplementation project(':test:fixtures:ec2-imds-fixture')
 
+  testImplementation project(':test:fixtures:aws-fixture-utils')
+  testImplementation project(':test:fixtures:ec2-imds-fixture')
+
   internalClusterTestImplementation project(':test:fixtures:ec2-imds-fixture')
 }
 
 tasks.named("dependencyLicenses").configure {
-  mapping from: /aws-java-sdk-.*/, to: 'aws-java-sdk'
-  mapping from: /jackson-.*/, to: 'jackson'
+  mapping from: 'annotations',              to: 'aws-sdk-2'
+  mapping from: 'apache-client',            to: 'aws-sdk-2'
+  mapping from: 'auth',                     to: 'aws-sdk-2'
+  mapping from: 'aws-core',                 to: 'aws-sdk-2'
+  mapping from: 'aws-query-protocol',       to: 'aws-sdk-2'
+  mapping from: 'checksums',                to: 'aws-sdk-2'
+  mapping from: 'checksums-spi',            to: 'aws-sdk-2'
+  mapping from: 'ec2',                      to: 'aws-sdk-2'
+  mapping from: 'endpoints-spi',            to: 'aws-sdk-2'
+  mapping from: 'http-auth',                to: 'aws-sdk-2'
+  mapping from: 'http-auth-aws',            to: 'aws-sdk-2'
+  mapping from: 'http-auth-spi',            to: 'aws-sdk-2'
+  mapping from: 'http-client-spi',          to: 'aws-sdk-2'
+  mapping from: 'identity-spi',             to: 'aws-sdk-2'
+  mapping from: 'imds',                     to: 'aws-sdk-2'
+  mapping from: 'json-utils',               to: 'aws-sdk-2'
+  mapping from: 'metrics-spi',              to: 'aws-sdk-2'
+  mapping from: 'profiles',                 to: 'aws-sdk-2'
+  mapping from: 'protocol-core',            to: 'aws-sdk-2'
+  mapping from: 'regions',                  to: 'aws-sdk-2'
+  mapping from: 'retries',                  to: 'aws-sdk-2'
+  mapping from: 'retries-spi',              to: 'aws-sdk-2'
+  mapping from: 'sdk-core',                 to: 'aws-sdk-2'
+  mapping from: 'third-party-jackson-core', to: 'aws-sdk-2'
+  mapping from: 'utils',                    to: 'aws-sdk-2'
 }
 
 esplugin.bundleSpec.from('config/discovery-ec2') {
@@ -68,19 +122,27 @@ tasks.register("writeTestJavaPolicy") {
           "permission org.bouncycastle.crypto.CryptoServicesPermission \"exportSecretKey\";",
           "permission org.bouncycastle.crypto.CryptoServicesPermission \"exportPrivateKey\";",
           "permission java.io.FilePermission \"\${javax.net.ssl.trustStore}\", \"read\";",
-          "permission java.util.PropertyPermission \"com.amazonaws.sdk.ec2MetadataServiceEndpointOverride\", \"write\";",
           "permission java.security.SecurityPermission \"getProperty.jdk.tls.disabledAlgorithms\";",
           "permission java.security.SecurityPermission \"getProperty.jdk.certpath.disabledAlgorithms\";",
           "permission java.security.SecurityPermission \"getProperty.keystore.type.compat\";",
           "permission java.security.SecurityPermission \"getProperty.org.bouncycastle.ec.max_f2m_field_size\";",
+          "permission java.util.PropertyPermission \"aws.ec2MetadataServiceEndpoint\", \"write\";",
+          "permission java.io.FilePermission \"\${user.home}/.aws/credentials\", \"read\";",
+          "permission java.io.FilePermission \"\${user.home}/.aws/config\", \"read\";",
+          "permission java.util.PropertyPermission \"http.proxyHost\", \"read\";",
+          "permission java.util.PropertyPermission \"aws.region\", \"read\";",
           "};"
         ].join("\n")
       )
     } else {
       javaPolicy.write(
         [
           "grant {",
-          "  permission java.util.PropertyPermission \"com.amazonaws.sdk.ec2MetadataServiceEndpointOverride\", \"write\";",
+          "permission java.util.PropertyPermission \"aws.ec2MetadataServiceEndpoint\", \"write\";",
+          "permission java.io.FilePermission \"\${user.home}/.aws/credentials\", \"read\";",
+          "permission java.io.FilePermission \"\${user.home}/.aws/config\", \"read\";",
+          "permission java.util.PropertyPermission \"http.proxyHost\", \"read\";",
+          "permission java.util.PropertyPermission \"aws.region\", \"read\";",
           "};"
         ].join("\n"))
     }
@@ -92,27 +154,44 @@ tasks.withType(Test).configureEach {
   // this is needed for insecure plugins, remove if possible!
   systemProperty 'tests.artifact', project.name
 
-  // this is needed to manipulate com.amazonaws.sdk.ec2MetadataServiceEndpointOverride system property
+  // this is needed to manipulate aws.ec2MetadataServiceEndpoint system property
   // it is better rather disable security manager at all with `systemProperty 'tests.security.manager', 'false'`
   if (buildParams.inFipsJvm){
     nonInputProperties.systemProperty 'java.security.policy', "=file://${buildDir}/tmp/java.policy"
   } else {
     nonInputProperties.systemProperty 'java.security.policy', "file://${buildDir}/tmp/java.policy"
   }
+
+  systemProperty 'aws.region', 'es-test-region'
 }
 
 tasks.named("thirdPartyAudit").configure {
   ignoreMissingClasses(
           // classes are missing
-          'com.amazonaws.jmespath.JmesPathExpression',
-          'com.amazonaws.jmespath.ObjectMapperSingleton',
           'javax.servlet.ServletContextEvent',
           'javax.servlet.ServletContextListener',
           'org.apache.avalon.framework.logger.Logger',
           'org.apache.log.Hierarchy',
           'org.apache.log.Logger',
           'javax.jms.Message',
-          'javax.xml.bind.DatatypeConverter',
-          'javax.xml.bind.JAXBContext'
+
+          // eventstream not used by the sync client
+          'software.amazon.eventstream.HeaderValue',
+          'software.amazon.eventstream.Message',
+          'software.amazon.eventstream.MessageDecoder',
+
+          // crt?
+          'software.amazon.awssdk.crt.auth.credentials.Credentials',
+          'software.amazon.awssdk.crt.auth.signing.AwsSigner',
+          'software.amazon.awssdk.crt.auth.signing.AwsSigningConfig',
+          'software.amazon.awssdk.crt.auth.signing.AwsSigningConfig$AwsSignatureType',
+          'software.amazon.awssdk.crt.auth.signing.AwsSigningConfig$AwsSignedBodyHeaderType',
+          'software.amazon.awssdk.crt.auth.signing.AwsSigningConfig$AwsSigningAlgorithm',
+          'software.amazon.awssdk.crt.auth.signing.AwsSigningResult',
+          'software.amazon.awssdk.crt.checksums.CRC32',
+          'software.amazon.awssdk.crt.checksums.CRC32C',
+          'software.amazon.awssdk.crt.http.HttpHeader',
+          'software.amazon.awssdk.crt.http.HttpRequest',
+          'software.amazon.awssdk.crt.http.HttpRequestBodyStream',
   )
 }