Add support for Apache HTTP client 5 in X-Pack SSL (#133895)

This extends the X-Pack SslProfile class to have support for HTTP Client 5's TlsStrategy class.

This is needed in order to support a migration from HTTP client 4 to HTTP client 5

Author: tvernum

build-tools-internal/version.properties

@@ -29,6 +29,8 @@ opensaml = 4.3.0
 # client dependencies
 httpclient        = 4.5.14
 httpcore          = 4.4.16
+httpclient5       = 5.5
+httpcore5         = 5.3.5
 httpasyncclient   = 4.1.5
 commonslogging    = 1.2
 commonscodec      = 1.15

gradle/verification-metadata.xml

@@ -2907,16 +2907,31 @@
             <sha256 value="a0a9dcd3696a6281f82e392d39aa674bf9662496605411de2a00d44435a7fb26" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.httpcomponents.client5" name="httpclient5" version="5.5">
+         <artifact name="httpclient5-5.5.jar">
+            <sha256 value="496b4b0e8d5f3a8139a5d2638486d304758bac3a9c39d76989f663cfd9354fc9" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.httpcomponents.core5" name="httpcore5" version="5.3.3">
          <artifact name="httpcore5-5.3.3.jar">
             <sha256 value="087b7ae9bde9d3518b4b5d06f3560d7fd0db04098655e76b64e791773847d503" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.httpcomponents.core5" name="httpcore5" version="5.3.5">
+         <artifact name="httpcore5-5.3.5.jar">
+            <sha256 value="34ce80396dcf2927406ddcd7c1c9063879220952ad0a2121c10103d3ad2cc3a4" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.httpcomponents.core5" name="httpcore5-h2" version="5.3.3">
          <artifact name="httpcore5-h2-5.3.3.jar">
             <sha256 value="a121f4b14ec525e54e29b9f5db7b93f4a97e088774e81c7143b5198f67d81bec" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.apache.httpcomponents.core5" name="httpcore5-h2" version="5.3.5">
+         <artifact name="httpcore5-h2-5.3.5.jar">
+            <sha256 value="2a661d8756fb298db7d561c2dabb5e79144b0dae6676735134ef6a8f4c3125e3" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.apache.james" name="apache-mime4j-core" version="0.8.13">
          <artifact name="apache-mime4j-core-0.8.13.jar">
             <sha256 value="00496c123926395d59e5dfdfc8342c607600c6c9e6e6dcab981a673b62481cdf" origin="Generated by Gradle"/>
@@ -4935,6 +4950,11 @@
             <sha256 value="2f2a92d410b268139d7d63b75ed25e21995cfe4100c19bf23577cfdbc8077bda" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="org.slf4j" name="slf4j-ext" version="2.0.6">
+         <artifact name="slf4j-ext-2.0.6.jar">
+            <sha256 value="0f6ef03bc0291899f3fb324baba0dee02fa8c6c1adc7b465f5b923ac70379efd" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="org.slf4j" name="slf4j-log4j12" version="1.7.10">
          <artifact name="slf4j-log4j12-1.7.10.jar">
             <sha256 value="2e4eebc6e346c92c417aa4e662738802645ef21c5eb4435132dc78d631f2eebb" origin="Generated by Gradle"/>

x-pack/plugin/core/build.gradle

@@ -32,6 +32,7 @@ esplugin {
 tasks.named("dependencyLicenses").configure {
   mapping from: /http.*/, to: 'httpclient' // pulled in by rest client
   mapping from: /commons-.*/, to: 'commons' // pulled in by rest client
+  mapping from: /slf4j-.*/, to: 'slf4j' 
 }
 
 configurations {
@@ -43,13 +44,30 @@ dependencies {
   compileOnly project(":server")
   api project(':libs:grok')
   api project(":libs:ssl-config")
+
   api "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   api "org.apache.httpcomponents:httpcore:${versions.httpcore}"
   api "org.apache.httpcomponents:httpcore-nio:${versions.httpcore}"
   api "org.apache.httpcomponents:httpasyncclient:${versions.httpasyncclient}"
+
+  api "org.apache.httpcomponents.client5:httpclient5:${versions.httpclient5}"
+  api "org.apache.httpcomponents.core5:httpcore5:${versions.httpcore5}"
+  api "org.apache.httpcomponents.core5:httpcore5-h2:${versions.httpcore5}"
+
+  // Ideally this would be `runtimeOnly` so that we don't accidentally write code against the SLF4j API
+  // However, some child plugins (like security) need to use slf4j directly in order to manage logging for their dependencies :(
+  // And due to the way our plugin loading handles java modules between dependent plugins, this plugin (x-pack-core) needs to declare
+  //  a module dependency on slf4j (`requires org.slf4j`) so that security can also be dependent on it
+  // And having a module dependency counts as using the jar, so we can't make it `runtimeOnly` :(
+
+  implementation "org.slf4j:slf4j-api:${versions.slf4j}"
+  runtimeOnly "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}"
+
   api "commons-logging:commons-logging:${versions.commonslogging}"
   api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
+
   api "commons-codec:commons-codec:${versions.commonscodec}"
+
   testImplementation project(path: ':modules:aggregations')
   testImplementation project(path: ':modules:data-streams')
   testImplementation project(':modules:mapper-extras')
@@ -60,9 +78,6 @@ dependencies {
   implementation project(":x-pack:plugin:core:template-resources")
 
   testImplementation "org.elasticsearch:mocksocket:${versions.mocksocket}"
-  testImplementation "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}"
-  // this might suffer from https://github.com/elastic/elasticsearch/issues/93714
-  testImplementation "org.slf4j:slf4j-api:${versions.slf4j}"
   testImplementation project(path: ':modules:reindex')
   testImplementation project(path: ':modules:parent-join')
   testImplementation project(path: ':modules:lang-mustache')
@@ -139,7 +154,11 @@ tasks.named("thirdPartyAudit").configure {
     //commons-logging provided dependencies
     'javax.servlet.ServletContextEvent',
     'javax.servlet.ServletContextListener',
-    'javax.jms.Message'
+    'javax.jms.Message',
+    // HttpClient5 can use Conscrypt (TLS using BoringSSL), but we don't want that
+    'org.conscrypt.Conscrypt',
+    // SLF4j via HttpClient5
+    'org.slf4j.ext.EventData'
   )
 }
 

x-pack/plugin/ent-search/licenses/log4j-slf4j-impl-LICENSE.txt renamed to x-pack/plugin/core/licenses/log4j-slf4j-impl-LICENSE.txt

@@ -23,6 +23,9 @@
     requires unboundid.ldapsdk;
     requires org.elasticsearch.tdigest;
     requires org.elasticsearch.xcore.templates;
+    requires org.apache.httpcomponents.client5.httpclient5;
+    requires org.apache.httpcomponents.core5.httpcore5;
+    requires org.slf4j;
 
     exports org.elasticsearch.index.engine.frozen;
     exports org.elasticsearch.license;

x-pack/plugin/ent-search/licenses/log4j-slf4j-impl-NOTICE.txt renamed to x-pack/plugin/core/licenses/log4j-slf4j-impl-NOTICE.txt

@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.ssl;
+
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.ssl.SslConfiguration;
+
+import java.util.List;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLParameters;
+
+public abstract class AbstractSslBuilder<T> {
+
+    public T build(SslConfiguration config, SSLContext sslContext) {
+        String[] ciphers = supportedCiphers(sslParameters(sslContext).getCipherSuites(), config.getCipherSuites(), false);
+        String[] supportedProtocols = config.supportedProtocols().toArray(Strings.EMPTY_ARRAY);
+        HostnameVerifier verifier;
+
+        if (config.verificationMode().isHostnameVerificationEnabled()) {
+            verifier = SSLIOSessionStrategy.getDefaultHostnameVerifier();
+        } else {
+            verifier = NoopHostnameVerifier.INSTANCE;
+        }
+
+        return build(sslContext, supportedProtocols, ciphers, verifier);
+    }
+
+    /**
+     * This method exists to simplify testing
+     */
+    String[] supportedCiphers(String[] supportedCiphers, List<String> requestedCiphers, boolean log) {
+        return SSLService.supportedCiphers(supportedCiphers, requestedCiphers, log);
+    }
+
+    /**
+     * The {@link SSLParameters} that are associated with the {@code sslContext}.
+     * <p>
+     * This method exists to simplify testing since {@link SSLContext#getSupportedSSLParameters()} is {@code final}.
+     *
+     * @param sslContext The SSL context for the current SSL settings
+     * @return Never {@code null}.
+     */
+    SSLParameters sslParameters(SSLContext sslContext) {
+        return sslContext.getSupportedSSLParameters();
+    }
+
+    abstract T build(SSLContext sslContext, String[] protocols, String[] ciphers, HostnameVerifier verifier);
+}

x-pack/plugin/ent-search/licenses/slf4j-api-LICENSE.txt renamed to x-pack/plugin/core/licenses/slf4j-LICENSE.txt

@@ -8,68 +8,32 @@
 package org.elasticsearch.xpack.core.ssl;
 
 import org.apache.http.HttpHost;
-import org.apache.http.conn.ssl.NoopHostnameVerifier;
 import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy;
 import org.apache.http.nio.reactor.IOSession;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.logging.LoggerMessageFormat;
-import org.elasticsearch.common.ssl.SslConfiguration;
 import org.elasticsearch.common.ssl.SslDiagnostics;
 
 import java.security.cert.Certificate;
 import java.security.cert.X509Certificate;
-import java.util.List;
 
 import javax.net.ssl.HostnameVerifier;
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLException;
-import javax.net.ssl.SSLParameters;
 import javax.net.ssl.SSLPeerUnverifiedException;
 import javax.net.ssl.SSLSession;
 import javax.security.auth.x500.X500Principal;
 
-public class SSLIOSessionStrategyBuilder {
+class SSLIOSessionStrategyBuilder extends AbstractSslBuilder<SSLIOSessionStrategy> {
 
     public static final SSLIOSessionStrategyBuilder INSTANCE = new SSLIOSessionStrategyBuilder();
 
-    public SSLIOSessionStrategy sslIOSessionStrategy(SslConfiguration config, SSLContext sslContext) {
-        String[] ciphers = supportedCiphers(sslParameters(sslContext).getCipherSuites(), config.getCipherSuites(), false);
-        String[] supportedProtocols = config.supportedProtocols().toArray(Strings.EMPTY_ARRAY);
-        HostnameVerifier verifier;
-
-        if (config.verificationMode().isHostnameVerificationEnabled()) {
-            verifier = SSLIOSessionStrategy.getDefaultHostnameVerifier();
-        } else {
-            verifier = NoopHostnameVerifier.INSTANCE;
-        }
-
-        return sslIOSessionStrategy(sslContext, supportedProtocols, ciphers, verifier);
-    }
-
-    /**
-     * This method exists to simplify testing
-     */
-    String[] supportedCiphers(String[] supportedCiphers, List<String> requestedCiphers, boolean log) {
-        return SSLService.supportedCiphers(supportedCiphers, requestedCiphers, log);
-    }
-
-    /**
-     * The {@link SSLParameters} that are associated with the {@code sslContext}.
-     * <p>
-     * This method exists to simplify testing since {@link SSLContext#getSupportedSSLParameters()} is {@code final}.
-     *
-     * @param sslContext The SSL context for the current SSL settings
-     * @return Never {@code null}.
-     */
-    SSLParameters sslParameters(SSLContext sslContext) {
-        return sslContext.getSupportedSSLParameters();
-    }
-
     /**
      * This method only exists to simplify testing because {@link SSLIOSessionStrategy} does
      * not expose any of the parameters that you give it.
      */
-    SSLIOSessionStrategy sslIOSessionStrategy(SSLContext sslContext, String[] protocols, String[] ciphers, HostnameVerifier verifier) {
+    @Override
+    SSLIOSessionStrategy build(SSLContext sslContext, String[] protocols, String[] ciphers, HostnameVerifier verifier) {
         return new SSLIOSessionStrategy(sslContext, protocols, ciphers, verifier) {
             @Override
             protected void verifySession(HttpHost host, IOSession iosession, SSLSession session) throws SSLException {