Allow adjustment of transport TLS handshake timeout

The default 10s TLS handshake timeout may be too short if there is some
bug causing event-loop latency, and this has more serious consequences
than the underlying performance issue (e.g. it prevents the cluster from
scaling up to work around the problem). With this commit we expose a
setting that allows the timeout to be configured, providing a workaround
in such cases.

Author: DaveCTurner

docs/reference/elasticsearch/configuration-reference/security-settings.md

@@ -1933,6 +1933,8 @@ You can configure the following TLS/SSL settings.
 `xpack.security.transport.ssl.trust_restrictions.x509_fields` ![logo cloud](https://doc-icons.s3.us-east-2.amazonaws.com/logo_cloud.svg "Supported on Elastic Cloud Hosted")
 :   Specifies which field(s) from the TLS certificate is used to match for the restricted trust management that is used for remote clusters connections. This should only be set when a self managed cluster can not create certificates that follow the Elastic Cloud pattern. The default value is ["subjectAltName.otherName.commonName"], the Elastic Cloud pattern. "subjectAltName.dnsName" is also supported and can be configured in addition to or in replacement of the default.
 
+`xpack.security.transport.ssl.handshake_timeout`
+:   Specifies the timeout for a TLS handshake when opening a transport connection. Defaults to `10s`.
 
 ### Transport TLS/SSL key and trusted certificate settings [security-transport-tls-ssl-key-trusted-certificate-settings]
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLService.java

@@ -29,6 +29,7 @@
 import org.elasticsearch.common.ssl.SslTrustConfig;
 import org.elasticsearch.common.util.Maps;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.common.socket.SocketAccess;
@@ -118,9 +119,16 @@ public class SSLService {
         Setting.Property.NodeScope
     );
 
+    private static final Setting<TimeValue> TRANSPORT_TLS_HANDSHAKE_TIMEOUT_SETTING = Setting.positiveTimeSetting(
+        "xpack.security.transport.ssl.handshake_timeout",
+        TimeValue.timeValueSeconds(10),
+        Setting.Property.NodeScope
+    );
+
     private final Environment env;
     private final Settings settings;
     private final boolean diagnoseTrustExceptions;
+    private final long handshakeTimeoutMillis;
 
     /**
      * This is a mapping from "context name" (in general use, the name of a setting key)
@@ -156,6 +164,7 @@ public SSLService(Environment environment, Map<String, SslConfiguration> sslConf
         this.env = environment;
         this.settings = env.settings();
         this.diagnoseTrustExceptions = DIAGNOSE_TRUST_EXCEPTIONS_SETTING.get(environment.settings());
+        this.handshakeTimeoutMillis = TRANSPORT_TLS_HANDSHAKE_TIMEOUT_SETTING.get(environment.settings()).millis();
         this.sslConfigurations = sslConfigurations;
         this.sslContexts = loadSslConfigurations(this.sslConfigurations);
     }
@@ -166,6 +175,7 @@ public SSLService(Settings settings, Environment environment) {
         this.env = environment;
         this.settings = env.settings();
         this.diagnoseTrustExceptions = DIAGNOSE_TRUST_EXCEPTIONS_SETTING.get(settings);
+        this.handshakeTimeoutMillis = TRANSPORT_TLS_HANDSHAKE_TIMEOUT_SETTING.get(settings).millis();
         this.sslConfigurations = getSSLConfigurations(env, this.settings);
         this.sslContexts = loadSslConfigurations(this.sslConfigurations);
     }
@@ -178,6 +188,7 @@ private SSLService(
         this.env = environment;
         this.settings = env.settings();
         this.diagnoseTrustExceptions = DIAGNOSE_TRUST_EXCEPTIONS_SETTING.get(environment.settings());
+        this.handshakeTimeoutMillis = TRANSPORT_TLS_HANDSHAKE_TIMEOUT_SETTING.get(environment.settings()).millis();
         this.sslConfigurations = sslConfigurations;
         this.sslContexts = sslContexts;
     }
@@ -214,6 +225,7 @@ SSLContextHolder sslContextHolder(SslConfiguration sslConfiguration) {
 
     public static void registerSettings(List<Setting<?>> settingList) {
         settingList.add(DIAGNOSE_TRUST_EXCEPTIONS_SETTING);
+        settingList.add(TRANSPORT_TLS_HANDSHAKE_TIMEOUT_SETTING);
     }
 
     /**
@@ -979,4 +991,8 @@ private static String sslContextAlgorithm(List<String> supportedProtocols) {
             "no supported SSL/TLS protocol was found in the configured supported protocols: " + supportedProtocols
         );
     }
+
+    public long getHandshakeTimeoutMillis() {
+        return handshakeTimeoutMillis;
+    }
 }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/core/security/transport/netty4/SecurityNetty4Transport.java

@@ -240,6 +240,7 @@ protected void initChannel(Channel ch) throws Exception {
             SSLEngine serverEngine = sslService.createSSLEngine(configuration, null, -1);
             serverEngine.setUseClientMode(false);
             final SslHandler sslHandler = new SslHandler(serverEngine);
+            sslHandler.setHandshakeTimeoutMillis(sslService.getHandshakeTimeoutMillis());
             ch.pipeline().addFirst("sslhandler", sslHandler);
             super.initChannel(ch);
             assert ch.pipeline().first() == sslHandler : "SSL handler must be first handler in pipeline";
@@ -340,6 +341,7 @@ public void connect(ChannelHandlerContext ctx, SocketAddress remoteAddress, Sock
             }
             final ChannelPromise connectPromise = ctx.newPromise();
             final SslHandler sslHandler = new SslHandler(sslEngine);
+            sslHandler.setHandshakeTimeoutMillis(sslService.getHandshakeTimeoutMillis());
             ctx.pipeline().replace(this, "ssl", sslHandler);
             final Future<?> handshakePromise = sslHandler.handshakeFuture();
             Netty4Utils.addListener(connectPromise, result -> {

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SimpleSecurityNetty4ServerTransportTests.java

@@ -9,8 +9,8 @@
 import io.netty.bootstrap.Bootstrap;
 import io.netty.channel.ChannelOption;
 import io.netty.channel.socket.nio.NioChannelOption;
-import io.netty.handler.ssl.SslHandshakeTimeoutException;
 
+import org.apache.logging.log4j.Level;
 import org.apache.lucene.util.Constants;
 import org.elasticsearch.ExceptionsHelper;
 import org.elasticsearch.TransportVersion;
@@ -35,13 +35,17 @@
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.PageCacheRecycler;
 import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.env.TestEnvironment;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.mocksocket.MockServerSocket;
+import org.elasticsearch.mocksocket.MockSocket;
+import org.elasticsearch.test.MockLog;
+import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.test.transport.StubbableTransport;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -65,6 +69,7 @@
 import org.elasticsearch.xpack.security.transport.SSLEngineUtils;
 import org.elasticsearch.xpack.security.transport.filter.IPFilter;
 
+import java.io.EOFException;
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.net.InetAddress;
@@ -94,6 +99,7 @@
 import javax.net.ssl.SNIServerName;
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLEngine;
+import javax.net.ssl.SSLHandshakeException;
 import javax.net.ssl.SSLParameters;
 import javax.net.ssl.SSLServerSocket;
 import javax.net.ssl.SSLServerSocketFactory;
@@ -902,7 +908,15 @@ public void testTcpHandshakeTimeout() throws IOException {
         }
     }
 
+    @TestLogging(reason = "inbound timeout is reported at TRACE", value = "org.elasticsearch.transport.netty4.ESLoggingHandler:TRACE")
     public void testTlsHandshakeTimeout() throws IOException {
+        runOutboundTlsHandshakeTimeoutTest(null);
+        runOutboundTlsHandshakeTimeoutTest(randomLongBetween(1, 500));
+        runInboundTlsHandshakeTimeoutTest(null);
+        runInboundTlsHandshakeTimeoutTest(randomLongBetween(1, 500));
+    }
+
+    private void runOutboundTlsHandshakeTimeoutTest(@Nullable /* to use the default */ Long handshakeTimeoutMillis) throws IOException {
         final CountDownLatch doneLatch = new CountDownLatch(1);
         try (ServerSocket socket = new MockServerSocket()) {
             socket.bind(getLocalEphemeral(), 1);
@@ -928,16 +942,56 @@ public void testTlsHandshakeTimeout() throws IOException {
                 TransportRequestOptions.Type.REG,
                 TransportRequestOptions.Type.STATE
             );
-            final var future = new TestPlainActionFuture<Releasable>();
-            serviceA.connectToNode(dummy, builder.build(), future);
-            final var ex = expectThrows(ExecutionException.class, ConnectTransportException.class, future::get); // long wait
-            assertEquals("[][" + dummy.getAddress() + "] connect_exception", ex.getMessage());
-            assertNotNull(ExceptionsHelper.unwrap(ex, SslHandshakeTimeoutException.class));
+            final ConnectTransportException exception;
+            final var transportSettings = Settings.builder();
+            if (handshakeTimeoutMillis == null) {
+                handshakeTimeoutMillis = 10000L; // default
+            } else {
+                transportSettings.put("xpack.security.transport.ssl.handshake_timeout", TimeValue.timeValueMillis(handshakeTimeoutMillis));
+            }
+            try (var service = buildService(getTestName(), version0, transportVersion0, transportSettings.build())) {
+                final var future = new TestPlainActionFuture<Releasable>();
+                service.connectToNode(dummy, builder.build(), future);
+                exception = expectThrows(ExecutionException.class, ConnectTransportException.class, future::get); // long wait
+                assertEquals("[][" + dummy.getAddress() + "] connect_exception", exception.getMessage());
+                assertThat(
+                    asInstanceOf(SSLHandshakeException.class, exception.getCause()).getMessage(),
+                    equalTo("handshake timed out after " + handshakeTimeoutMillis + "ms")
+                );
+            }
         } finally {
             doneLatch.countDown();
         }
     }
 
+    @SuppressForbidden(reason = "test needs a simple TCP connection")
+    private void runInboundTlsHandshakeTimeoutTest(@Nullable /* to use the default */ Long handshakeTimeoutMillis) throws IOException {
+        final var transportSettings = Settings.builder();
+        if (handshakeTimeoutMillis == null) {
+            handshakeTimeoutMillis = 10000L; // default
+        } else {
+            transportSettings.put("xpack.security.transport.ssl.handshake_timeout", TimeValue.timeValueMillis(handshakeTimeoutMillis));
+        }
+        try (
+            var service = buildService(getTestName(), version0, transportVersion0, transportSettings.build());
+            Socket clientSocket = new MockSocket();
+            MockLog mockLog = MockLog.capture("org.elasticsearch.transport.netty4.ESLoggingHandler")
+        ) {
+            mockLog.addExpectation(
+                new MockLog.SeenEventExpectation(
+                    "timeout event message",
+                    "org.elasticsearch.transport.netty4.ESLoggingHandler",
+                    Level.TRACE,
+                    "SslHandshakeTimeoutException: handshake timed out after " + handshakeTimeoutMillis + "ms"
+                )
+            );
+
+            clientSocket.connect(service.boundAddress().boundAddresses()[0].address());
+            expectThrows(EOFException.class, () -> clientSocket.getInputStream().skipNBytes(Long.MAX_VALUE));
+            mockLog.assertAllExpectationsMatched();
+        }
+    }
+
     public void testTcpHandshakeConnectionReset() throws IOException, InterruptedException {
         assumeFalse("Can't run in a FIPS JVM, TrustAllConfig is not a SunJSSE TrustManagers", inFipsJvm());
         SSLService sslService = createSSLService();