Skip to content

Commit e57a0d0

Browse files
DaveCTurnerDaveCTurner
authored
Allow adjustment of transport TLS handshake timeout (#130909)
The default 10s TLS handshake timeout may be too short if there is some bug causing event-loop latency, and this has more serious consequences than the underlying performance issue (e.g. it prevents the cluster from scaling up to work around the problem). With this commit we expose a setting that allows the timeout to be configured, providing a workaround in such cases.
1 parent 41c0ba0 commit e57a0d0

File tree

11 files changed

+179
-31
lines changed

11 files changed

+179
-31
lines changed

‎docs/changelog/130909.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 130909
2+
summary: Allow adjustment of transport TLS handshake timeout
3+
area: Network
4+
type: enhancement
5+
issues: []

‎docs/reference/elasticsearch/configuration-reference/security-settings.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1933,6 +1933,8 @@ You can configure the following TLS/SSL settings.
19331933
`xpack.security.transport.ssl.trust_restrictions.x509_fields` ![logo cloud](https://doc-icons.s3.us-east-2.amazonaws.com/logo_cloud.svg "Supported on Elastic Cloud Hosted")
19341934
: Specifies which field(s) from the TLS certificate is used to match for the restricted trust management that is used for remote clusters connections. This should only be set when a self managed cluster can not create certificates that follow the Elastic Cloud pattern. The default value is ["subjectAltName.otherName.commonName"], the Elastic Cloud pattern. "subjectAltName.dnsName" is also supported and can be configured in addition to or in replacement of the default.
19351935

1936+
`xpack.security.transport.ssl.handshake_timeout`
1937+
: Specifies the timeout for a TLS handshake when opening a transport connection. Defaults to `10s`.
19361938

19371939
### Transport TLS/SSL key and trusted certificate settings [security-transport-tls-ssl-key-trusted-certificate-settings]
19381940

@@ -2131,6 +2133,9 @@ You can configure the following TLS/SSL settings.
21312133

21322134
For more information, see Oracle’s [Java Cryptography Architecture documentation](https://docs.oracle.com/en/java/javase/11/security/oracle-providers.md#GUID-7093246A-31A3-4304-AC5F-5FB6400405E2).
21332135

2136+
`xpack.security.remote_cluster_server.ssl.handshake_timeout`
2137+
: Specifies the timeout for a TLS handshake when handling an inbound remote-cluster connection. Defaults to `10s`.
2138+
21342139

21352140
### Remote cluster server (API key based model) TLS/SSL key and trusted certificate settings [security-remote-cluster-server-tls-ssl-key-trusted-certificate-settings]
21362141

@@ -2260,6 +2265,9 @@ You can configure the following TLS/SSL settings.
22602265

22612266
For more information, see Oracle’s [Java Cryptography Architecture documentation](https://docs.oracle.com/en/java/javase/11/security/oracle-providers.md#GUID-7093246A-31A3-4304-AC5F-5FB6400405E2).
22622267

2268+
`xpack.security.remote_cluster_client.ssl.handshake_timeout`
2269+
: Specifies the timeout for a TLS handshake when opening a remote-cluster connection. Defaults to `10s`.
2270+
22632271

22642272
### Remote cluster client (API key based model) TLS/SSL key and trusted certificate settings [security-remote-cluster-client-tls-ssl-key-trusted-certificate-settings]
22652273

‎libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslConfiguration.java‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,8 @@ public record SslConfiguration(
4040
SslVerificationMode verificationMode,
4141
SslClientAuthenticationMode clientAuth,
4242
List<String> ciphers,
43-
List<String> supportedProtocols
43+
List<String> supportedProtocols,
44+
long handshakeTimeoutMillis
4445
) {
4546

4647
/**
@@ -71,7 +72,8 @@ public SslConfiguration(
7172
SslVerificationMode verificationMode,
7273
SslClientAuthenticationMode clientAuth,
7374
List<String> ciphers,
74-
List<String> supportedProtocols
75+
List<String> supportedProtocols,
76+
long handshakeTimeoutMillis
7577
) {
7678
this.settingPrefix = settingPrefix;
7779
this.explicitlyConfigured = explicitlyConfigured;
@@ -85,6 +87,10 @@ public SslConfiguration(
8587
this.keyConfig = Objects.requireNonNull(keyConfig, "key config cannot be null");
8688
this.verificationMode = Objects.requireNonNull(verificationMode, "verification mode cannot be null");
8789
this.clientAuth = Objects.requireNonNull(clientAuth, "client authentication cannot be null");
90+
if (handshakeTimeoutMillis < 1L) {
91+
throw new SslConfigException("handshake timeout must be at least 1ms");
92+
}
93+
this.handshakeTimeoutMillis = handshakeTimeoutMillis;
8894
this.ciphers = Collections.unmodifiableList(ciphers);
8995
this.supportedProtocols = Collections.unmodifiableList(supportedProtocols);
9096
}
@@ -164,11 +170,21 @@ public boolean equals(Object o) {
164170
&& this.verificationMode == that.verificationMode
165171
&& this.clientAuth == that.clientAuth
166172
&& Objects.equals(this.ciphers, that.ciphers)
167-
&& Objects.equals(this.supportedProtocols, that.supportedProtocols);
173+
&& Objects.equals(this.supportedProtocols, that.supportedProtocols)
174+
&& this.handshakeTimeoutMillis == that.handshakeTimeoutMillis;
168175
}
169176

170177
@Override
171178
public int hashCode() {
172-
return Objects.hash(settingPrefix, trustConfig, keyConfig, verificationMode, clientAuth, ciphers, supportedProtocols);
179+
return Objects.hash(
180+
settingPrefix,
181+
trustConfig,
182+
keyConfig,
183+
verificationMode,
184+
clientAuth,
185+
ciphers,
186+
supportedProtocols,
187+
handshakeTimeoutMillis
188+
);
173189
}
174190
}

‎libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslConfigurationKeys.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,10 @@ public class SslConfigurationKeys {
132132
* The use of this setting {@link #isDeprecated(String) is deprecated}.
133133
*/
134134
public static final String KEY_LEGACY_PASSPHRASE = "key_passphrase";
135+
/**
136+
* The timeout for TLS handshakes in this context.
137+
*/
138+
public static final String HANDSHAKE_TIMEOUT = "handshake_timeout";
135139

136140
private static final Set<String> DEPRECATED_KEYS = new HashSet<>(
137141
Arrays.asList(TRUSTSTORE_LEGACY_PASSWORD, KEYSTORE_LEGACY_PASSWORD, KEYSTORE_LEGACY_KEY_PASSWORD, KEY_LEGACY_PASSPHRASE)

‎libs/ssl-config/src/main/java/org/elasticsearch/common/ssl/SslConfigurationLoader.java‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
package org.elasticsearch.common.ssl;
1111

1212
import org.elasticsearch.core.Nullable;
13+
import org.elasticsearch.core.TimeValue;
1314

1415
import java.nio.file.Path;
1516
import java.security.KeyStore;
@@ -27,6 +28,7 @@
2728
import static org.elasticsearch.common.ssl.SslConfigurationKeys.CERTIFICATE_AUTHORITIES;
2829
import static org.elasticsearch.common.ssl.SslConfigurationKeys.CIPHERS;
2930
import static org.elasticsearch.common.ssl.SslConfigurationKeys.CLIENT_AUTH;
31+
import static org.elasticsearch.common.ssl.SslConfigurationKeys.HANDSHAKE_TIMEOUT;
3032
import static org.elasticsearch.common.ssl.SslConfigurationKeys.KEY;
3133
import static org.elasticsearch.common.ssl.SslConfigurationKeys.KEYSTORE_ALGORITHM;
3234
import static org.elasticsearch.common.ssl.SslConfigurationKeys.KEYSTORE_LEGACY_KEY_PASSWORD;
@@ -152,6 +154,8 @@ public abstract class SslConfigurationLoader {
152154
private static final char[] EMPTY_PASSWORD = new char[0];
153155
public static final List<X509Field> GLOBAL_DEFAULT_RESTRICTED_TRUST_FIELDS = List.of(X509Field.SAN_OTHERNAME_COMMONNAME);
154156

157+
public static final TimeValue DEFAULT_HANDSHAKE_TIMEOUT = TimeValue.timeValueSeconds(10);
158+
155159
private final String settingPrefix;
156160

157161
private SslTrustConfig defaultTrustConfig;
@@ -302,6 +306,11 @@ public SslConfiguration load(Path basePath) {
302306
X509Field::parseForRestrictedTrust,
303307
defaultRestrictedTrustFields
304308
);
309+
final long handshakeTimeoutMillis = resolveSetting(
310+
HANDSHAKE_TIMEOUT,
311+
s -> TimeValue.parseTimeValue(s, HANDSHAKE_TIMEOUT),
312+
DEFAULT_HANDSHAKE_TIMEOUT
313+
).millis();
305314

306315
final SslKeyConfig keyConfig = buildKeyConfig(basePath);
307316
final SslTrustConfig trustConfig = buildTrustConfig(basePath, verificationMode, keyConfig, Set.copyOf(trustRestrictionsX509Fields));
@@ -321,7 +330,8 @@ public SslConfiguration load(Path basePath) {
321330
verificationMode,
322331
clientAuth,
323332
ciphers,
324-
protocols
333+
protocols,
334+
handshakeTimeoutMillis
325335
);
326336
}
327337

‎libs/ssl-config/src/test/java/org/elasticsearch/common/ssl/SslConfigurationTests.java‎

Lines changed: 38 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ public void testBasicConstruction() {
3939
final SslClientAuthenticationMode clientAuth = randomFrom(SslClientAuthenticationMode.values());
4040
final List<String> ciphers = randomSubsetOf(randomIntBetween(1, DEFAULT_CIPHERS.size()), DEFAULT_CIPHERS);
4141
final List<String> protocols = randomSubsetOf(randomIntBetween(1, 4), VALID_PROTOCOLS);
42+
final long handshakeTimeoutMillis = randomHandshakeTimeoutMillis();
4243
final SslConfiguration configuration = new SslConfiguration(
4344
"test.ssl",
4445
true,
@@ -47,7 +48,8 @@ public void testBasicConstruction() {
4748
verificationMode,
4849
clientAuth,
4950
ciphers,
50-
protocols
51+
protocols,
52+
handshakeTimeoutMillis
5153
);
5254

5355
assertThat(configuration.trustConfig(), is(trustConfig));
@@ -56,13 +58,15 @@ public void testBasicConstruction() {
5658
assertThat(configuration.clientAuth(), is(clientAuth));
5759
assertThat(configuration.getCipherSuites(), is(ciphers));
5860
assertThat(configuration.supportedProtocols(), is(protocols));
61+
assertThat(configuration.handshakeTimeoutMillis(), is(handshakeTimeoutMillis));
5962

6063
assertThat(configuration.toString(), containsString("TEST-TRUST"));
6164
assertThat(configuration.toString(), containsString("TEST-KEY"));
6265
assertThat(configuration.toString(), containsString(verificationMode.toString()));
6366
assertThat(configuration.toString(), containsString(clientAuth.toString()));
6467
assertThat(configuration.toString(), containsString(randomFrom(ciphers)));
6568
assertThat(configuration.toString(), containsString(randomFrom(protocols)));
69+
assertThat(configuration.toString(), containsString("handshakeTimeoutMillis=" + handshakeTimeoutMillis));
6670
}
6771

6872
public void testEqualsAndHashCode() {
@@ -72,6 +76,7 @@ public void testEqualsAndHashCode() {
7276
final SslClientAuthenticationMode clientAuth = randomFrom(SslClientAuthenticationMode.values());
7377
final List<String> ciphers = randomSubsetOf(randomIntBetween(1, DEFAULT_CIPHERS.size() - 1), DEFAULT_CIPHERS);
7478
final List<String> protocols = randomSubsetOf(randomIntBetween(1, VALID_PROTOCOLS.length - 1), VALID_PROTOCOLS);
79+
final long handshakeTimeoutMillis = randomHandshakeTimeoutMillis();
7580
final SslConfiguration configuration = new SslConfiguration(
7681
"test.ssl",
7782
true,
@@ -80,7 +85,8 @@ public void testEqualsAndHashCode() {
8085
verificationMode,
8186
clientAuth,
8287
ciphers,
83-
protocols
88+
protocols,
89+
handshakeTimeoutMillis
8490
);
8591

8692
EqualsHashCodeTestUtils.checkEqualsAndHashCode(
@@ -93,14 +99,15 @@ public void testEqualsAndHashCode() {
9399
orig.verificationMode(),
94100
orig.clientAuth(),
95101
orig.getCipherSuites(),
96-
orig.supportedProtocols()
102+
orig.supportedProtocols(),
103+
orig.handshakeTimeoutMillis()
97104
),
98105
this::mutateSslConfiguration
99106
);
100107
}
101108

102109
private SslConfiguration mutateSslConfiguration(SslConfiguration orig) {
103-
return switch (randomIntBetween(1, 4)) {
110+
return switch (randomIntBetween(1, 5)) {
104111
case 1 -> new SslConfiguration(
105112
"test.ssl",
106113
true,
@@ -109,7 +116,8 @@ private SslConfiguration mutateSslConfiguration(SslConfiguration orig) {
109116
randomValueOtherThan(orig.verificationMode(), () -> randomFrom(SslVerificationMode.values())),
110117
orig.clientAuth(),
111118
orig.getCipherSuites(),
112-
orig.supportedProtocols()
119+
orig.supportedProtocols(),
120+
orig.handshakeTimeoutMillis()
113121
);
114122
case 2 -> new SslConfiguration(
115123
"test.ssl",
@@ -119,7 +127,8 @@ private SslConfiguration mutateSslConfiguration(SslConfiguration orig) {
119127
orig.verificationMode(),
120128
randomValueOtherThan(orig.clientAuth(), () -> randomFrom(SslClientAuthenticationMode.values())),
121129
orig.getCipherSuites(),
122-
orig.supportedProtocols()
130+
orig.supportedProtocols(),
131+
orig.handshakeTimeoutMillis()
123132
);
124133
case 3 -> new SslConfiguration(
125134
"test.ssl",
@@ -129,7 +138,19 @@ private SslConfiguration mutateSslConfiguration(SslConfiguration orig) {
129138
orig.verificationMode(),
130139
orig.clientAuth(),
131140
DEFAULT_CIPHERS,
132-
orig.supportedProtocols()
141+
orig.supportedProtocols(),
142+
orig.handshakeTimeoutMillis()
143+
);
144+
case 4 -> new SslConfiguration(
145+
"test.ssl",
146+
true,
147+
orig.trustConfig(),
148+
orig.keyConfig(),
149+
orig.verificationMode(),
150+
orig.clientAuth(),
151+
orig.getCipherSuites(),
152+
Arrays.asList(VALID_PROTOCOLS),
153+
orig.handshakeTimeoutMillis()
133154
);
134155
default -> new SslConfiguration(
135156
"test.ssl",
@@ -139,11 +160,16 @@ private SslConfiguration mutateSslConfiguration(SslConfiguration orig) {
139160
orig.verificationMode(),
140161
orig.clientAuth(),
141162
orig.getCipherSuites(),
142-
Arrays.asList(VALID_PROTOCOLS)
163+
orig.supportedProtocols(),
164+
randomValueOtherThan(orig.handshakeTimeoutMillis(), SslConfigurationTests::randomHandshakeTimeoutMillis)
143165
);
144166
};
145167
}
146168

169+
private static long randomHandshakeTimeoutMillis() {
170+
return randomLongBetween(1, 100000);
171+
}
172+
147173
public void testDependentFiles() {
148174
final SslTrustConfig trustConfig = Mockito.mock(SslTrustConfig.class);
149175
final SslKeyConfig keyConfig = Mockito.mock(SslKeyConfig.class);
@@ -155,7 +181,8 @@ public void testDependentFiles() {
155181
randomFrom(SslVerificationMode.values()),
156182
randomFrom(SslClientAuthenticationMode.values()),
157183
DEFAULT_CIPHERS,
158-
SslConfigurationLoader.DEFAULT_PROTOCOLS
184+
SslConfigurationLoader.DEFAULT_PROTOCOLS,
185+
randomHandshakeTimeoutMillis()
159186
);
160187

161188
final Path dir = createTempDir();
@@ -182,7 +209,8 @@ public void testBuildSslContext() {
182209
randomFrom(SslVerificationMode.values()),
183210
randomFrom(SslClientAuthenticationMode.values()),
184211
DEFAULT_CIPHERS,
185-
Collections.singletonList(protocol)
212+
Collections.singletonList(protocol),
213+
randomHandshakeTimeoutMillis()
186214
);
187215

188216
Mockito.when(trustConfig.createTrustManager()).thenReturn(null);

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -344,18 +344,14 @@ public static Setting<String> defaultStoredSecureTokenHashAlgorithmSetting(
344344
public static final SslClientAuthenticationMode REMOTE_CLUSTER_CLIENT_AUTH_DEFAULT = SslClientAuthenticationMode.NONE;
345345
public static final SslVerificationMode VERIFICATION_MODE_DEFAULT = SslVerificationMode.FULL;
346346

347-
// http specific settings
348347
public static final String HTTP_SSL_PREFIX = SecurityField.setting("http.ssl.");
349-
private static final SSLConfigurationSettings HTTP_SSL = SSLConfigurationSettings.withPrefix(HTTP_SSL_PREFIX, true);
350-
351-
// transport specific settings
352348
public static final String TRANSPORT_SSL_PREFIX = SecurityField.setting("transport.ssl.");
353-
private static final SSLConfigurationSettings TRANSPORT_SSL = SSLConfigurationSettings.withPrefix(TRANSPORT_SSL_PREFIX, true);
354-
355-
// remote cluster specific settings
356349
public static final String REMOTE_CLUSTER_SERVER_SSL_PREFIX = SecurityField.setting("remote_cluster_server.ssl.");
357350
public static final String REMOTE_CLUSTER_CLIENT_SSL_PREFIX = SecurityField.setting("remote_cluster_client.ssl.");
358351

352+
private static final SSLConfigurationSettings HTTP_SSL = SSLConfigurationSettings.withPrefix(HTTP_SSL_PREFIX, true);
353+
private static final SSLConfigurationSettings TRANSPORT_SSL = SSLConfigurationSettings.withPrefix(TRANSPORT_SSL_PREFIX, true);
354+
359355
private static final SSLConfigurationSettings REMOTE_CLUSTER_SERVER_SSL = SSLConfigurationSettings.withPrefix(
360356
REMOTE_CLUSTER_SERVER_SSL_PREFIX,
361357
false

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationSettings.java‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,12 @@
1313
import org.elasticsearch.common.settings.Settings;
1414
import org.elasticsearch.common.ssl.SslClientAuthenticationMode;
1515
import org.elasticsearch.common.ssl.SslConfigurationKeys;
16+
import org.elasticsearch.common.ssl.SslConfigurationLoader;
1617
import org.elasticsearch.common.ssl.SslVerificationMode;
1718
import org.elasticsearch.common.ssl.X509Field;
1819
import org.elasticsearch.common.util.CollectionUtils;
20+
import org.elasticsearch.core.TimeValue;
21+
import org.elasticsearch.xpack.core.XPackSettings;
1922
import org.elasticsearch.xpack.core.security.authc.RealmConfig;
2023

2124
import java.util.ArrayList;
@@ -30,6 +33,7 @@
3033
import javax.net.ssl.TrustManagerFactory;
3134

3235
import static org.elasticsearch.common.ssl.SslConfigurationLoader.GLOBAL_DEFAULT_RESTRICTED_TRUST_FIELDS;
36+
import static org.elasticsearch.xpack.core.XPackSettings.TRANSPORT_SSL_PREFIX;
3337

3438
/**
3539
* Bridges SSLConfiguration into the {@link Settings} framework, using {@link Setting} objects.
@@ -50,6 +54,7 @@ public class SSLConfigurationSettings {
5054
final Setting<List<String>> caPaths;
5155
final Setting<Optional<SslClientAuthenticationMode>> clientAuth;
5256
final Setting<Optional<SslVerificationMode>> verificationMode;
57+
final Setting<TimeValue> handshakeTimeout;
5358

5459
// public for PKI realm
5560
private final Setting<SecureString> legacyTruststorePassword;
@@ -223,6 +228,11 @@ public class SSLConfigurationSettings {
223228
public static final Function<String, Setting.AffixSetting<Optional<SslVerificationMode>>> VERIFICATION_MODE_SETTING_REALM =
224229
VERIFICATION_MODE::realm;
225230

231+
public static final SslSetting<TimeValue> HANDSHAKE_TIMEOUT = SslSetting.setting(
232+
SslConfigurationKeys.HANDSHAKE_TIMEOUT,
233+
key -> Setting.positiveTimeSetting(key, SslConfigurationLoader.DEFAULT_HANDSHAKE_TIMEOUT, Property.NodeScope)
234+
);
235+
226236
/**
227237
* @param prefix The prefix under which each setting should be defined. Must be either the empty string (<code>""</code>) or a string
228238
* ending in <code>"."</code>
@@ -246,6 +256,7 @@ private SSLConfigurationSettings(String prefix, boolean acceptNonSecurePasswords
246256
caPaths = CERT_AUTH_PATH.withPrefix(prefix);
247257
clientAuth = CLIENT_AUTH_SETTING.withPrefix(prefix);
248258
verificationMode = VERIFICATION_MODE.withPrefix(prefix);
259+
handshakeTimeout = HANDSHAKE_TIMEOUT.withPrefix(prefix);
249260

250261
final List<Setting<? extends Object>> enabled = CollectionUtils.arrayAsArrayList(
251262
ciphers,
@@ -270,6 +281,16 @@ private SSLConfigurationSettings(String prefix, boolean acceptNonSecurePasswords
270281
enabled.addAll(x509KeyPair.getEnabledSettings());
271282
disabled.addAll(x509KeyPair.getDisabledSettings());
272283

284+
if (TRANSPORT_SSL_PREFIX.equals(prefix)
285+
|| XPackSettings.REMOTE_CLUSTER_CLIENT_SSL_PREFIX.equals(prefix)
286+
|| XPackSettings.REMOTE_CLUSTER_SERVER_SSL_PREFIX.equals(prefix)) {
287+
enabled.add(handshakeTimeout);
288+
} else {
289+
// Today the handshake timeout is only adjustable for transport connections - see SecurityNetty4Transport. In principle we
290+
// could extend this to other contexts too, we just haven't done so yet.
291+
disabled.add(handshakeTimeout);
292+
}
293+
273294
this.enabledSettings = Collections.unmodifiableList(enabled);
274295
this.disabledSettings = Collections.unmodifiableList(disabled);
275296
}
@@ -327,7 +348,8 @@ private static Collection<SslSetting<?>> settings() {
327348
CERT,
328349
CERT_AUTH_PATH,
329350
CLIENT_AUTH_SETTING,
330-
VERIFICATION_MODE
351+
VERIFICATION_MODE,
352+
HANDSHAKE_TIMEOUT
331353
);
332354
}
333355

0 commit comments

Comments
 (0)