Skip to content

Commit 7ff4352

Browse files
ywangdywangd
authored
Ensure RCS client connection use remote cluster specific TCP settings (#94423)
RCS remote cluster has separate tcp settings, e.g. remote_cluster.tcp.keepalive. They default to the corresponding main transport settings, but can also have different values. This PR ensures that if different values are specified, they are used for remote cluster client connections.
1 parent 12ab625 commit 7ff4352

File tree

4 files changed

+571
-28
lines changed

4 files changed

+571
-28
lines changed

‎modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Transport.java

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,9 @@ public class Netty4Transport extends TcpTransport {
8888
);
8989

9090
public static final Setting<Integer> NETTY_BOSS_COUNT = intSetting("transport.netty.boss_count", 1, 1, Property.NodeScope);
91+
public static final ChannelOption<Integer> OPTION_TCP_KEEP_IDLE = NioChannelOption.of(NetUtils.getTcpKeepIdleSocketOption());
92+
public static final ChannelOption<Integer> OPTION_TCP_KEEP_INTERVAL = NioChannelOption.of(NetUtils.getTcpKeepIntervalSocketOption());
93+
public static final ChannelOption<Integer> OPTION_TCP_KEEP_COUNT = NioChannelOption.of(NetUtils.getTcpKeepCountSocketOption());
9194

9295
private final SharedGroupFactory sharedGroupFactory;
9396
private final RecvByteBufAllocator recvByteBufAllocator;
@@ -160,24 +163,21 @@ private Bootstrap createClientBootstrap(SharedGroupFactory.SharedGroup sharedGro
160163
bootstrap.channel(NettyAllocator.getChannelType());
161164
bootstrap.option(ChannelOption.ALLOCATOR, NettyAllocator.getAllocator());
162165

166+
// The TCP options are re-configured for client connections to RCS remote clusters
167+
// If how options are configured is changed here, please also update RemoteClusterClientBootstrapOptions#configure
168+
// which is used inside SecurityNetty4Transport#getClientBootstrap
163169
bootstrap.option(ChannelOption.TCP_NODELAY, TransportSettings.TCP_NO_DELAY.get(settings));
164170
bootstrap.option(ChannelOption.SO_KEEPALIVE, TransportSettings.TCP_KEEP_ALIVE.get(settings));
165171
if (TransportSettings.TCP_KEEP_ALIVE.get(settings)) {
166172
// Note that Netty logs a warning if it can't set the option
167173
if (TransportSettings.TCP_KEEP_IDLE.get(settings) >= 0) {
168-
bootstrap.option(NioChannelOption.of(NetUtils.getTcpKeepIdleSocketOption()), TransportSettings.TCP_KEEP_IDLE.get(settings));
174+
bootstrap.option(OPTION_TCP_KEEP_IDLE, TransportSettings.TCP_KEEP_IDLE.get(settings));
169175
}
170176
if (TransportSettings.TCP_KEEP_INTERVAL.get(settings) >= 0) {
171-
bootstrap.option(
172-
NioChannelOption.of(NetUtils.getTcpKeepIntervalSocketOption()),
173-
TransportSettings.TCP_KEEP_INTERVAL.get(settings)
174-
);
177+
bootstrap.option(OPTION_TCP_KEEP_INTERVAL, TransportSettings.TCP_KEEP_INTERVAL.get(settings));
175178
}
176179
if (TransportSettings.TCP_KEEP_COUNT.get(settings) >= 0) {
177-
bootstrap.option(
178-
NioChannelOption.of(NetUtils.getTcpKeepCountSocketOption()),
179-
TransportSettings.TCP_KEEP_COUNT.get(settings)
180-
);
180+
bootstrap.option(OPTION_TCP_KEEP_COUNT, TransportSettings.TCP_KEEP_COUNT.get(settings));
181181
}
182182
}
183183

@@ -278,7 +278,7 @@ protected ChannelHandler getClientChannelInitializer(DiscoveryNode node, Connect
278278
@Override
279279
protected Netty4TcpChannel initiateChannel(DiscoveryNode node, ConnectionProfile connectionProfile) throws IOException {
280280
InetSocketAddress address = node.getAddress().address();
281-
Bootstrap bootstrapWithHandler = clientBootstrap.clone();
281+
Bootstrap bootstrapWithHandler = getClientBootstrap(connectionProfile);
282282
bootstrapWithHandler.handler(getClientChannelInitializer(node, connectionProfile));
283283
bootstrapWithHandler.remoteAddress(address);
284284
ChannelFuture connectFuture = bootstrapWithHandler.connect();
@@ -301,6 +301,10 @@ protected Netty4TcpChannel initiateChannel(DiscoveryNode node, ConnectionProfile
301301
return nettyChannel;
302302
}
303303

304+
protected Bootstrap getClientBootstrap(ConnectionProfile connectionProfile) {
305+
return clientBootstrap.clone();
306+
}
307+
304308
@Override
305309
protected Netty4TcpServerChannel bind(String name, InetSocketAddress address) {
306310
Channel channel = serverBootstraps.get(name).bind(address).syncUninterruptibly().channel();

‎x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/core/security/transport/netty4/SecurityNetty4Transport.java

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,11 @@
66
*/
77
package org.elasticsearch.xpack.core.security.transport.netty4;
88

9+
import io.netty.bootstrap.Bootstrap;
910
import io.netty.channel.Channel;
1011
import io.netty.channel.ChannelHandler;
1112
import io.netty.channel.ChannelHandlerContext;
13+
import io.netty.channel.ChannelOption;
1214
import io.netty.channel.ChannelOutboundHandlerAdapter;
1315
import io.netty.channel.ChannelPromise;
1416
import io.netty.handler.ssl.SslHandler;
@@ -22,11 +24,13 @@
2224
import org.elasticsearch.common.network.NetworkService;
2325
import org.elasticsearch.common.settings.Settings;
2426
import org.elasticsearch.common.ssl.SslConfiguration;
27+
import org.elasticsearch.common.unit.ByteSizeValue;
2528
import org.elasticsearch.common.util.PageCacheRecycler;
2629
import org.elasticsearch.indices.breaker.CircuitBreakerService;
2730
import org.elasticsearch.threadpool.ThreadPool;
2831
import org.elasticsearch.transport.ConnectTransportException;
2932
import org.elasticsearch.transport.ConnectionProfile;
33+
import org.elasticsearch.transport.RemoteClusterPortSettings;
3034
import org.elasticsearch.transport.TcpChannel;
3135
import org.elasticsearch.transport.TransportSettings;
3236
import org.elasticsearch.transport.netty4.Netty4Transport;
@@ -66,6 +70,7 @@ public class SecurityNetty4Transport extends Netty4Transport {
6670
private final boolean remoteClusterPortEnabled;
6771
private final boolean remoteClusterServerSslEnabled;
6872
private final SslConfiguration remoteClusterClientSslConfiguration;
73+
private final RemoteClusterClientBootstrapOptions remoteClusterClientBootstrapOptions;
6974

7075
public SecurityNetty4Transport(
7176
final Settings settings,
@@ -104,6 +109,7 @@ public SecurityNetty4Transport(
104109
} else {
105110
this.remoteClusterClientSslConfiguration = null;
106111
}
112+
this.remoteClusterClientBootstrapOptions = RemoteClusterClientBootstrapOptions.fromSettings(settings);
107113
}
108114

109115
@Override
@@ -143,6 +149,21 @@ protected ChannelHandler getClientChannelInitializer(DiscoveryNode node, Connect
143149
return new SecurityClientChannelInitializer(node, connectionProfile);
144150
}
145151

152+
@Override
153+
protected Bootstrap getClientBootstrap(ConnectionProfile connectionProfile) {
154+
final Bootstrap bootstrap = super.getClientBootstrap(connectionProfile);
155+
if (false == REMOTE_CLUSTER_PROFILE.equals(connectionProfile.getTransportProfile())
156+
|| remoteClusterClientBootstrapOptions.isEmpty()) {
157+
return bootstrap;
158+
}
159+
160+
logger.trace("reconfiguring client bootstrap for remote cluster client connection");
161+
// Only client connections to a new RCS remote cluster can have transport profile of _remote_cluster
162+
// All other client connections use the default transport profile regardless of the transport profile used on the server side.
163+
remoteClusterClientBootstrapOptions.configure(bootstrap);
164+
return bootstrap;
165+
}
166+
146167
@Override
147168
public void onException(TcpChannel channel, Exception e) {
148169
exceptionHandler.accept(channel, e);
@@ -279,4 +300,163 @@ public void connect(ChannelHandlerContext ctx, SocketAddress remoteAddress, Sock
279300
super.connect(ctx, remoteAddress, localAddress, connectPromise);
280301
}
281302
}
303+
304+
// This class captures the differences of client side TCP network settings between default and _remote_cluster transport profiles.
305+
// A field will be null if there is no difference between associated settings of the two profiles. It has a non-null value only
306+
// when the _remote_cluster profile has a different value from the default profile.
307+
record RemoteClusterClientBootstrapOptions(
308+
Boolean tcpNoDelay,
309+
Boolean tcpKeepAlive,
310+
Integer tcpKeepIdle,
311+
Integer tcpKeepInterval,
312+
Integer tcpKeepCount,
313+
ByteSizeValue tcpSendBufferSize,
314+
ByteSizeValue tcpReceiveBufferSize,
315+
Boolean tcpReuseAddress
316+
) {
317+
318+
boolean isEmpty() {
319+
return tcpNoDelay == null
320+
&& tcpKeepAlive == null
321+
&& tcpKeepIdle == null
322+
&& tcpKeepInterval == null
323+
&& tcpKeepCount == null
324+
&& tcpSendBufferSize == null
325+
&& tcpReceiveBufferSize == null
326+
&& tcpReuseAddress == null;
327+
}
328+
329+
void configure(Bootstrap bootstrap) {
330+
if (tcpNoDelay != null) {
331+
bootstrap.option(ChannelOption.TCP_NODELAY, tcpNoDelay);
332+
}
333+
334+
if (tcpKeepAlive != null) {
335+
bootstrap.option(ChannelOption.SO_KEEPALIVE, tcpKeepAlive);
336+
if (tcpKeepAlive) {
337+
// Note that Netty logs a warning if it can't set the option
338+
if (tcpKeepIdle != null) {
339+
if (tcpKeepIdle >= 0) {
340+
bootstrap.option(OPTION_TCP_KEEP_IDLE, tcpKeepIdle);
341+
} else {
342+
bootstrap.option(OPTION_TCP_KEEP_IDLE, null);
343+
}
344+
}
345+
if (tcpKeepInterval != null) {
346+
if (tcpKeepInterval >= 0) {
347+
bootstrap.option(OPTION_TCP_KEEP_INTERVAL, tcpKeepInterval);
348+
} else {
349+
bootstrap.option(OPTION_TCP_KEEP_INTERVAL, null);
350+
}
351+
}
352+
if (tcpKeepCount != null) {
353+
if (tcpKeepCount >= 0) {
354+
bootstrap.option(OPTION_TCP_KEEP_COUNT, tcpKeepCount);
355+
} else {
356+
bootstrap.option(OPTION_TCP_KEEP_COUNT, null);
357+
}
358+
}
359+
} else {
360+
bootstrap.option(OPTION_TCP_KEEP_IDLE, null);
361+
bootstrap.option(OPTION_TCP_KEEP_INTERVAL, null);
362+
bootstrap.option(OPTION_TCP_KEEP_COUNT, null);
363+
}
364+
}
365+
366+
if (tcpSendBufferSize != null) {
367+
if (tcpSendBufferSize.getBytes() > 0) {
368+
bootstrap.option(ChannelOption.SO_SNDBUF, Math.toIntExact(tcpSendBufferSize.getBytes()));
369+
} else {
370+
bootstrap.option(ChannelOption.SO_SNDBUF, null);
371+
}
372+
}
373+
374+
if (tcpReceiveBufferSize != null) {
375+
if (tcpReceiveBufferSize.getBytes() > 0) {
376+
bootstrap.option(ChannelOption.SO_RCVBUF, Math.toIntExact(tcpReceiveBufferSize.getBytes()));
377+
} else {
378+
bootstrap.option(ChannelOption.SO_RCVBUF, null);
379+
}
380+
}
381+
382+
if (tcpReuseAddress != null) {
383+
bootstrap.option(ChannelOption.SO_REUSEADDR, tcpReuseAddress);
384+
}
385+
}
386+
387+
static RemoteClusterClientBootstrapOptions fromSettings(Settings settings) {
388+
Boolean tcpNoDelay = RemoteClusterPortSettings.TCP_NO_DELAY.get(settings);
389+
if (tcpNoDelay == TransportSettings.TCP_NO_DELAY.get(settings)) {
390+
tcpNoDelay = null;
391+
}
392+
393+
// It is possible that both default and _remote_cluster enable keepAlive but have different
394+
// values for either keepIdle, keepInterval or keepCount. In this case, we need have a
395+
// non-null value for keepAlive even it is the same between default and _remote_cluster.
396+
Boolean tcpKeepAlive = RemoteClusterPortSettings.TCP_KEEP_ALIVE.get(settings);
397+
Integer tcpKeepIdle = RemoteClusterPortSettings.TCP_KEEP_IDLE.get(settings);
398+
Integer tcpKeepInterval = RemoteClusterPortSettings.TCP_KEEP_INTERVAL.get(settings);
399+
Integer tcpKeepCount = RemoteClusterPortSettings.TCP_KEEP_COUNT.get(settings);
400+
final Boolean defaultTcpKeepAlive = TransportSettings.TCP_KEEP_ALIVE.get(settings);
401+
402+
if (tcpKeepAlive) {
403+
if (defaultTcpKeepAlive) {
404+
// Both profiles have keepAlive enabled, we need to check whether any keepIdle, keepInterval, keepCount is different
405+
if (tcpKeepIdle.equals(TransportSettings.TCP_KEEP_IDLE.get(settings))) {
406+
tcpKeepIdle = null;
407+
}
408+
if (tcpKeepInterval.equals(TransportSettings.TCP_KEEP_INTERVAL.get(settings))) {
409+
tcpKeepInterval = null;
410+
}
411+
if (tcpKeepCount.equals(TransportSettings.TCP_KEEP_COUNT.get(settings))) {
412+
tcpKeepCount = null;
413+
}
414+
if (tcpKeepIdle == null && tcpKeepInterval == null && tcpKeepCount == null) {
415+
// If keepIdle, keepInterval, keepCount are all identical, keepAlive can be null as well.
416+
// That is no need to update anything keepXxx related
417+
tcpKeepAlive = null;
418+
}
419+
}
420+
} else {
421+
if (false == defaultTcpKeepAlive) {
422+
tcpKeepAlive = null;
423+
}
424+
// _remote_cluster has keepAlive disabled, all other keepXxx has no reason to exist
425+
tcpKeepIdle = null;
426+
tcpKeepInterval = null;
427+
tcpKeepCount = null;
428+
}
429+
430+
assert (tcpKeepAlive == null && tcpKeepIdle == null && tcpKeepInterval == null && tcpKeepCount == null)
431+
|| (tcpKeepAlive == false && tcpKeepIdle == null && tcpKeepInterval == null && tcpKeepCount == null)
432+
|| (tcpKeepAlive && (tcpKeepIdle != null || tcpKeepInterval != null || tcpKeepCount != null))
433+
: "keepAlive == true must be accompanied with either keepIdle, keepInterval or keepCount change";
434+
435+
ByteSizeValue tcpSendBufferSize = RemoteClusterPortSettings.TCP_SEND_BUFFER_SIZE.get(settings);
436+
if (tcpSendBufferSize.equals(TransportSettings.TCP_SEND_BUFFER_SIZE.get(settings))) {
437+
tcpSendBufferSize = null;
438+
}
439+
440+
ByteSizeValue tcpReceiveBufferSize = RemoteClusterPortSettings.TCP_RECEIVE_BUFFER_SIZE.get(settings);
441+
if (tcpReceiveBufferSize.equals(TransportSettings.TCP_RECEIVE_BUFFER_SIZE.get(settings))) {
442+
tcpReceiveBufferSize = null;
443+
}
444+
445+
Boolean tcpReuseAddress = RemoteClusterPortSettings.TCP_REUSE_ADDRESS.get(settings);
446+
if (tcpReuseAddress == TransportSettings.TCP_REUSE_ADDRESS.get(settings)) {
447+
tcpReuseAddress = null;
448+
}
449+
450+
return new RemoteClusterClientBootstrapOptions(
451+
tcpNoDelay,
452+
tcpKeepAlive,
453+
tcpKeepIdle,
454+
tcpKeepInterval,
455+
tcpKeepCount,
456+
tcpSendBufferSize,
457+
tcpReceiveBufferSize,
458+
tcpReuseAddress
459+
);
460+
}
461+
}
282462
}

0 commit comments

Comments
 (0)