Skip to content

Commit 51e9d6f

Browse files
original-brownbearoriginal-brownbear
authored
Revert Serializing Outbound Transport Messages on IO Threads (#64632) (#64654)
Serializing outbound transport message on the IO loop was introduced in #56961. Unfortunately it turns out that this is incompatible with assumptions made by CCR code here: https://github.com/elastic/elasticsearch/blob/f22ddf822e24bb17f1772c3bacb7ee97a00339b8/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/repositories/GetCcrRestoreFileChunkAction.java#L60-L61 and that are not easy to work around on short notice. Raising reverting this move (as a temporary solution, it's still a valuable change long-term) as a blocker therefore as this seriously affects the stability of the initial phase of the CCR following by causing corrupted bytes to be send to the follower.
1 parent 9e4105e commit 51e9d6f

File tree

7 files changed

+47
-85
lines changed

7 files changed

+47
-85
lines changed

‎modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4MessageChannelHandler.java‎

Lines changed: 15 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -33,11 +33,9 @@
3333
import org.elasticsearch.common.util.PageCacheRecycler;
3434
import org.elasticsearch.threadpool.ThreadPool;
3535
import org.elasticsearch.transport.InboundPipeline;
36-
import org.elasticsearch.transport.OutboundHandler;
3736
import org.elasticsearch.transport.Transport;
3837
import org.elasticsearch.transport.Transports;
3938

40-
import java.io.IOException;
4139
import java.nio.channels.ClosedChannelException;
4240
import java.util.ArrayDeque;
4341
import java.util.Queue;
@@ -93,15 +91,15 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
9391

9492
@Override
9593
public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) {
96-
assert msg instanceof OutboundHandler.SendContext;
94+
assert msg instanceof ByteBuf;
9795
assert Transports.assertDefaultThreadContext(transport.getThreadPool().getThreadContext());
98-
final boolean queued = queuedWrites.offer(new WriteOperation((OutboundHandler.SendContext) msg, promise));
96+
final boolean queued = queuedWrites.offer(new WriteOperation((ByteBuf) msg, promise));
9997
assert queued;
10098
assert Transports.assertDefaultThreadContext(transport.getThreadPool().getThreadContext());
10199
}
102100

103101
@Override
104-
public void channelWritabilityChanged(ChannelHandlerContext ctx) throws IOException {
102+
public void channelWritabilityChanged(ChannelHandlerContext ctx) {
105103
assert Transports.assertDefaultThreadContext(transport.getThreadPool().getThreadContext());
106104
if (ctx.channel().isWritable()) {
107105
doFlush(ctx);
@@ -110,7 +108,7 @@ public void channelWritabilityChanged(ChannelHandlerContext ctx) throws IOExcept
110108
}
111109

112110
@Override
113-
public void flush(ChannelHandlerContext ctx) throws IOException {
111+
public void flush(ChannelHandlerContext ctx) {
114112
assert Transports.assertDefaultThreadContext(transport.getThreadPool().getThreadContext());
115113
Channel channel = ctx.channel();
116114
if (channel.isWritable() || channel.isActive() == false) {
@@ -126,7 +124,7 @@ public void channelInactive(ChannelHandlerContext ctx) throws Exception {
126124
super.channelInactive(ctx);
127125
}
128126

129-
private void doFlush(ChannelHandlerContext ctx) throws IOException {
127+
private void doFlush(ChannelHandlerContext ctx) {
130128
assert ctx.executor().inEventLoop();
131129
final Channel channel = ctx.channel();
132130
if (channel.isActive() == false) {
@@ -144,25 +142,24 @@ private void doFlush(ChannelHandlerContext ctx) throws IOException {
144142
break;
145143
}
146144
final WriteOperation write = currentWrite;
147-
final ByteBuf currentBuffer = write.buffer();
148-
if (currentBuffer.readableBytes() == 0) {
145+
if (write.buf.readableBytes() == 0) {
149146
write.promise.trySuccess();
150147
currentWrite = null;
151148
continue;
152149
}
153-
final int readableBytes = currentBuffer.readableBytes();
150+
final int readableBytes = write.buf.readableBytes();
154151
final int bufferSize = Math.min(readableBytes, 1 << 18);
155-
final int readerIndex = currentBuffer.readerIndex();
152+
final int readerIndex = write.buf.readerIndex();
156153
final boolean sliced = readableBytes != bufferSize;
157154
final ByteBuf writeBuffer;
158155
if (sliced) {
159-
writeBuffer = currentBuffer.retainedSlice(readerIndex, bufferSize);
160-
currentBuffer.readerIndex(readerIndex + bufferSize);
156+
writeBuffer = write.buf.retainedSlice(readerIndex, bufferSize);
157+
write.buf.readerIndex(readerIndex + bufferSize);
161158
} else {
162-
writeBuffer = currentBuffer;
159+
writeBuffer = write.buf;
163160
}
164161
final ChannelFuture writeFuture = ctx.write(writeBuffer);
165-
if (sliced == false || currentBuffer.readableBytes() == 0) {
162+
if (sliced == false || write.buf.readableBytes() == 0) {
166163
currentWrite = null;
167164
writeFuture.addListener(future -> {
168165
assert ctx.executor().inEventLoop();
@@ -197,24 +194,13 @@ private void failQueuedWrites() {
197194

198195
private static final class WriteOperation {
199196

200-
private ByteBuf buf;
201-
202-
private OutboundHandler.SendContext context;
197+
private final ByteBuf buf;
203198

204199
private final ChannelPromise promise;
205200

206-
WriteOperation(OutboundHandler.SendContext context, ChannelPromise promise) {
207-
this.context = context;
201+
WriteOperation(ByteBuf buf, ChannelPromise promise) {
202+
this.buf = buf;
208203
this.promise = promise;
209204
}
210-
211-
ByteBuf buffer() throws IOException {
212-
if (buf == null) {
213-
buf = Netty4Utils.toByteBuf(context.get());
214-
context = null;
215-
}
216-
assert context == null;
217-
return buf;
218-
}
219205
}
220206
}

‎modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4TcpChannel.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@
2525
import org.elasticsearch.ExceptionsHelper;
2626
import org.elasticsearch.action.ActionListener;
2727
import org.elasticsearch.common.Nullable;
28+
import org.elasticsearch.common.bytes.BytesReference;
2829
import org.elasticsearch.common.concurrent.CompletableContext;
29-
import org.elasticsearch.transport.OutboundHandler;
3030
import org.elasticsearch.transport.TcpChannel;
3131
import org.elasticsearch.transport.TransportException;
3232

@@ -142,11 +142,11 @@ public InetSocketAddress getRemoteAddress() {
142142
}
143143

144144
@Override
145-
public void sendMessage(OutboundHandler.SendContext sendContext) {
146-
channel.writeAndFlush(sendContext, addPromise(sendContext, channel));
145+
public void sendMessage(BytesReference reference, ActionListener<Void> listener) {
146+
channel.writeAndFlush(Netty4Utils.toByteBuf(reference), addPromise(listener, channel));
147147

148148
if (channel.eventLoop().isShutdown()) {
149-
sendContext.onFailure(new TransportException("Cannot send message, event loop is shutting down."));
149+
listener.onFailure(new TransportException("Cannot send message, event loop is shutting down."));
150150
}
151151
}
152152

‎plugins/transport-nio/src/main/java/org/elasticsearch/transport/nio/NioTcpChannel.java‎

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,8 @@
2222
import org.elasticsearch.action.ActionListener;
2323
import org.elasticsearch.common.bytes.BytesReference;
2424
import org.elasticsearch.nio.NioSocketChannel;
25-
import org.elasticsearch.transport.OutboundHandler;
2625
import org.elasticsearch.transport.TcpChannel;
2726

28-
import java.io.IOException;
2927
import java.nio.channels.SocketChannel;
3028

3129
public class NioTcpChannel extends NioSocketChannel implements TcpChannel {
@@ -40,16 +38,8 @@ public NioTcpChannel(boolean isServer, String profile, SocketChannel socketChann
4038
this.profile = profile;
4139
}
4240

43-
@Override
44-
public void sendMessage(OutboundHandler.SendContext sendContext) {
45-
final BytesReference message;
46-
try {
47-
message = sendContext.get();
48-
} catch (IOException e) {
49-
sendContext.onFailure(e);
50-
return;
51-
}
52-
getContext().sendMessage(BytesReference.toByteBuffers(message), ActionListener.toBiConsumer(sendContext));
41+
public void sendMessage(BytesReference reference, ActionListener<Void> listener) {
42+
getContext().sendMessage(BytesReference.toByteBuffers(reference), ActionListener.toBiConsumer(listener));
5343
}
5444

5545
@Override

‎server/src/main/java/org/elasticsearch/transport/OutboundHandler.java‎

Lines changed: 15 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@
4242
import java.io.IOException;
4343
import java.util.Set;
4444

45-
public final class OutboundHandler {
45+
final class OutboundHandler {
4646

4747
private static final Logger logger = LogManager.getLogger(OutboundHandler.class);
4848

@@ -66,7 +66,12 @@ public final class OutboundHandler {
6666

6767
void sendBytes(TcpChannel channel, BytesReference bytes, ActionListener<Void> listener) {
6868
SendContext sendContext = new SendContext(channel, () -> bytes, listener);
69-
internalSend(channel, sendContext);
69+
try {
70+
internalSend(channel, sendContext);
71+
} catch (IOException e) {
72+
// This should not happen as the bytes are already serialized
73+
throw new AssertionError(e);
74+
}
7075
}
7176

7277
/**
@@ -120,17 +125,17 @@ private void sendMessage(TcpChannel channel, OutboundMessage networkMessage, Act
120125
internalSend(channel, sendContext);
121126
}
122127

123-
private void internalSend(TcpChannel channel, SendContext sendContext) {
128+
private void internalSend(TcpChannel channel, SendContext sendContext) throws IOException {
124129
channel.getChannelStats().markAccessed(threadPool.relativeTimeInMillis());
130+
BytesReference reference = sendContext.get();
125131
// stash thread context so that channel event loop is not polluted by thread context
126132
try (ThreadContext.StoredContext existing = threadPool.getThreadContext().stashContext()) {
127-
channel.sendMessage(sendContext);
133+
channel.sendMessage(reference, sendContext);
128134
} catch (RuntimeException ex) {
129135
sendContext.onFailure(ex);
130136
CloseableChannel.closeChannel(channel);
131137
throw ex;
132138
}
133-
134139
}
135140

136141
void setMessageListener(TransportMessageListener listener) {
@@ -143,7 +148,7 @@ void setMessageListener(TransportMessageListener listener) {
143148

144149
private static class MessageSerializer implements CheckedSupplier<BytesReference, IOException>, Releasable {
145150

146-
private OutboundMessage message;
151+
private final OutboundMessage message;
147152
private final BigArrays bigArrays;
148153
private volatile ReleasableBytesStreamOutput bytesStreamOutput;
149154

@@ -154,12 +159,8 @@ private MessageSerializer(OutboundMessage message, BigArrays bigArrays) {
154159

155160
@Override
156161
public BytesReference get() throws IOException {
157-
try {
158-
bytesStreamOutput = new ReleasableBytesStreamOutput(bigArrays);
159-
return message.serialize(bytesStreamOutput);
160-
} finally {
161-
message = null;
162-
}
162+
bytesStreamOutput = new ReleasableBytesStreamOutput(bigArrays);
163+
return message.serialize(bytesStreamOutput);
163164
}
164165

165166
@Override
@@ -168,10 +169,10 @@ public void close() {
168169
}
169170
}
170171

171-
public class SendContext extends NotifyOnceListener<Void> implements CheckedSupplier<BytesReference, IOException> {
172+
private class SendContext extends NotifyOnceListener<Void> implements CheckedSupplier<BytesReference, IOException> {
172173

173174
private final TcpChannel channel;
174-
private CheckedSupplier<BytesReference, IOException> messageSupplier;
175+
private final CheckedSupplier<BytesReference, IOException> messageSupplier;
175176
private final ActionListener<Void> listener;
176177
private final Releasable optionalReleasable;
177178
private long messageSize = -1;
@@ -189,13 +190,10 @@ private SendContext(TcpChannel channel, CheckedSupplier<BytesReference, IOExcept
189190
this.optionalReleasable = optionalReleasable;
190191
}
191192

192-
@Override
193193
public BytesReference get() throws IOException {
194194
BytesReference message;
195195
try {
196-
assert messageSupplier != null;
197196
message = messageSupplier.get();
198-
messageSupplier = null;
199197
messageSize = message.length();
200198
TransportLogger.logOutboundMessage(channel, message);
201199
return message;
@@ -214,7 +212,6 @@ protected void innerOnResponse(Void v) {
214212

215213
@Override
216214
protected void innerOnFailure(Exception e) {
217-
messageSupplier = null;
218215
if (NetworkExceptionHelper.isCloseConnectionException(e)) {
219216
logger.debug(() -> new ParameterizedMessage("send message failed [channel: {}]", channel), e);
220217
} else {

‎server/src/main/java/org/elasticsearch/transport/TcpChannel.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
package org.elasticsearch.transport;
2121

2222
import org.elasticsearch.action.ActionListener;
23+
import org.elasticsearch.common.bytes.BytesReference;
2324
import org.elasticsearch.common.network.CloseableChannel;
2425
import org.elasticsearch.common.unit.TimeValue;
2526

@@ -58,11 +59,13 @@ public interface TcpChannel extends CloseableChannel {
5859
InetSocketAddress getRemoteAddress();
5960

6061
/**
61-
* Sends a tcp message to the channel.
62+
* Sends a tcp message to the channel. The listener will be executed once the send process has been
63+
* completed.
6264
*
63-
* @param sendContext Send Context
65+
* @param reference to send to channel
66+
* @param listener to execute upon send completion
6467
*/
65-
void sendMessage(OutboundHandler.SendContext sendContext);
68+
void sendMessage(BytesReference reference, ActionListener<Void> listener);
6669

6770
/**
6871
* Adds a listener that will be executed when the channel is connected. If the channel is still

‎test/framework/src/main/java/org/elasticsearch/transport/FakeTcpChannel.java‎

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
import org.elasticsearch.common.bytes.BytesReference;
2323
import org.elasticsearch.common.concurrent.CompletableContext;
2424

25-
import java.io.IOException;
2625
import java.net.InetSocketAddress;
2726
import java.util.concurrent.atomic.AtomicReference;
2827

@@ -89,14 +88,9 @@ public InetSocketAddress getRemoteAddress() {
8988
}
9089

9190
@Override
92-
public void sendMessage(OutboundHandler.SendContext sendContext) {
93-
try {
94-
messageCaptor.set(sendContext.get());
95-
} catch (IOException e) {
96-
sendContext.onFailure(e);
97-
return;
98-
}
99-
listenerCaptor.set(sendContext);
91+
public void sendMessage(BytesReference reference, ActionListener<Void> listener) {
92+
messageCaptor.set(reference);
93+
listenerCaptor.set(listener);
10094
}
10195

10296
@Override

‎test/framework/src/main/java/org/elasticsearch/transport/nio/MockNioTransport.java‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,6 @@
5555
import org.elasticsearch.threadpool.ThreadPool;
5656
import org.elasticsearch.transport.ConnectionProfile;
5757
import org.elasticsearch.transport.InboundPipeline;
58-
import org.elasticsearch.transport.OutboundHandler;
5958
import org.elasticsearch.transport.StatsTracker;
6059
import org.elasticsearch.transport.TcpChannel;
6160
import org.elasticsearch.transport.TcpServerChannel;
@@ -366,15 +365,8 @@ public ChannelStats getChannelStats() {
366365
}
367366

368367
@Override
369-
public void sendMessage(OutboundHandler.SendContext sendContext) {
370-
final BytesReference message;
371-
try {
372-
message = sendContext.get();
373-
} catch (IOException e) {
374-
sendContext.onFailure(e);
375-
return;
376-
}
377-
getContext().sendMessage(BytesReference.toByteBuffers(message), ActionListener.toBiConsumer(sendContext));
368+
public void sendMessage(BytesReference reference, ActionListener<Void> listener) {
369+
getContext().sendMessage(BytesReference.toByteBuffers(reference), ActionListener.toBiConsumer(listener));
378370
}
379371
}
380372

0 commit comments

Comments
 (0)