Skip to content

Commit 138e3c7

Browse files
dnhatndnhatn
authored
Ensure CCR partial reads never overuse buffer (#58620)
When the documents are large, a follower can receive a partial response because the requesting range of operations is capped by max_read_request_size instead of max_read_request_operation_count. In this case, the follower will continue reading the subsequent ranges without checking the remaining size of the buffer. The buffer then can use more memory than max_write_buffer_size and even causes OOM. Backport of #58620
1 parent f57743e commit 138e3c7

File tree

3 files changed

+83
-11
lines changed

3 files changed

+83
-11
lines changed

‎x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTask.java‎

Lines changed: 25 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@ public abstract class ShardFollowNodeTask extends AllocatedPersistentTask {
9090
private long failedWriteRequests = 0;
9191
private long operationWritten = 0;
9292
private long lastFetchTime = -1;
93+
private final Queue<Tuple<Long, Long>> partialReadRequests = new PriorityQueue<>(Comparator.comparing(Tuple::v1));
9394
private final Queue<Translog.Operation> buffer = new PriorityQueue<>(Comparator.comparing(Translog.Operation::seqNo));
9495
private long bufferSizeInBytes = 0;
9596
private final LinkedHashMap<Long, Tuple<AtomicInteger, ElasticsearchException>> fetchExceptions;
@@ -175,6 +176,20 @@ synchronized void coordinateReads() {
175176

176177
LOGGER.trace("{} coordinate reads, lastRequestedSeqNo={}, leaderGlobalCheckpoint={}",
177178
params.getFollowShardId(), lastRequestedSeqNo, leaderGlobalCheckpoint);
179+
assert partialReadRequests.size() <= params.getMaxOutstandingReadRequests() :
180+
"too many partial read requests [" + partialReadRequests + "]";
181+
while (hasReadBudget() && partialReadRequests.isEmpty() == false) {
182+
final Tuple<Long, Long> range = partialReadRequests.remove();
183+
assert range.v1() <= range.v2() && range.v2() <= lastRequestedSeqNo :
184+
"invalid partial range [" + range.v1() + "," + range.v2() + "]; last requested seq_no [" + lastRequestedSeqNo + "]";
185+
final long fromSeqNo = range.v1();
186+
final long maxRequiredSeqNo = range.v2();
187+
final int requestOpCount = Math.toIntExact(maxRequiredSeqNo - fromSeqNo + 1);
188+
LOGGER.trace("{}[{} ongoing reads] continue partial read request from_seqno={} max_required_seqno={} batch_count={}",
189+
params.getFollowShardId(), numOutstandingReads, fromSeqNo, maxRequiredSeqNo, requestOpCount);
190+
numOutstandingReads++;
191+
sendShardChangesRequest(fromSeqNo, requestOpCount, maxRequiredSeqNo);
192+
}
178193
final int maxReadRequestOperationCount = params.getMaxReadRequestOperationCount();
179194
while (hasReadBudget() && lastRequestedSeqNo < leaderGlobalCheckpoint) {
180195
final long from = lastRequestedSeqNo + 1;
@@ -190,8 +205,8 @@ synchronized void coordinateReads() {
190205
LOGGER.trace("{}[{} ongoing reads] read from_seqno={} max_required_seqno={} batch_count={}",
191206
params.getFollowShardId(), numOutstandingReads, from, maxRequiredSeqNo, requestOpCount);
192207
numOutstandingReads++;
193-
sendShardChangesRequest(from, requestOpCount, maxRequiredSeqNo);
194208
lastRequestedSeqNo = maxRequiredSeqNo;
209+
sendShardChangesRequest(from, requestOpCount, maxRequiredSeqNo);
195210
}
196211

197212
if (numOutstandingReads == 0 && hasReadBudget()) {
@@ -207,6 +222,9 @@ synchronized void coordinateReads() {
207222

208223
private boolean hasReadBudget() {
209224
assert Thread.holdsLock(this);
225+
// TODO: To ensure that we never overuse the buffer, we need to
226+
// - Overestimate the size and count of the responses of the outstanding request when calculating the budget
227+
// - Limit the size and count of next read requests by the remaining size and count of the buffer
210228
if (numOutstandingReads >= params.getMaxOutstandingReadRequests()) {
211229
LOGGER.trace("{} no new reads, maximum number of concurrent reads have been reached [{}]",
212230
params.getFollowShardId(), numOutstandingReads);
@@ -216,7 +234,7 @@ private boolean hasReadBudget() {
216234
LOGGER.trace("{} no new reads, buffer size limit has been reached [{}]", params.getFollowShardId(), bufferSizeInBytes);
217235
return false;
218236
}
219-
if (buffer.size() > params.getMaxWriteBufferCount()) {
237+
if (buffer.size() >= params.getMaxWriteBufferCount()) {
220238
LOGGER.trace("{} no new reads, buffer count limit has been reached [{}]", params.getFollowShardId(), buffer.size());
221239
return false;
222240
}
@@ -359,16 +377,13 @@ synchronized void innerHandleReadResponse(long from, long maxRequiredSeqNo, Shar
359377
"] is larger than the global checkpoint [" + leaderGlobalCheckpoint + "]";
360378
coordinateWrites();
361379
}
362-
if (newFromSeqNo <= maxRequiredSeqNo && isStopped() == false) {
363-
int newSize = Math.toIntExact(maxRequiredSeqNo - newFromSeqNo + 1);
364-
LOGGER.trace("{} received [{}] ops, still missing [{}/{}], continuing to read...",
380+
if (newFromSeqNo <= maxRequiredSeqNo) {
381+
LOGGER.trace("{} received [{}] operations, enqueue partial read request [{}/{}]",
365382
params.getFollowShardId(), response.getOperations().length, newFromSeqNo, maxRequiredSeqNo);
366-
sendShardChangesRequest(newFromSeqNo, newSize, maxRequiredSeqNo);
367-
} else {
368-
// read is completed, decrement
369-
numOutstandingReads--;
370-
coordinateReads();
383+
partialReadRequests.add(Tuple.tuple(newFromSeqNo, maxRequiredSeqNo));
371384
}
385+
numOutstandingReads--;
386+
coordinateReads();
372387
}
373388

374389
private void sendBulkShardOperationsRequest(List<Translog.Operation> operations, long leaderMaxSeqNoOfUpdatesOrDeletes,

‎x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
import org.elasticsearch.common.collect.ImmutableOpenMap;
4343
import org.elasticsearch.common.network.NetworkModule;
4444
import org.elasticsearch.common.settings.Settings;
45+
import org.elasticsearch.common.unit.ByteSizeValue;
4546
import org.elasticsearch.common.unit.TimeValue;
4647
import org.elasticsearch.common.xcontent.XContentBuilder;
4748
import org.elasticsearch.core.internal.io.IOUtils;
@@ -480,6 +481,8 @@ public static PutFollowAction.Request putFollow(String leaderIndex, String follo
480481
request.setFollowerIndex(followerIndex);
481482
request.getParameters().setMaxRetryDelay(TimeValue.timeValueMillis(10));
482483
request.getParameters().setReadPollTimeout(TimeValue.timeValueMillis(10));
484+
request.getParameters().setMaxReadRequestSize(new ByteSizeValue(between(1, 32 * 1024 * 1024)));
485+
request.getParameters().setMaxReadRequestOperationCount(between(1, 10000));
483486
request.waitForActiveShards(waitForActiveShards);
484487
return request;
485488
}

‎x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/ShardFollowNodeTaskTests.java‎

Lines changed: 55 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
package org.elasticsearch.xpack.ccr.action;
77

88
import org.elasticsearch.ElasticsearchException;
9+
import org.elasticsearch.action.ActionListener;
910
import org.elasticsearch.common.UUIDs;
1011
import org.elasticsearch.common.collect.Tuple;
1112
import org.elasticsearch.common.settings.Settings;
@@ -77,6 +78,7 @@ public class ShardFollowNodeTaskTests extends ESTestCase {
7778
private Queue<Long> followerGlobalCheckpoints;
7879
private Queue<Long> maxSeqNos;
7980
private Queue<Integer> responseSizes;
81+
private Queue<ActionListener<BulkShardOperationsResponse>> pendingBulkShardRequests;
8082

8183
public void testCoordinateReads() {
8284
ShardFollowTaskParams params = new ShardFollowTaskParams();
@@ -597,6 +599,55 @@ public void testReceiveNothingExpectedSomething() {
597599
assertThat(status.leaderGlobalCheckpoint(), equalTo(63L));
598600
}
599601

602+
public void testHandlePartialResponses() {
603+
ShardFollowTaskParams params = new ShardFollowTaskParams();
604+
params.maxReadRequestOperationCount = 10;
605+
params.maxOutstandingReadRequests = 2;
606+
params.maxOutstandingWriteRequests = 1;
607+
params.maxWriteBufferCount = 3;
608+
609+
ShardFollowNodeTask task = createShardFollowTask(params);
610+
startTask(task, 99, -1);
611+
612+
task.coordinateReads();
613+
assertThat(shardChangesRequests.size(), equalTo(2));
614+
assertThat(shardChangesRequests.get(0)[0], equalTo(0L));
615+
assertThat(shardChangesRequests.get(0)[1], equalTo(10L));
616+
assertThat(shardChangesRequests.get(1)[0], equalTo(10L));
617+
assertThat(shardChangesRequests.get(1)[1], equalTo(10L));
618+
619+
task.innerHandleReadResponse(0L, 9L, generateShardChangesResponse(0L, 5L, 0L, 0L, 99L));
620+
assertThat(pendingBulkShardRequests, hasSize(1));
621+
assertThat("continue the partial request", shardChangesRequests, hasSize(3));
622+
assertThat(shardChangesRequests.get(2)[0], equalTo(6L));
623+
assertThat(shardChangesRequests.get(2)[1], equalTo(4L));
624+
assertThat(pendingBulkShardRequests, hasSize(1));
625+
task.innerHandleReadResponse(10, 19L, generateShardChangesResponse(10L, 17L, 0L, 0L, 99L));
626+
assertThat("do not continue partial reads as the buffer is full", shardChangesRequests, hasSize(3));
627+
task.innerHandleReadResponse(6L, 9L, generateShardChangesResponse(6L, 8L, 0L, 0L, 99L));
628+
assertThat("do not continue partial reads as the buffer is full", shardChangesRequests, hasSize(3));
629+
pendingBulkShardRequests.remove().onResponse(new BulkShardOperationsResponse());
630+
assertThat(pendingBulkShardRequests, hasSize(1));
631+
632+
assertThat("continue two partial requests as the buffer is empty after sending", shardChangesRequests, hasSize(5));
633+
assertThat(shardChangesRequests.get(3)[0], equalTo(9L));
634+
assertThat(shardChangesRequests.get(3)[1], equalTo(1L));
635+
assertThat(shardChangesRequests.get(4)[0], equalTo(18L));
636+
assertThat(shardChangesRequests.get(4)[1], equalTo(2L));
637+
638+
task.innerHandleReadResponse(18L, 19L, generateShardChangesResponse(18L, 19L, 0L, 0L, 99L));
639+
assertThat("start new range as the buffer has empty slots", shardChangesRequests, hasSize(6));
640+
assertThat(shardChangesRequests.get(5)[0], equalTo(20L));
641+
assertThat(shardChangesRequests.get(5)[1], equalTo(10L));
642+
643+
task.innerHandleReadResponse(9L, 9L, generateShardChangesResponse(9L, 9L, 0L, 0L, 99L));
644+
assertThat("do not start new range as the buffer is full", shardChangesRequests, hasSize(6));
645+
pendingBulkShardRequests.remove().onResponse(new BulkShardOperationsResponse());
646+
assertThat("start new range as the buffer is empty after sending", shardChangesRequests, hasSize(7));
647+
assertThat(shardChangesRequests.get(6)[0], equalTo(30L));
648+
assertThat(shardChangesRequests.get(6)[1], equalTo(10L));
649+
}
650+
600651
public void testMappingUpdate() {
601652
ShardFollowTaskParams params = new ShardFollowTaskParams();
602653
params.maxReadRequestOperationCount = 64;
@@ -909,7 +960,7 @@ public void testMaxWriteRequestSize() {
909960

910961
ShardChangesAction.Response response = generateShardChangesResponse(0, 63, 0L, 0L, 64L);
911962
// Also invokes coordinatesWrites()
912-
task.innerHandleReadResponse(0L, 64L, response);
963+
task.innerHandleReadResponse(0L, 63L, response);
913964

914965
assertThat(bulkShardOperationRequests.size(), equalTo(64));
915966
}
@@ -1033,6 +1084,7 @@ private ShardFollowNodeTask createShardFollowTask(ShardFollowTaskParams params)
10331084
followerGlobalCheckpoints = new LinkedList<>();
10341085
maxSeqNos = new LinkedList<>();
10351086
responseSizes = new LinkedList<>();
1087+
pendingBulkShardRequests = new LinkedList<>();
10361088
return new ShardFollowNodeTask(
10371089
1L, "type", ShardFollowTask.NAME, "description", null, Collections.emptyMap(), followTask, scheduler, System::nanoTime) {
10381090

@@ -1082,6 +1134,8 @@ protected void innerSendBulkShardOperationsRequest(
10821134
response.setGlobalCheckpoint(followerGlobalCheckpoint);
10831135
response.setMaxSeqNo(followerGlobalCheckpoint);
10841136
handler.accept(response);
1137+
} else {
1138+
pendingBulkShardRequests.add(ActionListener.wrap(handler::accept, errorHandler));
10851139
}
10861140
}
10871141

0 commit comments

Comments
 (0)