-
Notifications
You must be signed in to change notification settings - Fork 25.6k
Add support for delegating write to split target #136241
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 33 commits
53a338d
064f3ec
984063e
b93104a
9b79168
12e46af
770e0f1
3e72e7a
0cf593c
826d11a
a793d5c
b153030
0dc91f1
9dc5079
a9c9885
c2c5490
df26a61
636225d
4584a23
c9c32d8
fe1d3c3
fd43a75
7aca078
f1c4b2d
0c16c45
5fcc140
c14623a
9946884
646bdbe
7c5235a
701d19c
ed015ee
d3648e9
5ba4e73
aa8505e
dc0fb02
6cc838c
292f8cc
83dbaf6
2ba6abd
4ebc4d0
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,104 @@ | ||
/* | ||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
* or more contributor license agreements. Licensed under the "Elastic License | ||
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side | ||
* Public License v 1"; you may not use this file except in compliance with, at | ||
* your election, the "Elastic License 2.0", the "GNU Affero General Public | ||
* License v3.0 only", or the "Server Side Public License, v 1". | ||
*/ | ||
|
||
package org.elasticsearch.action.bulk; | ||
|
||
import org.elasticsearch.action.DocWriteRequest; | ||
import org.elasticsearch.cluster.metadata.ProjectMetadata; | ||
import org.elasticsearch.cluster.routing.IndexRouting; | ||
import org.elasticsearch.core.Tuple; | ||
import org.elasticsearch.index.Index; | ||
import org.elasticsearch.index.shard.ShardId; | ||
|
||
import java.util.ArrayList; | ||
import java.util.HashMap; | ||
import java.util.List; | ||
import java.util.Map; | ||
|
||
public final class ShardBulkSplitHelper { | ||
|
||
private ShardBulkSplitHelper() {} | ||
|
||
public static Map<ShardId, BulkShardRequest> splitRequests(BulkShardRequest request, ProjectMetadata project) { | ||
Tim-Brooks marked this conversation as resolved.
Show resolved
Hide resolved
|
||
final ShardId sourceShardId = request.shardId(); | ||
final Index index = sourceShardId.getIndex(); | ||
IndexRouting indexRouting = IndexRouting.fromIndexMetadata(project.getIndexSafe(index)); | ||
|
||
Map<ShardId, List<BulkItemRequest>> requestsByShard = new HashMap<>(); | ||
Map<ShardId, BulkShardRequest> bulkRequestsPerShard = new HashMap<>(); | ||
|
||
// Iterate through the items in the input request and split them based on the | ||
// current resharding-split state. | ||
BulkItemRequest[] items = request.items(); | ||
if (items.length == 0) { // Nothing to split | ||
return Map.of(sourceShardId, request); | ||
} | ||
|
||
for (BulkItemRequest bulkItemRequest : items) { | ||
DocWriteRequest<?> docWriteRequest = bulkItemRequest.request(); | ||
int newShardId = docWriteRequest.rerouteAtSourceDuringResharding(indexRouting); | ||
List<BulkItemRequest> shardRequests = requestsByShard.computeIfAbsent( | ||
new ShardId(index, newShardId), | ||
shardNum -> new ArrayList<>() | ||
); | ||
shardRequests.add(new BulkItemRequest(bulkItemRequest.id(), bulkItemRequest.request())); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. should we assert anything about the shard id in There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
||
} | ||
|
||
// All items belong to either the source shard or target shard. | ||
if (requestsByShard.size() == 1) { | ||
Map.Entry<ShardId, List<BulkItemRequest>> entry = requestsByShard.entrySet().iterator().next(); | ||
// Return the original request if no items were split to target. | ||
if (entry.getKey().equals(sourceShardId)) { | ||
Tim-Brooks marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
return Map.of(sourceShardId, request); | ||
} | ||
} | ||
|
||
for (Map.Entry<ShardId, List<BulkItemRequest>> entry : requestsByShard.entrySet()) { | ||
final ShardId shardId = entry.getKey(); | ||
final List<BulkItemRequest> requests = entry.getValue(); | ||
BulkShardRequest bulkShardRequest = new BulkShardRequest( | ||
shardId, | ||
request.getRefreshPolicy(), | ||
requests.toArray(new BulkItemRequest[0]), | ||
request.isSimulated() | ||
); | ||
bulkRequestsPerShard.put(shardId, bulkShardRequest); | ||
} | ||
return bulkRequestsPerShard; | ||
} | ||
|
||
public static Tuple<BulkShardResponse, Exception> combineResponses( | ||
BulkShardRequest originalRequest, | ||
Map<ShardId, BulkShardRequest> splitRequests, | ||
Map<ShardId, Tuple<BulkShardResponse, Exception>> responses | ||
Tim-Brooks marked this conversation as resolved.
Show resolved
Hide resolved
|
||
) { | ||
BulkItemResponse[] bulkItemResponses = new BulkItemResponse[originalRequest.items().length]; | ||
for (Map.Entry<ShardId, Tuple<BulkShardResponse, Exception>> entry : responses.entrySet()) { | ||
ShardId shardId = entry.getKey(); | ||
Tuple<BulkShardResponse, Exception> value = entry.getValue(); | ||
Exception exception = value.v2(); | ||
if (exception != null) { | ||
BulkShardRequest bulkShardRequest = splitRequests.get(shardId); | ||
for (BulkItemRequest item : bulkShardRequest.items()) { | ||
DocWriteRequest<?> request = item.request(); | ||
BulkItemResponse.Failure failure = new BulkItemResponse.Failure(item.index(), request.id(), exception); | ||
bulkItemResponses[item.id()] = BulkItemResponse.failure(item.id(), request.opType(), failure); | ||
} | ||
} else { | ||
for (BulkItemResponse bulkItemResponse : value.v1().getResponses()) { | ||
bulkItemResponses[bulkItemResponse.getItemId()] = bulkItemResponse; | ||
} | ||
} | ||
} | ||
BulkShardResponse bulkShardResponse = new BulkShardResponse(originalRequest.shardId(), bulkItemResponses); | ||
// TODO: Decide how to handle | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think you looked at consumers of this? If so could you leave a breadcrumb to your investigation? If not, we should ticket that and link the ticket here. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
||
bulkShardResponse.setShardInfo(responses.get(originalRequest.shardId()).v1().getShardInfo()); | ||
return new Tuple<>(bulkShardResponse, null); | ||
} | ||
} |
Uh oh!
There was an error while loading. Please reload this page.