|
10 | 10 | package org.elasticsearch.reindex.management; |
11 | 11 |
|
12 | 12 | import org.apache.lucene.util.SetOnce; |
| 13 | +import org.elasticsearch.ElasticsearchException; |
| 14 | +import org.elasticsearch.action.ActionListener; |
| 15 | +import org.elasticsearch.action.ActionRequest; |
| 16 | +import org.elasticsearch.action.ActionResponse; |
13 | 17 | import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskResponse; |
| 18 | +import org.elasticsearch.action.bulk.BulkRequest; |
| 19 | +import org.elasticsearch.action.bulk.TransportBulkAction; |
14 | 20 | import org.elasticsearch.action.get.GetResponse; |
| 21 | +import org.elasticsearch.action.support.ActionFilter; |
| 22 | +import org.elasticsearch.action.support.ActionFilterChain; |
15 | 23 | import org.elasticsearch.client.Request; |
16 | 24 | import org.elasticsearch.client.Response; |
17 | 25 | import org.elasticsearch.client.RestClient; |
|
25 | 33 | import org.elasticsearch.http.HttpServerTransport; |
26 | 34 | import org.elasticsearch.index.reindex.BulkByScrollTask; |
27 | 35 | import org.elasticsearch.index.reindex.ReindexAction; |
| 36 | +import org.elasticsearch.node.Node; |
28 | 37 | import org.elasticsearch.node.ShutdownPrepareService; |
| 38 | +import org.elasticsearch.plugins.ActionPlugin; |
29 | 39 | import org.elasticsearch.plugins.Plugin; |
30 | 40 | import org.elasticsearch.plugins.PluginsService; |
31 | 41 | import org.elasticsearch.reindex.ReindexMetrics; |
|
34 | 44 | import org.elasticsearch.reindex.TransportReindexAction; |
35 | 45 | import org.elasticsearch.rest.root.MainRestPlugin; |
36 | 46 | import org.elasticsearch.tasks.RawTaskStatus; |
| 47 | +import org.elasticsearch.tasks.Task; |
37 | 48 | import org.elasticsearch.tasks.TaskId; |
38 | 49 | import org.elasticsearch.tasks.TaskInfo; |
39 | 50 | import org.elasticsearch.tasks.TaskResult; |
@@ -97,7 +108,13 @@ public class ReindexRelocationIT extends ESIntegTestCase { |
97 | 108 |
|
98 | 109 | @Override |
99 | 110 | protected Collection<Class<? extends Plugin>> nodePlugins() { |
100 | | - return Arrays.asList(ReindexPlugin.class, ReindexManagementPlugin.class, MainRestPlugin.class, TestTelemetryPlugin.class); |
| 111 | + return Arrays.asList( |
| 112 | + ReindexPlugin.class, |
| 113 | + ReindexManagementPlugin.class, |
| 114 | + MainRestPlugin.class, |
| 115 | + TestTelemetryPlugin.class, |
| 116 | + BlockTasksWritePlugin.class |
| 117 | + ); |
101 | 118 | } |
102 | 119 |
|
103 | 120 | @Override |
@@ -230,6 +247,105 @@ private void testReindexRelocation( |
230 | 247 | assertExpectedNumberOfDocumentsInDestinationIndex(); |
231 | 248 | } |
232 | 249 |
|
| 250 | + /** |
| 251 | + * Verifies that the destination node writes the source task result to {@code .tasks} during relocation, so the chain link is preserved |
| 252 | + * even when the source node cannot write. The test uses {@link BlockTasksWritePlugin} to block all {@code .tasks} writes on the source |
| 253 | + * node, so only the destination's write (in {@code Reindexer.storeRelocationSourceTaskResult}) succeeds. |
| 254 | + */ |
| 255 | + public void testDestinationWritesSourceTaskResultToTasksIndex() throws Exception { |
| 256 | + assumeTrue("reindex resilience is enabled", ReindexPlugin.REINDEX_RESILIENCE_ENABLED); |
| 257 | + final int shards = randomIntBetween(1, 5); |
| 258 | + |
| 259 | + final String nodeAName = internalCluster().startNode( |
| 260 | + NodeRoles.onlyRoles(Set.of(DiscoveryNodeRole.DATA_ROLE, DiscoveryNodeRole.MASTER_ROLE)) |
| 261 | + ); |
| 262 | + final String nodeAId = nodeIdByName(nodeAName); |
| 263 | + final String nodeBName = internalCluster().startNode( |
| 264 | + NodeRoles.onlyRoles(Set.of(DiscoveryNodeRole.DATA_ROLE, DiscoveryNodeRole.MASTER_ROLE)) |
| 265 | + ); |
| 266 | + ensureStableCluster(2); |
| 267 | + |
| 268 | + createIndexPinnedToNodeName(SOURCE_INDEX, nodeAName, shards); |
| 269 | + createIndexPinnedToNodeName(DEST_INDEX, nodeAName, shards); |
| 270 | + indexRandom(true, SOURCE_INDEX, numberOfDocumentsThatTakes60SecondsToIngest); |
| 271 | + ensureGreen(SOURCE_INDEX, DEST_INDEX); |
| 272 | + |
| 273 | + final TaskId originalTaskId = startAsyncThrottledLocalReindexOnNode(nodeBName, 1); |
| 274 | + assertBusy(() -> { |
| 275 | + final TaskResult running = getRunningReindex(originalTaskId); |
| 276 | + assertThat(running.getTask().taskId().getNodeId(), equalTo(nodeIdByName(nodeBName))); |
| 277 | + }); |
| 278 | + |
| 279 | + assertFalse(".tasks index should not exist before shutdown", indexExists(TaskResultsService.TASK_INDEX)); |
| 280 | + |
| 281 | + // Block .tasks writes on the source node so only the destination's write can succeed. |
| 282 | + BlockTasksWritePlugin.blockedNodeName = nodeBName; |
| 283 | + try { |
| 284 | + shutdownNodeNameAndRelocate(nodeBName); |
| 285 | + |
| 286 | + final TaskId relocatedTaskId = assertOriginalTaskEndStateInTasksIndexAndGetRelocatedTaskId( |
| 287 | + originalTaskId, |
| 288 | + nodeAId, |
| 289 | + localReindexDescription(), |
| 290 | + 1, |
| 291 | + shards |
| 292 | + ); |
| 293 | + |
| 294 | + unthrottleReindex(relocatedTaskId); |
| 295 | + assertRelocatedTaskExpectedEndState(relocatedTaskId, localReindexDescription(), 1, shards); |
| 296 | + assertExpectedNumberOfDocumentsInDestinationIndex(); |
| 297 | + } finally { |
| 298 | + BlockTasksWritePlugin.blockedNodeName = null; |
| 299 | + } |
| 300 | + } |
| 301 | + |
| 302 | + /** |
| 303 | + * Test plugin that blocks {@code .tasks} index writes on a specific node. |
| 304 | + * Used to verify the destination writes the source task result during relocation. |
| 305 | + */ |
| 306 | + public static class BlockTasksWritePlugin extends Plugin implements ActionPlugin { |
| 307 | + static volatile String blockedNodeName = null; |
| 308 | + private volatile String myNodeName; |
| 309 | + |
| 310 | + @Override |
| 311 | + public Collection<Object> createComponents(PluginServices services) { |
| 312 | + myNodeName = Node.NODE_NAME_SETTING.get(services.environment().settings()); |
| 313 | + return List.of(); |
| 314 | + } |
| 315 | + |
| 316 | + @Override |
| 317 | + public List<ActionFilter> getActionFilters() { |
| 318 | + return List.of(new ActionFilter() { |
| 319 | + @Override |
| 320 | + public int order() { |
| 321 | + return Integer.MIN_VALUE; |
| 322 | + } |
| 323 | + |
| 324 | + @Override |
| 325 | + public <Request extends ActionRequest, Response extends ActionResponse> void apply( |
| 326 | + Task task, |
| 327 | + String action, |
| 328 | + Request request, |
| 329 | + ActionListener<Response> listener, |
| 330 | + ActionFilterChain<Request, Response> chain |
| 331 | + ) { |
| 332 | + if (myNodeName != null && myNodeName.equals(blockedNodeName) && isTasksIndexWrite(action, request)) { |
| 333 | + listener.onFailure(new ElasticsearchException("blocked .tasks write on [" + myNodeName + "] for testing")); |
| 334 | + return; |
| 335 | + } |
| 336 | + chain.proceed(task, action, request, listener); |
| 337 | + } |
| 338 | + |
| 339 | + private boolean isTasksIndexWrite(String action, ActionRequest request) { |
| 340 | + if (action.equals(TransportBulkAction.NAME) && request instanceof BulkRequest bulkRequest) { |
| 341 | + return bulkRequest.requests().stream().anyMatch(r -> TaskResultsService.TASK_INDEX.equals(r.index())); |
| 342 | + } |
| 343 | + return false; |
| 344 | + } |
| 345 | + }); |
| 346 | + } |
| 347 | + } |
| 348 | + |
233 | 349 | private void shutdownNodeNameAndRelocate(final String nodeName) throws Exception { |
234 | 350 | // testing assumption: .tasks should not exist yet — it's created when the task result is stored during relocation |
235 | 351 | assertFalse(".tasks index should not exist before shutdown", indexExists(TaskResultsService.TASK_INDEX)); |
|
0 commit comments