Skip to content

Commit 1a10500

Browse files
parkertimminsparkertimmins
committed
Remove read-only within reindex ds index action
Also, remove read-only and metadata blocks at start, so that read-only/closed indices can be opened, and so that read-only can be added to indices with only metadata blocked.
1 parent f323967 commit 1a10500

File tree

4 files changed

+51
-102
lines changed

4 files changed

+51
-102
lines changed

‎x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java‎

Lines changed: 18 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77

88
package org.elasticsearch.xpack.migrate.action;
99

10-
import org.elasticsearch.ElasticsearchException;
1110
import org.elasticsearch.ResourceNotFoundException;
1211
import org.elasticsearch.action.DocWriteRequest;
1312
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
@@ -16,7 +15,6 @@
1615
import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
1716
import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
1817
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
19-
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
2018
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
2119
import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest;
2220
import org.elasticsearch.action.admin.indices.template.delete.TransportDeleteIndexTemplateAction;
@@ -26,11 +24,9 @@
2624
import org.elasticsearch.action.index.IndexRequest;
2725
import org.elasticsearch.action.ingest.PutPipelineRequest;
2826
import org.elasticsearch.action.ingest.PutPipelineTransportAction;
29-
import org.elasticsearch.cluster.block.ClusterBlockException;
3027
import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
3128
import org.elasticsearch.cluster.metadata.IndexMetadata;
3229
import org.elasticsearch.cluster.metadata.MappingMetadata;
33-
import org.elasticsearch.cluster.metadata.MetadataIndexStateService;
3430
import org.elasticsearch.cluster.metadata.Template;
3531
import org.elasticsearch.common.bytes.BytesArray;
3632
import org.elasticsearch.common.compress.CompressedXContent;
@@ -49,7 +45,6 @@
4945
import org.elasticsearch.xcontent.XContentType;
5046
import org.elasticsearch.xpack.migrate.MigratePlugin;
5147
import org.elasticsearch.xpack.migrate.MigrateTemplateRegistry;
52-
import org.junit.After;
5348
import org.junit.Before;
5449

5550
import java.io.IOException;
@@ -64,27 +59,16 @@
6459
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
6560
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
6661
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
67-
import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
6862
import static org.hamcrest.Matchers.equalTo;
6963

7064
public class ReindexDatastreamIndexTransportActionIT extends ESIntegTestCase {
7165

72-
private String sourceIndex;
73-
7466
@Before
7567
private void setup() throws Exception {
76-
sourceIndex = null;
7768
deletePipeline(MigrateTemplateRegistry.REINDEX_DATA_STREAM_PIPELINE_NAME);
7869
assertBusy(() -> { assertTrue(getPipelines(MigrateTemplateRegistry.REINDEX_DATA_STREAM_PIPELINE_NAME).isFound()); });
7970
}
8071

81-
@After
82-
private void cleanup() {
83-
if (sourceIndex != null) {
84-
cleanupMetadataBlocks(sourceIndex);
85-
}
86-
}
87-
8872
private static final String MAPPING = """
8973
{
9074
"_doc":{
@@ -121,7 +105,7 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
121105
""";
122106

123107
public void testTimestamp0AddedIfMissing() {
124-
sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
108+
var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
125109
safeGet(indicesAdmin().create(new CreateIndexRequest(sourceIndex)));
126110

127111
// add doc without timestamp
@@ -146,7 +130,7 @@ public void testTimestamp0AddedIfMissing() {
146130

147131
public void testTimestampNotAddedIfExists() {
148132

149-
sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
133+
var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
150134
safeGet(indicesAdmin().create(new CreateIndexRequest(sourceIndex)));
151135

152136
// add doc with timestamp
@@ -196,7 +180,7 @@ public void testCustomReindexPipeline() {
196180

197181
safeGet(clusterAdmin().execute(PutPipelineTransportAction.TYPE, putRequest));
198182

199-
sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
183+
var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
200184
safeGet(indicesAdmin().create(new CreateIndexRequest(sourceIndex)));
201185

202186
// add doc with timestamp
@@ -223,7 +207,7 @@ public void testCustomReindexPipeline() {
223207

224208
public void testDestIndexDeletedIfExists() throws Exception {
225209
// empty source index
226-
sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
210+
var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
227211
safeGet(indicesAdmin().create(new CreateIndexRequest(sourceIndex)));
228212

229213
// dest index with docs
@@ -242,7 +226,7 @@ public void testDestIndexDeletedIfExists() throws Exception {
242226
}
243227

244228
public void testDestIndexNameSet_noDotPrefix() throws Exception {
245-
sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
229+
var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
246230
safeGet(indicesAdmin().create(new CreateIndexRequest(sourceIndex)));
247231

248232
// call reindex
@@ -255,7 +239,7 @@ public void testDestIndexNameSet_noDotPrefix() throws Exception {
255239
}
256240

257241
public void testDestIndexNameSet_withDotPrefix() {
258-
sourceIndex = "." + randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
242+
var sourceIndex = "." + randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
259243
safeGet(indicesAdmin().create(new CreateIndexRequest(sourceIndex)));
260244

261245
// call reindex
@@ -270,10 +254,16 @@ public void testDestIndexNameSet_withDotPrefix() {
270254
public void testDestIndexContainsDocs() {
271255
// source index with docs
272256
var numDocs = randomIntBetween(1, 100);
273-
sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
257+
var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
274258
safeGet(indicesAdmin().create(new CreateIndexRequest(sourceIndex)));
275259
indexDocs(sourceIndex, numDocs);
276260

261+
var settings = Settings.builder()
262+
.put(IndexMetadata.SETTING_BLOCKS_METADATA, randomBoolean())
263+
.put(IndexMetadata.SETTING_READ_ONLY, randomBoolean())
264+
.build();
265+
safeGet(indicesAdmin().updateSettings(new UpdateSettingsRequest(settings, sourceIndex)));
266+
277267
// call reindex
278268
var response = safeGet(
279269
client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex))
@@ -284,29 +274,6 @@ public void testDestIndexContainsDocs() {
284274
assertHitCount(prepareSearch(response.getDestIndex()).setSize(0), numDocs);
285275
}
286276

287-
public void testSetSourceToReadOnly() throws Exception {
288-
var settings = randomBoolean() ? Settings.builder().put(IndexMetadata.SETTING_READ_ONLY, true).build() : Settings.EMPTY;
289-
290-
// empty source index
291-
sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
292-
safeGet(indicesAdmin().create(new CreateIndexRequest(sourceIndex, settings)));
293-
294-
// call reindex
295-
safeGet(client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)));
296-
297-
// Assert that source index is now read-only but not verified read-only
298-
GetSettingsResponse getSettingsResponse = safeGet(admin().indices().getSettings(new GetSettingsRequest().indices(sourceIndex)));
299-
assertTrue(parseBoolean(getSettingsResponse.getSetting(sourceIndex, IndexMetadata.SETTING_READ_ONLY)));
300-
assertFalse(
301-
parseBoolean(getSettingsResponse.getSetting(sourceIndex, MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING.getKey()))
302-
);
303-
304-
// assert that write to source fails
305-
var indexReq = new IndexRequest(sourceIndex).source(jsonBuilder().startObject().field("field", "1").endObject());
306-
expectThrows(ClusterBlockException.class, client().index(indexReq));
307-
assertHitCount(prepareSearch(sourceIndex).setSize(0), 0);
308-
}
309-
310277
public void testMissingSourceIndex() {
311278
var nonExistentSourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
312279
expectThrows(
@@ -319,7 +286,7 @@ public void testSettingsAddedBeforeReindex() {
319286
// start with a static setting
320287
var numShards = randomIntBetween(1, 10);
321288
var staticSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards).build();
322-
sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
289+
var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
323290
safeGet(indicesAdmin().create(new CreateIndexRequest(sourceIndex, staticSettings)));
324291

325292
// update with a dynamic setting
@@ -344,7 +311,7 @@ public void testSettingsAddedBeforeReindex() {
344311
}
345312

346313
public void testMappingsAddedToDestIndex() {
347-
sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
314+
var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
348315
safeGet(indicesAdmin().create(new CreateIndexRequest(sourceIndex).mapping(MAPPING)));
349316

350317
// call reindex
@@ -364,32 +331,8 @@ public void testMappingsAddedToDestIndex() {
364331
assertEquals("text", XContentMapValues.extractValue("properties.foo1.type", destMappings));
365332
}
366333

367-
public void testFailIfMetadataBlockSet() {
368-
sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
369-
var settings = Settings.builder().put(IndexMetadata.SETTING_BLOCKS_METADATA, true).build();
370-
safeGet(indicesAdmin().create(new CreateIndexRequest(sourceIndex, settings)));
371-
372-
ElasticsearchException e = expectThrows(
373-
ElasticsearchException.class,
374-
client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex))
375-
);
376-
assertTrue(e.getMessage().contains("Cannot reindex index") || e.getCause().getMessage().equals("Cannot reindex index"));
377-
}
378-
379-
public void testFailIfReadBlockSet() {
380-
sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
381-
var settings = Settings.builder().put(IndexMetadata.SETTING_BLOCKS_READ, true).build();
382-
safeGet(indicesAdmin().create(new CreateIndexRequest(sourceIndex, settings)));
383-
384-
ElasticsearchException e = expectThrows(
385-
ElasticsearchException.class,
386-
client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex))
387-
);
388-
assertTrue(e.getMessage().contains("Cannot reindex index") || e.getCause().getMessage().equals("Cannot reindex index"));
389-
}
390-
391334
public void testReadOnlyBlocksNotAddedBack() {
392-
sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
335+
var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
393336
var settings = Settings.builder()
394337
.put(IndexMetadata.SETTING_READ_ONLY, randomBoolean())
395338
.put(IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE, randomBoolean())
@@ -419,7 +362,7 @@ public void testUpdateSettingsDefaultsRestored() {
419362
indicesAdmin().execute(TransportDeleteIndexTemplateAction.TYPE, new DeleteIndexTemplateRequest("random_index_template"))
420363
);
421364

422-
sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
365+
var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
423366
assertAcked(indicesAdmin().create(new CreateIndexRequest(sourceIndex)));
424367

425368
// call reindex
@@ -458,7 +401,7 @@ public void testSettingsAndMappingsFromTemplate() throws IOException {
458401
request.indexTemplate(template);
459402
safeGet(client().execute(TransportPutComposableIndexTemplateAction.TYPE, request));
460403

461-
sourceIndex = "logs-" + randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
404+
var sourceIndex = "logs-" + randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
462405
safeGet(indicesAdmin().create(new CreateIndexRequest(sourceIndex)));
463406

464407
{

‎x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java‎

Lines changed: 29 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.elasticsearch.action.admin.indices.readonly.TransportAddIndexBlockAction;
2525
import org.elasticsearch.action.admin.indices.refresh.RefreshAction;
2626
import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
27+
import org.elasticsearch.action.admin.indices.settings.put.TransportUpdateSettingsAction;
2728
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
2829
import org.elasticsearch.action.bulk.BulkItemResponse;
2930
import org.elasticsearch.action.search.SearchRequest;
@@ -55,10 +56,12 @@
5556
import org.elasticsearch.xpack.core.deprecation.DeprecatedIndexPredicate;
5657
import org.elasticsearch.xpack.migrate.MigrateTemplateRegistry;
5758

59+
import java.util.Arrays;
5860
import java.util.Locale;
5961
import java.util.Map;
6062
import java.util.Objects;
6163

64+
import static org.elasticsearch.cluster.metadata.IndexMetadata.APIBlock.METADATA;
6265
import static org.elasticsearch.cluster.metadata.IndexMetadata.APIBlock.READ_ONLY;
6366

6467
public class ReindexDataStreamIndexTransportAction extends HandledTransportAction<
@@ -145,19 +148,10 @@ protected void doExecute(
145148
);
146149
}
147150

148-
if (settingsBefore.getAsBoolean(IndexMetadata.SETTING_BLOCKS_READ, false)) {
149-
var errorMessage = String.format(Locale.ROOT, "Cannot reindex index [%s] which has a read block.", destIndexName);
150-
listener.onFailure(new ElasticsearchException(errorMessage));
151-
return;
152-
}
153-
if (settingsBefore.getAsBoolean(IndexMetadata.SETTING_BLOCKS_METADATA, false)) {
154-
var errorMessage = String.format(Locale.ROOT, "Cannot reindex index [%s] which has a metadata block.", destIndexName);
155-
listener.onFailure(new ElasticsearchException(errorMessage));
156-
return;
157-
}
158151
final boolean wasClosed = isClosed(sourceIndex);
159-
SubscribableListener.<AcknowledgedResponse>newForked(l -> setReadOnly(sourceIndexName, l, taskId))
152+
SubscribableListener.<AcknowledgedResponse>newForked(l -> removeMetadataBlocks(sourceIndexName, taskId, l))
160153
.<OpenIndexResponse>andThen(l -> openIndexIfClosed(sourceIndexName, wasClosed, l, taskId))
154+
.<AcknowledgedResponse>andThen(l -> setReadOnly(sourceIndexName, l, taskId))
161155
.<BroadcastResponse>andThen(l -> refresh(sourceIndexName, l, taskId))
162156
.<AcknowledgedResponse>andThen(l -> deleteDestIfExists(destIndexName, l, taskId))
163157
.<AcknowledgedResponse>andThen(l -> createIndex(sourceIndex, destIndexName, l, taskId))
@@ -166,6 +160,7 @@ protected void doExecute(
166160
.<AcknowledgedResponse>andThen(l -> copyIndexMetadataToDest(sourceIndexName, destIndexName, l, taskId))
167161
.<AcknowledgedResponse>andThen(l -> sanityCheck(sourceIndexName, destIndexName, l, taskId))
168162
.<CloseIndexResponse>andThen(l -> closeIndexIfWasClosed(destIndexName, wasClosed, l, taskId))
163+
.<AcknowledgedResponse>andThen(l -> removeAPIBlocks(sourceIndexName, taskId, l, READ_ONLY))
169164
.andThenApply(ignored -> new ReindexDataStreamIndexAction.Response(destIndexName))
170165
.addListener(listener);
171166
}
@@ -397,6 +392,29 @@ private void addBlockToIndex(
397392
client.admin().indices().execute(TransportAddIndexBlockAction.TYPE, addIndexBlockRequest, listener);
398393
}
399394

395+
/**
396+
* All metadata blocks need to be removed at the start for the following reasons:
397+
* 1) If the source index has a metadata only block, the read-only block can't be added.
398+
* 2) If the source index is read-only and closed, it can't be opened.
399+
*/
400+
private void removeMetadataBlocks(String indexName, TaskId parentTaskId, ActionListener<AcknowledgedResponse> listener) {
401+
logger.debug("Removing metadata blocks from index [{}]", indexName);
402+
removeAPIBlocks(indexName, parentTaskId, listener, METADATA, READ_ONLY);
403+
}
404+
405+
private void removeAPIBlocks(
406+
String indexName,
407+
TaskId parentTaskId,
408+
ActionListener<AcknowledgedResponse> listener,
409+
IndexMetadata.APIBlock... blocks
410+
) {
411+
Settings.Builder settings = Settings.builder();
412+
Arrays.stream(blocks).forEach(b -> settings.putNull(b.settingName()));
413+
var updateSettingsRequest = new UpdateSettingsRequest(settings.build(), indexName);
414+
updateSettingsRequest.setParentTask(parentTaskId);
415+
client.execute(TransportUpdateSettingsAction.TYPE, updateSettingsRequest, listener);
416+
}
417+
400418
private void getIndexDocCount(String index, TaskId parentTaskId, ActionListener<Long> listener) {
401419
SearchRequest countRequest = new SearchRequest(index);
402420
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().size(0).trackTotalHits(true);

‎x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/task/ReindexDataStreamPersistentTaskExecutor.java‎

Lines changed: 1 addition & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,6 @@
1515
import org.elasticsearch.action.admin.indices.delete.TransportDeleteIndexAction;
1616
import org.elasticsearch.action.admin.indices.rollover.RolloverAction;
1717
import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
18-
import org.elasticsearch.action.admin.indices.settings.put.TransportUpdateSettingsAction;
19-
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
2018
import org.elasticsearch.action.datastreams.GetDataStreamAction;
2119
import org.elasticsearch.action.datastreams.ModifyDataStreamsAction;
2220
import org.elasticsearch.action.support.CountDownActionListener;
@@ -25,10 +23,8 @@
2523
import org.elasticsearch.client.internal.Client;
2624
import org.elasticsearch.cluster.metadata.DataStream;
2725
import org.elasticsearch.cluster.metadata.DataStreamAction;
28-
import org.elasticsearch.cluster.metadata.IndexMetadata;
2926
import org.elasticsearch.cluster.service.ClusterService;
3027
import org.elasticsearch.common.settings.Setting;
31-
import org.elasticsearch.common.settings.Settings;
3228
import org.elasticsearch.core.Nullable;
3329
import org.elasticsearch.core.TimeValue;
3430
import org.elasticsearch.index.Index;
@@ -220,9 +216,7 @@ private void maybeProcessNextIndex(
220216
l -> client.execute(ReindexDataStreamIndexAction.INSTANCE, reindexDataStreamIndexRequest, l)
221217
)
222218
.<AcknowledgedResponse>andThen(
223-
(l, result) -> removeReadOnlyBlock(index.getName(), parentTaskId, l.delegateFailure((delegate, response) -> {
224-
updateDataStream(sourceDataStream, index.getName(), result.getDestIndex(), delegate, parentTaskId);
225-
}))
219+
(l, result) -> updateDataStream(sourceDataStream, index.getName(), result.getDestIndex(), l, parentTaskId)
226220
)
227221
.<AcknowledgedResponse>andThen(l -> deleteIndex(index.getName(), parentTaskId, l))
228222
.addListener(ActionListener.wrap(unused -> {
@@ -252,15 +246,6 @@ private void updateDataStream(
252246
client.execute(ModifyDataStreamsAction.INSTANCE, modifyDataStreamRequest, listener);
253247
}
254248

255-
private void removeReadOnlyBlock(String indexName, TaskId parentTaskId, ActionListener<AcknowledgedResponse> listener) {
256-
var updateSettingsRequest = new UpdateSettingsRequest(
257-
Settings.builder().putNull(IndexMetadata.SETTING_READ_ONLY).build(),
258-
indexName
259-
);
260-
updateSettingsRequest.setParentTask(parentTaskId);
261-
client.execute(TransportUpdateSettingsAction.TYPE, updateSettingsRequest, listener);
262-
}
263-
264249
private void deleteIndex(String indexName, TaskId parentTaskId, ActionListener<AcknowledgedResponse> listener) {
265250
DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(indexName);
266251
deleteIndexRequest.setParentTask(parentTaskId);

‎x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -548,6 +548,9 @@ private void upgradeDataStream(String dataStreamName, int numRolloversOnOldClust
548548
if (randomBoolean()) {
549549
closeIndex(oldIndexName);
550550
}
551+
if (randomBoolean()) {
552+
assertOK(client().performRequest(new Request("PUT", oldIndexName + "/_block/read_only")));
553+
}
551554
}
552555
Request reindexRequest = new Request("POST", "/_migration/reindex");
553556
reindexRequest.setJsonEntity(Strings.format("""

0 commit comments

Comments
 (0)