diff --git a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java index e3b73d0aaa5cb..7a4a9f4a6157d 100644 --- a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java +++ b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.migrate.action; -import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; @@ -16,7 +15,6 @@ import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest; -import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest; import org.elasticsearch.action.admin.indices.template.delete.TransportDeleteIndexTemplateAction; @@ -26,11 +24,9 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.ingest.PutPipelineRequest; import org.elasticsearch.action.ingest.PutPipelineTransportAction; -import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.MappingMetadata; -import org.elasticsearch.cluster.metadata.MetadataIndexStateService; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.compress.CompressedXContent; @@ -63,7 +59,6 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; -import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.equalTo; public class ReindexDatastreamIndexTransportActionIT extends ESIntegTestCase { @@ -243,8 +238,7 @@ public void testDestIndexNameSet_noDotPrefix() throws Exception { assertEquals(expectedDestIndexName, response.getDestIndex()); } - public void testDestIndexNameSet_withDotPrefix() throws Exception { - + public void testDestIndexNameSet_withDotPrefix() { var sourceIndex = "." + randomAlphaOfLength(20).toLowerCase(Locale.ROOT); safeGet(indicesAdmin().create(new CreateIndexRequest(sourceIndex))); @@ -257,13 +251,19 @@ public void testDestIndexNameSet_withDotPrefix() throws Exception { assertEquals(expectedDestIndexName, response.getDestIndex()); } - public void testDestIndexContainsDocs() throws Exception { + public void testDestIndexContainsDocs() { // source index with docs var numDocs = randomIntBetween(1, 100); var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); safeGet(indicesAdmin().create(new CreateIndexRequest(sourceIndex))); indexDocs(sourceIndex, numDocs); + var settings = Settings.builder() + .put(IndexMetadata.SETTING_BLOCKS_METADATA, randomBoolean()) + .put(IndexMetadata.SETTING_READ_ONLY, randomBoolean()) + .build(); + safeGet(indicesAdmin().updateSettings(new UpdateSettingsRequest(settings, sourceIndex))); + // call reindex var response = safeGet( client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) @@ -274,29 +274,6 @@ public void testDestIndexContainsDocs() throws Exception { assertHitCount(prepareSearch(response.getDestIndex()).setSize(0), numDocs); } - public void testSetSourceToBlockWrites() throws Exception { - var settings = randomBoolean() ? Settings.builder().put(IndexMetadata.SETTING_BLOCKS_WRITE, true).build() : Settings.EMPTY; - - // empty source index - var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); - safeGet(indicesAdmin().create(new CreateIndexRequest(sourceIndex, settings))); - - // call reindex - safeGet(client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex))); - - // Assert that source index is now read-only but not verified read-only - GetSettingsResponse getSettingsResponse = safeGet(admin().indices().getSettings(new GetSettingsRequest().indices(sourceIndex))); - assertTrue(parseBoolean(getSettingsResponse.getSetting(sourceIndex, IndexMetadata.SETTING_BLOCKS_WRITE))); - assertFalse( - parseBoolean(getSettingsResponse.getSetting(sourceIndex, MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING.getKey())) - ); - - // assert that write to source fails - var indexReq = new IndexRequest(sourceIndex).source(jsonBuilder().startObject().field("field", "1").endObject()); - expectThrows(ClusterBlockException.class, client().index(indexReq)); - assertHitCount(prepareSearch(sourceIndex).setSize(0), 0); - } - public void testMissingSourceIndex() { var nonExistentSourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); expectThrows( @@ -354,34 +331,6 @@ public void testMappingsAddedToDestIndex() { assertEquals("text", XContentMapValues.extractValue("properties.foo1.type", destMappings)); } - public void testFailIfMetadataBlockSet() { - var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); - var settings = Settings.builder().put(IndexMetadata.SETTING_BLOCKS_METADATA, true).build(); - safeGet(indicesAdmin().create(new CreateIndexRequest(sourceIndex, settings))); - - ElasticsearchException e = expectThrows( - ElasticsearchException.class, - client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) - ); - assertTrue(e.getMessage().contains("Cannot reindex index") || e.getCause().getMessage().equals("Cannot reindex index")); - - cleanupMetadataBlocks(sourceIndex); - } - - public void testFailIfReadBlockSet() { - var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); - var settings = Settings.builder().put(IndexMetadata.SETTING_BLOCKS_READ, true).build(); - safeGet(indicesAdmin().create(new CreateIndexRequest(sourceIndex, settings))); - - ElasticsearchException e = expectThrows( - ElasticsearchException.class, - client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)) - ); - assertTrue(e.getMessage().contains("Cannot reindex index") || e.getCause().getMessage().equals("Cannot reindex index")); - - cleanupMetadataBlocks(sourceIndex); - } - public void testReadOnlyBlocksNotAddedBack() { var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); var settings = Settings.builder() @@ -401,7 +350,6 @@ public void testReadOnlyBlocksNotAddedBack() { assertFalse(parseBoolean(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE))); assertFalse(parseBoolean(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_BLOCKS_WRITE))); - cleanupMetadataBlocks(sourceIndex); cleanupMetadataBlocks(destIndex); } @@ -588,9 +536,8 @@ private static void cleanupMetadataBlocks(String index) { var settings = Settings.builder() .putNull(IndexMetadata.SETTING_READ_ONLY) .putNull(IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE) - .putNull(IndexMetadata.SETTING_BLOCKS_METADATA) - .build(); - safeGet(indicesAdmin().updateSettings(new UpdateSettingsRequest(settings, index))); + .putNull(IndexMetadata.SETTING_BLOCKS_METADATA); + updateIndexSettings(settings, index); } private static void indexDocs(String index, int numDocs) { diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java index 31fdcbe074c13..a80ee9a24e7d6 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.admin.indices.readonly.TransportAddIndexBlockAction; import org.elasticsearch.action.admin.indices.refresh.RefreshAction; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.settings.put.TransportUpdateSettingsAction; import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.search.SearchRequest; @@ -55,11 +56,13 @@ import org.elasticsearch.xpack.core.deprecation.DeprecatedIndexPredicate; import org.elasticsearch.xpack.migrate.MigrateTemplateRegistry; +import java.util.Arrays; import java.util.Locale; import java.util.Map; import java.util.Objects; -import static org.elasticsearch.cluster.metadata.IndexMetadata.APIBlock.WRITE; +import static org.elasticsearch.cluster.metadata.IndexMetadata.APIBlock.METADATA; +import static org.elasticsearch.cluster.metadata.IndexMetadata.APIBlock.READ_ONLY; public class ReindexDataStreamIndexTransportAction extends HandledTransportAction< ReindexDataStreamIndexAction.Request, @@ -145,19 +148,10 @@ protected void doExecute( ); } - if (settingsBefore.getAsBoolean(IndexMetadata.SETTING_BLOCKS_READ, false)) { - var errorMessage = String.format(Locale.ROOT, "Cannot reindex index [%s] which has a read block.", destIndexName); - listener.onFailure(new ElasticsearchException(errorMessage)); - return; - } - if (settingsBefore.getAsBoolean(IndexMetadata.SETTING_BLOCKS_METADATA, false)) { - var errorMessage = String.format(Locale.ROOT, "Cannot reindex index [%s] which has a metadata block.", destIndexName); - listener.onFailure(new ElasticsearchException(errorMessage)); - return; - } final boolean wasClosed = isClosed(sourceIndex); - SubscribableListener.newForked(l -> setBlockWrites(sourceIndexName, l, taskId)) + SubscribableListener.newForked(l -> removeMetadataBlocks(sourceIndexName, taskId, l)) .andThen(l -> openIndexIfClosed(sourceIndexName, wasClosed, l, taskId)) + .andThen(l -> setReadOnly(sourceIndexName, l, taskId)) .andThen(l -> refresh(sourceIndexName, l, taskId)) .andThen(l -> deleteDestIfExists(destIndexName, l, taskId)) .andThen(l -> createIndex(sourceIndex, destIndexName, l, taskId)) @@ -166,6 +160,7 @@ protected void doExecute( .andThen(l -> copyIndexMetadataToDest(sourceIndexName, destIndexName, l, taskId)) .andThen(l -> sanityCheck(sourceIndexName, destIndexName, l, taskId)) .andThen(l -> closeIndexIfWasClosed(destIndexName, wasClosed, l, taskId)) + .andThen(l -> removeAPIBlocks(sourceIndexName, taskId, l, READ_ONLY)) .andThenApply(ignored -> new ReindexDataStreamIndexAction.Response(destIndexName)) .addListener(listener); } @@ -201,9 +196,9 @@ private static boolean isClosed(IndexMetadata indexMetadata) { return indexMetadata.getState().equals(IndexMetadata.State.CLOSE); } - private void setBlockWrites(String sourceIndexName, ActionListener listener, TaskId parentTaskId) { - logger.debug("Setting write block on source index [{}]", sourceIndexName); - addBlockToIndex(WRITE, sourceIndexName, new ActionListener<>() { + private void setReadOnly(String sourceIndexName, ActionListener listener, TaskId parentTaskId) { + logger.debug("Setting read-only on source index [{}]", sourceIndexName); + addBlockToIndex(READ_ONLY, sourceIndexName, new ActionListener<>() { @Override public void onResponse(AddIndexBlockResponse response) { if (response.isAcknowledged()) { @@ -397,6 +392,29 @@ private void addBlockToIndex( client.admin().indices().execute(TransportAddIndexBlockAction.TYPE, addIndexBlockRequest, listener); } + /** + * All metadata blocks need to be removed at the start for the following reasons: + * 1) If the source index has a metadata only block, the read-only block can't be added. + * 2) If the source index is read-only and closed, it can't be opened. + */ + private void removeMetadataBlocks(String indexName, TaskId parentTaskId, ActionListener listener) { + logger.debug("Removing metadata blocks from index [{}]", indexName); + removeAPIBlocks(indexName, parentTaskId, listener, METADATA, READ_ONLY); + } + + private void removeAPIBlocks( + String indexName, + TaskId parentTaskId, + ActionListener listener, + IndexMetadata.APIBlock... blocks + ) { + Settings.Builder settings = Settings.builder(); + Arrays.stream(blocks).forEach(b -> settings.putNull(b.settingName())); + var updateSettingsRequest = new UpdateSettingsRequest(settings.build(), indexName); + updateSettingsRequest.setParentTask(parentTaskId); + client.execute(TransportUpdateSettingsAction.TYPE, updateSettingsRequest, listener); + } + private void getIndexDocCount(String index, TaskId parentTaskId, ActionListener listener) { SearchRequest countRequest = new SearchRequest(index); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().size(0).trackTotalHits(true); diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java index 1d320f97a41a2..77ab71b5db6e3 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/DataStreamsUpgradeIT.java @@ -548,6 +548,9 @@ private void upgradeDataStream(String dataStreamName, int numRolloversOnOldClust if (randomBoolean()) { closeIndex(oldIndexName); } + if (randomBoolean()) { + assertOK(client().performRequest(new Request("PUT", oldIndexName + "/_block/read_only"))); + } } Request reindexRequest = new Request("POST", "/_migration/reindex"); reindexRequest.setJsonEntity(Strings.format("""