Various Datastream Reindex Fixes (elastic#121376)

* Fix incorrect value of reindex_required flag on ignored index warning
* Datastream reindex now uses unverified write block to allow retrying failed reindex

Author: lukewhiting

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java

@@ -102,7 +102,7 @@ static DeprecationIssue ignoredOldIndicesCheck(DataStream dataStream, ClusterSta
                     + "OK to remain read-only after upgrade",
                 false,
                 ofEntries(
-                    entry("reindex_required", true),
+                    entry("reindex_required", false),
                     entry("total_backing_indices", backingIndices.size()),
                     entry("ignored_indices_requiring_upgrade_count", ignoredIndices.size()),
                     entry("ignored_indices_requiring_upgrade", ignoredIndices)

x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationCheckerTests.java

@@ -290,7 +290,7 @@ public void testOldIndicesIgnoredWarningCheck() {
                 + "OK to remain read-only after upgrade",
             false,
             ofEntries(
-                entry("reindex_required", true),
+                entry("reindex_required", false),
                 entry("total_backing_indices", oldIndexCount + newIndexCount),
                 entry("ignored_indices_requiring_upgrade_count", expectedIndices.size()),
                 entry("ignored_indices_requiring_upgrade", expectedIndices)

x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java

@@ -16,6 +16,7 @@
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
 import org.elasticsearch.action.admin.indices.rollover.RolloverRequest;
 import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
+import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
 import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest;
 import org.elasticsearch.action.admin.indices.template.delete.TransportDeleteIndexTemplateAction;
@@ -27,6 +28,7 @@
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.MappingMetadata;
+import org.elasticsearch.cluster.metadata.MetadataIndexStateService;
 import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
@@ -50,6 +52,7 @@
 import java.util.Locale;
 import java.util.Map;
 
+import static java.lang.Boolean.parseBoolean;
 import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.DEFAULT_TIMESTAMP_FIELD;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
@@ -147,6 +150,13 @@ public void testSetSourceToBlockWrites() throws Exception {
         // call reindex
         client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)).actionGet();
 
+        // Assert that source index is now read-only but not verified read-only
+        GetSettingsResponse getSettingsResponse = admin().indices().getSettings(new GetSettingsRequest().indices(sourceIndex)).actionGet();
+        assertTrue(parseBoolean(getSettingsResponse.getSetting(sourceIndex, IndexMetadata.SETTING_BLOCKS_WRITE)));
+        assertFalse(
+            parseBoolean(getSettingsResponse.getSetting(sourceIndex, MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING.getKey()))
+        );
+
         // assert that write to source fails
         var indexReq = new IndexRequest(sourceIndex).source(jsonBuilder().startObject().field("field", "1").endObject());
         assertThrows(ClusterBlockException.class, () -> client().index(indexReq).actionGet());
@@ -253,9 +263,9 @@ public void testReadOnlyBlocksNotAddedBack() {
             .getDestIndex();
 
         var settingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(destIndex)).actionGet();
-        assertFalse(Boolean.parseBoolean(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_READ_ONLY)));
-        assertFalse(Boolean.parseBoolean(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE)));
-        assertFalse(Boolean.parseBoolean(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_BLOCKS_WRITE)));
+        assertFalse(parseBoolean(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_READ_ONLY)));
+        assertFalse(parseBoolean(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE)));
+        assertFalse(parseBoolean(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_BLOCKS_WRITE)));
 
         cleanupMetadataBlocks(sourceIndex);
         cleanupMetadataBlocks(destIndex);

x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java

@@ -342,6 +342,7 @@ private void addBlockToIndex(
         TaskId parentTaskId
     ) {
         AddIndexBlockRequest addIndexBlockRequest = new AddIndexBlockRequest(block, index);
+        addIndexBlockRequest.markVerified(false);
         addIndexBlockRequest.setParentTask(parentTaskId);
         client.admin().indices().execute(TransportAddIndexBlockAction.TYPE, addIndexBlockRequest, listener);
     }