Merge branch 'main' into refactor-ilm-tests

Author: nielsbauman

docs/changelog/131706.yaml

@@ -0,0 +1,5 @@
+pr: 131706
+summary: Add extension points to remediate index metadata in during snapshot restore
+area: Snapshot/Restore
+type: enhancement
+issues: []

docs/reference/elasticsearch/rest-apis/reindex-indices.md

@@ -35,6 +35,11 @@ You can learn how to:
 - [Reindex with an ingest pipeline](#reindex-with-an-ingest-pipeline)
 - [Reindex from remote](#reindex-from-remote)
 
+**Troubleshooting**
+- [Monitor reindex tasks](#monitor-reindex-tasks)
+- [Diagnose node failures](#diagnose-node-failures)
+- [Version conflicts](#version-conflicts)
+
 ## Basic reindexing example
 
 Use the Reindex API to copy all documents from one index to another.
@@ -731,4 +736,50 @@ POST _reindex
 Reindex from remote supports configurable SSL settings.
 These must be specified in the `elasticsearch.yml` file, with the exception of the secure settings, which you add in the {{es}} keystore.
 It is not possible to configure SSL in the body of the reindex API request.
-Refer to [Reindex settings](/reference/elasticsearch/configuration-reference/index-management-settings.md#reindex-settings).
+Refer to [Reindex settings](/reference/elasticsearch/configuration-reference/index-management-settings.md#reindex-settings).
+
+## Monitor reindex tasks [monitor-reindex-tasks]
+
+When run asynchronously with `wait_for_completion=false`, a reindex task can be monitored with the task management API:
+```console
+GET _tasks/r1A2WoRbTwKZ516z6NEs5A:36619
+```
+% TEST[catch:missing]
+
+To view all currently running reindex tasks:
+```console
+GET _tasks?actions=*reindex
+```
+
+You can also cancel a running reindex task:
+```console
+POST _tasks/r1A2WoRbTwKZ516z6NEs5A:36619/_cancel
+```
+
+## Diagnose node failures [diagnose-node-failures]
+
+Node crashes can sometimes be caused by insufficient disk space. To check disk allocation across your cluster:
+```console
+GET _cat/allocation?v
+```
+
+## Version conflicts [version-conflicts]
+
+By default, version conflicts abort the reindexing process.
+To continue reindexing in the case of conflicts, set `conflicts` to `proceed`.
+This may be necessary when retrying a failed reindex operation, as the destination index could be left in a partial state.
+
+```console
+POST _reindex
+{
+  "source": {
+    "index": "my-index-000001"
+  },
+  "dest": {
+    "index": "my-new-index-000001",
+    "op_type": "create"
+  },
+  "conflicts": "proceed"
+}
+```
+% TEST[setup:my_index]

server/src/internalClusterTest/java/org/elasticsearch/snapshots/RemediateSnapshotIT.java

@@ -0,0 +1,109 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.snapshots;
+
+import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
+import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.plugins.Plugin;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.stream.Stream;
+
+import static org.elasticsearch.index.IndexSettings.INDEX_SEARCH_IDLE_AFTER;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
+
+public class RemediateSnapshotIT extends AbstractSnapshotIntegTestCase {
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return Stream.concat(Stream.of(RemediateSnapshotTestPlugin.class), super.nodePlugins().stream()).toList();
+    }
+
+    public void testRemediationOnRestore() {
+        Client client = client();
+
+        createRepository("test-repo", "fs");
+
+        createIndex("test-idx-1", "test-idx-2", "test-idx-3");
+        ensureGreen();
+
+        GetIndexResponse getIndexResponse = client.admin()
+            .indices()
+            .prepareGetIndex(TEST_REQUEST_TIMEOUT)
+            .setIndices("test-idx-1", "test-idx-2", "test-idx-3")
+            .get();
+
+        assertThat(
+            INDEX_SEARCH_IDLE_AFTER.get(getIndexResponse.settings().get("test-idx-1")),
+            equalTo(INDEX_SEARCH_IDLE_AFTER.getDefault(Settings.EMPTY))
+        );
+        assertThat(
+            INDEX_SEARCH_IDLE_AFTER.get(getIndexResponse.settings().get("test-idx-2")),
+            equalTo(INDEX_SEARCH_IDLE_AFTER.getDefault(Settings.EMPTY))
+        );
+        assertThat(
+            INDEX_SEARCH_IDLE_AFTER.get(getIndexResponse.settings().get("test-idx-3")),
+            equalTo(INDEX_SEARCH_IDLE_AFTER.getDefault(Settings.EMPTY))
+        );
+
+        indexRandomDocs("test-idx-1", 100);
+        indexRandomDocs("test-idx-2", 100);
+
+        createSnapshot("test-repo", "test-snap", Arrays.asList("test-idx-1", "test-idx-2"));
+
+        logger.info("--> close snapshot indices");
+        client.admin().indices().prepareClose("test-idx-1", "test-idx-2").get();
+
+        logger.info("--> restore indices");
+        RestoreSnapshotResponse restoreSnapshotResponse = client.admin()
+            .cluster()
+            .prepareRestoreSnapshot(TEST_REQUEST_TIMEOUT, "test-repo", "test-snap")
+            .setWaitForCompletion(true)
+            .get();
+        assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
+
+        GetIndexResponse getIndexResponseAfter = client.admin()
+            .indices()
+            .prepareGetIndex(TEST_REQUEST_TIMEOUT)
+            .setIndices("test-idx-1", "test-idx-2", "test-idx-3")
+            .get();
+
+        assertDocCount("test-idx-1", 100L);
+        assertThat(INDEX_SEARCH_IDLE_AFTER.get(getIndexResponseAfter.settings().get("test-idx-1")), equalTo(TimeValue.timeValueMinutes(2)));
+        assertDocCount("test-idx-2", 100L);
+        assertThat(INDEX_SEARCH_IDLE_AFTER.get(getIndexResponseAfter.settings().get("test-idx-2")), equalTo(TimeValue.timeValueMinutes(2)));
+    }
+
+    /**
+     * Dummy plugin to load SPI function off of
+     */
+    public static class RemediateSnapshotTestPlugin extends Plugin {}
+
+    public static class RemediateSnapshotTestTransformer implements IndexMetadataRestoreTransformer {
+        @Override
+        public IndexMetadata updateIndexMetadata(IndexMetadata original) {
+            // Set a property to something mild, outside its default
+            return IndexMetadata.builder(original)
+                .settings(
+                    Settings.builder()
+                        .put(original.getSettings())
+                        .put(INDEX_SEARCH_IDLE_AFTER.getKey(), TimeValue.timeValueMinutes(2))
+                        .build()
+                )
+                .build();
+        }
+    }
+}

server/src/internalClusterTest/resources/META-INF/services/org.elasticsearch.snapshots.IndexMetadataRestoreTransformer

@@ -0,0 +1,10 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the "Elastic License
+# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+# Public License v 1"; you may not use this file except in compliance with, at
+# your election, the "Elastic License 2.0", the "GNU Affero General Public
+# License v3.0 only", or the "Server Side Public License, v 1".
+#
+
+org.elasticsearch.snapshots.RemediateSnapshotIT$RemediateSnapshotTestTransformer

server/src/main/java/org/elasticsearch/node/NodeConstruction.java

@@ -205,6 +205,8 @@
 import org.elasticsearch.search.SearchUtils;
 import org.elasticsearch.search.aggregations.support.AggregationUsageService;
 import org.elasticsearch.shutdown.PluginShutdownService;
+import org.elasticsearch.snapshots.IndexMetadataRestoreTransformer;
+import org.elasticsearch.snapshots.IndexMetadataRestoreTransformer.NoOpRestoreTransformer;
 import org.elasticsearch.snapshots.InternalSnapshotsInfoService;
 import org.elasticsearch.snapshots.RepositoryIntegrityHealthIndicatorService;
 import org.elasticsearch.snapshots.RestoreService;
@@ -1160,7 +1162,8 @@ public Map<String, String> queryFields() {
             indicesService,
             fileSettingsService,
             threadPool,
-            projectResolver.supportsMultipleProjects()
+            projectResolver.supportsMultipleProjects(),
+            pluginsService.loadSingletonServiceProvider(IndexMetadataRestoreTransformer.class, NoOpRestoreTransformer::getInstance)
         );
 
         DiscoveryModule discoveryModule = createDiscoveryModule(

server/src/main/java/org/elasticsearch/snapshots/IndexMetadataRestoreTransformer.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.snapshots;
+
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+
+/**
+ * A temporary interface meant to allow a plugin to provide remedial logic for index metadata being restored from a snapshot. This was
+ * added to address an allocation issue and will eventually be removed once any affected snapshots age out.
+ */
+public interface IndexMetadataRestoreTransformer {
+    IndexMetadata updateIndexMetadata(IndexMetadata original);
+
+    /**
+     * A default implementation of {@link IndexMetadataRestoreTransformer} which does nothing
+     */
+    final class NoOpRestoreTransformer implements IndexMetadataRestoreTransformer {
+        public static final NoOpRestoreTransformer INSTANCE = new NoOpRestoreTransformer();
+
+        public static NoOpRestoreTransformer getInstance() {
+            return INSTANCE;
+        }
+
+        private NoOpRestoreTransformer() {}
+
+        @Override
+        public IndexMetadata updateIndexMetadata(IndexMetadata original) {
+            return original;
+        }
+    }
+}

server/src/main/java/org/elasticsearch/snapshots/RestoreService.java

@@ -206,6 +206,8 @@ public final class RestoreService implements ClusterStateApplier {
 
     private final Executor snapshotMetaExecutor;
 
+    private final IndexMetadataRestoreTransformer indexMetadataRestoreTransformer;
+
     private volatile boolean refreshRepositoryUuidOnRestore;
 
     public RestoreService(
@@ -219,7 +221,8 @@ public RestoreService(
         IndicesService indicesService,
         FileSettingsService fileSettingsService,
         ThreadPool threadPool,
-        boolean deserializeProjectMetadata
+        boolean deserializeProjectMetadata,
+        IndexMetadataRestoreTransformer indexMetadataRestoreTransformer
     ) {
         this.clusterService = clusterService;
         this.repositoriesService = repositoriesService;
@@ -240,6 +243,7 @@ public RestoreService(
         this.refreshRepositoryUuidOnRestore = REFRESH_REPO_UUID_ON_RESTORE_SETTING.get(clusterService.getSettings());
         clusterService.getClusterSettings()
             .addSettingsUpdateConsumer(REFRESH_REPO_UUID_ON_RESTORE_SETTING, this::setRefreshRepositoryUuidOnRestore);
+        this.indexMetadataRestoreTransformer = indexMetadataRestoreTransformer;
     }
 
     /**
@@ -521,6 +525,8 @@ private void startRestore(
         }
         for (IndexId indexId : repositoryData.resolveIndices(requestedIndicesIncludingSystem).values()) {
             IndexMetadata snapshotIndexMetaData = repository.getSnapshotIndexMetaData(repositoryData, snapshotId, indexId);
+            // Update the snapshot index metadata before adding it to the metadata
+            snapshotIndexMetaData = indexMetadataRestoreTransformer.updateIndexMetadata(snapshotIndexMetaData);
             if (snapshotIndexMetaData.isSystem()) {
                 if (requestIndices.contains(indexId.getName())) {
                     explicitlyRequestedSystemIndices.add(indexId.getName());

server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java

@@ -2703,7 +2703,8 @@ public boolean clusterHasFeature(ClusterState state, NodeFeature feature) {
                     indicesService,
                     mock(FileSettingsService.class),
                     threadPool,
-                    false
+                    false,
+                    IndexMetadataRestoreTransformer.NoOpRestoreTransformer.getInstance()
                 );
                 actions.put(
                     TransportPutMappingAction.TYPE,