[8.18] Move system indices migration to migrate plugin (elastic#123551) (elastic#123934)

* Move system indices migration to migrate plugin (elastic#123551)

It seems the best way to fix elastic#122949 is to use existing data stream reindex API. However, this API is located in the migrate x-pack plugin. This commit moves the system indices migration logic (REST handlers, transport actions, and task) to the migrate plugin.

(cherry picked from commit 0a769c8)

* Restore tests

Author: alexey-ivanov-es

modules/reindex/build.gradle

@@ -45,9 +45,6 @@ dependencies {
   clusterModules project(':modules:lang-painless')
   clusterModules project(':modules:parent-join')
   clusterModules project(":modules:rest-root")
-
-  internalClusterTestImplementation project(':modules:lang-painless')
-  internalClusterTestImplementation project(':modules:lang-painless:spi')
 }
 
 restResources {

modules/reindex/src/main/plugin-metadata/plugin-security.policy

@@ -10,9 +10,6 @@
 grant {
    // reindex opens socket connections using the rest client
    permission java.net.SocketPermission "*", "connect";
-
-  // needed for Painless to generate runtime classes
-  permission java.lang.RuntimePermission "createClassLoader";
 };
 
 grant codeBase "${codebase.elasticsearch-rest-client}" {

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java

@@ -11,9 +11,9 @@
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 
-import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.test.XContentTestUtils;
 import org.junit.BeforeClass;
 
@@ -112,7 +112,7 @@ public void testGetFeatureUpgradeStatus() throws Exception {
                 // for the next major version upgrade (see e.g. #93666). Trying to express this with features may be problematic, so we
                 // want to keep using versions here. We also assume that for non-semantic version migrations are not required.
                 boolean migrationNeeded = parseLegacyVersion(getOldClusterVersion()).map(
-                    v -> v.before(TransportGetFeatureUpgradeStatusAction.NO_UPGRADE_REQUIRED_VERSION)
+                    v -> v.before(SystemIndices.NO_UPGRADE_REQUIRED_VERSION)
                 ).orElse(false);
                 if (migrationNeeded) {
                     assertThat(feature.get("migration_status"), equalTo("MIGRATION_NEEDED"));

rest-api-spec/build.gradle

@@ -276,4 +276,6 @@ tasks.named("yamlRestTestV7CompatTransform").configure({ task ->
   task.skipTest("search.vectors/130_knn_query_nested_search/nested kNN search inner_hits size > 1", "waiting for #118774 backport")
   task.skipTest("search.vectors/110_knn_query_with_filter/PRE_FILTER: pre-filter across multiple aliases", "waiting for #118774 backport")
   task.skipTest("search.vectors/160_knn_query_missing_params/kNN search in a dis_max query - missing num_candidates", "waiting for #118774 backport")
+  task.skipTest("migration/10_get_feature_upgrade_status/Get feature upgrade status", "Moved to plugin")
+  task.skipTest("migration/20_post_feature_upgrade/Get feature upgrade status", "Moved to plugin")
 })

server/src/main/java/module-info.java

@@ -62,7 +62,6 @@
     exports org.elasticsearch.action.admin.cluster.coordination;
     exports org.elasticsearch.action.admin.cluster.desirednodes;
     exports org.elasticsearch.action.admin.cluster.health;
-    exports org.elasticsearch.action.admin.cluster.migration;
     exports org.elasticsearch.action.admin.cluster.node.capabilities;
     exports org.elasticsearch.action.admin.cluster.node.hotthreads;
     exports org.elasticsearch.action.admin.cluster.node.info;
@@ -380,7 +379,6 @@
     exports org.elasticsearch.tasks;
     exports org.elasticsearch.threadpool;
     exports org.elasticsearch.transport;
-    exports org.elasticsearch.upgrades;
     exports org.elasticsearch.usage;
     exports org.elasticsearch.watcher;
 

server/src/main/java/org/elasticsearch/action/ActionModule.java

@@ -26,10 +26,6 @@
 import org.elasticsearch.action.admin.cluster.desirednodes.TransportUpdateDesiredNodesAction;
 import org.elasticsearch.action.admin.cluster.desirednodes.UpdateDesiredNodesAction;
 import org.elasticsearch.action.admin.cluster.health.TransportClusterHealthAction;
-import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusAction;
-import org.elasticsearch.action.admin.cluster.migration.PostFeatureUpgradeAction;
-import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction;
-import org.elasticsearch.action.admin.cluster.migration.TransportPostFeatureUpgradeAction;
 import org.elasticsearch.action.admin.cluster.node.capabilities.TransportNodesCapabilitiesAction;
 import org.elasticsearch.action.admin.cluster.node.features.TransportNodesFeaturesAction;
 import org.elasticsearch.action.admin.cluster.node.hotthreads.TransportNodesHotThreadsAction;
@@ -283,7 +279,6 @@
 import org.elasticsearch.rest.action.admin.cluster.RestDeleteStoredScriptAction;
 import org.elasticsearch.rest.action.admin.cluster.RestGetDesiredBalanceAction;
 import org.elasticsearch.rest.action.admin.cluster.RestGetDesiredNodesAction;
-import org.elasticsearch.rest.action.admin.cluster.RestGetFeatureUpgradeStatusAction;
 import org.elasticsearch.rest.action.admin.cluster.RestGetRepositoriesAction;
 import org.elasticsearch.rest.action.admin.cluster.RestGetScriptContextAction;
 import org.elasticsearch.rest.action.admin.cluster.RestGetScriptLanguageAction;
@@ -297,7 +292,6 @@
 import org.elasticsearch.rest.action.admin.cluster.RestNodesStatsAction;
 import org.elasticsearch.rest.action.admin.cluster.RestNodesUsageAction;
 import org.elasticsearch.rest.action.admin.cluster.RestPendingClusterTasksAction;
-import org.elasticsearch.rest.action.admin.cluster.RestPostFeatureUpgradeAction;
 import org.elasticsearch.rest.action.admin.cluster.RestPrevalidateNodeRemovalAction;
 import org.elasticsearch.rest.action.admin.cluster.RestPutRepositoryAction;
 import org.elasticsearch.rest.action.admin.cluster.RestPutStoredScriptAction;
@@ -671,10 +665,7 @@ public <Request extends ActionRequest, Response extends ActionResponse> void reg
         actions.register(TransportSnapshotsStatusAction.TYPE, TransportSnapshotsStatusAction.class);
         actions.register(SnapshottableFeaturesAction.INSTANCE, TransportSnapshottableFeaturesAction.class);
         actions.register(ResetFeatureStateAction.INSTANCE, TransportResetFeatureStateAction.class);
-        actions.register(GetFeatureUpgradeStatusAction.INSTANCE, TransportGetFeatureUpgradeStatusAction.class);
-        actions.register(PostFeatureUpgradeAction.INSTANCE, TransportPostFeatureUpgradeAction.class);
         actions.register(TransportGetShardSnapshotAction.TYPE, TransportGetShardSnapshotAction.class);
-
         actions.register(IndicesStatsAction.INSTANCE, TransportIndicesStatsAction.class);
         actions.register(IndicesSegmentsAction.INSTANCE, TransportIndicesSegmentsAction.class);
         actions.register(TransportIndicesShardStoresAction.TYPE, TransportIndicesShardStoresAction.class);
@@ -872,8 +863,6 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster, Predicate<
         registerHandler.accept(new RestSnapshotsStatusAction());
         registerHandler.accept(new RestSnapshottableFeaturesAction());
         registerHandler.accept(new RestResetFeatureStateAction());
-        registerHandler.accept(new RestGetFeatureUpgradeStatusAction());
-        registerHandler.accept(new RestPostFeatureUpgradeAction());
         registerHandler.accept(new RestGetIndicesAction());
         registerHandler.accept(new RestIndicesStatsAction());
         registerHandler.accept(new RestIndicesSegmentsAction());

server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java

@@ -142,7 +142,7 @@ protected void masterOperation(
                 listener.onFailure(new IllegalStateException(message));
                 return;
             }
-            updateRequest = buildSystemIndexUpdateRequest(request, cause, descriptor);
+            updateRequest = buildManagedSystemIndexUpdateRequest(request, cause, descriptor);
         } else {
             updateRequest = buildUpdateRequest(request, cause, indexName, resolvedAt);
         }
@@ -174,30 +174,54 @@ private CreateIndexClusterStateUpdateRequest buildUpdateRequest(
             .waitForActiveShards(request.waitForActiveShards());
     }
 
-    private static CreateIndexClusterStateUpdateRequest buildSystemIndexUpdateRequest(
+    private static CreateIndexClusterStateUpdateRequest buildManagedSystemIndexUpdateRequest(
         CreateIndexRequest request,
         String cause,
         SystemIndexDescriptor descriptor
     ) {
-        final Settings settings = Objects.requireNonNullElse(descriptor.getSettings(), Settings.EMPTY);
+        boolean indexMigrationInProgress = cause.equals(SystemIndices.MIGRATE_SYSTEM_INDEX_CAUSE)
+            && request.index().endsWith(SystemIndices.UPGRADED_INDEX_SUFFIX);
 
+        final Settings settings;
+        final String mappings;
         final Set<Alias> aliases;
-        if (descriptor.getAliasName() == null) {
+        final String indexName;
+
+        // if we are migrating a system index to a new index, we use settings/mappings/index name from the request,
+        // since it was created by SystemIndexMigrator
+        if (indexMigrationInProgress) {
+            settings = request.settings();
+            mappings = request.mappings();
+            indexName = request.index();
+            // we will update alias later on
             aliases = Set.of();
         } else {
-            aliases = Set.of(new Alias(descriptor.getAliasName()).isHidden(true).writeIndex(true));
-        }
+            settings = Objects.requireNonNullElse(descriptor.getSettings(), Settings.EMPTY);
+            mappings = descriptor.getMappings();
+
+            if (descriptor.getAliasName() == null) {
+                aliases = Set.of();
+            } else {
+                aliases = Set.of(new Alias(descriptor.getAliasName()).isHidden(true).writeIndex(true));
+            }
 
-        // Throw an error if we are trying to directly create a system index other than the primary system index (or the alias)
-        if (request.index().equals(descriptor.getPrimaryIndex()) == false && request.index().equals(descriptor.getAliasName()) == false) {
-            throw new IllegalArgumentException(
-                "Cannot create system index with name " + request.index() + "; descriptor primary index is " + descriptor.getPrimaryIndex()
-            );
+            // Throw an error if we are trying to directly create a system index other
+            // than the primary system index (or the alias, or we are migrating the index)
+            if (request.index().equals(descriptor.getPrimaryIndex()) == false
+                && request.index().equals(descriptor.getAliasName()) == false) {
+                throw new IllegalArgumentException(
+                    "Cannot create system index with name "
+                        + request.index()
+                        + "; descriptor primary index is "
+                        + descriptor.getPrimaryIndex()
+                );
+            }
+            indexName = descriptor.getPrimaryIndex();
         }
 
-        return new CreateIndexClusterStateUpdateRequest(cause, descriptor.getPrimaryIndex(), request.index()).aliases(aliases)
+        return new CreateIndexClusterStateUpdateRequest(cause, indexName, request.index()).aliases(aliases)
             .waitForActiveShards(ActiveShardCount.ALL)
-            .mappings(descriptor.getMappings())
+            .mappings(mappings)
             .settings(settings);
     }
 }

server/src/main/java/org/elasticsearch/cluster/ClusterModule.java

@@ -84,7 +84,6 @@
 import org.elasticsearch.tasks.TaskResultsService;
 import org.elasticsearch.telemetry.TelemetryProvider;
 import org.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.upgrades.FeatureMigrationResults;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ParseField;
 
@@ -235,7 +234,6 @@ public static List<Entry> getNamedWriteables() {
         );
         registerMetadataCustom(entries, DataStreamMetadata.TYPE, DataStreamMetadata::new, DataStreamMetadata::readDiffFrom);
         registerMetadataCustom(entries, NodesShutdownMetadata.TYPE, NodesShutdownMetadata::new, NodesShutdownMetadata::readDiffFrom);
-        registerMetadataCustom(entries, FeatureMigrationResults.TYPE, FeatureMigrationResults::new, FeatureMigrationResults::readDiffFrom);
         registerMetadataCustom(entries, DesiredNodesMetadata.TYPE, DesiredNodesMetadata::new, DesiredNodesMetadata::readDiffFrom);
         registerMetadataCustom(
             entries,

server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java

@@ -14,7 +14,6 @@
 import org.apache.lucene.util.automaton.Operations;
 import org.apache.lucene.util.automaton.RegExp;
 import org.elasticsearch.Version;
-import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction;
 import org.elasticsearch.action.admin.indices.create.AutoCreateAction;
 import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -152,7 +151,7 @@ public class SystemIndexDescriptor implements IndexPatternMatcher, Comparable<Sy
 
     /**
      * An optional reindexing script to use when migrating an index created
-     * before {@link TransportGetFeatureUpgradeStatusAction#NO_UPGRADE_REQUIRED_INDEX_VERSION}.
+     * before {@link SystemIndices#NO_UPGRADE_REQUIRED_INDEX_VERSION}.
      * This script can be used to modify documents before they are added to the new index.
      * For example, it can be used to remove deprecated fields from the index.
      * <br>

server/src/main/java/org/elasticsearch/indices/SystemIndices.java

@@ -16,6 +16,7 @@
 import org.apache.lucene.util.automaton.CharacterRunAutomaton;
 import org.apache.lucene.util.automaton.MinimizationOperations;
 import org.apache.lucene.util.automaton.Operations;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.cluster.snapshots.features.ResetFeatureStateResponse.ResetFeatureStateStatus;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
@@ -37,6 +38,8 @@
 import org.elasticsearch.core.Predicates;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.plugins.SystemIndexPlugin;
 import org.elasticsearch.snapshots.SnapshotsService;
 
@@ -111,7 +114,16 @@
 public class SystemIndices {
     public static final String SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY = "_system_index_access_allowed";
     public static final String EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY = "_external_system_index_access_origin";
-    public static final String UPGRADED_INDEX_SUFFIX = "-reindexed-for-9";
+
+    /**
+     * These versions should be set to current major and current major's index version
+     */
+    public static final Version NO_UPGRADE_REQUIRED_VERSION = Version.V_8_0_0;
+    public static final IndexVersion NO_UPGRADE_REQUIRED_INDEX_VERSION = IndexVersions.V_8_0_0;
+    public static final String MIGRATE_SYSTEM_INDEX_CAUSE = "migrate-system-index";
+
+    private static final int UPGRADED_TO_VERSION = NO_UPGRADE_REQUIRED_VERSION.major + 1;
+    public static final String UPGRADED_INDEX_SUFFIX = "-reindexed-for-" + UPGRADED_TO_VERSION;
 
     private static final Automaton EMPTY = Automata.makeEmpty();
 
@@ -347,6 +359,7 @@ public ExecutorSelector getExecutorSelector() {
 
     /**
      * Finds a single matching {@link SystemIndexDescriptor}, if any, for the given index name.
+     * Does not take into account system data streams and their backing indices.
      * @param name the name of the index
      * @return The matching {@link SystemIndexDescriptor} or {@code null} if no descriptor is found
      */
@@ -355,7 +368,7 @@ public ExecutorSelector getExecutorSelector() {
     }
 
     @Nullable
-    static SystemIndexDescriptor findMatchingDescriptor(SystemIndexDescriptor[] indexDescriptors, String name) {
+    private static SystemIndexDescriptor findMatchingDescriptor(SystemIndexDescriptor[] indexDescriptors, String name) {
         SystemIndexDescriptor matchingDescriptor = null;
         for (SystemIndexDescriptor systemIndexDescriptor : indexDescriptors) {
             if (systemIndexDescriptor.matchesIndexPattern(name)) {