Add support for specifying reindexing script for system index migration (#119001)

Adds support for setting a reindexing script for system index migration during /_migration/system_features task.
Script is set using SystemIndexDescriptor.Builder.setMigrationScript method.

Author: alexey-ivanov-es

docs/changelog/119001.yaml

@@ -0,0 +1,5 @@
+pr: 119001
+summary: Add support for specifying reindexing script for system index migration
+area: Infra/Core
+type: enhancement
+issues: []

server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptor.java

@@ -51,6 +51,7 @@ public class TestSystemIndexDescriptor extends SystemIndexDescriptor {
             INDEX_NAME,
             0,
             "stack",
+            null,
             Type.INTERNAL_MANAGED,
             List.of(),
             List.of(),
@@ -70,6 +71,7 @@ public class TestSystemIndexDescriptor extends SystemIndexDescriptor {
             name,
             0,
             "stack",
+            null,
             Type.INTERNAL_MANAGED,
             List.of(),
             List.of(),

server/src/main/java/org/elasticsearch/action/admin/cluster/migration/TransportGetFeatureUpgradeStatusAction.java

@@ -53,7 +53,7 @@ public class TransportGetFeatureUpgradeStatusAction extends TransportMasterNodeA
     GetFeatureUpgradeStatusResponse> {
 
     /**
-     * Once all feature migrations for 8.x -> 9.x have been tested, we can bump this to Version.V_8_0_0
+     * Once all feature migrations for 9.x -> 10.x have been tested, we can bump this to Version.V_9_0_0
      */
     public static final Version NO_UPGRADE_REQUIRED_VERSION = Version.V_8_0_0;
     public static final IndexVersion NO_UPGRADE_REQUIRED_INDEX_VERSION = IndexVersions.V_8_0_0;

server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java

@@ -13,6 +13,7 @@
 import org.apache.lucene.util.automaton.CharacterRunAutomaton;
 import org.apache.lucene.util.automaton.Operations;
 import org.apache.lucene.util.automaton.RegExp;
+import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction;
 import org.elasticsearch.action.admin.indices.create.AutoCreateAction;
 import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -145,6 +146,24 @@ public class SystemIndexDescriptor implements IndexPatternMatcher, Comparable<Sy
     /** For internally-managed indices, specifies the origin to use when creating or updating the index */
     private final String origin;
 
+    /**
+     * An optional reindexing script to use when migrating an index created
+     * before {@link TransportGetFeatureUpgradeStatusAction#NO_UPGRADE_REQUIRED_INDEX_VERSION}.
+     * This script can be used to modify documents before they are added to the new index.
+     * For example, it can be used to remove deprecated fields from the index.
+     * <br>
+     * Note: the script usually should only exist in  the versions supporting migration to the next major release -
+     * specifically, the last (two) minors of the current major.
+     * It should be created once the last minor branch has diverged from the next major branch (main).
+     * This ensures the script is available only in the versions where it is needed
+     * and avoids removing and maintaining it in the next major branch.
+     * For example: In order to migrate an index created in v7 when upgrading to v9,
+     * the script should be in the v8 minors supporting upgrade to v9 - 8.18 and 8.19.
+     * <br>
+     * See: <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-reindex.html#reindex-scripts">Reindex scripts</a>
+     */
+    private final String migrationScript;
+
     /** Mapping version from the descriptor */
     private final MappingsVersion mappingsVersion;
 
@@ -197,6 +216,7 @@ public class SystemIndexDescriptor implements IndexPatternMatcher, Comparable<Sy
      * @param origin the client origin to use when creating this index. Internal system indices must not provide an origin, while external
      *               system indices must do so.
      * @param type The {@link Type} of system index
+     * @param migrationScript The script to apply when migrating this system index, or null
      * @param allowedElasticProductOrigins A list of allowed origin values that should be allowed access in the case of external system
      *                                     indices
      * @param priorSystemIndexDescriptors A list of system index descriptors that describe the same index in a way that is compatible with
@@ -212,6 +232,7 @@ protected SystemIndexDescriptor(
         String aliasName,
         int indexFormat,
         String origin,
+        String migrationScript,
         Type type,
         List<String> allowedElasticProductOrigins,
         List<SystemIndexDescriptor> priorSystemIndexDescriptors,
@@ -346,6 +367,7 @@ protected SystemIndexDescriptor(
 
         this.description = description;
         this.mappings = mappings;
+        this.migrationScript = migrationScript;
 
         settings = Objects.isNull(settings) ? Settings.EMPTY : settings;
 
@@ -557,6 +579,10 @@ public ExecutorNames getThreadPoolNames() {
         return this.executorNames;
     }
 
+    public String getMigrationScript() {
+        return migrationScript;
+    }
+
     public static Builder builder() {
         return new Builder();
     }
@@ -659,6 +685,7 @@ public static class Builder {
         private String aliasName = null;
         private int indexFormat = 0;
         private String origin = null;
+        private String migrationScript;
         private Type type = Type.INTERNAL_MANAGED;
         private List<String> allowedElasticProductOrigins = List.of();
         private List<SystemIndexDescriptor> priorSystemIndexDescriptors = List.of();
@@ -721,6 +748,11 @@ public Builder setOrigin(String origin) {
             return this;
         }
 
+        public Builder setMigrationScript(String migrationScript) {
+            this.migrationScript = migrationScript;
+            return this;
+        }
+
         public Builder setType(Type type) {
             this.type = type;
             return this;
@@ -765,6 +797,7 @@ public SystemIndexDescriptor build() {
                 aliasName,
                 indexFormat,
                 origin,
+                migrationScript,
                 type,
                 allowedElasticProductOrigins,
                 priorSystemIndexDescriptors,

server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationInfo.java

@@ -44,6 +44,7 @@ class SystemIndexMigrationInfo implements Comparable<SystemIndexMigrationInfo> {
     private final Settings settings;
     private final String mapping;
     private final String origin;
+    private final String migrationScript;
     private final SystemIndices.Feature owningFeature;
     private final boolean allowsTemplates;
 
@@ -57,6 +58,7 @@ private SystemIndexMigrationInfo(
         Settings settings,
         String mapping,
         String origin,
+        String migrationScript,
         SystemIndices.Feature owningFeature,
         boolean allowsTemplates
     ) {
@@ -65,6 +67,7 @@ private SystemIndexMigrationInfo(
         this.settings = settings;
         this.mapping = mapping;
         this.origin = origin;
+        this.migrationScript = migrationScript;
         this.owningFeature = owningFeature;
         this.allowsTemplates = allowsTemplates;
     }
@@ -118,6 +121,10 @@ String getOrigin() {
         return origin;
     }
 
+    String getMigrationScript() {
+        return migrationScript;
+    }
+
     /**
      * By default, system indices should not be affected by user defined templates, so this
      * method should return false in almost all cases. At the moment certain Kibana indices use
@@ -217,6 +224,7 @@ static SystemIndexMigrationInfo build(
             settings,
             mapping,
             descriptor.getOrigin(),
+            descriptor.getMigrationScript(),
             feature,
             descriptor.allowsTemplates()
         );

server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrator.java

@@ -45,6 +45,7 @@
 import org.elasticsearch.index.reindex.ReindexRequest;
 import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.persistent.AllocatedPersistentTask;
+import org.elasticsearch.script.Script;
 import org.elasticsearch.tasks.TaskId;
 
 import java.util.LinkedList;
@@ -563,6 +564,10 @@ private void reindex(SystemIndexMigrationInfo migrationInfo, ActionListener<Bulk
         reindexRequest.setSourceIndices(migrationInfo.getCurrentIndexName());
         reindexRequest.setDestIndex(migrationInfo.getNextIndexName());
         reindexRequest.setRefresh(true);
+        String migrationScript = migrationInfo.getMigrationScript();
+        if (Strings.isNullOrEmpty(migrationScript) == false) {
+            reindexRequest.setScript(Script.parse(migrationScript));
+        }
         migrationInfo.createClient(baseClient).execute(ReindexAction.INSTANCE, reindexRequest, listener);
     }