Skip to content

Commit e4aae6a

Browse files
alexey-ivanov-esalexey-ivanov-es
committed
Move system indices migration to migrate plugin
It seems the best way to fix #122949 is to use existing data stream reindex API. However, this API is located in the migrate x-pack plugin. This commit moves the system indices migration logic (REST handlers, transport actions, and task) to the migrate plugin. Port of #123551
1 parent cac356a commit e4aae6a

File tree

49 files changed

+369
-429
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+369
-429
lines changed

‎modules/reindex/build.gradle‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,6 @@ dependencies {
4444
clusterModules project(':modules:lang-painless')
4545
clusterModules project(':modules:parent-join')
4646
clusterModules project(":modules:rest-root")
47-
48-
internalClusterTestImplementation project(':modules:lang-painless')
49-
internalClusterTestImplementation project(':modules:lang-painless:spi')
5047
}
5148

5249
restResources {

‎modules/reindex/src/main/plugin-metadata/plugin-security.policy‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,6 @@
1010
grant {
1111
// reindex opens socket connections using the rest client
1212
permission java.net.SocketPermission "*", "connect";
13-
14-
// needed for Painless to generate runtime classes
15-
permission java.lang.RuntimePermission "createClassLoader";
1613
};
1714

1815
grant codeBase "${codebase.elasticsearch-rest-client}" {

‎qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,9 @@
1111

1212
import com.carrotsearch.randomizedtesting.annotations.Name;
1313

14-
import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction;
1514
import org.elasticsearch.client.Request;
1615
import org.elasticsearch.client.ResponseException;
16+
import org.elasticsearch.indices.SystemIndices;
1717
import org.elasticsearch.test.XContentTestUtils;
1818
import org.junit.BeforeClass;
1919

@@ -112,7 +112,7 @@ public void testGetFeatureUpgradeStatus() throws Exception {
112112
// for the next major version upgrade (see e.g. #93666). Trying to express this with features may be problematic, so we
113113
// want to keep using versions here. We also assume that for non-semantic version migrations are not required.
114114
boolean migrationNeeded = parseLegacyVersion(getOldClusterVersion()).map(
115-
v -> v.before(TransportGetFeatureUpgradeStatusAction.NO_UPGRADE_REQUIRED_VERSION)
115+
v -> v.before(SystemIndices.NO_UPGRADE_REQUIRED_VERSION)
116116
).orElse(false);
117117
if (migrationNeeded) {
118118
assertThat(feature.get("migration_status"), equalTo("MIGRATION_NEEDED"));

‎server/src/main/java/module-info.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,6 @@
6363
exports org.elasticsearch.action.admin.cluster.coordination;
6464
exports org.elasticsearch.action.admin.cluster.desirednodes;
6565
exports org.elasticsearch.action.admin.cluster.health;
66-
exports org.elasticsearch.action.admin.cluster.migration;
6766
exports org.elasticsearch.action.admin.cluster.node.capabilities;
6867
exports org.elasticsearch.action.admin.cluster.node.hotthreads;
6968
exports org.elasticsearch.action.admin.cluster.node.info;
@@ -385,7 +384,6 @@
385384
exports org.elasticsearch.tasks;
386385
exports org.elasticsearch.threadpool;
387386
exports org.elasticsearch.transport;
388-
exports org.elasticsearch.upgrades;
389387
exports org.elasticsearch.usage;
390388
exports org.elasticsearch.watcher;
391389

‎server/src/main/java/org/elasticsearch/action/ActionModule.java‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,6 @@
2626
import org.elasticsearch.action.admin.cluster.desirednodes.TransportUpdateDesiredNodesAction;
2727
import org.elasticsearch.action.admin.cluster.desirednodes.UpdateDesiredNodesAction;
2828
import org.elasticsearch.action.admin.cluster.health.TransportClusterHealthAction;
29-
import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusAction;
30-
import org.elasticsearch.action.admin.cluster.migration.PostFeatureUpgradeAction;
31-
import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction;
32-
import org.elasticsearch.action.admin.cluster.migration.TransportPostFeatureUpgradeAction;
3329
import org.elasticsearch.action.admin.cluster.node.capabilities.TransportNodesCapabilitiesAction;
3430
import org.elasticsearch.action.admin.cluster.node.features.TransportNodesFeaturesAction;
3531
import org.elasticsearch.action.admin.cluster.node.hotthreads.TransportNodesHotThreadsAction;
@@ -286,7 +282,6 @@
286282
import org.elasticsearch.rest.action.admin.cluster.RestDeleteStoredScriptAction;
287283
import org.elasticsearch.rest.action.admin.cluster.RestGetDesiredBalanceAction;
288284
import org.elasticsearch.rest.action.admin.cluster.RestGetDesiredNodesAction;
289-
import org.elasticsearch.rest.action.admin.cluster.RestGetFeatureUpgradeStatusAction;
290285
import org.elasticsearch.rest.action.admin.cluster.RestGetRepositoriesAction;
291286
import org.elasticsearch.rest.action.admin.cluster.RestGetScriptContextAction;
292287
import org.elasticsearch.rest.action.admin.cluster.RestGetScriptLanguageAction;
@@ -300,7 +295,6 @@
300295
import org.elasticsearch.rest.action.admin.cluster.RestNodesStatsAction;
301296
import org.elasticsearch.rest.action.admin.cluster.RestNodesUsageAction;
302297
import org.elasticsearch.rest.action.admin.cluster.RestPendingClusterTasksAction;
303-
import org.elasticsearch.rest.action.admin.cluster.RestPostFeatureUpgradeAction;
304298
import org.elasticsearch.rest.action.admin.cluster.RestPrevalidateNodeRemovalAction;
305299
import org.elasticsearch.rest.action.admin.cluster.RestPutRepositoryAction;
306300
import org.elasticsearch.rest.action.admin.cluster.RestPutStoredScriptAction;
@@ -681,10 +675,7 @@ public <Request extends ActionRequest, Response extends ActionResponse> void reg
681675
actions.register(TransportSnapshotsStatusAction.TYPE, TransportSnapshotsStatusAction.class);
682676
actions.register(SnapshottableFeaturesAction.INSTANCE, TransportSnapshottableFeaturesAction.class);
683677
actions.register(ResetFeatureStateAction.INSTANCE, TransportResetFeatureStateAction.class);
684-
actions.register(GetFeatureUpgradeStatusAction.INSTANCE, TransportGetFeatureUpgradeStatusAction.class);
685-
actions.register(PostFeatureUpgradeAction.INSTANCE, TransportPostFeatureUpgradeAction.class);
686678
actions.register(TransportGetShardSnapshotAction.TYPE, TransportGetShardSnapshotAction.class);
687-
688679
actions.register(IndicesStatsAction.INSTANCE, TransportIndicesStatsAction.class);
689680
actions.register(IndicesSegmentsAction.INSTANCE, TransportIndicesSegmentsAction.class);
690681
actions.register(TransportIndicesShardStoresAction.TYPE, TransportIndicesShardStoresAction.class);
@@ -882,8 +873,6 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster, Predicate<
882873
registerHandler.accept(new RestSnapshotsStatusAction());
883874
registerHandler.accept(new RestSnapshottableFeaturesAction());
884875
registerHandler.accept(new RestResetFeatureStateAction());
885-
registerHandler.accept(new RestGetFeatureUpgradeStatusAction());
886-
registerHandler.accept(new RestPostFeatureUpgradeAction());
887876
registerHandler.accept(new RestGetIndicesAction());
888877
registerHandler.accept(new RestIndicesStatsAction());
889878
registerHandler.accept(new RestIndicesSegmentsAction());

‎server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java‎

Lines changed: 37 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,7 @@ protected void masterOperation(
148148
listener.onFailure(new IllegalStateException(message));
149149
return;
150150
}
151-
updateRequest = buildSystemIndexUpdateRequest(request, cause, descriptor, projectId);
151+
updateRequest = buildManagedSystemIndexUpdateRequest(request, cause, descriptor, projectId);
152152
} else {
153153
updateRequest = buildUpdateRequest(request, cause, indexName, resolvedAt, projectId);
154154
}
@@ -181,31 +181,55 @@ private CreateIndexClusterStateUpdateRequest buildUpdateRequest(
181181
.waitForActiveShards(request.waitForActiveShards());
182182
}
183183

184-
private static CreateIndexClusterStateUpdateRequest buildSystemIndexUpdateRequest(
184+
private static CreateIndexClusterStateUpdateRequest buildManagedSystemIndexUpdateRequest(
185185
CreateIndexRequest request,
186186
String cause,
187187
SystemIndexDescriptor descriptor,
188188
ProjectId projectId
189189
) {
190-
final Settings settings = Objects.requireNonNullElse(descriptor.getSettings(), Settings.EMPTY);
190+
boolean indexMigrationInProgress = cause.equals(SystemIndices.MIGRATE_SYSTEM_INDEX_CAUSE)
191+
&& request.index().endsWith(SystemIndices.UPGRADED_INDEX_SUFFIX);
191192

193+
final Settings settings;
194+
final String mappings;
192195
final Set<Alias> aliases;
193-
if (descriptor.getAliasName() == null) {
196+
final String indexName;
197+
198+
// if we are migrating a system index to a new index, we use settings/mappings/index name from the request,
199+
// since it was created by SystemIndexMigrator
200+
if (indexMigrationInProgress) {
201+
settings = request.settings();
202+
mappings = request.mappings();
203+
indexName = request.index();
204+
// we will update alias later on
194205
aliases = Set.of();
195206
} else {
196-
aliases = Set.of(new Alias(descriptor.getAliasName()).isHidden(true).writeIndex(true));
197-
}
207+
settings = Objects.requireNonNullElse(descriptor.getSettings(), Settings.EMPTY);
208+
mappings = descriptor.getMappings();
209+
210+
if (descriptor.getAliasName() == null) {
211+
aliases = Set.of();
212+
} else {
213+
aliases = Set.of(new Alias(descriptor.getAliasName()).isHidden(true).writeIndex(true));
214+
}
198215

199-
// Throw an error if we are trying to directly create a system index other than the primary system index (or the alias)
200-
if (request.index().equals(descriptor.getPrimaryIndex()) == false && request.index().equals(descriptor.getAliasName()) == false) {
201-
throw new IllegalArgumentException(
202-
"Cannot create system index with name " + request.index() + "; descriptor primary index is " + descriptor.getPrimaryIndex()
203-
);
216+
// Throw an error if we are trying to directly create a system index other
217+
// than the primary system index (or the alias, or we are migrating the index)
218+
if (request.index().equals(descriptor.getPrimaryIndex()) == false
219+
&& request.index().equals(descriptor.getAliasName()) == false) {
220+
throw new IllegalArgumentException(
221+
"Cannot create system index with name "
222+
+ request.index()
223+
+ "; descriptor primary index is "
224+
+ descriptor.getPrimaryIndex()
225+
);
226+
}
227+
indexName = descriptor.getPrimaryIndex();
204228
}
205229

206-
return new CreateIndexClusterStateUpdateRequest(cause, projectId, descriptor.getPrimaryIndex(), request.index()).aliases(aliases)
230+
return new CreateIndexClusterStateUpdateRequest(cause, projectId, indexName, request.index()).aliases(aliases)
207231
.waitForActiveShards(ActiveShardCount.ALL)
208-
.mappings(descriptor.getMappings())
232+
.mappings(mappings)
209233
.settings(settings);
210234
}
211235
}

‎server/src/main/java/org/elasticsearch/cluster/ClusterModule.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,6 @@
8787
import org.elasticsearch.tasks.TaskResultsService;
8888
import org.elasticsearch.telemetry.TelemetryProvider;
8989
import org.elasticsearch.threadpool.ThreadPool;
90-
import org.elasticsearch.upgrades.FeatureMigrationResults;
9190
import org.elasticsearch.xcontent.NamedXContentRegistry;
9291
import org.elasticsearch.xcontent.ParseField;
9392

@@ -254,7 +253,6 @@ public static List<Entry> getNamedWriteables() {
254253
ComposableIndexTemplateMetadata::readDiffFrom
255254
);
256255
registerProjectCustom(entries, DataStreamMetadata.TYPE, DataStreamMetadata::new, DataStreamMetadata::readDiffFrom);
257-
registerProjectCustom(entries, FeatureMigrationResults.TYPE, FeatureMigrationResults::new, FeatureMigrationResults::readDiffFrom);
258256
registerMetadataCustom(entries, NodesShutdownMetadata.TYPE, NodesShutdownMetadata::new, NodesShutdownMetadata::readDiffFrom);
259257
registerMetadataCustom(entries, DesiredNodesMetadata.TYPE, DesiredNodesMetadata::new, DesiredNodesMetadata::readDiffFrom);
260258
registerProjectCustom(

‎server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
1414
import org.apache.lucene.util.automaton.Operations;
1515
import org.apache.lucene.util.automaton.RegExp;
16-
import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction;
1716
import org.elasticsearch.action.admin.indices.create.AutoCreateAction;
1817
import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
1918
import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -148,7 +147,7 @@ public class SystemIndexDescriptor implements IndexPatternMatcher, Comparable<Sy
148147

149148
/**
150149
* An optional reindexing script to use when migrating an index created
151-
* before {@link TransportGetFeatureUpgradeStatusAction#NO_UPGRADE_REQUIRED_INDEX_VERSION}.
150+
* before {@link SystemIndices#NO_UPGRADE_REQUIRED_INDEX_VERSION}.
152151
* This script can be used to modify documents before they are added to the new index.
153152
* For example, it can be used to remove deprecated fields from the index.
154153
* <br>

‎server/src/main/java/org/elasticsearch/indices/SystemIndices.java‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@
1515
import org.apache.lucene.util.automaton.Automaton;
1616
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
1717
import org.apache.lucene.util.automaton.Operations;
18+
import org.elasticsearch.Version;
1819
import org.elasticsearch.action.ActionListener;
19-
import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction;
2020
import org.elasticsearch.action.admin.cluster.snapshots.features.ResetFeatureStateResponse.ResetFeatureStateStatus;
2121
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
2222
import org.elasticsearch.action.admin.indices.delete.TransportDeleteIndexAction;
@@ -36,7 +36,10 @@
3636
import org.elasticsearch.core.Nullable;
3737
import org.elasticsearch.core.Predicates;
3838
import org.elasticsearch.core.Tuple;
39+
import org.elasticsearch.core.UpdateForV10;
3940
import org.elasticsearch.index.Index;
41+
import org.elasticsearch.index.IndexVersion;
42+
import org.elasticsearch.index.IndexVersions;
4043
import org.elasticsearch.plugins.SystemIndexPlugin;
4144
import org.elasticsearch.snapshots.SnapshotsService;
4245

@@ -111,7 +114,16 @@
111114
public class SystemIndices {
112115
public static final String SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY = "_system_index_access_allowed";
113116
public static final String EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY = "_external_system_index_access_origin";
114-
private static final int UPGRADED_TO_VERSION = TransportGetFeatureUpgradeStatusAction.NO_UPGRADE_REQUIRED_VERSION.major + 1;
117+
118+
/**
119+
* These versions should be set to current major and current major's index version
120+
*/
121+
@UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA)
122+
public static final Version NO_UPGRADE_REQUIRED_VERSION = Version.V_9_0_0;
123+
public static final IndexVersion NO_UPGRADE_REQUIRED_INDEX_VERSION = IndexVersions.UPGRADE_TO_LUCENE_10_0_0;
124+
public static final String MIGRATE_SYSTEM_INDEX_CAUSE = "migrate-system-index";
125+
126+
private static final int UPGRADED_TO_VERSION = NO_UPGRADE_REQUIRED_VERSION.major + 1;
115127
public static final String UPGRADED_INDEX_SUFFIX = "-reindexed-for-" + UPGRADED_TO_VERSION;
116128

117129
private static final Automaton EMPTY = Automata.makeEmpty();
@@ -346,6 +358,7 @@ public ExecutorSelector getExecutorSelector() {
346358

347359
/**
348360
* Finds a single matching {@link SystemIndexDescriptor}, if any, for the given index name.
361+
* Does not take into account system data streams and their backing indices.
349362
* @param name the name of the index
350363
* @return The matching {@link SystemIndexDescriptor} or {@code null} if no descriptor is found
351364
*/
@@ -354,7 +367,7 @@ public ExecutorSelector getExecutorSelector() {
354367
}
355368

356369
@Nullable
357-
static SystemIndexDescriptor findMatchingDescriptor(SystemIndexDescriptor[] indexDescriptors, String name) {
370+
private static SystemIndexDescriptor findMatchingDescriptor(SystemIndexDescriptor[] indexDescriptors, String name) {
358371
SystemIndexDescriptor matchingDescriptor = null;
359372
for (SystemIndexDescriptor systemIndexDescriptor : indexDescriptors) {
360373
if (systemIndexDescriptor.matchesIndexPattern(name)) {

‎server/src/main/java/org/elasticsearch/node/NodeConstruction.java‎

Lines changed: 4 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -220,7 +220,6 @@
220220
import org.elasticsearch.threadpool.internal.BuiltInExecutorBuilders;
221221
import org.elasticsearch.transport.Transport;
222222
import org.elasticsearch.transport.TransportService;
223-
import org.elasticsearch.upgrades.SystemIndexMigrationExecutor;
224223
import org.elasticsearch.usage.UsageService;
225224
import org.elasticsearch.watcher.ResourceWatcherService;
226225
import org.elasticsearch.xcontent.NamedXContentRegistry;
@@ -595,8 +594,7 @@ private void createClientAndRegistries(Settings settings, ThreadPool threadPool,
595594
IndicesModule.getNamedWriteables().stream(),
596595
searchModule.getNamedWriteables().stream(),
597596
pluginsService.flatMap(Plugin::getNamedWriteables),
598-
ClusterModule.getNamedWriteables().stream(),
599-
SystemIndexMigrationExecutor.getNamedWriteables().stream()
597+
ClusterModule.getNamedWriteables().stream()
600598
).flatMap(Function.identity()).toList()
601599
);
602600
xContentRegistry = new NamedXContentRegistry(
@@ -606,7 +604,6 @@ private void createClientAndRegistries(Settings settings, ThreadPool threadPool,
606604
searchModule.getNamedXContents().stream(),
607605
pluginsService.flatMap(Plugin::getNamedXContent),
608606
ClusterModule.getNamedXWriteables().stream(),
609-
SystemIndexMigrationExecutor.getNamedXContentParsers().stream(),
610607
HealthNodeTaskExecutor.getNamedXContentParsers().stream()
611608
).flatMap(Function.identity()).toList()
612609
);
@@ -1161,17 +1158,7 @@ public Map<String, String> searchFields() {
11611158

11621159
final ShutdownPrepareService shutdownPrepareService = new ShutdownPrepareService(settings, httpServerTransport, terminationHandler);
11631160

1164-
modules.add(
1165-
loadPersistentTasksService(
1166-
settingsModule,
1167-
clusterService,
1168-
threadPool,
1169-
systemIndices,
1170-
clusterModule.getIndexNameExpressionResolver(),
1171-
metadataUpdateSettingsService,
1172-
metadataCreateIndexService
1173-
)
1174-
);
1161+
modules.add(loadPersistentTasksService(settingsModule, clusterService, threadPool, clusterModule.getIndexNameExpressionResolver()));
11751162

11761163
modules.add(
11771164
loadPluginShutdownService(clusterService),
@@ -1674,27 +1661,16 @@ private Module loadPersistentTasksService(
16741661
SettingsModule settingsModule,
16751662
ClusterService clusterService,
16761663
ThreadPool threadPool,
1677-
SystemIndices systemIndices,
1678-
IndexNameExpressionResolver indexNameExpressionResolver,
1679-
MetadataUpdateSettingsService metadataUpdateSettingsService,
1680-
MetadataCreateIndexService metadataCreateIndexService
1664+
IndexNameExpressionResolver indexNameExpressionResolver
16811665
) {
16821666
PersistentTasksService persistentTasksService = new PersistentTasksService(clusterService, threadPool, client);
1683-
SystemIndexMigrationExecutor systemIndexMigrationExecutor = new SystemIndexMigrationExecutor(
1684-
client,
1685-
clusterService,
1686-
systemIndices,
1687-
metadataUpdateSettingsService,
1688-
metadataCreateIndexService,
1689-
settingsModule.getIndexScopedSettings()
1690-
);
16911667
HealthNodeTaskExecutor healthNodeTaskExecutor = HealthNodeTaskExecutor.create(
16921668
clusterService,
16931669
persistentTasksService,
16941670
settingsModule.getSettings(),
16951671
clusterService.getClusterSettings()
16961672
);
1697-
Stream<PersistentTasksExecutor<?>> builtinTaskExecutors = Stream.of(systemIndexMigrationExecutor, healthNodeTaskExecutor);
1673+
Stream<PersistentTasksExecutor<?>> builtinTaskExecutors = Stream.of(healthNodeTaskExecutor);
16981674

16991675
Stream<PersistentTasksExecutor<?>> pluginTaskExecutors = pluginsService.filterPlugins(PersistentTaskPlugin.class)
17001676
.map(p -> p.getPersistentTasksExecutor(clusterService, threadPool, client, settingsModule, indexNameExpressionResolver))

0 commit comments

Comments
 (0)