Skip to content

Commit 4ed9347

Browse files
alexey-ivanov-esalexey-ivanov-es
authored
[9.0] Move system indices migration to migrate plugin (elastic#124783)
It seems the best way to fix elastic#122949 is to use existing data stream reindex API. However, this API is located in the migrate x-pack plugin. This commit moves the system indices migration logic (REST handlers, transport actions, and task) to the migrate plugin. Port of elastic#123551
1 parent c67b4d4 commit 4ed9347

File tree

50 files changed

+368
-422
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+368
-422
lines changed

‎modules/reindex/build.gradle‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,6 @@ dependencies {
4444
clusterModules project(':modules:lang-painless')
4545
clusterModules project(':modules:parent-join')
4646
clusterModules project(":modules:rest-root")
47-
48-
internalClusterTestImplementation project(':modules:lang-painless')
49-
internalClusterTestImplementation project(':modules:lang-painless:spi')
5047
}
5148

5249
restResources {

‎modules/reindex/src/main/plugin-metadata/plugin-security.policy‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,6 @@
1010
grant {
1111
// reindex opens socket connections using the rest client
1212
permission java.net.SocketPermission "*", "connect";
13-
14-
// needed for Painless to generate runtime classes
15-
permission java.lang.RuntimePermission "createClassLoader";
1613
};
1714

1815
grant codeBase "${codebase.elasticsearch-rest-client}" {

‎qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/FeatureUpgradeIT.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,9 @@
1111

1212
import com.carrotsearch.randomizedtesting.annotations.Name;
1313

14-
import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction;
1514
import org.elasticsearch.client.Request;
1615
import org.elasticsearch.client.ResponseException;
16+
import org.elasticsearch.indices.SystemIndices;
1717
import org.elasticsearch.test.XContentTestUtils;
1818
import org.junit.BeforeClass;
1919

@@ -112,7 +112,7 @@ public void testGetFeatureUpgradeStatus() throws Exception {
112112
// for the next major version upgrade (see e.g. #93666). Trying to express this with features may be problematic, so we
113113
// want to keep using versions here. We also assume that for non-semantic version migrations are not required.
114114
boolean migrationNeeded = parseLegacyVersion(getOldClusterVersion()).map(
115-
v -> v.before(TransportGetFeatureUpgradeStatusAction.NO_UPGRADE_REQUIRED_VERSION)
115+
v -> v.before(SystemIndices.NO_UPGRADE_REQUIRED_VERSION)
116116
).orElse(false);
117117
if (migrationNeeded) {
118118
assertThat(feature.get("migration_status"), equalTo("MIGRATION_NEEDED"));

‎server/src/main/java/module-info.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,6 @@
6363
exports org.elasticsearch.action.admin.cluster.coordination;
6464
exports org.elasticsearch.action.admin.cluster.desirednodes;
6565
exports org.elasticsearch.action.admin.cluster.health;
66-
exports org.elasticsearch.action.admin.cluster.migration;
6766
exports org.elasticsearch.action.admin.cluster.node.capabilities;
6867
exports org.elasticsearch.action.admin.cluster.node.hotthreads;
6968
exports org.elasticsearch.action.admin.cluster.node.info;
@@ -383,7 +382,6 @@
383382
exports org.elasticsearch.tasks;
384383
exports org.elasticsearch.threadpool;
385384
exports org.elasticsearch.transport;
386-
exports org.elasticsearch.upgrades;
387385
exports org.elasticsearch.usage;
388386
exports org.elasticsearch.watcher;
389387

‎server/src/main/java/org/elasticsearch/action/ActionModule.java‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,6 @@
2626
import org.elasticsearch.action.admin.cluster.desirednodes.TransportUpdateDesiredNodesAction;
2727
import org.elasticsearch.action.admin.cluster.desirednodes.UpdateDesiredNodesAction;
2828
import org.elasticsearch.action.admin.cluster.health.TransportClusterHealthAction;
29-
import org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusAction;
30-
import org.elasticsearch.action.admin.cluster.migration.PostFeatureUpgradeAction;
31-
import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction;
32-
import org.elasticsearch.action.admin.cluster.migration.TransportPostFeatureUpgradeAction;
3329
import org.elasticsearch.action.admin.cluster.node.capabilities.TransportNodesCapabilitiesAction;
3430
import org.elasticsearch.action.admin.cluster.node.features.TransportNodesFeaturesAction;
3531
import org.elasticsearch.action.admin.cluster.node.hotthreads.TransportNodesHotThreadsAction;
@@ -283,7 +279,6 @@
283279
import org.elasticsearch.rest.action.admin.cluster.RestDeleteStoredScriptAction;
284280
import org.elasticsearch.rest.action.admin.cluster.RestGetDesiredBalanceAction;
285281
import org.elasticsearch.rest.action.admin.cluster.RestGetDesiredNodesAction;
286-
import org.elasticsearch.rest.action.admin.cluster.RestGetFeatureUpgradeStatusAction;
287282
import org.elasticsearch.rest.action.admin.cluster.RestGetRepositoriesAction;
288283
import org.elasticsearch.rest.action.admin.cluster.RestGetScriptContextAction;
289284
import org.elasticsearch.rest.action.admin.cluster.RestGetScriptLanguageAction;
@@ -297,7 +292,6 @@
297292
import org.elasticsearch.rest.action.admin.cluster.RestNodesStatsAction;
298293
import org.elasticsearch.rest.action.admin.cluster.RestNodesUsageAction;
299294
import org.elasticsearch.rest.action.admin.cluster.RestPendingClusterTasksAction;
300-
import org.elasticsearch.rest.action.admin.cluster.RestPostFeatureUpgradeAction;
301295
import org.elasticsearch.rest.action.admin.cluster.RestPrevalidateNodeRemovalAction;
302296
import org.elasticsearch.rest.action.admin.cluster.RestPutRepositoryAction;
303297
import org.elasticsearch.rest.action.admin.cluster.RestPutStoredScriptAction;
@@ -669,10 +663,7 @@ public <Request extends ActionRequest, Response extends ActionResponse> void reg
669663
actions.register(TransportSnapshotsStatusAction.TYPE, TransportSnapshotsStatusAction.class);
670664
actions.register(SnapshottableFeaturesAction.INSTANCE, TransportSnapshottableFeaturesAction.class);
671665
actions.register(ResetFeatureStateAction.INSTANCE, TransportResetFeatureStateAction.class);
672-
actions.register(GetFeatureUpgradeStatusAction.INSTANCE, TransportGetFeatureUpgradeStatusAction.class);
673-
actions.register(PostFeatureUpgradeAction.INSTANCE, TransportPostFeatureUpgradeAction.class);
674666
actions.register(TransportGetShardSnapshotAction.TYPE, TransportGetShardSnapshotAction.class);
675-
676667
actions.register(IndicesStatsAction.INSTANCE, TransportIndicesStatsAction.class);
677668
actions.register(IndicesSegmentsAction.INSTANCE, TransportIndicesSegmentsAction.class);
678669
actions.register(TransportIndicesShardStoresAction.TYPE, TransportIndicesShardStoresAction.class);
@@ -870,8 +861,6 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster, Predicate<
870861
registerHandler.accept(new RestSnapshotsStatusAction());
871862
registerHandler.accept(new RestSnapshottableFeaturesAction());
872863
registerHandler.accept(new RestResetFeatureStateAction());
873-
registerHandler.accept(new RestGetFeatureUpgradeStatusAction());
874-
registerHandler.accept(new RestPostFeatureUpgradeAction());
875864
registerHandler.accept(new RestGetIndicesAction());
876865
registerHandler.accept(new RestIndicesStatsAction());
877866
registerHandler.accept(new RestIndicesSegmentsAction());

‎server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java‎

Lines changed: 37 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,7 @@ protected void masterOperation(
141141
listener.onFailure(new IllegalStateException(message));
142142
return;
143143
}
144-
updateRequest = buildSystemIndexUpdateRequest(request, cause, descriptor);
144+
updateRequest = buildManagedSystemIndexUpdateRequest(request, cause, descriptor);
145145
} else {
146146
updateRequest = buildUpdateRequest(request, cause, indexName, resolvedAt);
147147
}
@@ -173,30 +173,54 @@ private CreateIndexClusterStateUpdateRequest buildUpdateRequest(
173173
.waitForActiveShards(request.waitForActiveShards());
174174
}
175175

176-
private static CreateIndexClusterStateUpdateRequest buildSystemIndexUpdateRequest(
176+
private static CreateIndexClusterStateUpdateRequest buildManagedSystemIndexUpdateRequest(
177177
CreateIndexRequest request,
178178
String cause,
179179
SystemIndexDescriptor descriptor
180180
) {
181-
final Settings settings = Objects.requireNonNullElse(descriptor.getSettings(), Settings.EMPTY);
181+
boolean indexMigrationInProgress = cause.equals(SystemIndices.MIGRATE_SYSTEM_INDEX_CAUSE)
182+
&& request.index().endsWith(SystemIndices.UPGRADED_INDEX_SUFFIX);
182183

184+
final Settings settings;
185+
final String mappings;
183186
final Set<Alias> aliases;
184-
if (descriptor.getAliasName() == null) {
187+
final String indexName;
188+
189+
// if we are migrating a system index to a new index, we use settings/mappings/index name from the request,
190+
// since it was created by SystemIndexMigrator
191+
if (indexMigrationInProgress) {
192+
settings = request.settings();
193+
mappings = request.mappings();
194+
indexName = request.index();
195+
// we will update alias later on
185196
aliases = Set.of();
186197
} else {
187-
aliases = Set.of(new Alias(descriptor.getAliasName()).isHidden(true).writeIndex(true));
188-
}
198+
settings = Objects.requireNonNullElse(descriptor.getSettings(), Settings.EMPTY);
199+
mappings = descriptor.getMappings();
200+
201+
if (descriptor.getAliasName() == null) {
202+
aliases = Set.of();
203+
} else {
204+
aliases = Set.of(new Alias(descriptor.getAliasName()).isHidden(true).writeIndex(true));
205+
}
189206

190-
// Throw an error if we are trying to directly create a system index other than the primary system index (or the alias)
191-
if (request.index().equals(descriptor.getPrimaryIndex()) == false && request.index().equals(descriptor.getAliasName()) == false) {
192-
throw new IllegalArgumentException(
193-
"Cannot create system index with name " + request.index() + "; descriptor primary index is " + descriptor.getPrimaryIndex()
194-
);
207+
// Throw an error if we are trying to directly create a system index other
208+
// than the primary system index (or the alias, or we are migrating the index)
209+
if (request.index().equals(descriptor.getPrimaryIndex()) == false
210+
&& request.index().equals(descriptor.getAliasName()) == false) {
211+
throw new IllegalArgumentException(
212+
"Cannot create system index with name "
213+
+ request.index()
214+
+ "; descriptor primary index is "
215+
+ descriptor.getPrimaryIndex()
216+
);
217+
}
218+
indexName = descriptor.getPrimaryIndex();
195219
}
196220

197-
return new CreateIndexClusterStateUpdateRequest(cause, descriptor.getPrimaryIndex(), request.index()).aliases(aliases)
221+
return new CreateIndexClusterStateUpdateRequest(cause, indexName, request.index()).aliases(aliases)
198222
.waitForActiveShards(ActiveShardCount.ALL)
199-
.mappings(descriptor.getMappings())
223+
.mappings(mappings)
200224
.settings(settings);
201225
}
202226
}

‎server/src/main/java/org/elasticsearch/cluster/ClusterModule.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,6 @@
8585
import org.elasticsearch.tasks.TaskResultsService;
8686
import org.elasticsearch.telemetry.TelemetryProvider;
8787
import org.elasticsearch.threadpool.ThreadPool;
88-
import org.elasticsearch.upgrades.FeatureMigrationResults;
8988
import org.elasticsearch.xcontent.NamedXContentRegistry;
9089
import org.elasticsearch.xcontent.ParseField;
9190

@@ -244,7 +243,6 @@ public static List<Entry> getNamedWriteables() {
244243
);
245244
registerMetadataCustom(entries, DataStreamMetadata.TYPE, DataStreamMetadata::new, DataStreamMetadata::readDiffFrom);
246245
registerMetadataCustom(entries, NodesShutdownMetadata.TYPE, NodesShutdownMetadata::new, NodesShutdownMetadata::readDiffFrom);
247-
registerMetadataCustom(entries, FeatureMigrationResults.TYPE, FeatureMigrationResults::new, FeatureMigrationResults::readDiffFrom);
248246
registerMetadataCustom(entries, DesiredNodesMetadata.TYPE, DesiredNodesMetadata::new, DesiredNodesMetadata::readDiffFrom);
249247
registerMetadataCustom(
250248
entries,

‎server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
1414
import org.apache.lucene.util.automaton.Operations;
1515
import org.apache.lucene.util.automaton.RegExp;
16-
import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction;
1716
import org.elasticsearch.action.admin.indices.create.AutoCreateAction;
1817
import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
1918
import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -148,7 +147,7 @@ public class SystemIndexDescriptor implements IndexPatternMatcher, Comparable<Sy
148147

149148
/**
150149
* An optional reindexing script to use when migrating an index created
151-
* before {@link TransportGetFeatureUpgradeStatusAction#NO_UPGRADE_REQUIRED_INDEX_VERSION}.
150+
* before {@link SystemIndices#NO_UPGRADE_REQUIRED_INDEX_VERSION}.
152151
* This script can be used to modify documents before they are added to the new index.
153152
* For example, it can be used to remove deprecated fields from the index.
154153
* <br>

‎server/src/main/java/org/elasticsearch/indices/SystemIndices.java‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@
1515
import org.apache.lucene.util.automaton.Automaton;
1616
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
1717
import org.apache.lucene.util.automaton.Operations;
18+
import org.elasticsearch.Version;
1819
import org.elasticsearch.action.ActionListener;
19-
import org.elasticsearch.action.admin.cluster.migration.TransportGetFeatureUpgradeStatusAction;
2020
import org.elasticsearch.action.admin.cluster.snapshots.features.ResetFeatureStateResponse.ResetFeatureStateStatus;
2121
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
2222
import org.elasticsearch.action.admin.indices.delete.TransportDeleteIndexAction;
@@ -36,7 +36,10 @@
3636
import org.elasticsearch.core.Nullable;
3737
import org.elasticsearch.core.Predicates;
3838
import org.elasticsearch.core.Tuple;
39+
import org.elasticsearch.core.UpdateForV10;
3940
import org.elasticsearch.index.Index;
41+
import org.elasticsearch.index.IndexVersion;
42+
import org.elasticsearch.index.IndexVersions;
4043
import org.elasticsearch.plugins.SystemIndexPlugin;
4144
import org.elasticsearch.snapshots.SnapshotsService;
4245

@@ -111,7 +114,16 @@
111114
public class SystemIndices {
112115
public static final String SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY = "_system_index_access_allowed";
113116
public static final String EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY = "_external_system_index_access_origin";
114-
private static final int UPGRADED_TO_VERSION = TransportGetFeatureUpgradeStatusAction.NO_UPGRADE_REQUIRED_VERSION.major + 1;
117+
118+
/**
119+
* These versions should be set to current major and current major's index version
120+
*/
121+
@UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA)
122+
public static final Version NO_UPGRADE_REQUIRED_VERSION = Version.V_9_0_0;
123+
public static final IndexVersion NO_UPGRADE_REQUIRED_INDEX_VERSION = IndexVersions.UPGRADE_TO_LUCENE_10_0_0;
124+
125+
public static final String MIGRATE_SYSTEM_INDEX_CAUSE = "migrate-system-index";
126+
private static final int UPGRADED_TO_VERSION = NO_UPGRADE_REQUIRED_VERSION.major + 1;
115127
public static final String UPGRADED_INDEX_SUFFIX = "-reindexed-for-" + UPGRADED_TO_VERSION;
116128

117129
private static final Automaton EMPTY = Automata.makeEmpty();
@@ -346,6 +358,7 @@ public ExecutorSelector getExecutorSelector() {
346358

347359
/**
348360
* Finds a single matching {@link SystemIndexDescriptor}, if any, for the given index name.
361+
* Does not take into account system data streams and their backing indices.
349362
* @param name the name of the index
350363
* @return The matching {@link SystemIndexDescriptor} or {@code null} if no descriptor is found
351364
*/
@@ -354,7 +367,7 @@ public ExecutorSelector getExecutorSelector() {
354367
}
355368

356369
@Nullable
357-
static SystemIndexDescriptor findMatchingDescriptor(SystemIndexDescriptor[] indexDescriptors, String name) {
370+
private static SystemIndexDescriptor findMatchingDescriptor(SystemIndexDescriptor[] indexDescriptors, String name) {
358371
SystemIndexDescriptor matchingDescriptor = null;
359372
for (SystemIndexDescriptor systemIndexDescriptor : indexDescriptors) {
360373
if (systemIndexDescriptor.matchesIndexPattern(name)) {

‎server/src/main/java/org/elasticsearch/node/NodeConstruction.java‎

Lines changed: 4 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -216,7 +216,6 @@
216216
import org.elasticsearch.threadpool.internal.BuiltInExecutorBuilders;
217217
import org.elasticsearch.transport.Transport;
218218
import org.elasticsearch.transport.TransportService;
219-
import org.elasticsearch.upgrades.SystemIndexMigrationExecutor;
220219
import org.elasticsearch.usage.UsageService;
221220
import org.elasticsearch.watcher.ResourceWatcherService;
222221
import org.elasticsearch.xcontent.NamedXContentRegistry;
@@ -599,8 +598,7 @@ private void createClientAndRegistries(Settings settings, ThreadPool threadPool,
599598
IndicesModule.getNamedWriteables().stream(),
600599
searchModule.getNamedWriteables().stream(),
601600
pluginsService.flatMap(Plugin::getNamedWriteables),
602-
ClusterModule.getNamedWriteables().stream(),
603-
SystemIndexMigrationExecutor.getNamedWriteables().stream()
601+
ClusterModule.getNamedWriteables().stream()
604602
).flatMap(Function.identity()).toList()
605603
);
606604
xContentRegistry = new NamedXContentRegistry(
@@ -610,7 +608,6 @@ private void createClientAndRegistries(Settings settings, ThreadPool threadPool,
610608
searchModule.getNamedXContents().stream(),
611609
pluginsService.flatMap(Plugin::getNamedXContent),
612610
ClusterModule.getNamedXWriteables().stream(),
613-
SystemIndexMigrationExecutor.getNamedXContentParsers().stream(),
614611
HealthNodeTaskExecutor.getNamedXContentParsers().stream()
615612
).flatMap(Function.identity()).toList()
616613
);
@@ -1154,11 +1151,8 @@ public Map<String, String> searchFields() {
11541151
settingsModule,
11551152
clusterService,
11561153
threadPool,
1157-
systemIndices,
11581154
featureService,
1159-
clusterModule.getIndexNameExpressionResolver(),
1160-
metadataUpdateSettingsService,
1161-
metadataCreateIndexService
1155+
clusterModule.getIndexNameExpressionResolver()
11621156
)
11631157
);
11641158

@@ -1651,29 +1645,18 @@ private Module loadPersistentTasksService(
16511645
SettingsModule settingsModule,
16521646
ClusterService clusterService,
16531647
ThreadPool threadPool,
1654-
SystemIndices systemIndices,
16551648
FeatureService featureService,
1656-
IndexNameExpressionResolver indexNameExpressionResolver,
1657-
MetadataUpdateSettingsService metadataUpdateSettingsService,
1658-
MetadataCreateIndexService metadataCreateIndexService
1649+
IndexNameExpressionResolver indexNameExpressionResolver
16591650
) {
16601651
PersistentTasksService persistentTasksService = new PersistentTasksService(clusterService, threadPool, client);
1661-
SystemIndexMigrationExecutor systemIndexMigrationExecutor = new SystemIndexMigrationExecutor(
1662-
client,
1663-
clusterService,
1664-
systemIndices,
1665-
metadataUpdateSettingsService,
1666-
metadataCreateIndexService,
1667-
settingsModule.getIndexScopedSettings()
1668-
);
16691652
HealthNodeTaskExecutor healthNodeTaskExecutor = HealthNodeTaskExecutor.create(
16701653
clusterService,
16711654
persistentTasksService,
16721655
featureService,
16731656
settingsModule.getSettings(),
16741657
clusterService.getClusterSettings()
16751658
);
1676-
Stream<PersistentTasksExecutor<?>> builtinTaskExecutors = Stream.of(systemIndexMigrationExecutor, healthNodeTaskExecutor);
1659+
Stream<PersistentTasksExecutor<?>> builtinTaskExecutors = Stream.of(healthNodeTaskExecutor);
16771660

16781661
Stream<PersistentTasksExecutor<?>> pluginTaskExecutors = pluginsService.filterPlugins(PersistentTaskPlugin.class)
16791662
.map(p -> p.getPersistentTasksExecutor(clusterService, threadPool, client, settingsModule, indexNameExpressionResolver))

0 commit comments

Comments
 (0)