Skip to content

Commit 5e2fac3

Browse files
dnhatndnhatn
authored
Add remove index setting command (#109276) (#109328)
The new subcommand elasticsearch-node remove-index-settings can be used to remove index settings from the cluster state in case where it contains incompatible index settings that prevent the cluster from forming. This tool can cause data loss and its use should be your last resort. Relates #96075
1 parent 692a1a2 commit 5e2fac3

File tree

5 files changed

+326
-0
lines changed

5 files changed

+326
-0
lines changed

‎docs/changelog/109276.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 109276
2+
summary: Add remove index setting command
3+
area: Infra/Settings
4+
type: enhancement
5+
issues: []

‎docs/reference/commands/node-tool.asciidoc‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,10 @@ This tool has a number of modes:
3131
from the cluster state in case where it contains incompatible settings that
3232
prevent the cluster from forming.
3333

34+
* `elasticsearch-node remove-index-settings` can be used to remove index settings
35+
from the cluster state in case where it contains incompatible index settings that
36+
prevent the cluster from forming.
37+
3438
* `elasticsearch-node remove-customs` can be used to remove custom metadata
3539
from the cluster state in case where it contains broken metadata that
3640
prevents the cluster state from being loaded.
@@ -103,6 +107,26 @@ The intended use is:
103107
* Repeat for all other master-eligible nodes
104108
* Start the nodes
105109

110+
[discrete]
111+
==== Removing index settings
112+
113+
There may be situations where an index contains index settings
114+
that prevent the cluster from forming. Since the cluster cannot form,
115+
it is not possible to remove these settings using the
116+
<<indices-update-settings>> API.
117+
118+
The `elasticsearch-node remove-index-settings` tool allows you to forcefully remove
119+
those index settings from the on-disk cluster state. The tool takes a
120+
list of index settings as parameters that should be removed, and also supports
121+
wildcard patterns.
122+
123+
The intended use is:
124+
125+
* Stop the node
126+
* Run `elasticsearch-node remove-index-settings name-of-index-setting-to-remove` on the node
127+
* Repeat for all nodes
128+
* Start the nodes
129+
106130
[discrete]
107131
==== Removing custom metadata from the cluster state
108132

@@ -433,6 +457,37 @@ You can also use wildcards to remove multiple settings, for example using
433457
node$ ./bin/elasticsearch-node remove-settings xpack.monitoring.*
434458
----
435459

460+
[discrete]
461+
==== Removing index settings
462+
463+
If your indices contain index settings that prevent the cluster
464+
from forming, you can run the following command to remove one
465+
or more index settings.
466+
467+
[source,txt]
468+
----
469+
node$ ./bin/elasticsearch-node remove-index-settings index.my_plugin.foo
470+
471+
WARNING: Elasticsearch MUST be stopped before running this tool.
472+
473+
You should only run this tool if you have incompatible index settings in the
474+
cluster state that prevent the cluster from forming.
475+
This tool can cause data loss and its use should be your last resort.
476+
477+
Do you want to proceed?
478+
479+
Confirm [y/N] y
480+
481+
Index settings were successfully removed from the cluster state
482+
----
483+
484+
You can also use wildcards to remove multiple index settings, for example using
485+
486+
[source,txt]
487+
----
488+
node$ ./bin/elasticsearch-node remove-index-settings index.my_plugin.*
489+
----
490+
436491
[discrete]
437492
==== Removing custom metadata from the cluster state
438493

‎server/src/internalClusterTest/java/org/elasticsearch/cluster/coordination/RemoveIndexSettingsCommandIT.java‎

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0 and the Server Side Public License, v 1; you may not use this file except
5+
* in compliance with, at your election, the Elastic License 2.0 or the Server
6+
* Side Public License, v 1.
7+
*/
8+
package org.elasticsearch.cluster.coordination;
9+
10+
import joptsimple.OptionSet;
11+
12+
import org.elasticsearch.ElasticsearchException;
13+
import org.elasticsearch.cli.MockTerminal;
14+
import org.elasticsearch.cli.UserException;
15+
import org.elasticsearch.common.collect.ImmutableOpenMap;
16+
import org.elasticsearch.common.settings.Setting;
17+
import org.elasticsearch.common.settings.Settings;
18+
import org.elasticsearch.common.util.CollectionUtils;
19+
import org.elasticsearch.env.Environment;
20+
import org.elasticsearch.env.TestEnvironment;
21+
import org.elasticsearch.plugins.Plugin;
22+
import org.elasticsearch.test.ESIntegTestCase;
23+
24+
import java.util.Arrays;
25+
import java.util.Collection;
26+
import java.util.List;
27+
28+
import static org.hamcrest.Matchers.containsString;
29+
import static org.hamcrest.Matchers.equalTo;
30+
import static org.hamcrest.Matchers.not;
31+
32+
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
33+
public class RemoveIndexSettingsCommandIT extends ESIntegTestCase {
34+
35+
static final Setting<Integer> FOO = Setting.intSetting("index.foo", 1, Setting.Property.IndexScope, Setting.Property.Dynamic);
36+
static final Setting<Integer> BAR = Setting.intSetting("index.bar", 2, Setting.Property.IndexScope, Setting.Property.Final);
37+
38+
public static class ExtraSettingsPlugin extends Plugin {
39+
@Override
40+
public List<Setting<?>> getSettings() {
41+
return Arrays.asList(FOO, BAR);
42+
}
43+
}
44+
45+
@Override
46+
protected Collection<Class<? extends Plugin>> nodePlugins() {
47+
return CollectionUtils.appendToCopy(super.nodePlugins(), ExtraSettingsPlugin.class);
48+
}
49+
50+
public void testRemoveSettingsAbortedByUser() throws Exception {
51+
internalCluster().setBootstrapMasterNodeIndex(0);
52+
String node = internalCluster().startNode();
53+
createIndex("test-index", Settings.builder().put(FOO.getKey(), 101).put(BAR.getKey(), 102).build());
54+
ensureYellow("test-index");
55+
Settings dataPathSettings = internalCluster().dataPathSettings(node);
56+
ensureStableCluster(1);
57+
internalCluster().stopRandomDataNode();
58+
59+
Settings nodeSettings = Settings.builder().put(internalCluster().getDefaultSettings()).put(dataPathSettings).build();
60+
ElasticsearchException error = expectThrows(
61+
ElasticsearchException.class,
62+
() -> removeIndexSettings(TestEnvironment.newEnvironment(nodeSettings), true, "index.foo")
63+
);
64+
assertThat(error.getMessage(), equalTo(ElasticsearchNodeCommand.ABORTED_BY_USER_MSG));
65+
internalCluster().startNode(nodeSettings);
66+
}
67+
68+
public void testRemoveSettingsSuccessful() throws Exception {
69+
internalCluster().setBootstrapMasterNodeIndex(0);
70+
String node = internalCluster().startNode();
71+
Settings dataPathSettings = internalCluster().dataPathSettings(node);
72+
73+
int numIndices = randomIntBetween(1, 10);
74+
int[] barValues = new int[numIndices];
75+
for (int i = 0; i < numIndices; i++) {
76+
String index = "test-index-" + i;
77+
barValues[i] = between(1, 1000);
78+
createIndex(index, Settings.builder().put(FOO.getKey(), between(1, 1000)).put(BAR.getKey(), barValues[i]).build());
79+
}
80+
int moreIndices = randomIntBetween(1, 10);
81+
for (int i = 0; i < moreIndices; i++) {
82+
createIndex("more-index-" + i, Settings.EMPTY);
83+
}
84+
internalCluster().stopNode(node);
85+
86+
Environment environment = TestEnvironment.newEnvironment(
87+
Settings.builder().put(internalCluster().getDefaultSettings()).put(dataPathSettings).build()
88+
);
89+
90+
MockTerminal terminal = removeIndexSettings(environment, false, "index.foo");
91+
assertThat(terminal.getOutput(), containsString(RemoveIndexSettingsCommand.SETTINGS_REMOVED_MSG));
92+
for (int i = 0; i < numIndices; i++) {
93+
assertThat(terminal.getOutput(), containsString("Index setting [index.foo] will be removed from index [[test-index-" + i));
94+
}
95+
for (int i = 0; i < moreIndices; i++) {
96+
assertThat(terminal.getOutput(), not(containsString("Index setting [index.foo] will be removed from index [[more-index-" + i)));
97+
}
98+
Settings nodeSettings = Settings.builder().put(internalCluster().getDefaultSettings()).put(dataPathSettings).build();
99+
internalCluster().startNode(nodeSettings);
100+
101+
ImmutableOpenMap<String, Settings> getIndexSettings = client().admin()
102+
.indices()
103+
.prepareGetSettings("test-index-*")
104+
.get()
105+
.getIndexToSettings();
106+
for (int i = 0; i < numIndices; i++) {
107+
String index = "test-index-" + i;
108+
Settings indexSettings = getIndexSettings.get(index);
109+
assertFalse(indexSettings.hasValue("index.foo"));
110+
assertThat(indexSettings.get("index.bar"), equalTo(Integer.toString(barValues[i])));
111+
}
112+
getIndexSettings = client().admin().indices().prepareGetSettings("more-index-*").get().getIndexToSettings();
113+
for (int i = 0; i < moreIndices; i++) {
114+
assertNotNull(getIndexSettings.get("more-index-" + i));
115+
}
116+
}
117+
118+
public void testSettingDoesNotMatch() throws Exception {
119+
internalCluster().setBootstrapMasterNodeIndex(0);
120+
String node = internalCluster().startNode();
121+
createIndex("test-index", Settings.builder().put(FOO.getKey(), 101).put(BAR.getKey(), 102).build());
122+
ensureYellow("test-index");
123+
Settings dataPathSettings = internalCluster().dataPathSettings(node);
124+
ensureStableCluster(1);
125+
internalCluster().stopRandomDataNode();
126+
127+
Settings nodeSettings = Settings.builder().put(internalCluster().getDefaultSettings()).put(dataPathSettings).build();
128+
UserException error = expectThrows(
129+
UserException.class,
130+
() -> removeIndexSettings(TestEnvironment.newEnvironment(nodeSettings), true, "index.not_foo")
131+
);
132+
assertThat(error.getMessage(), containsString("No index setting matching [index.not_foo] were found on this node"));
133+
internalCluster().startNode(nodeSettings);
134+
}
135+
136+
private MockTerminal executeCommand(ElasticsearchNodeCommand command, Environment environment, boolean abort, String... args)
137+
throws Exception {
138+
final MockTerminal terminal = new MockTerminal();
139+
final OptionSet options = command.getParser().parse(args);
140+
final String input;
141+
142+
if (abort) {
143+
input = randomValueOtherThanMany(c -> c.equalsIgnoreCase("y"), () -> randomAlphaOfLength(1));
144+
} else {
145+
input = randomBoolean() ? "y" : "Y";
146+
}
147+
148+
terminal.addTextInput(input);
149+
150+
try {
151+
command.execute(terminal, options, environment);
152+
} finally {
153+
assertThat(terminal.getOutput(), containsString(ElasticsearchNodeCommand.STOP_WARNING_MSG));
154+
}
155+
156+
return terminal;
157+
}
158+
159+
private MockTerminal removeIndexSettings(Environment environment, boolean abort, String... args) throws Exception {
160+
final MockTerminal terminal = executeCommand(new RemoveIndexSettingsCommand(), environment, abort, args);
161+
assertThat(terminal.getOutput(), containsString(RemoveIndexSettingsCommand.CONFIRMATION_MSG));
162+
assertThat(terminal.getOutput(), containsString(RemoveIndexSettingsCommand.SETTINGS_REMOVED_MSG));
163+
return terminal;
164+
}
165+
}

‎server/src/main/java/org/elasticsearch/cluster/coordination/NodeToolCli.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ public NodeToolCli() {
3030
subcommands.put("detach-cluster", new DetachClusterCommand());
3131
subcommands.put("override-version", new OverrideNodeVersionCommand());
3232
subcommands.put("remove-settings", new RemoveSettingsCommand());
33+
subcommands.put("remove-index-settings", new RemoveIndexSettingsCommand());
3334
subcommands.put("remove-customs", new RemoveCustomsCommand());
3435
}
3536

‎server/src/main/java/org/elasticsearch/cluster/coordination/RemoveIndexSettingsCommand.java‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0 and the Server Side Public License, v 1; you may not use this file except
5+
* in compliance with, at your election, the Elastic License 2.0 or the Server
6+
* Side Public License, v 1.
7+
*/
8+
package org.elasticsearch.cluster.coordination;
9+
10+
import joptsimple.OptionSet;
11+
import joptsimple.OptionSpec;
12+
13+
import org.elasticsearch.cli.ExitCodes;
14+
import org.elasticsearch.cli.Terminal;
15+
import org.elasticsearch.cli.UserException;
16+
import org.elasticsearch.cluster.ClusterState;
17+
import org.elasticsearch.cluster.metadata.IndexMetadata;
18+
import org.elasticsearch.cluster.metadata.Metadata;
19+
import org.elasticsearch.common.regex.Regex;
20+
import org.elasticsearch.common.settings.Settings;
21+
import org.elasticsearch.core.Tuple;
22+
import org.elasticsearch.env.Environment;
23+
import org.elasticsearch.gateway.PersistedClusterStateService;
24+
25+
import java.io.IOException;
26+
import java.nio.file.Path;
27+
import java.util.List;
28+
29+
public class RemoveIndexSettingsCommand extends ElasticsearchNodeCommand {
30+
31+
static final String SETTINGS_REMOVED_MSG = "Index settings were successfully removed from the cluster state";
32+
static final String CONFIRMATION_MSG = DELIMITER
33+
+ "\n"
34+
+ "You should only run this tool if you have incompatible index settings in the\n"
35+
+ "cluster state that prevent the cluster from forming.\n"
36+
+ "This tool can cause data loss and its use should be your last resort.\n"
37+
+ "\n"
38+
+ "Do you want to proceed?\n";
39+
40+
private final OptionSpec<String> arguments;
41+
42+
public RemoveIndexSettingsCommand() {
43+
super("Removes index settings from the cluster state");
44+
arguments = parser.nonOptions("index setting names");
45+
}
46+
47+
@Override
48+
protected void processDataPaths(Terminal terminal, Path[] dataPaths, int nodeLockId, OptionSet options, Environment env)
49+
throws IOException, UserException {
50+
final List<String> settingsToRemove = arguments.values(options);
51+
if (settingsToRemove.isEmpty()) {
52+
throw new UserException(ExitCodes.USAGE, "Must supply at least one index setting to remove");
53+
}
54+
55+
final PersistedClusterStateService persistedClusterStateService = createPersistedClusterStateService(env.settings(), dataPaths);
56+
57+
terminal.println(Terminal.Verbosity.VERBOSE, "Loading cluster state");
58+
final Tuple<Long, ClusterState> termAndClusterState = loadTermAndClusterState(persistedClusterStateService, env);
59+
final ClusterState oldClusterState = termAndClusterState.v2();
60+
final Metadata.Builder newMetadataBuilder = Metadata.builder(oldClusterState.metadata());
61+
int changes = 0;
62+
for (IndexMetadata indexMetadata : oldClusterState.metadata()) {
63+
Settings oldSettings = indexMetadata.getSettings();
64+
Settings.Builder newSettings = Settings.builder().put(oldSettings);
65+
boolean removed = false;
66+
for (String settingToRemove : settingsToRemove) {
67+
for (String settingKey : oldSettings.keySet()) {
68+
if (Regex.simpleMatch(settingToRemove, settingKey)) {
69+
terminal.println(
70+
"Index setting [" + settingKey + "] will be removed from index [" + indexMetadata.getIndex() + "]"
71+
);
72+
newSettings.remove(settingKey);
73+
removed = true;
74+
}
75+
}
76+
}
77+
if (removed) {
78+
newMetadataBuilder.put(IndexMetadata.builder(indexMetadata).settings(newSettings));
79+
changes++;
80+
}
81+
}
82+
if (changes == 0) {
83+
throw new UserException(ExitCodes.USAGE, "No index setting matching " + settingsToRemove + " were found on this node");
84+
}
85+
86+
final ClusterState newClusterState = ClusterState.builder(oldClusterState).metadata(newMetadataBuilder).build();
87+
terminal.println(
88+
Terminal.Verbosity.VERBOSE,
89+
"[old cluster state = " + oldClusterState + ", new cluster state = " + newClusterState + "]"
90+
);
91+
92+
confirm(terminal, CONFIRMATION_MSG);
93+
94+
try (PersistedClusterStateService.Writer writer = persistedClusterStateService.createWriter()) {
95+
writer.writeFullStateAndCommit(termAndClusterState.v1(), newClusterState);
96+
}
97+
98+
terminal.println(SETTINGS_REMOVED_MSG);
99+
}
100+
}

0 commit comments

Comments
 (0)