Skip to content

Commit b9f0075

Browse files
cbueschercbuescher
authored
Add test for deprecated index settings in N-2 indices (#122493) (#123981)
This change adds a new rolling upgrade test that checks having certain deprecated index settings present in an N-2 index doesn't prevent the index from being opened in the current version. The deprecated settings specifically tested here are: * index.mapper.dynamic * index.max_adjacency_matrix_filters * index.force_memory_term_dictionary * index.soft_deletes.enabled
1 parent 0d0a723 commit b9f0075

File tree

3 files changed

+241
-81
lines changed

3 files changed

+241
-81
lines changed

‎qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/RollingUpgradeDeprecatedSettingsIT.java‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.lucene;
11+
12+
import io.netty.handler.codec.http.HttpMethod;
13+
14+
import org.apache.http.HttpHost;
15+
import org.elasticsearch.client.Request;
16+
import org.elasticsearch.client.RestClient;
17+
import org.elasticsearch.client.RestClientBuilder;
18+
import org.elasticsearch.cluster.metadata.IndexMetadata;
19+
import org.elasticsearch.common.settings.Settings;
20+
import org.elasticsearch.index.IndexSettings;
21+
import org.elasticsearch.index.mapper.MapperService;
22+
import org.elasticsearch.index.store.Store;
23+
import org.elasticsearch.test.cluster.util.Version;
24+
25+
import java.io.IOException;
26+
import java.util.List;
27+
28+
import static org.hamcrest.Matchers.equalTo;
29+
30+
public class RollingUpgradeDeprecatedSettingsIT extends RollingUpgradeIndexCompatibilityTestCase {
31+
32+
static {
33+
clusterConfig = config -> config.setting("xpack.license.self_generated.type", "trial");
34+
}
35+
36+
public RollingUpgradeDeprecatedSettingsIT(List<Version> nodesVersions) {
37+
super(nodesVersions);
38+
}
39+
40+
/**
41+
* Creates an index on N-2, upgrades to N -1 and marks as read-only, then remains searchable during rolling upgrades.
42+
*/
43+
@SuppressWarnings("deprecation")
44+
public void testIndexUpgrade() throws Exception {
45+
final String index = suffix("index-rolling-upgraded");
46+
final int numDocs = 2543;
47+
48+
// setup index with deprecated index settings on N-2
49+
if (isFullyUpgradedTo(VERSION_MINUS_2)) {
50+
createIndexLenient(
51+
client(),
52+
index,
53+
Settings.builder()
54+
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
55+
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
56+
// add some index settings deprecated in v7
57+
.put(MapperService.INDEX_MAPPER_DYNAMIC_SETTING.getKey(), true)
58+
.put(IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING.getKey(), 100)
59+
.put(Store.FORCE_RAM_TERM_DICT.getKey(), false)
60+
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false)
61+
.build()
62+
);
63+
indexDocs(index, numDocs);
64+
return;
65+
}
66+
assertThat(indexVersion(index), equalTo(VERSION_MINUS_2));
67+
ensureGreen(index);
68+
69+
if (isIndexClosed(index) == false) {
70+
assertDocCount(client(), index, numDocs);
71+
}
72+
73+
if (isFullyUpgradedTo(VERSION_MINUS_1)) {
74+
final var maybeClose = randomBoolean();
75+
if (maybeClose) {
76+
logger.debug("--> closing index [{}] before upgrade", index);
77+
closeIndex(index);
78+
}
79+
80+
addAndAssertIndexBlocks(index, maybeClose);
81+
return;
82+
}
83+
84+
if (nodesVersions().values().stream().anyMatch(v -> v.onOrAfter(VERSION_CURRENT))) {
85+
final var isClosed = isIndexClosed(index);
86+
assertAndModifyIndexBlocks(index, isClosed);
87+
ensureWriteBlock(index, isClosed);
88+
89+
if (isClosed) {
90+
logger.debug("--> re-opening index [{}] after upgrade", index);
91+
openIndex(index);
92+
ensureGreen(index);
93+
}
94+
95+
assertThat(indexVersion(index), equalTo(VERSION_MINUS_2));
96+
assertDocCount(client(), index, numDocs);
97+
98+
updateRandomIndexSettings(index);
99+
updateRandomMappings(index);
100+
101+
if (randomBoolean()) {
102+
logger.debug("--> random closing of index [{}] before upgrade", index);
103+
closeIndex(index);
104+
ensureGreen(index);
105+
}
106+
}
107+
}
108+
109+
private static void createIndexLenient(RestClient client, String name, Settings settings) throws IOException {
110+
final Request request = newXContentRequest(HttpMethod.PUT, "/" + name, (builder, params) -> {
111+
builder.startObject("settings");
112+
settings.toXContent(builder, params);
113+
builder.endObject();
114+
return builder;
115+
});
116+
client.performRequest(request);
117+
}
118+
119+
/**
120+
* Builds a REST client that will tolerate warnings in the response headers. The default
121+
* is to throw an exception.
122+
*/
123+
@Override
124+
protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException {
125+
RestClientBuilder builder = RestClient.builder(hosts);
126+
configureClient(builder, settings);
127+
builder.setStrictDeprecationMode(false);
128+
return builder.build();
129+
}
130+
}

‎qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/RollingUpgradeIndexCompatibilityTestCase.java‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,15 +14,28 @@
1414
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
1515
import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering;
1616

17+
import org.elasticsearch.cluster.metadata.IndexMetadata;
18+
import org.elasticsearch.common.settings.Settings;
1719
import org.elasticsearch.test.cluster.util.Version;
1820

1921
import java.util.Comparator;
2022
import java.util.List;
2123
import java.util.stream.Stream;
2224

25+
import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_READ_ONLY_BLOCK;
26+
import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_WRITE_BLOCK;
27+
import static org.elasticsearch.cluster.metadata.MetadataIndexStateService.INDEX_CLOSED_BLOCK;
28+
import static org.elasticsearch.cluster.metadata.MetadataIndexStateService.VERIFIED_BEFORE_CLOSE_SETTING;
29+
import static org.elasticsearch.cluster.metadata.MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING;
2330
import static org.elasticsearch.test.cluster.util.Version.CURRENT;
31+
import static org.hamcrest.Matchers.allOf;
32+
import static org.hamcrest.Matchers.contains;
33+
import static org.hamcrest.Matchers.either;
2434
import static org.hamcrest.Matchers.equalTo;
35+
import static org.hamcrest.Matchers.hasItem;
2536
import static org.hamcrest.Matchers.hasSize;
37+
import static org.hamcrest.Matchers.is;
38+
import static org.hamcrest.Matchers.not;
2639
import static org.hamcrest.Matchers.notNullValue;
2740

2841
/**
@@ -86,6 +99,101 @@ protected void maybeUpgrade() throws Exception {
8699
}
87100
}
88101

102+
protected void addAndAssertIndexBlocks(String index, boolean maybeClose) throws Exception {
103+
final var randomBlocks = randomFrom(
104+
List.of(IndexMetadata.APIBlock.WRITE, IndexMetadata.APIBlock.READ_ONLY),
105+
List.of(IndexMetadata.APIBlock.READ_ONLY),
106+
List.of(IndexMetadata.APIBlock.WRITE)
107+
);
108+
for (var randomBlock : randomBlocks) {
109+
addIndexBlock(index, randomBlock);
110+
assertThat(indexBlocks(index), hasItem(randomBlock.getBlock()));
111+
}
112+
113+
assertThat(indexBlocks(index), maybeClose ? hasItem(INDEX_CLOSED_BLOCK) : not(hasItem(INDEX_CLOSED_BLOCK)));
114+
assertIndexSetting(index, VERIFIED_BEFORE_CLOSE_SETTING, is(maybeClose));
115+
assertIndexSetting(index, VERIFIED_READ_ONLY_SETTING, is(true));
116+
return;
117+
}
118+
119+
/**
120+
* assert that index has either a read-only block or write block, if index is closed also a cloded-block.
121+
* In case both blocks are present, randomly modify one of them.
122+
*/
123+
protected void assertAndModifyIndexBlocks(String index, boolean isClosed) throws Exception {
124+
logger.debug("--> upgraded index [{}] is now in [{}] state", index, isClosed ? "closed" : "open");
125+
assertThat(
126+
indexBlocks(index),
127+
allOf(
128+
either(hasItem(INDEX_READ_ONLY_BLOCK)).or(hasItem(INDEX_WRITE_BLOCK)),
129+
isClosed ? hasItem(INDEX_CLOSED_BLOCK) : not(hasItem(INDEX_CLOSED_BLOCK))
130+
)
131+
);
132+
assertIndexSetting(index, VERIFIED_BEFORE_CLOSE_SETTING, is(isClosed));
133+
assertIndexSetting(index, VERIFIED_READ_ONLY_SETTING, is(true));
134+
135+
var blocks = indexBlocks(index).stream().filter(c -> c.equals(INDEX_WRITE_BLOCK) || c.equals(INDEX_READ_ONLY_BLOCK)).toList();
136+
if (blocks.size() == 2) {
137+
switch (randomInt(2)) {
138+
case 0:
139+
updateIndexSettings(
140+
index,
141+
Settings.builder()
142+
.putNull(IndexMetadata.APIBlock.WRITE.settingName())
143+
.put(IndexMetadata.APIBlock.READ_ONLY.settingName(), true)
144+
);
145+
assertThat(
146+
indexBlocks(index),
147+
isClosed ? contains(INDEX_CLOSED_BLOCK, INDEX_READ_ONLY_BLOCK) : contains(INDEX_READ_ONLY_BLOCK)
148+
);
149+
break;
150+
case 1:
151+
updateIndexSettings(
152+
index,
153+
Settings.builder()
154+
.putNull(IndexMetadata.APIBlock.READ_ONLY.settingName())
155+
.put(IndexMetadata.APIBlock.WRITE.settingName(), true)
156+
);
157+
assertThat(
158+
indexBlocks(index),
159+
isClosed ? contains(INDEX_CLOSED_BLOCK, INDEX_WRITE_BLOCK) : contains(INDEX_WRITE_BLOCK)
160+
);
161+
break;
162+
case 2:
163+
updateIndexSettings(index, Settings.builder().put(IndexMetadata.APIBlock.READ_ONLY.settingName(), false));
164+
assertThat(
165+
indexBlocks(index),
166+
isClosed ? contains(INDEX_CLOSED_BLOCK, INDEX_WRITE_BLOCK) : contains(INDEX_WRITE_BLOCK)
167+
);
168+
break;
169+
default:
170+
throw new AssertionError();
171+
}
172+
}
173+
}
174+
175+
/**
176+
* ensure we have a write-block. If this is currently not the case, set it.
177+
*/
178+
protected void ensureWriteBlock(String index, boolean isClosed) throws Exception {
179+
List<org.elasticsearch.cluster.block.ClusterBlock> blocks = indexBlocks(index).stream()
180+
.filter(c -> c.equals(INDEX_WRITE_BLOCK) || c.equals(INDEX_READ_ONLY_BLOCK))
181+
.toList();
182+
if (blocks.contains(INDEX_READ_ONLY_BLOCK)) {
183+
logger.debug("--> read_only API block can be replaced by a write block (required for the remaining tests)");
184+
updateIndexSettings(
185+
index,
186+
Settings.builder()
187+
.putNull(IndexMetadata.APIBlock.READ_ONLY.settingName())
188+
.put(IndexMetadata.APIBlock.WRITE.settingName(), true)
189+
);
190+
}
191+
192+
assertIndexSetting(index, VERIFIED_READ_ONLY_SETTING, is(true));
193+
assertIndexSetting(index, VERIFIED_BEFORE_CLOSE_SETTING, is(isClosed));
194+
assertThat(indexBlocks(index), isClosed ? contains(INDEX_CLOSED_BLOCK, INDEX_WRITE_BLOCK) : contains(INDEX_WRITE_BLOCK));
195+
}
196+
89197
/**
90198
* Execute the test suite with the parameters provided by the {@link #parameters()} in nodes versions order.
91199
*/

‎qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/RollingUpgradeLuceneIndexCompatibilityTestCase.java‎

Lines changed: 3 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -18,19 +18,14 @@
1818

1919
import java.util.List;
2020

21-
import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_READ_ONLY_BLOCK;
2221
import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_WRITE_BLOCK;
2322
import static org.elasticsearch.cluster.metadata.MetadataIndexStateService.INDEX_CLOSED_BLOCK;
2423
import static org.elasticsearch.cluster.metadata.MetadataIndexStateService.VERIFIED_BEFORE_CLOSE_SETTING;
2524
import static org.elasticsearch.cluster.metadata.MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING;
26-
import static org.hamcrest.Matchers.allOf;
2725
import static org.hamcrest.Matchers.contains;
2826
import static org.hamcrest.Matchers.containsString;
29-
import static org.hamcrest.Matchers.either;
3027
import static org.hamcrest.Matchers.equalTo;
31-
import static org.hamcrest.Matchers.hasItem;
3228
import static org.hamcrest.Matchers.is;
33-
import static org.hamcrest.Matchers.not;
3429

3530
public class RollingUpgradeLuceneIndexCompatibilityTestCase extends RollingUpgradeIndexCompatibilityTestCase {
3631

@@ -73,88 +68,15 @@ public void testIndexUpgrade() throws Exception {
7368
closeIndex(index);
7469
}
7570

76-
final var randomBlocks = randomFrom(
77-
List.of(IndexMetadata.APIBlock.WRITE, IndexMetadata.APIBlock.READ_ONLY),
78-
List.of(IndexMetadata.APIBlock.READ_ONLY),
79-
List.of(IndexMetadata.APIBlock.WRITE)
80-
);
81-
for (var randomBlock : randomBlocks) {
82-
addIndexBlock(index, randomBlock);
83-
assertThat(indexBlocks(index), hasItem(randomBlock.getBlock()));
84-
}
85-
86-
assertThat(indexBlocks(index), maybeClose ? hasItem(INDEX_CLOSED_BLOCK) : not(hasItem(INDEX_CLOSED_BLOCK)));
87-
assertIndexSetting(index, VERIFIED_BEFORE_CLOSE_SETTING, is(maybeClose));
88-
assertIndexSetting(index, VERIFIED_READ_ONLY_SETTING, is(true));
71+
addAndAssertIndexBlocks(index, maybeClose);
8972
return;
9073
}
9174

9275
if (nodesVersions().values().stream().anyMatch(v -> v.onOrAfter(VERSION_CURRENT))) {
9376
final var isClosed = isIndexClosed(index);
94-
logger.debug("--> upgraded index [{}] is now in [{}] state", index, isClosed ? "closed" : "open");
95-
assertThat(
96-
indexBlocks(index),
97-
allOf(
98-
either(hasItem(INDEX_READ_ONLY_BLOCK)).or(hasItem(INDEX_WRITE_BLOCK)),
99-
isClosed ? hasItem(INDEX_CLOSED_BLOCK) : not(hasItem(INDEX_CLOSED_BLOCK))
100-
)
101-
);
102-
assertIndexSetting(index, VERIFIED_BEFORE_CLOSE_SETTING, is(isClosed));
103-
assertIndexSetting(index, VERIFIED_READ_ONLY_SETTING, is(true));
77+
assertAndModifyIndexBlocks(index, isClosed);
10478

105-
var blocks = indexBlocks(index).stream().filter(c -> c.equals(INDEX_WRITE_BLOCK) || c.equals(INDEX_READ_ONLY_BLOCK)).toList();
106-
if (blocks.size() == 2) {
107-
switch (randomInt(2)) {
108-
case 0:
109-
updateIndexSettings(
110-
index,
111-
Settings.builder()
112-
.putNull(IndexMetadata.APIBlock.WRITE.settingName())
113-
.put(IndexMetadata.APIBlock.READ_ONLY.settingName(), true)
114-
);
115-
assertThat(
116-
indexBlocks(index),
117-
isClosed ? contains(INDEX_CLOSED_BLOCK, INDEX_READ_ONLY_BLOCK) : contains(INDEX_READ_ONLY_BLOCK)
118-
);
119-
break;
120-
case 1:
121-
updateIndexSettings(
122-
index,
123-
Settings.builder()
124-
.putNull(IndexMetadata.APIBlock.READ_ONLY.settingName())
125-
.put(IndexMetadata.APIBlock.WRITE.settingName(), true)
126-
);
127-
assertThat(
128-
indexBlocks(index),
129-
isClosed ? contains(INDEX_CLOSED_BLOCK, INDEX_WRITE_BLOCK) : contains(INDEX_WRITE_BLOCK)
130-
);
131-
break;
132-
case 2:
133-
updateIndexSettings(index, Settings.builder().put(IndexMetadata.APIBlock.READ_ONLY.settingName(), false));
134-
assertThat(
135-
indexBlocks(index),
136-
isClosed ? contains(INDEX_CLOSED_BLOCK, INDEX_WRITE_BLOCK) : contains(INDEX_WRITE_BLOCK)
137-
);
138-
break;
139-
default:
140-
throw new AssertionError();
141-
}
142-
}
143-
144-
blocks = indexBlocks(index).stream().filter(c -> c.equals(INDEX_WRITE_BLOCK) || c.equals(INDEX_READ_ONLY_BLOCK)).toList();
145-
if (blocks.contains(INDEX_READ_ONLY_BLOCK)) {
146-
logger.debug("--> read_only API block can be replaced by a write block (required for the remaining tests)");
147-
updateIndexSettings(
148-
index,
149-
Settings.builder()
150-
.putNull(IndexMetadata.APIBlock.READ_ONLY.settingName())
151-
.put(IndexMetadata.APIBlock.WRITE.settingName(), true)
152-
);
153-
}
154-
155-
assertIndexSetting(index, VERIFIED_READ_ONLY_SETTING, is(true));
156-
assertIndexSetting(index, VERIFIED_BEFORE_CLOSE_SETTING, is(isClosed));
157-
assertThat(indexBlocks(index), isClosed ? contains(INDEX_CLOSED_BLOCK, INDEX_WRITE_BLOCK) : contains(INDEX_WRITE_BLOCK));
79+
ensureWriteBlock(index, isClosed);
15880

15981
if (isClosed) {
16082
logger.debug("--> re-opening index [{}] after upgrade", index);

0 commit comments

Comments
 (0)