Skip to content

Commit c62ee88

Browse files
tlrxtlrx
authored
Add rolling upgrade BWC tests for N-2 index compatibility (#119468)
This change adds rolling upgrade BWC tests for searchable snapshots in N-2 version, so that they are tested in a mixed versions cluster. The tests do not use replicas because it may or may not be assigned depending on which node the primary is assigned on (ie, if the cluster has 1 new node and 2 old node and the primary shard is assigned to the new node, then a replica cannot be assigned until at least another node is upgraded). It also renames the existing full cluster restart BWC tests and mutualizes some common code in AbstractIndexCompatibilityTestCase. Relates ES-10432
1 parent d2d0636 commit c62ee88

6 files changed

+380
-72
lines changed

‎qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractLuceneIndexCompatibilityTestCase.java‎ renamed to ‎qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractIndexCompatibilityTestCase.java‎

Lines changed: 58 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,6 @@
99

1010
package org.elasticsearch.lucene;
1111

12-
import com.carrotsearch.randomizedtesting.TestMethodAndParams;
13-
import com.carrotsearch.randomizedtesting.annotations.Name;
14-
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
15-
import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering;
16-
1712
import org.elasticsearch.client.Request;
1813
import org.elasticsearch.common.settings.Settings;
1914
import org.elasticsearch.core.Strings;
@@ -22,16 +17,18 @@
2217
import org.elasticsearch.test.cluster.local.distribution.DistributionType;
2318
import org.elasticsearch.test.cluster.util.Version;
2419
import org.elasticsearch.test.rest.ESRestTestCase;
20+
import org.junit.After;
2521
import org.junit.Before;
2622
import org.junit.ClassRule;
2723
import org.junit.rules.RuleChain;
2824
import org.junit.rules.TemporaryFolder;
2925
import org.junit.rules.TestRule;
3026

31-
import java.util.Comparator;
27+
import java.io.IOException;
28+
import java.util.HashMap;
3229
import java.util.Locale;
30+
import java.util.Map;
3331
import java.util.stream.IntStream;
34-
import java.util.stream.Stream;
3532

3633
import static org.elasticsearch.test.cluster.util.Version.CURRENT;
3734
import static org.elasticsearch.test.cluster.util.Version.fromString;
@@ -41,24 +38,21 @@
4138
import static org.hamcrest.Matchers.is;
4239
import static org.hamcrest.Matchers.notNullValue;
4340

44-
/**
45-
* Test suite for Lucene indices backward compatibility with N-2 versions. The test suite creates a cluster in N-2 version, then upgrades it
46-
* to N-1 version and finally upgrades it to the current version. Test methods are executed after each upgrade.
47-
*/
48-
@TestCaseOrdering(AbstractLuceneIndexCompatibilityTestCase.TestCaseOrdering.class)
49-
public abstract class AbstractLuceneIndexCompatibilityTestCase extends ESRestTestCase {
41+
public abstract class AbstractIndexCompatibilityTestCase extends ESRestTestCase {
5042

5143
protected static final Version VERSION_MINUS_2 = fromString(System.getProperty("tests.minimum.index.compatible"));
5244
protected static final Version VERSION_MINUS_1 = fromString(System.getProperty("tests.minimum.wire.compatible"));
5345
protected static final Version VERSION_CURRENT = CURRENT;
5446

55-
protected static TemporaryFolder REPOSITORY_PATH = new TemporaryFolder();
47+
protected static final int NODES = 3;
48+
49+
private static TemporaryFolder REPOSITORY_PATH = new TemporaryFolder();
5650

5751
protected static LocalClusterConfigProvider clusterConfig = c -> {};
5852
private static ElasticsearchCluster cluster = ElasticsearchCluster.local()
5953
.distribution(DistributionType.DEFAULT)
6054
.version(VERSION_MINUS_2)
61-
.nodes(2)
55+
.nodes(NODES)
6256
.setting("path.repo", () -> REPOSITORY_PATH.getRoot().getPath())
6357
.setting("xpack.security.enabled", "false")
6458
.setting("xpack.ml.enabled", "false")
@@ -71,15 +65,44 @@ public abstract class AbstractLuceneIndexCompatibilityTestCase extends ESRestTes
7165

7266
private static boolean upgradeFailed = false;
7367

74-
private final Version clusterVersion;
68+
@Before
69+
public final void maybeUpgradeBeforeTest() throws Exception {
70+
// We want to use this test suite for the V9 upgrade, but we are not fully committed to necessarily having N-2 support
71+
// in V10, so we add a check here to ensure we'll revisit this decision once V10 exists.
72+
assertThat("Explicit check that N-2 version is Elasticsearch 7", VERSION_MINUS_2.getMajor(), equalTo(7));
73+
74+
if (upgradeFailed == false) {
75+
try {
76+
maybeUpgrade();
77+
} catch (Exception e) {
78+
upgradeFailed = true;
79+
throw e;
80+
}
81+
}
7582

76-
public AbstractLuceneIndexCompatibilityTestCase(@Name("cluster") Version clusterVersion) {
77-
this.clusterVersion = clusterVersion;
83+
// Skip remaining tests if upgrade failed
84+
assumeFalse("Cluster upgrade failed", upgradeFailed);
7885
}
7986

80-
@ParametersFactory
81-
public static Iterable<Object[]> parameters() {
82-
return Stream.of(VERSION_MINUS_2, VERSION_MINUS_1, CURRENT).map(v -> new Object[] { v }).toList();
87+
protected abstract void maybeUpgrade() throws Exception;
88+
89+
@After
90+
public final void deleteSnapshotBlobCache() throws IOException {
91+
// TODO ES-10475: The .snapshot-blob-cache created in legacy version can block upgrades, we should probably delete it automatically
92+
try {
93+
var request = new Request("DELETE", "/.snapshot-blob-cache");
94+
request.setOptions(
95+
expectWarnings(
96+
"this request accesses system indices: [.snapshot-blob-cache], but in a future major version, "
97+
+ "direct access to system indices will be prevented by default"
98+
)
99+
);
100+
adminClient().performRequest(request);
101+
} catch (IOException e) {
102+
if (isNotFoundResponseException(e) == false) {
103+
throw e;
104+
}
105+
}
83106
}
84107

85108
@Override
@@ -92,26 +115,8 @@ protected boolean preserveClusterUponCompletion() {
92115
return true;
93116
}
94117

95-
@Before
96-
public void maybeUpgrade() throws Exception {
97-
// We want to use this test suite for the V9 upgrade, but we are not fully committed to necessarily having N-2 support
98-
// in V10, so we add a check here to ensure we'll revisit this decision once V10 exists.
99-
assertThat("Explicit check that N-2 version is Elasticsearch 7", VERSION_MINUS_2.getMajor(), equalTo(7));
100-
101-
var currentVersion = clusterVersion();
102-
if (currentVersion.before(clusterVersion)) {
103-
try {
104-
cluster.upgradeToVersion(clusterVersion);
105-
closeClients();
106-
initClient();
107-
} catch (Exception e) {
108-
upgradeFailed = true;
109-
throw e;
110-
}
111-
}
112-
113-
// Skip remaining tests if upgrade failed
114-
assumeFalse("Cluster upgrade failed", upgradeFailed);
118+
protected ElasticsearchCluster cluster() {
119+
return cluster;
115120
}
116121

117122
protected String suffix(String name) {
@@ -124,12 +129,18 @@ protected Settings repositorySettings() {
124129
.build();
125130
}
126131

127-
protected static Version clusterVersion() throws Exception {
128-
var response = assertOK(client().performRequest(new Request("GET", "/")));
129-
var responseBody = createFromResponse(response);
130-
var version = Version.fromString(responseBody.evaluate("version.number").toString());
131-
assertThat("Failed to retrieve cluster version", version, notNullValue());
132-
return version;
132+
protected static Map<String, Version> nodesVersions() throws Exception {
133+
var nodesInfos = getNodesInfo(adminClient());
134+
assertThat(nodesInfos.size(), equalTo(NODES));
135+
var versions = new HashMap<String, Version>();
136+
for (var nodeInfos : nodesInfos.values()) {
137+
versions.put((String) nodeInfos.get("name"), Version.fromString((String) nodeInfos.get("version")));
138+
}
139+
return versions;
140+
}
141+
142+
protected static boolean isFullyUpgradedTo(Version version) throws Exception {
143+
return nodesVersions().values().stream().allMatch(v -> v.equals(version));
133144
}
134145

135146
protected static Version indexVersion(String indexName) throws Exception {
@@ -181,16 +192,4 @@ protected static void restoreIndex(String repository, String snapshot, String in
181192
assertThat(responseBody.evaluate("snapshot.shards.total"), equalTo((int) responseBody.evaluate("snapshot.shards.failed")));
182193
assertThat(responseBody.evaluate("snapshot.shards.successful"), equalTo(0));
183194
}
184-
185-
/**
186-
* Execute the test suite with the parameters provided by the {@link #parameters()} in version order.
187-
*/
188-
public static class TestCaseOrdering implements Comparator<TestMethodAndParams> {
189-
@Override
190-
public int compare(TestMethodAndParams o1, TestMethodAndParams o2) {
191-
var version1 = (Version) o1.getInstanceArguments().get(0);
192-
var version2 = (Version) o2.getInstanceArguments().get(0);
193-
return version1.compareTo(version2);
194-
}
195-
}
196195
}

‎qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartIndexCompatibilityTestCase.java‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.lucene;
11+
12+
import com.carrotsearch.randomizedtesting.TestMethodAndParams;
13+
import com.carrotsearch.randomizedtesting.annotations.Name;
14+
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
15+
import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering;
16+
17+
import org.elasticsearch.test.cluster.util.Version;
18+
19+
import java.util.Comparator;
20+
import java.util.stream.Stream;
21+
22+
import static org.elasticsearch.test.cluster.util.Version.CURRENT;
23+
import static org.hamcrest.Matchers.equalTo;
24+
25+
/**
26+
* Test suite for Lucene indices backward compatibility with N-2 versions after full cluster restart upgrades. The test suite creates a
27+
* cluster in N-2 version, then upgrades it to N-1 version and finally upgrades it to the current version. Test methods are executed after
28+
* each upgrade.
29+
*/
30+
@TestCaseOrdering(FullClusterRestartIndexCompatibilityTestCase.TestCaseOrdering.class)
31+
public abstract class FullClusterRestartIndexCompatibilityTestCase extends AbstractIndexCompatibilityTestCase {
32+
33+
private final Version clusterVersion;
34+
35+
public FullClusterRestartIndexCompatibilityTestCase(@Name("cluster") Version clusterVersion) {
36+
this.clusterVersion = clusterVersion;
37+
}
38+
39+
@ParametersFactory
40+
public static Iterable<Object[]> parameters() {
41+
return Stream.of(VERSION_MINUS_2, VERSION_MINUS_1, CURRENT).map(v -> new Object[] { v }).toList();
42+
}
43+
44+
@Override
45+
protected void maybeUpgrade() throws Exception {
46+
if (nodesVersions().values().stream().anyMatch(version -> version.before(clusterVersion))) {
47+
cluster().upgradeToVersion(clusterVersion);
48+
closeClients();
49+
initClient();
50+
}
51+
assertThat(isFullyUpgradedTo(clusterVersion), equalTo(true));
52+
}
53+
54+
/**
55+
* Execute the test suite with the parameters provided by the {@link #parameters()} in version order.
56+
*/
57+
public static class TestCaseOrdering implements Comparator<TestMethodAndParams> {
58+
@Override
59+
public int compare(TestMethodAndParams o1, TestMethodAndParams o2) {
60+
var version1 = (Version) o1.getInstanceArguments().get(0);
61+
var version2 = (Version) o2.getInstanceArguments().get(0);
62+
return version1.compareTo(version2);
63+
}
64+
}
65+
}

‎qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/LuceneCompatibilityIT.java‎ renamed to ‎qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartLuceneIndexCompatibilityIT.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,13 @@
2323
import static org.hamcrest.Matchers.allOf;
2424
import static org.hamcrest.Matchers.equalTo;
2525

26-
public class LuceneCompatibilityIT extends AbstractLuceneIndexCompatibilityTestCase {
26+
public class FullClusterRestartLuceneIndexCompatibilityIT extends FullClusterRestartIndexCompatibilityTestCase {
2727

2828
static {
2929
clusterConfig = config -> config.setting("xpack.license.self_generated.type", "trial");
3030
}
3131

32-
public LuceneCompatibilityIT(Version version) {
32+
public FullClusterRestartLuceneIndexCompatibilityIT(Version version) {
3333
super(version);
3434
}
3535

@@ -42,7 +42,7 @@ public void testRestoreIndex() throws Exception {
4242
final String index = suffix("index");
4343
final int numDocs = 1234;
4444

45-
if (VERSION_MINUS_2.equals(clusterVersion())) {
45+
if (isFullyUpgradedTo(VERSION_MINUS_2)) {
4646
logger.debug("--> registering repository [{}]", repository);
4747
registerRepository(client(), repository, FsRepository.TYPE, true, repositorySettings());
4848

@@ -65,7 +65,7 @@ public void testRestoreIndex() throws Exception {
6565
return;
6666
}
6767

68-
if (VERSION_MINUS_1.equals(clusterVersion())) {
68+
if (isFullyUpgradedTo(VERSION_MINUS_1)) {
6969
ensureGreen(index);
7070

7171
assertThat(indexVersion(index), equalTo(VERSION_MINUS_2));
@@ -76,7 +76,7 @@ public void testRestoreIndex() throws Exception {
7676
return;
7777
}
7878

79-
if (VERSION_CURRENT.equals(clusterVersion())) {
79+
if (isFullyUpgradedTo(VERSION_CURRENT)) {
8080
var restoredIndex = suffix("index-restored");
8181
logger.debug("--> restoring index [{}] as [{}]", index, restoredIndex);
8282

‎qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/SearchableSnapshotCompatibilityIT.java‎ renamed to ‎qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/FullClusterRestartSearchableSnapshotIndexCompatibilityIT.java‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,15 @@
1717

1818
import static org.hamcrest.Matchers.equalTo;
1919

20-
public class SearchableSnapshotCompatibilityIT extends AbstractLuceneIndexCompatibilityTestCase {
20+
public class FullClusterRestartSearchableSnapshotIndexCompatibilityIT extends FullClusterRestartIndexCompatibilityTestCase {
2121

2222
static {
2323
clusterConfig = config -> config.setting("xpack.license.self_generated.type", "trial")
2424
.setting("xpack.searchable.snapshot.shared_cache.size", "16MB")
2525
.setting("xpack.searchable.snapshot.shared_cache.region_size", "256KB");
2626
}
2727

28-
public SearchableSnapshotCompatibilityIT(Version version) {
28+
public FullClusterRestartSearchableSnapshotIndexCompatibilityIT(Version version) {
2929
super(version);
3030
}
3131

@@ -38,7 +38,7 @@ public void testSearchableSnapshot() throws Exception {
3838
final String index = suffix("index");
3939
final int numDocs = 1234;
4040

41-
if (VERSION_MINUS_2.equals(clusterVersion())) {
41+
if (isFullyUpgradedTo(VERSION_MINUS_2)) {
4242
logger.debug("--> registering repository [{}]", repository);
4343
registerRepository(client(), repository, FsRepository.TYPE, true, repositorySettings());
4444

@@ -61,7 +61,7 @@ public void testSearchableSnapshot() throws Exception {
6161
return;
6262
}
6363

64-
if (VERSION_MINUS_1.equals(clusterVersion())) {
64+
if (isFullyUpgradedTo(VERSION_MINUS_1)) {
6565
ensureGreen(index);
6666

6767
assertThat(indexVersion(index), equalTo(VERSION_MINUS_2));
@@ -72,7 +72,7 @@ public void testSearchableSnapshot() throws Exception {
7272
return;
7373
}
7474

75-
if (VERSION_CURRENT.equals(clusterVersion())) {
75+
if (isFullyUpgradedTo(VERSION_CURRENT)) {
7676
var mountedIndex = suffix("index-mounted");
7777
logger.debug("--> mounting index [{}] as [{}]", index, mountedIndex);
7878
mountIndex(repository, snapshot, index, randomBoolean(), mountedIndex);
@@ -98,7 +98,7 @@ public void testSearchableSnapshotUpgrade() throws Exception {
9898
final String index = suffix("index");
9999
final int numDocs = 4321;
100100

101-
if (VERSION_MINUS_2.equals(clusterVersion())) {
101+
if (isFullyUpgradedTo(VERSION_MINUS_2)) {
102102
logger.debug("--> registering repository [{}]", repository);
103103
registerRepository(client(), repository, FsRepository.TYPE, true, repositorySettings());
104104

@@ -124,7 +124,7 @@ public void testSearchableSnapshotUpgrade() throws Exception {
124124
return;
125125
}
126126

127-
if (VERSION_MINUS_1.equals(clusterVersion())) {
127+
if (isFullyUpgradedTo(VERSION_MINUS_1)) {
128128
logger.debug("--> mounting index [{}] as [{}]", index, mountedIndex);
129129
mountIndex(repository, snapshot, index, randomBoolean(), mountedIndex);
130130

@@ -135,7 +135,7 @@ public void testSearchableSnapshotUpgrade() throws Exception {
135135
return;
136136
}
137137

138-
if (VERSION_CURRENT.equals(clusterVersion())) {
138+
if (isFullyUpgradedTo(VERSION_CURRENT)) {
139139
ensureGreen(mountedIndex);
140140

141141
assertThat(indexVersion(mountedIndex), equalTo(VERSION_MINUS_2));

0 commit comments

Comments
 (0)