Skip to content

Commit cd2ee4c

Browse files
drempapisdrempapis
authored
Archive-Index upgrade compatibility (#118599)
Add tests for archive indices and N-2 compatibility.
1 parent d7490ed commit cd2ee4c

File tree

9 files changed

+484
-0
lines changed

9 files changed

+484
-0
lines changed

‎docs/changelog/118599.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 118599
2+
summary: Archive-Index upgrade compatibility
3+
area: Search
4+
type: enhancement
5+
issues: []

‎x-pack/qa/repository-old-versions-compatibility/build.gradle‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
apply plugin: 'elasticsearch.internal-java-rest-test'
10+
apply plugin: 'elasticsearch.internal-test-artifact'
11+
apply plugin: 'elasticsearch.bwc-test'
12+
13+
buildParams.bwcVersions.withLatestReadOnlyIndexCompatible { bwcVersion ->
14+
tasks.named("javaRestTest").configure {
15+
systemProperty("tests.minimum.index.compatible", bwcVersion)
16+
usesBwcDistribution(bwcVersion)
17+
enabled = true
18+
}
19+
}
20+
21+
tasks.withType(Test).configureEach {
22+
// CI doesn't like it when there's multiple clusters running at once
23+
maxParallelForks = 1
24+
}
25+

‎x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/AbstractUpgradeCompatibilityTestCase.java‎

Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.oldrepos;
9+
10+
import com.carrotsearch.randomizedtesting.TestMethodAndParams;
11+
import com.carrotsearch.randomizedtesting.annotations.Name;
12+
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
13+
import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering;
14+
15+
import org.apache.http.util.EntityUtils;
16+
import org.elasticsearch.client.Request;
17+
import org.elasticsearch.client.Response;
18+
import org.elasticsearch.client.RestClient;
19+
import org.elasticsearch.common.settings.Settings;
20+
import org.elasticsearch.repositories.fs.FsRepository;
21+
import org.elasticsearch.test.cluster.ElasticsearchCluster;
22+
import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider;
23+
import org.elasticsearch.test.cluster.local.distribution.DistributionType;
24+
import org.elasticsearch.test.cluster.util.Version;
25+
import org.elasticsearch.test.rest.ESRestTestCase;
26+
import org.junit.Before;
27+
import org.junit.ClassRule;
28+
import org.junit.rules.RuleChain;
29+
import org.junit.rules.TemporaryFolder;
30+
import org.junit.rules.TestRule;
31+
32+
import java.io.IOException;
33+
import java.io.OutputStream;
34+
import java.net.URISyntaxException;
35+
import java.nio.file.Files;
36+
import java.nio.file.Path;
37+
import java.nio.file.Paths;
38+
import java.util.Comparator;
39+
import java.util.Objects;
40+
import java.util.stream.Stream;
41+
import java.util.zip.ZipEntry;
42+
import java.util.zip.ZipInputStream;
43+
44+
import static org.elasticsearch.test.cluster.util.Version.CURRENT;
45+
import static org.elasticsearch.test.cluster.util.Version.fromString;
46+
import static org.elasticsearch.test.rest.ObjectPath.createFromResponse;
47+
import static org.hamcrest.Matchers.equalTo;
48+
import static org.hamcrest.Matchers.notNullValue;
49+
50+
@TestCaseOrdering(AbstractUpgradeCompatibilityTestCase.TestCaseOrdering.class)
51+
public abstract class AbstractUpgradeCompatibilityTestCase extends ESRestTestCase {
52+
53+
protected static final Version VERSION_MINUS_2 = fromString(System.getProperty("tests.minimum.index.compatible"));
54+
protected static final Version VERSION_MINUS_1 = fromString(System.getProperty("tests.minimum.wire.compatible"));
55+
protected static final Version VERSION_CURRENT = CURRENT;
56+
57+
protected static TemporaryFolder REPOSITORY_PATH = new TemporaryFolder();
58+
59+
protected static LocalClusterConfigProvider clusterConfig = c -> {};
60+
private static ElasticsearchCluster cluster = ElasticsearchCluster.local()
61+
.distribution(DistributionType.DEFAULT)
62+
.version(VERSION_MINUS_1)
63+
.nodes(2)
64+
.setting("xpack.security.enabled", "false")
65+
.setting("xpack.ml.enabled", "false")
66+
.setting("path.repo", () -> REPOSITORY_PATH.getRoot().getPath())
67+
.apply(() -> clusterConfig)
68+
.build();
69+
70+
@ClassRule
71+
public static TestRule ruleChain = RuleChain.outerRule(REPOSITORY_PATH).around(cluster);
72+
73+
private static boolean upgradeFailed = false;
74+
75+
private final Version clusterVersion;
76+
77+
public AbstractUpgradeCompatibilityTestCase(@Name("cluster") Version clusterVersion) {
78+
this.clusterVersion = clusterVersion;
79+
}
80+
81+
@ParametersFactory
82+
public static Iterable<Object[]> parameters() {
83+
return Stream.of(VERSION_MINUS_1, CURRENT).map(v -> new Object[] { v }).toList();
84+
}
85+
86+
@Override
87+
protected String getTestRestCluster() {
88+
return cluster.getHttpAddresses();
89+
}
90+
91+
@Override
92+
protected boolean preserveClusterUponCompletion() {
93+
return true;
94+
}
95+
96+
/**
97+
* This method verifies the currentVersion against the clusterVersion and performs a "full cluster restart" upgrade if the current
98+
* is before clusterVersion. The cluster version is fetched externally and is controlled by the gradle setup.
99+
*
100+
* @throws Exception
101+
*/
102+
@Before
103+
public void maybeUpgrade() throws Exception {
104+
// We want to use this test suite for the V9 upgrade, but we are not fully committed to necessarily having N-2 support
105+
// in V10, so we add a check here to ensure we'll revisit this decision once V10 exists.
106+
assertThat("Explicit check that N-2 version is Elasticsearch 7", VERSION_MINUS_2.getMajor(), equalTo(7));
107+
108+
var currentVersion = clusterVersion();
109+
if (currentVersion.before(clusterVersion)) {
110+
try {
111+
cluster.upgradeToVersion(clusterVersion);
112+
closeClients();
113+
initClient();
114+
} catch (Exception e) {
115+
upgradeFailed = true;
116+
throw e;
117+
}
118+
}
119+
120+
// Skip remaining tests if upgrade failed
121+
assumeFalse("Cluster upgrade failed", upgradeFailed);
122+
}
123+
124+
protected static Version clusterVersion() throws Exception {
125+
var response = assertOK(client().performRequest(new Request("GET", "/")));
126+
var responseBody = createFromResponse(response);
127+
var version = Version.fromString(responseBody.evaluate("version.number").toString());
128+
assertThat("Failed to retrieve cluster version", version, notNullValue());
129+
return version;
130+
}
131+
132+
/**
133+
* Execute the test suite with the parameters provided by the {@link #parameters()} in version order.
134+
*/
135+
public static class TestCaseOrdering implements Comparator<TestMethodAndParams> {
136+
@Override
137+
public int compare(TestMethodAndParams o1, TestMethodAndParams o2) {
138+
var version1 = (Version) o1.getInstanceArguments().get(0);
139+
var version2 = (Version) o2.getInstanceArguments().get(0);
140+
return version1.compareTo(version2);
141+
}
142+
}
143+
144+
public final void verifyCompatibility(String version) throws Exception {
145+
final String repository = "repository";
146+
final String snapshot = "snapshot";
147+
final String index = "index";
148+
final int numDocs = 5;
149+
150+
String repositoryPath = REPOSITORY_PATH.getRoot().getPath();
151+
152+
if (VERSION_MINUS_1.equals(clusterVersion())) {
153+
assertEquals(VERSION_MINUS_1, clusterVersion());
154+
assertTrue(getIndices(client()).isEmpty());
155+
156+
// Copy a snapshot of an index with 5 documents
157+
copySnapshotFromResources(repositoryPath, version);
158+
registerRepository(client(), repository, FsRepository.TYPE, true, Settings.builder().put("location", repositoryPath).build());
159+
recover(client(), repository, snapshot, index);
160+
161+
assertTrue(getIndices(client()).contains(index));
162+
assertDocCount(client(), index, numDocs);
163+
164+
return;
165+
}
166+
167+
if (VERSION_CURRENT.equals(clusterVersion())) {
168+
assertEquals(VERSION_CURRENT, clusterVersion());
169+
assertTrue(getIndices(client()).contains(index));
170+
assertDocCount(client(), index, numDocs);
171+
}
172+
}
173+
174+
public abstract void recover(RestClient restClient, String repository, String snapshot, String index) throws Exception;
175+
176+
private static String getIndices(RestClient client) throws IOException {
177+
final Request request = new Request("GET", "_cat/indices");
178+
Response response = client.performRequest(request);
179+
return EntityUtils.toString(response.getEntity());
180+
}
181+
182+
private static void copySnapshotFromResources(String repositoryPath, String version) throws IOException, URISyntaxException {
183+
Path zipFilePath = Paths.get(
184+
Objects.requireNonNull(AbstractUpgradeCompatibilityTestCase.class.getClassLoader().getResource("snapshot_v" + version + ".zip"))
185+
.toURI()
186+
);
187+
unzip(zipFilePath, Paths.get(repositoryPath));
188+
}
189+
190+
private static void unzip(Path zipFilePath, Path outputDir) throws IOException {
191+
try (ZipInputStream zipIn = new ZipInputStream(Files.newInputStream(zipFilePath))) {
192+
ZipEntry entry;
193+
while ((entry = zipIn.getNextEntry()) != null) {
194+
Path outputPath = outputDir.resolve(entry.getName());
195+
if (entry.isDirectory()) {
196+
Files.createDirectories(outputPath);
197+
} else {
198+
Files.createDirectories(outputPath.getParent());
199+
try (OutputStream out = Files.newOutputStream(outputPath)) {
200+
byte[] buffer = new byte[1024];
201+
int len;
202+
while ((len = zipIn.read(buffer)) > 0) {
203+
out.write(buffer, 0, len);
204+
}
205+
}
206+
}
207+
zipIn.closeEntry();
208+
}
209+
}
210+
}
211+
}

‎x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/ArchiveIndexTestCase.java‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.oldrepos.archiveindex;
9+
10+
import org.elasticsearch.client.Request;
11+
import org.elasticsearch.client.RestClient;
12+
import org.elasticsearch.common.Strings;
13+
import org.elasticsearch.oldrepos.AbstractUpgradeCompatibilityTestCase;
14+
import org.elasticsearch.test.cluster.util.Version;
15+
16+
import static org.elasticsearch.test.rest.ObjectPath.createFromResponse;
17+
18+
/**
19+
* Test suite for Archive indices backward compatibility with N-2 versions.
20+
* The test suite creates a cluster in the N-1 version, where N is the current version.
21+
* Restores snapshots from old-clusters (version 5/6) and upgrades it to the current version.
22+
* Test methods are executed after each upgrade.
23+
*
24+
* For example the test suite creates a cluster of version 8, then restores a snapshot of an index created
25+
* when deployed ES version 5/6. The cluster then upgrades to version 9, verifying that the archive index
26+
* is successfully restored.
27+
*/
28+
public class ArchiveIndexTestCase extends AbstractUpgradeCompatibilityTestCase {
29+
30+
static {
31+
clusterConfig = config -> config.setting("xpack.license.self_generated.type", "trial");
32+
}
33+
34+
public ArchiveIndexTestCase(Version version) {
35+
super(version);
36+
}
37+
38+
/**
39+
* Overrides the snapshot-restore operation for archive-indices scenario.
40+
*/
41+
@Override
42+
public void recover(RestClient client, String repository, String snapshot, String index) throws Exception {
43+
var request = new Request("POST", "/_snapshot/" + repository + "/" + snapshot + "/_restore");
44+
request.addParameter("wait_for_completion", "true");
45+
request.setJsonEntity(Strings.format("""
46+
{
47+
"indices": "%s",
48+
"include_global_state": false,
49+
"rename_pattern": "(.+)",
50+
"include_aliases": false
51+
}""", index));
52+
createFromResponse(client.performRequest(request));
53+
}
54+
}

‎x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion5IT.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.oldrepos.archiveindex;
9+
10+
import org.elasticsearch.test.cluster.util.Version;
11+
12+
public class RestoreFromVersion5IT extends ArchiveIndexTestCase {
13+
14+
public RestoreFromVersion5IT(Version version) {
15+
super(version);
16+
}
17+
18+
public void testArchiveIndex() throws Exception {
19+
verifyCompatibility("5");
20+
}
21+
}

‎x-pack/qa/repository-old-versions-compatibility/src/javaRestTest/java/org/elasticsearch/oldrepos/archiveindex/RestoreFromVersion6IT.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.oldrepos.archiveindex;
9+
10+
import org.elasticsearch.test.cluster.util.Version;
11+
12+
public class RestoreFromVersion6IT extends ArchiveIndexTestCase {
13+
14+
public RestoreFromVersion6IT(Version version) {
15+
super(version);
16+
}
17+
18+
public void testArchiveIndex() throws Exception {
19+
verifyCompatibility("6");
20+
}
21+
}

0 commit comments

Comments
 (0)