Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
da8f1e2
Add testing and refactoring
drempapis Dec 12, 2024
713384f
Update code
drempapis Dec 12, 2024
f69c5ff
update code
drempapis Dec 12, 2024
f311c3d
update code
drempapis Dec 12, 2024
78083bc
Merge branch 'main' into test/archive-index-version-compatibility
drempapis Dec 13, 2024
0b80ca6
spotless execute
drempapis Dec 13, 2024
387bf82
Merge branch 'test/archive-index-version-compatibility' of github.com…
drempapis Dec 13, 2024
5cfa128
Merge branch 'main' into test/archive-index-version-compatibility
drempapis Dec 16, 2024
6a3cceb
Move project to other package and slight refactoring
drempapis Dec 16, 2024
6c54c50
apply spotless
drempapis Dec 16, 2024
9a63102
update after review
drempapis Dec 16, 2024
476328f
update after review
drempapis Dec 16, 2024
884a9e0
update after review
drempapis Dec 16, 2024
79960b3
Merge branch 'main' into test/archive-index-version-compatibility
drempapis Dec 16, 2024
2fcb1ca
update after review
drempapis Dec 17, 2024
e8d6fee
Merge branch 'main' into test/archive-index-version-compatibility
drempapis Dec 17, 2024
acec518
update code
drempapis Dec 17, 2024
b566c30
Merge branch 'main' into test/archive-index-version-compatibility
drempapis Dec 17, 2024
60ecd0b
Merge branch 'main' into test/archive-index-version-compatibility
drempapis Dec 17, 2024
cbd9a4a
Merge branch 'main' into test/archive-index-version-compatibility
drempapis Dec 17, 2024
df26c76
Update after review
drempapis Dec 17, 2024
cd15012
Merge branch 'main' into test/archive-index-version-compatibility
drempapis Dec 17, 2024
10a227b
Merge branch 'main' into test/archive-index-version-compatibility
drempapis Dec 18, 2024
146334d
Merge branch 'main' into test/archive-index-version-compatibility
drempapis Jan 2, 2025
ece34cc
[CI] Auto commit changes from spotless
Jan 2, 2025
c395ddf
update after review
drempapis Jan 2, 2025
1362eae
erge branch 'test/archive-index-version-compatibility' of github.com:…
drempapis Jan 2, 2025
07f0553
Revert code
drempapis Jan 2, 2025
817afbb
Add changelog
drempapis Jan 2, 2025
81333dd
revert code
drempapis Jan 2, 2025
8b6068c
Merge branch 'main' into test/archive-index-version-compatibility
drempapis Jan 2, 2025
57384ae
Merge branch 'main' into test/archive-index-version-compatibility
drempapis Jan 2, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/changelog/118599.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
pr: 118599
summary: Archive-Index upgrade compatibility
area: Search
type: enhancement
issues: []
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe that perhaps the changelog entry is not needed given that this became a test only change. It should probably be removed.

Copy link
Contributor Author

@drempapis drempapis Jan 7, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I accidentally added it and got notified. I created this tiny pr to fix it #119634

25 changes: 25 additions & 0 deletions x-pack/qa/repository-old-versions-compatibility/build.gradle
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the "Elastic License
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
* Public License v 1"; you may not use this file except in compliance with, at
* your election, the "Elastic License 2.0", the "GNU Affero General Public
* License v3.0 only", or the "Server Side Public License, v 1".
*/
apply plugin: 'elasticsearch.internal-java-rest-test'
apply plugin: 'elasticsearch.internal-test-artifact'
apply plugin: 'elasticsearch.bwc-test'

buildParams.bwcVersions.withLatestReadOnlyIndexCompatible { bwcVersion ->
tasks.named("javaRestTest").configure {
systemProperty("tests.minimum.index.compatible", bwcVersion)
usesBwcDistribution(bwcVersion)
enabled = true
}
}

tasks.withType(Test).configureEach {
// CI doesn't like it when there's multiple clusters running at once
maxParallelForks = 1
}

Original file line number Diff line number Diff line change
@@ -0,0 +1,211 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

package org.elasticsearch.oldrepos;

import com.carrotsearch.randomizedtesting.TestMethodAndParams;
import com.carrotsearch.randomizedtesting.annotations.Name;
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering;

import org.apache.http.util.EntityUtils;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.repositories.fs.FsRepository;
import org.elasticsearch.test.cluster.ElasticsearchCluster;
import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider;
import org.elasticsearch.test.cluster.local.distribution.DistributionType;
import org.elasticsearch.test.cluster.util.Version;
import org.elasticsearch.test.rest.ESRestTestCase;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.rules.RuleChain;
import org.junit.rules.TemporaryFolder;
import org.junit.rules.TestRule;

import java.io.IOException;
import java.io.OutputStream;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Comparator;
import java.util.Objects;
import java.util.stream.Stream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

import static org.elasticsearch.test.cluster.util.Version.CURRENT;
import static org.elasticsearch.test.cluster.util.Version.fromString;
import static org.elasticsearch.test.rest.ObjectPath.createFromResponse;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.notNullValue;

@TestCaseOrdering(AbstractUpgradeCompatibilityTestCase.TestCaseOrdering.class)
public abstract class AbstractUpgradeCompatibilityTestCase extends ESRestTestCase {

protected static final Version VERSION_MINUS_2 = fromString(System.getProperty("tests.minimum.index.compatible"));
protected static final Version VERSION_MINUS_1 = fromString(System.getProperty("tests.minimum.wire.compatible"));
protected static final Version VERSION_CURRENT = CURRENT;

protected static TemporaryFolder REPOSITORY_PATH = new TemporaryFolder();

protected static LocalClusterConfigProvider clusterConfig = c -> {};
private static ElasticsearchCluster cluster = ElasticsearchCluster.local()
.distribution(DistributionType.DEFAULT)
.version(VERSION_MINUS_1)
.nodes(2)
.setting("xpack.security.enabled", "false")
.setting("xpack.ml.enabled", "false")
.setting("path.repo", () -> REPOSITORY_PATH.getRoot().getPath())
.apply(() -> clusterConfig)
.build();

@ClassRule
public static TestRule ruleChain = RuleChain.outerRule(REPOSITORY_PATH).around(cluster);

private static boolean upgradeFailed = false;

private final Version clusterVersion;

public AbstractUpgradeCompatibilityTestCase(@Name("cluster") Version clusterVersion) {
this.clusterVersion = clusterVersion;
}

@ParametersFactory
public static Iterable<Object[]> parameters() {
return Stream.of(VERSION_MINUS_1, CURRENT).map(v -> new Object[] { v }).toList();
}

@Override
protected String getTestRestCluster() {
return cluster.getHttpAddresses();
}

@Override
protected boolean preserveClusterUponCompletion() {
return true;
}

/**
* This method verifies the currentVersion against the clusterVersion and performs a "full cluster restart" upgrade if the current
* is before clusterVersion. The cluster version is fetched externally and is controlled by the gradle setup.
*
* @throws Exception
*/
@Before
public void maybeUpgrade() throws Exception {
// We want to use this test suite for the V9 upgrade, but we are not fully committed to necessarily having N-2 support
// in V10, so we add a check here to ensure we'll revisit this decision once V10 exists.
assertThat("Explicit check that N-2 version is Elasticsearch 7", VERSION_MINUS_2.getMajor(), equalTo(7));

var currentVersion = clusterVersion();
if (currentVersion.before(clusterVersion)) {
try {
cluster.upgradeToVersion(clusterVersion);
closeClients();
initClient();
} catch (Exception e) {
upgradeFailed = true;
throw e;
}
}

// Skip remaining tests if upgrade failed
assumeFalse("Cluster upgrade failed", upgradeFailed);
}

protected static Version clusterVersion() throws Exception {
var response = assertOK(client().performRequest(new Request("GET", "/")));
var responseBody = createFromResponse(response);
var version = Version.fromString(responseBody.evaluate("version.number").toString());
assertThat("Failed to retrieve cluster version", version, notNullValue());
return version;
}

/**
* Execute the test suite with the parameters provided by the {@link #parameters()} in version order.
*/
public static class TestCaseOrdering implements Comparator<TestMethodAndParams> {
@Override
public int compare(TestMethodAndParams o1, TestMethodAndParams o2) {
var version1 = (Version) o1.getInstanceArguments().get(0);
var version2 = (Version) o2.getInstanceArguments().get(0);
return version1.compareTo(version2);
}
}

public final void verifyCompatibility(String version) throws Exception {
final String repository = "repository";
final String snapshot = "snapshot";
final String index = "index";
final int numDocs = 5;

String repositoryPath = REPOSITORY_PATH.getRoot().getPath();

if (VERSION_MINUS_1.equals(clusterVersion())) {
assertEquals(VERSION_MINUS_1, clusterVersion());
assertTrue(getIndices(client()).isEmpty());

// Copy a snapshot of an index with 5 documents
copySnapshotFromResources(repositoryPath, version);
registerRepository(client(), repository, FsRepository.TYPE, true, Settings.builder().put("location", repositoryPath).build());
recover(client(), repository, snapshot, index);

assertTrue(getIndices(client()).contains(index));
assertDocCount(client(), index, numDocs);

return;
}

if (VERSION_CURRENT.equals(clusterVersion())) {
assertEquals(VERSION_CURRENT, clusterVersion());
assertTrue(getIndices(client()).contains(index));
assertDocCount(client(), index, numDocs);
}
}

public abstract void recover(RestClient restClient, String repository, String snapshot, String index) throws Exception;

private static String getIndices(RestClient client) throws IOException {
final Request request = new Request("GET", "_cat/indices");
Response response = client.performRequest(request);
return EntityUtils.toString(response.getEntity());
}

private static void copySnapshotFromResources(String repositoryPath, String version) throws IOException, URISyntaxException {
Path zipFilePath = Paths.get(
Objects.requireNonNull(AbstractUpgradeCompatibilityTestCase.class.getClassLoader().getResource("snapshot_v" + version + ".zip"))
.toURI()
);
unzip(zipFilePath, Paths.get(repositoryPath));
}

private static void unzip(Path zipFilePath, Path outputDir) throws IOException {
try (ZipInputStream zipIn = new ZipInputStream(Files.newInputStream(zipFilePath))) {
ZipEntry entry;
while ((entry = zipIn.getNextEntry()) != null) {
Path outputPath = outputDir.resolve(entry.getName());
if (entry.isDirectory()) {
Files.createDirectories(outputPath);
} else {
Files.createDirectories(outputPath.getParent());
try (OutputStream out = Files.newOutputStream(outputPath)) {
byte[] buffer = new byte[1024];
int len;
while ((len = zipIn.read(buffer)) > 0) {
out.write(buffer, 0, len);
}
}
}
zipIn.closeEntry();
}
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

package org.elasticsearch.oldrepos.archiveindex;

import org.elasticsearch.client.Request;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.common.Strings;
import org.elasticsearch.oldrepos.AbstractUpgradeCompatibilityTestCase;
import org.elasticsearch.test.cluster.util.Version;

import static org.elasticsearch.test.rest.ObjectPath.createFromResponse;

/**
* Test suite for Archive indices backward compatibility with N-2 versions.
* The test suite creates a cluster in the N-1 version, where N is the current version.
* Restores snapshots from old-clusters (version 5/6) and upgrades it to the current version.
* Test methods are executed after each upgrade.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it would help to provide an example with concrete versions as well., for clarity

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll do so, ty!

*
* For example the test suite creates a cluster of version 8, then restores a snapshot of an index created
* when deployed ES version 5/6. The cluster then upgrades to version 9, verifying that the archive index
* is successfully restored.
*/
public class ArchiveIndexTestCase extends AbstractUpgradeCompatibilityTestCase {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think I could use some upgrade mention in this class name as well given it's what it tests? You probably wanted to make a distinction between archive indices restored from snapshot and those mounted as searchable snapshots?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Correct, that was my intention. I considered that the Upgrade functionality was implicitly implied by the parent class, where this one denotes that the test is for ArchiveIndex. The same logic has been implemented for SearchableSnapshot in this pr


static {
clusterConfig = config -> config.setting("xpack.license.self_generated.type", "trial");
}

public ArchiveIndexTestCase(Version version) {
super(version);
}

/**
* Overrides the snapshot-restore operation for archive-indices scenario.
*/
@Override
public void recover(RestClient client, String repository, String snapshot, String index) throws Exception {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would call this method importArchiveIndex or something along those lines.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Using the template pattern, this template placeholder will be used for the archive-index and searchable-snapshots tests. This method implements a restore operation for archive-index tests and a mount operation for searchable snapshots. Yes, by changing the name to importIndex fits better.

var request = new Request("POST", "/_snapshot/" + repository + "/" + snapshot + "/_restore");
request.addParameter("wait_for_completion", "true");
request.setJsonEntity(Strings.format("""
{
"indices": "%s",
"include_global_state": false,
"rename_pattern": "(.+)",
"include_aliases": false
}""", index));
createFromResponse(client.performRequest(request));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

package org.elasticsearch.oldrepos.archiveindex;

import org.elasticsearch.test.cluster.util.Version;

public class RestoreFromVersion5IT extends ArchiveIndexTestCase {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Because the Gradle setup is not flexible, I needed to divide the different base versions into separate classes. For each case, we need a fresh cluster with version Current-1.


public RestoreFromVersion5IT(Version version) {
super(version);
}

public void testArchiveIndex() throws Exception {
verifyCompatibility("5");
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

package org.elasticsearch.oldrepos.archiveindex;

import org.elasticsearch.test.cluster.util.Version;

public class RestoreFromVersion6IT extends ArchiveIndexTestCase {

public RestoreFromVersion6IT(Version version) {
super(version);
}

public void testArchiveIndex() throws Exception {
verifyCompatibility("6");
}
}
Loading