Skip to content

Commit 61dc931

Browse files
authored
Add lucene version compatibility tests (#121104)
This commit adds compatibility tests that target ES revisions that align with specific Lucene versions. In this case, we are intending to upgrade from Lucene 10.0 to 10.1. Since no on-prem Elasticsearch release exists with 10.0, we need another method to ensure compatibility with Lucene 10.0 indicies. The work here is a bit hacky since all our compatibility testing infrastructure is centered around versions and we're now effectively doing compatibility tests between two different revisions of Elasticsearch that both report the same version. Ideally this specific testing would be replaced by unit tests, rather that reusing our full cluster restart tests for this purpose. We'll also want to bump the commit referenced in the CI pipelines here to align with the last commit using Lucene 10.0.
1 parent 3128d66 commit 61dc931

File tree

22 files changed

+173
-16
lines changed

22 files changed

+173
-16
lines changed

.buildkite/pipelines/intake.template.yml

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,27 @@ steps:
6363
buildDirectory: /dev/shm/bk
6464
env:
6565
BWC_VERSION: "{{matrix.BWC_VERSION}}"
66+
- group: lucene-compat
67+
steps:
68+
- label: "{{matrix.LUCENE_VERSION}} / lucene-compat"
69+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=$$ES_VERSION -Dtests.bwc.refspec.main=$$ES_COMMIT luceneBwcTest
70+
timeout_in_minutes: 300
71+
matrix:
72+
setup:
73+
LUCENE_VERSION:
74+
- "10.0.0"
75+
ES_VERSION:
76+
- "9.0.0"
77+
ES_COMMIT:
78+
- "b2cc9d9b8f00ee621f93ddca07ea9c671aab1578" # update to match last commit before lucene bump
79+
agents:
80+
provider: gcp
81+
image: family/elasticsearch-ubuntu-2004
82+
machineType: custom-32-98304
83+
buildDirectory: /dev/shm/bk
84+
env:
85+
ES_VERSION: "{{matrix.ES_VERSION}}"
86+
ES_COMMIT: "{{matrix.ES_COMMIT}}"
6687
- label: rest-compat
6788
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkRestCompat
6889
timeout_in_minutes: 300

.buildkite/pipelines/intake.yml

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,27 @@ steps:
6464
buildDirectory: /dev/shm/bk
6565
env:
6666
BWC_VERSION: "{{matrix.BWC_VERSION}}"
67+
- group: lucene-compat
68+
steps:
69+
- label: "{{matrix.LUCENE_VERSION}} / lucene-compat"
70+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=$$ES_VERSION -Dtests.bwc.refspec.main=$$ES_COMMIT luceneBwcTest
71+
timeout_in_minutes: 300
72+
matrix:
73+
setup:
74+
LUCENE_VERSION:
75+
- "10.0.0"
76+
ES_VERSION:
77+
- "9.0.0"
78+
ES_COMMIT:
79+
- "b2cc9d9b8f00ee621f93ddca07ea9c671aab1578" # update to match last commit before lucene bump
80+
agents:
81+
provider: gcp
82+
image: family/elasticsearch-ubuntu-2004
83+
machineType: custom-32-98304
84+
buildDirectory: /dev/shm/bk
85+
env:
86+
ES_VERSION: "{{matrix.ES_VERSION}}"
87+
ES_COMMIT: "{{matrix.ES_COMMIT}}"
6788
- label: rest-compat
6889
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints checkRestCompat
6990
timeout_in_minutes: 300

.buildkite/pipelines/periodic.template.yml

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,27 @@ steps:
201201
image: family/elasticsearch-ubuntu-2004
202202
machineType: n2-standard-8
203203
buildDirectory: /dev/shm/bk
204+
- group: lucene-compat
205+
steps:
206+
- label: "{{matrix.LUCENE_VERSION}} / lucene-compat"
207+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=$$ES_VERSION -Dtests.bwc.refspec.main=$$ES_COMMIT luceneBwcTest
208+
timeout_in_minutes: 300
209+
matrix:
210+
setup:
211+
LUCENE_VERSION:
212+
- "10.0.0"
213+
ES_VERSION:
214+
- "9.0.0"
215+
ES_COMMIT:
216+
- "b2cc9d9b8f00ee621f93ddca07ea9c671aab1578" # update to match last commit before lucene bump
217+
agents:
218+
provider: gcp
219+
image: family/elasticsearch-ubuntu-2004
220+
machineType: custom-32-98304
221+
buildDirectory: /dev/shm/bk
222+
env:
223+
ES_VERSION: "{{matrix.ES_VERSION}}"
224+
ES_COMMIT: "{{matrix.ES_COMMIT}}"
204225
- label: Upload Snyk Dependency Graph
205226
command: .ci/scripts/run-gradle.sh uploadSnykDependencyGraph -PsnykTargetReference=$BUILDKITE_BRANCH
206227
env:

.buildkite/pipelines/periodic.yml

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -582,6 +582,27 @@ steps:
582582
image: family/elasticsearch-ubuntu-2004
583583
machineType: n2-standard-8
584584
buildDirectory: /dev/shm/bk
585+
- group: lucene-compat
586+
steps:
587+
- label: "{{matrix.LUCENE_VERSION}} / lucene-compat"
588+
command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=$$ES_VERSION -Dtests.bwc.refspec.main=$$ES_COMMIT luceneBwcTest
589+
timeout_in_minutes: 300
590+
matrix:
591+
setup:
592+
LUCENE_VERSION:
593+
- "10.0.0"
594+
ES_VERSION:
595+
- "9.0.0"
596+
ES_COMMIT:
597+
- "b2cc9d9b8f00ee621f93ddca07ea9c671aab1578" # update to match last commit before lucene bump
598+
agents:
599+
provider: gcp
600+
image: family/elasticsearch-ubuntu-2004
601+
machineType: custom-32-98304
602+
buildDirectory: /dev/shm/bk
603+
env:
604+
ES_VERSION: "{{matrix.ES_VERSION}}"
605+
ES_COMMIT: "{{matrix.ES_COMMIT}}"
585606
- label: Upload Snyk Dependency Graph
586607
command: .ci/scripts/run-gradle.sh uploadSnykDependencyGraph -PsnykTargetReference=$BUILDKITE_BRANCH
587608
env:

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,24 @@ public void apply(Project project) {
8686
fileSystemOperations
8787
);
8888
});
89+
90+
// Also set up the "main" project which is just used for arbitrary overrides. See InternalDistributionDownloadPlugin.
91+
if (System.getProperty("tests.bwc.main.version") != null) {
92+
configureBwcProject(
93+
project.project(":distribution:bwc:main"),
94+
buildParams,
95+
new BwcVersions.UnreleasedVersionInfo(
96+
Version.fromString(System.getProperty("tests.bwc.main.version")),
97+
"main",
98+
":distribution:bwc:main"
99+
),
100+
providerFactory,
101+
objectFactory,
102+
toolChainService,
103+
isCi,
104+
fileSystemOperations
105+
);
106+
}
89107
}
90108

91109
private static void configureBwcProject(

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,29 @@ private void registerInternalDistributionResolutions(List<DistributionResolution
102102
}
103103
return null;
104104
}));
105+
106+
// Distribution resolution for "override" versions. This allows for building from source for any version, including the current
107+
// version of existing released versions from a commit form the main branch. This is done by passing certain system properties, ex:
108+
//
109+
// -Dtests.bwc.refspec.main=deadbeef -Dtests.bwc.main.version=9.0.0
110+
//
111+
// The 'test.bwc.main.version' property should map to the version returned by the commit referenced in 'tests.bwc.refspec.main'.
112+
resolutions.add(new DistributionResolution("override", (project, distribution) -> {
113+
String versionProperty = System.getProperty("tests.bwc.main.version");
114+
// We use this phony version as a placeholder for the real version
115+
if (distribution.getVersion().equals("0.0.0")) {
116+
BwcVersions.UnreleasedVersionInfo unreleasedVersionInfo = new BwcVersions.UnreleasedVersionInfo(
117+
Version.fromString(versionProperty),
118+
"main",
119+
":distribution:bwc:main"
120+
);
121+
String projectConfig = getProjectConfig(distribution, unreleasedVersionInfo);
122+
return new ProjectBasedDistributionDependency(
123+
(config) -> projectDependency(project.getDependencies(), unreleasedVersionInfo.gradleProjectPath(), projectConfig)
124+
);
125+
}
126+
return null;
127+
}));
105128
}
106129

107130
private boolean isCurrentVersion(ElasticsearchDistribution distribution) {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -223,7 +223,7 @@ public Void call(Object... args) {
223223
}
224224

225225
Version version = (Version) args[0];
226-
boolean isReleased = bwcVersions.unreleasedInfo(version) == null;
226+
boolean isReleased = bwcVersions.unreleasedInfo(version) == null && version.toString().equals("0.0.0") == false;
227227
String versionString = version.toString();
228228
ElasticsearchDistribution bwcDistro = createDistribution(project, "bwc_" + versionString, versionString);
229229

build-tools/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,9 @@
2525

2626
import java.util.ArrayList;
2727
import java.util.Collections;
28+
import java.util.Comparator;
2829
import java.util.List;
30+
import java.util.Objects;
2931

3032
import javax.inject.Inject;
3133

@@ -141,8 +143,9 @@ private void registerDistributionDependencies(Project project, ElasticsearchDist
141143

142144
private DistributionDependency resolveDependencyNotation(Project project, ElasticsearchDistribution distro) {
143145
return distributionsResolutionStrategies.stream()
146+
.sorted(Comparator.comparing(DistributionResolution::getPriority).reversed())
144147
.map(r -> r.getResolver().resolve(project, distro))
145-
.filter(d -> d != null)
148+
.filter(Objects::nonNull)
146149
.findFirst()
147150
.orElseGet(() -> DistributionDependency.of(dependencyNotation(distro)));
148151
}

distribution/bwc/main/build.gradle

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
// This project is used only for overriding bwc distributions. See InternalDistributionDownloadPlugin for details.

modules/ingest-geoip/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.elasticsearch.test.cluster.ElasticsearchCluster;
2424
import org.elasticsearch.test.cluster.FeatureFlag;
2525
import org.elasticsearch.test.cluster.local.distribution.DistributionType;
26+
import org.elasticsearch.test.cluster.util.Version;
2627
import org.elasticsearch.test.rest.ObjectPath;
2728
import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus;
2829
import org.elasticsearch.upgrades.ParameterizedFullClusterRestartTestCase;
@@ -54,7 +55,7 @@ public class FullClusterRestartIT extends ParameterizedFullClusterRestartTestCas
5455

5556
private static final ElasticsearchCluster cluster = ElasticsearchCluster.local()
5657
.distribution(DistributionType.DEFAULT)
57-
.version(getOldClusterTestVersion())
58+
.version(Version.fromString(OLD_CLUSTER_VERSION))
5859
.nodes(2)
5960
.setting("ingest.geoip.downloader.endpoint", () -> fixture.getAddress(), s -> useFixture)
6061
.setting("xpack.security.enabled", useSecurity ? "true" : "false")

0 commit comments

Comments
 (0)