Skip to content

Commit dd78b74

Browse files
jozalajozala
authored
Merge branch '9.0' into backport/9.0/pr-133633
2 parents 6ca592a + d2d2f44 commit dd78b74

File tree

36 files changed

+254
-101
lines changed

36 files changed

+254
-101
lines changed

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy‎

Lines changed: 41 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
5151
def "resolves expanded bwc versions from source"() {
5252
given:
5353
internalBuild()
54-
bwcMajor1ProjectSetup()
54+
bwcProjectSetup("major1")
5555
buildFile << """
5656
apply plugin: 'elasticsearch.internal-distribution-download'
5757
@@ -81,7 +81,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
8181
def "fails on resolving bwc versions with no bundled jdk"() {
8282
given:
8383
internalBuild()
84-
bwcMajor1ProjectSetup()
84+
bwcProjectSetup("major1")
8585
buildFile << """
8686
apply plugin: 'elasticsearch.internal-distribution-download'
8787
@@ -105,12 +105,47 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
105105
"without a bundled JDK is not supported.")
106106
}
107107

108-
private void bwcMajor1ProjectSetup() {
108+
def "resolves detached version from source"() {
109+
given:
110+
internalBuild()
111+
bwcProjectSetup("main")
112+
buildFile << """
113+
apply plugin: 'elasticsearch.internal-distribution-download'
114+
115+
System.setProperty("tests.bwc.main.version", "9.2.0")
116+
117+
elasticsearch_distributions {
118+
test_distro {
119+
version = "9.2.0"
120+
type = "archive"
121+
platform = "linux"
122+
architecture = Architecture.current();
123+
detachedVersion = true
124+
}
125+
}
126+
127+
tasks.register("setupDistro", Sync) {
128+
from(elasticsearch_distributions.test_distro)
129+
into("build/distro")
130+
}
131+
"""
132+
133+
when:
134+
def result = gradleRunner("setupDistro").build()
135+
136+
then:
137+
result.task(":distribution:bwc:main:buildBwcExpandedTask").outcome == TaskOutcome.SUCCESS
138+
result.task(":setupDistro").outcome == TaskOutcome.SUCCESS
139+
assertExtractedDistroIsCreated("distribution/bwc/main/build/install/elastic-distro",
140+
'bwc-marker.txt')
141+
}
142+
143+
private void bwcProjectSetup(String bwcProjectName) {
109144
settingsFile << """
110-
include ':distribution:bwc:major1'
145+
include ':distribution:bwc:$bwcProjectName'
111146
"""
112-
def bwcSubProjectFolder = testProjectDir.newFolder("distribution", "bwc", "major1")
113-
new File(bwcSubProjectFolder, 'bwc-marker.txt') << "bwc=major1"
147+
def bwcSubProjectFolder = testProjectDir.newFolder("distribution", "bwc", bwcProjectName)
148+
new File(bwcSubProjectFolder, 'bwc-marker.txt') << "bwc=$bwcProjectName"
114149
new File(bwcSubProjectFolder, 'build.gradle') << """
115150
apply plugin:'base'
116151

‎build-tools-internal/src/main/groovy/elasticsearch.bc-upgrade-test.gradle‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,18 @@ import org.elasticsearch.gradle.Version
1111
import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
1212

1313
tasks.register("bcUpgradeTest", StandaloneRestIntegTestTask) {
14-
// We use a phony version here as the real version is provided via `tests.bwc.main.version` system property
15-
usesBwcDistribution(Version.fromString("0.0.0"))
16-
systemProperty("tests.old_cluster_version", "0.0.0")
17-
onlyIf("tests.bwc.main.version system property exists") { System.getProperty("tests.bwc.main.version") != null }
14+
project.gradle.taskGraph.whenReady { allTasks ->
15+
if (allTasks.hasTask(getPath())) {
16+
if (System.getProperty("tests.bwc.refspec.main") == null) {
17+
throw new GradleException("You must set the `tests.bwc.refspec.main` system property to run bcUpgradeTest")
18+
}
19+
if (System.getProperty("tests.bwc.main.version") == null) {
20+
throw new GradleException("You must set the `tests.bwc.main.version` system property to run bcUpgradeTest")
21+
}
22+
}
23+
}
24+
if (System.getProperty("tests.bwc.refspec.main") != null && System.getProperty("tests.bwc.main.version") != null) {
25+
usesBwcDistributionFromRef(System.getProperty("tests.bwc.refspec.main"), Version.fromString(System.getProperty("tests.bwc.main.version")))
26+
systemProperty("tests.old_cluster_version", System.getProperty("tests.bwc.main.version"))
27+
}
1828
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java‎

Lines changed: 8 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ public void apply(Project project) {
7474
*/
7575
private void registerInternalDistributionResolutions(List<DistributionResolution> resolutions, Provider<BwcVersions> bwcVersions) {
7676
resolutions.add(new DistributionResolution("local-build", (project, distribution) -> {
77-
if (isCurrentVersion(distribution)) {
77+
if (isCurrentVersion(distribution) && distribution.isDetachedVersion() == false) {
7878
// non-external project, so depend on local build
7979
return new ProjectBasedDistributionDependency(
8080
config -> projectDependency(project.getDependencies(), distributionProjectPath(distribution), config)
@@ -86,7 +86,7 @@ private void registerInternalDistributionResolutions(List<DistributionResolution
8686
resolutions.add(new DistributionResolution("bwc", (project, distribution) -> {
8787
BwcVersions.UnreleasedVersionInfo unreleasedInfo = bwcVersions.get()
8888
.unreleasedInfo(Version.fromString(distribution.getVersion()));
89-
if (unreleasedInfo != null) {
89+
if (unreleasedInfo != null && distribution.isDetachedVersion() == false) {
9090
if (distribution.getBundledJdk() == false) {
9191
throw new GradleException(
9292
"Configuring a snapshot bwc distribution ('"
@@ -103,21 +103,17 @@ private void registerInternalDistributionResolutions(List<DistributionResolution
103103
return null;
104104
}));
105105

106-
// Distribution resolution for "override" versions. This allows for building from source for any version, including the current
106+
// Distribution resolution for "detached" versions. This allows for building from source for any version, including the current
107107
// version of existing released versions from a commit form the main branch. This is done by passing certain system properties, ex:
108108
//
109109
// -Dtests.bwc.refspec.main=deadbeef -Dtests.bwc.main.version=9.0.0
110110
//
111-
// The 'test.bwc.main.version' property should map to the version returned by the commit referenced in 'tests.bwc.refspec.main'.
112-
resolutions.add(new DistributionResolution("override", (project, distribution) -> {
113-
String versionProperty = System.getProperty("tests.bwc.main.version");
114-
// We use this phony version as a placeholder for the real version
115-
if (distribution.getVersion().equals("0.0.0")) {
116-
if (versionProperty == null) {
117-
throw new GradleException("System property 'tests.bwc.main.version' expected for building bwc version.");
118-
}
111+
// The distribution version set by 'test.bwc.main.version' property should map to the version returned by the commit
112+
// referenced in 'tests.bwc.refspec.main'.
113+
resolutions.add(new DistributionResolution("detached", (project, distribution) -> {
114+
if (distribution.isDetachedVersion()) {
119115
BwcVersions.UnreleasedVersionInfo unreleasedVersionInfo = new BwcVersions.UnreleasedVersionInfo(
120-
Version.fromString(versionProperty),
116+
Version.fromString(distribution.getVersion()),
121117
"main",
122118
":distribution:bwc:main"
123119
);

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java‎

Lines changed: 66 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -106,13 +106,15 @@ public void apply(Project project) {
106106
ElasticsearchDistribution defaultDistro = createDistribution(
107107
project,
108108
DEFAULT_REST_INTEG_TEST_DISTRO,
109-
VersionProperties.getElasticsearch()
109+
VersionProperties.getElasticsearch(),
110+
false
110111
);
111112
ElasticsearchDistribution integTestDistro = createDistribution(
112113
project,
113114
INTEG_TEST_REST_INTEG_TEST_DISTRO,
114115
VersionProperties.getElasticsearch(),
115-
ElasticsearchDistributionTypes.INTEG_TEST_ZIP
116+
ElasticsearchDistributionTypes.INTEG_TEST_ZIP,
117+
false
116118
);
117119

118120
// Create configures for module and plugin dependencies
@@ -229,21 +231,11 @@ public Void call(Object... args) {
229231
}
230232

231233
Version version = (Version) args[0];
232-
boolean isReleased = bwcVersions.unreleasedInfo(version) == null && version.toString().equals("0.0.0") == false;
234+
boolean isReleased = bwcVersions.unreleasedInfo(version) == null;
233235
String versionString = version.toString();
234-
ElasticsearchDistribution bwcDistro = createDistribution(project, "bwc_" + versionString, versionString);
235-
236-
if (jdkIsIncompatibleWithOS(Version.fromString(versionString))) {
237-
var toolChainService = project.getExtensions().getByType(JavaToolchainService.class);
238-
var fallbackJdk17Launcher = toolChainService.launcherFor(spec -> {
239-
spec.getVendor().set(JvmVendorSpec.ADOPTIUM);
240-
spec.getLanguageVersion().set(JavaLanguageVersion.of(17));
241-
});
242-
task.environment(
243-
"ES_FALLBACK_JAVA_HOME",
244-
fallbackJdk17Launcher.get().getMetadata().getInstallationPath().getAsFile().getPath()
245-
);
246-
}
236+
ElasticsearchDistribution bwcDistro = createDistribution(project, "bwc_" + versionString, versionString, false);
237+
238+
handleJdkIncompatibleWithOS(version, project, task);
247239
task.dependsOn(bwcDistro);
248240
registerDistributionInputs(task, bwcDistro);
249241

@@ -252,16 +244,63 @@ public Void call(Object... args) {
252244
providerFactory.provider(() -> bwcDistro.getExtracted().getSingleFile().getPath())
253245
);
254246

255-
if (version.getMajor() > 0 && version.before(bwcVersions.getMinimumWireCompatibleVersion())) {
247+
if (version.before(bwcVersions.getMinimumWireCompatibleVersion())) {
256248
// If we are upgrade testing older versions we also need to upgrade to 7.last
257249
this.call(bwcVersions.getMinimumWireCompatibleVersion());
258250
}
259251
return null;
260252
}
261253
});
254+
255+
task.getExtensions().getExtraProperties().set("usesBwcDistributionFromRef", new Closure<Void>(task) {
256+
@Override
257+
public Void call(Object... args) {
258+
if (args.length != 2 || args[0] instanceof String == false || args[1] instanceof Version == false) {
259+
throw new IllegalArgumentException("Expected arguments (String refSpec, org.elasticsearch.gradle.Version version)");
260+
}
261+
262+
String refSpec = (String) args[0];
263+
Version version = (Version) args[1];
264+
boolean isDetachedVersion = true;
265+
String versionString = version.toString();
266+
267+
ElasticsearchDistribution bwcDistro = createDistribution(project, "bwc_" + refSpec, versionString, isDetachedVersion);
268+
handleJdkIncompatibleWithOS(version, project, task);
269+
270+
task.dependsOn(bwcDistro);
271+
registerDistributionInputs(task, bwcDistro);
272+
273+
nonInputSystemProperties.systemProperty(
274+
BWC_SNAPSHOT_DISTRIBUTION_SYSPROP_PREFIX + versionString,
275+
providerFactory.provider(() -> bwcDistro.getExtracted().getSingleFile().getPath())
276+
);
277+
return null;
278+
}
279+
});
262280
});
263281
}
264282

283+
/**
284+
* Older distributions ship with openjdk versions that are not compatible with newer kernels of ubuntu 24.04 and later
285+
* Therefore we pass explicitly the runtime java to use the adoptium jdk that is maintained longer and compatible
286+
* with newer kernels.
287+
* 8.10.4 is the last version shipped with jdk < 21. We configure these cluster to run with jdk 17 adoptium as 17 was
288+
* the last LTS release before 21
289+
*/
290+
private static void handleJdkIncompatibleWithOS(Version version, Project project, StandaloneRestIntegTestTask task) {
291+
if (jdkIsIncompatibleWithOS(version)) {
292+
var toolChainService = project.getExtensions().getByType(JavaToolchainService.class);
293+
var fallbackJdk17Launcher = toolChainService.launcherFor(spec -> {
294+
spec.getVendor().set(JvmVendorSpec.ADOPTIUM);
295+
spec.getLanguageVersion().set(JavaLanguageVersion.of(17));
296+
});
297+
task.environment(
298+
"ES_FALLBACK_JAVA_HOME",
299+
fallbackJdk17Launcher.get().getMetadata().getInstallationPath().getAsFile().getPath()
300+
);
301+
}
302+
}
303+
265304
private void copyDependencies(Project project, DependencySet dependencies, Configuration configuration) {
266305
configuration.getDependencies()
267306
.stream()
@@ -270,16 +309,23 @@ private void copyDependencies(Project project, DependencySet dependencies, Confi
270309
.forEach(dependencies::add);
271310
}
272311

273-
private ElasticsearchDistribution createDistribution(Project project, String name, String version) {
274-
return createDistribution(project, name, version, null);
312+
private ElasticsearchDistribution createDistribution(Project project, String name, String version, boolean detachedVersion) {
313+
return createDistribution(project, name, version, null, detachedVersion);
275314
}
276315

277-
private ElasticsearchDistribution createDistribution(Project project, String name, String version, ElasticsearchDistributionType type) {
316+
private ElasticsearchDistribution createDistribution(
317+
Project project,
318+
String name,
319+
String version,
320+
ElasticsearchDistributionType type,
321+
boolean detachedVersion
322+
) {
278323
NamedDomainObjectContainer<ElasticsearchDistribution> distributions = DistributionDownloadPlugin.getContainer(project);
279324
ElasticsearchDistribution maybeDistro = distributions.findByName(name);
280325
if (maybeDistro == null) {
281326
return distributions.create(name, distro -> {
282327
distro.setVersion(version);
328+
distro.setDetachedVersion(detachedVersion);
283329
distro.setArchitecture(Architecture.current());
284330
if (type != null) {
285331
distro.setType(type);

‎build-tools/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ public String toString() {
4949

5050
private final Property<Architecture> architecture;
5151
private final Property<String> version;
52+
private final Property<Boolean> detachedVersion;
5253
private final Property<ElasticsearchDistributionType> type;
5354
private final Property<Platform> platform;
5455
private final Property<Boolean> bundledJdk;
@@ -69,6 +70,7 @@ public String toString() {
6970
this.configuration = fileConfiguration;
7071
this.architecture = objectFactory.property(Architecture.class);
7172
this.version = objectFactory.property(String.class).convention(VersionProperties.getElasticsearch());
73+
this.detachedVersion = objectFactory.property(Boolean.class).convention(false);
7274
this.type = objectFactory.property(ElasticsearchDistributionType.class);
7375
this.type.convention(ElasticsearchDistributionTypes.ARCHIVE);
7476
this.platform = objectFactory.property(Platform.class);
@@ -91,6 +93,19 @@ public void setVersion(String version) {
9193
this.version.set(version);
9294
}
9395

96+
/**
97+
* Informs if the version is not tied to any Elasticsearch release and is a custom build.
98+
* This is true when the distribution is not from HEAD but also not any known released version.
99+
* In that case the detached source build needs to be prepared by `usedBwcDistributionFromRef(ref, version)`.
100+
*/
101+
public boolean isDetachedVersion() {
102+
return detachedVersion.get();
103+
}
104+
105+
public void setDetachedVersion(boolean detachedVersion) {
106+
this.detachedVersion.set(detachedVersion);
107+
}
108+
94109
public Platform getPlatform() {
95110
return platform.getOrNull();
96111
}

‎modules/ingest-geoip/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
import org.elasticsearch.test.cluster.ElasticsearchCluster;
1818
import org.elasticsearch.test.cluster.FeatureFlag;
1919
import org.elasticsearch.test.cluster.local.distribution.DistributionType;
20-
import org.elasticsearch.test.cluster.util.Version;
2120
import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus;
2221
import org.elasticsearch.upgrades.ParameterizedFullClusterRestartTestCase;
2322
import org.junit.ClassRule;
@@ -41,7 +40,7 @@ public class FullClusterRestartIT extends ParameterizedFullClusterRestartTestCas
4140

4241
private static final ElasticsearchCluster cluster = ElasticsearchCluster.local()
4342
.distribution(DistributionType.DEFAULT)
44-
.version(Version.fromString(OLD_CLUSTER_VERSION))
43+
.version(OLD_CLUSTER_VERSION, isOldClusterDetachedVersion())
4544
.nodes(2)
4645
.setting("ingest.geoip.downloader.endpoint", () -> fixture.getAddress(), s -> useFixture)
4746
.setting("xpack.security.enabled", "false")

‎qa/full-cluster-restart/build.gradle‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,20 @@ buildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName ->
2323
}
2424

2525
tasks.register("luceneBwcTest", StandaloneRestIntegTestTask) {
26-
// We use a phony version here as the real version is provided via `tests.bwc.main.version` system property
27-
usesBwcDistribution(Version.fromString("0.0.0"))
28-
systemProperty("tests.old_cluster_version", "0.0.0")
29-
onlyIf("tests.bwc.main.version system property exists") { System.getProperty("tests.bwc.main.version") != null }
26+
project.gradle.taskGraph.whenReady { allTasks ->
27+
if (allTasks.hasTask(getPath())) {
28+
if (System.getProperty("tests.bwc.refspec.main") == null) {
29+
throw new GradleException("You must set the `tests.bwc.refspec.main` system property to run luceneBwcTest")
30+
}
31+
if (System.getProperty("tests.bwc.main.version") == null) {
32+
throw new GradleException("You must set the `tests.bwc.main.version` system property to run luceneBwcTest")
33+
}
34+
}
35+
}
36+
if (System.getProperty("tests.bwc.refspec.main") != null && System.getProperty("tests.bwc.main.version") != null) {
37+
usesBwcDistributionFromRef(System.getProperty("tests.bwc.refspec.main"), Version.fromString(System.getProperty("tests.bwc.main.version")))
38+
systemProperty("tests.old_cluster_version", System.getProperty("tests.bwc.main.version"))
39+
}
3040
}
3141

3242
tasks.named("bcUpgradeTest").configure {

‎qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartArchivedSettingsIT.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
import org.elasticsearch.test.cluster.FeatureFlag;
2222
import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider;
2323
import org.elasticsearch.test.cluster.local.distribution.DistributionType;
24-
import org.elasticsearch.test.cluster.util.Version;
2524
import org.elasticsearch.test.rest.ObjectPath;
2625
import org.junit.ClassRule;
2726
import org.junit.rules.RuleChain;
@@ -45,7 +44,7 @@ public class FullClusterRestartArchivedSettingsIT extends ParameterizedFullClust
4544

4645
private static ElasticsearchCluster cluster = ElasticsearchCluster.local()
4746
.distribution(DistributionType.DEFAULT)
48-
.version(Version.fromString(OLD_CLUSTER_VERSION))
47+
.version(OLD_CLUSTER_VERSION, isOldClusterDetachedVersion())
4948
.nodes(2)
5049
.setting("path.repo", () -> repoDirectory.getRoot().getPath())
5150
.setting("xpack.security.enabled", "false")

‎qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartDownsampleIT.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ private static ElasticsearchCluster buildCluster() {
5050
Version oldVersion = Version.fromString(OLD_CLUSTER_VERSION);
5151
var cluster = ElasticsearchCluster.local()
5252
.distribution(DistributionType.DEFAULT)
53-
.version(Version.fromString(OLD_CLUSTER_VERSION))
53+
.version(OLD_CLUSTER_VERSION, isOldClusterDetachedVersion())
5454
.nodes(2)
5555
.setting("xpack.security.enabled", "false")
5656
.setting("indices.lifecycle.poll_interval", "5s")

0 commit comments

Comments
 (0)