[Test] Replace 0.0.0 version with "detached" property on ElasticsearchDistibution (#134584) (#135924)

Introduce a new `detached` property to indicate if the version is a custom build
not tied to any release. This was previously handled by specifying
version as 0.0.0, but caused problems when comparing versions (e.g., to
decide on configuration).

(cherry picked from commit 664a604)

# Conflicts:
#	qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/AbstractRollingUpgradeWithSecurityTestCase.java
#	test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
#	x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/StandardToLogsDbIndexModeRollingUpgradeIT.java

Author: jozala

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy

@@ -51,7 +51,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
     def "resolves expanded bwc versions from source"() {
         given:
         internalBuild()
-        bwcMajor1ProjectSetup()
+        bwcProjectSetup("major1")
         buildFile << """
             apply plugin: 'elasticsearch.internal-distribution-download'
 
@@ -81,7 +81,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
     def "fails on resolving bwc versions with no bundled jdk"() {
         given:
         internalBuild()
-        bwcMajor1ProjectSetup()
+        bwcProjectSetup("major1")
         buildFile << """
             apply plugin: 'elasticsearch.internal-distribution-download'
 
@@ -105,12 +105,47 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
                 "without a bundled JDK is not supported.")
     }
 
-    private void bwcMajor1ProjectSetup() {
+    def "resolves detached version from source"() {
+        given:
+        internalBuild()
+        bwcProjectSetup("main")
+        buildFile << """
+            apply plugin: 'elasticsearch.internal-distribution-download'
+
+            System.setProperty("tests.bwc.main.version", "9.2.0")
+
+            elasticsearch_distributions {
+              test_distro {
+                  version = "9.2.0"
+                  type = "archive"
+                  platform = "linux"
+                  architecture = Architecture.current();
+                  detachedVersion = true
+              }
+            }
+
+            tasks.register("setupDistro", Sync) {
+                from(elasticsearch_distributions.test_distro)
+                into("build/distro")
+            }
+        """
+
+        when:
+        def result = gradleRunner("setupDistro").build()
+
+        then:
+        result.task(":distribution:bwc:main:buildBwcExpandedTask").outcome == TaskOutcome.SUCCESS
+        result.task(":setupDistro").outcome == TaskOutcome.SUCCESS
+        assertExtractedDistroIsCreated("distribution/bwc/main/build/install/elastic-distro",
+            'bwc-marker.txt')
+    }
+
+    private void bwcProjectSetup(String bwcProjectName) {
         settingsFile << """
-        include ':distribution:bwc:major1'
+        include ':distribution:bwc:$bwcProjectName'
         """
-        def bwcSubProjectFolder = testProjectDir.newFolder("distribution", "bwc", "major1")
-        new File(bwcSubProjectFolder, 'bwc-marker.txt') << "bwc=major1"
+        def bwcSubProjectFolder = testProjectDir.newFolder("distribution", "bwc", bwcProjectName)
+        new File(bwcSubProjectFolder, 'bwc-marker.txt') << "bwc=$bwcProjectName"
         new File(bwcSubProjectFolder, 'build.gradle') << """
             apply plugin:'base'
 

build-tools-internal/src/main/groovy/elasticsearch.bc-upgrade-test.gradle

@@ -11,8 +11,18 @@ import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
 
 tasks.register("bcUpgradeTest", StandaloneRestIntegTestTask) {
-  // We use a phony version here as the real version is provided via `tests.bwc.main.version` system property
-  usesBwcDistribution(Version.fromString("0.0.0"))
-  systemProperty("tests.old_cluster_version", "0.0.0")
-  onlyIf("tests.bwc.main.version system property exists") { System.getProperty("tests.bwc.main.version") != null }
+  project.gradle.taskGraph.whenReady { allTasks ->
+    if (allTasks.hasTask(getPath())) {
+      if (System.getProperty("tests.bwc.refspec.main") == null) {
+        throw new GradleException("You must set the `tests.bwc.refspec.main` system property to run bcUpgradeTest")
+      }
+      if (System.getProperty("tests.bwc.main.version") == null) {
+        throw new GradleException("You must set the `tests.bwc.main.version` system property to run bcUpgradeTest")
+      }
+    }
+  }
+  if (System.getProperty("tests.bwc.refspec.main") != null && System.getProperty("tests.bwc.main.version") != null) {
+    usesBwcDistributionFromRef(System.getProperty("tests.bwc.refspec.main"), Version.fromString(System.getProperty("tests.bwc.main.version")))
+    systemProperty("tests.old_cluster_version", System.getProperty("tests.bwc.main.version"))
+  }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java

@@ -74,7 +74,7 @@ public void apply(Project project) {
      */
     private void registerInternalDistributionResolutions(List<DistributionResolution> resolutions, Provider<BwcVersions> bwcVersions) {
         resolutions.add(new DistributionResolution("local-build", (project, distribution) -> {
-            if (isCurrentVersion(distribution)) {
+            if (isCurrentVersion(distribution) && distribution.isDetachedVersion() == false) {
                 // non-external project, so depend on local build
                 return new ProjectBasedDistributionDependency(
                     config -> projectDependency(project.getDependencies(), distributionProjectPath(distribution), config)
@@ -86,7 +86,7 @@ private void registerInternalDistributionResolutions(List<DistributionResolution
         resolutions.add(new DistributionResolution("bwc", (project, distribution) -> {
             BwcVersions.UnreleasedVersionInfo unreleasedInfo = bwcVersions.get()
                 .unreleasedInfo(Version.fromString(distribution.getVersion()));
-            if (unreleasedInfo != null) {
+            if (unreleasedInfo != null && distribution.isDetachedVersion() == false) {
                 if (distribution.getBundledJdk() == false) {
                     throw new GradleException(
                         "Configuring a snapshot bwc distribution ('"
@@ -103,21 +103,17 @@ private void registerInternalDistributionResolutions(List<DistributionResolution
             return null;
         }));
 
-        // Distribution resolution for "override" versions. This allows for building from source for any version, including the current
+        // Distribution resolution for "detached" versions. This allows for building from source for any version, including the current
         // version of existing released versions from a commit form the main branch. This is done by passing certain system properties, ex:
         //
         // -Dtests.bwc.refspec.main=deadbeef -Dtests.bwc.main.version=9.0.0
         //
-        // The 'test.bwc.main.version' property should map to the version returned by the commit referenced in 'tests.bwc.refspec.main'.
-        resolutions.add(new DistributionResolution("override", (project, distribution) -> {
-            String versionProperty = System.getProperty("tests.bwc.main.version");
-            // We use this phony version as a placeholder for the real version
-            if (distribution.getVersion().equals("0.0.0")) {
-                if (versionProperty == null) {
-                    throw new GradleException("System property 'tests.bwc.main.version' expected for building bwc version.");
-                }
+        // The distribution version set by 'test.bwc.main.version' property should map to the version returned by the commit
+        // referenced in 'tests.bwc.refspec.main'.
+        resolutions.add(new DistributionResolution("detached", (project, distribution) -> {
+            if (distribution.isDetachedVersion()) {
                 BwcVersions.UnreleasedVersionInfo unreleasedVersionInfo = new BwcVersions.UnreleasedVersionInfo(
-                    Version.fromString(versionProperty),
+                    Version.fromString(distribution.getVersion()),
                     "main",
                     ":distribution:bwc:main"
                 );

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java

@@ -106,13 +106,15 @@ public void apply(Project project) {
         ElasticsearchDistribution defaultDistro = createDistribution(
             project,
             DEFAULT_REST_INTEG_TEST_DISTRO,
-            VersionProperties.getElasticsearch()
+            VersionProperties.getElasticsearch(),
+            false
         );
         ElasticsearchDistribution integTestDistro = createDistribution(
             project,
             INTEG_TEST_REST_INTEG_TEST_DISTRO,
             VersionProperties.getElasticsearch(),
-            ElasticsearchDistributionTypes.INTEG_TEST_ZIP
+            ElasticsearchDistributionTypes.INTEG_TEST_ZIP,
+            false
         );
 
         // Create configures for module and plugin dependencies
@@ -229,21 +231,11 @@ public Void call(Object... args) {
                     }
 
                     Version version = (Version) args[0];
-                    boolean isReleased = bwcVersions.unreleasedInfo(version) == null && version.toString().equals("0.0.0") == false;
+                    boolean isReleased = bwcVersions.unreleasedInfo(version) == null;
                     String versionString = version.toString();
-                    ElasticsearchDistribution bwcDistro = createDistribution(project, "bwc_" + versionString, versionString);
-
-                    if (jdkIsIncompatibleWithOS(Version.fromString(versionString))) {
-                        var toolChainService = project.getExtensions().getByType(JavaToolchainService.class);
-                        var fallbackJdk17Launcher = toolChainService.launcherFor(spec -> {
-                            spec.getVendor().set(JvmVendorSpec.ADOPTIUM);
-                            spec.getLanguageVersion().set(JavaLanguageVersion.of(17));
-                        });
-                        task.environment(
-                            "ES_FALLBACK_JAVA_HOME",
-                            fallbackJdk17Launcher.get().getMetadata().getInstallationPath().getAsFile().getPath()
-                        );
-                    }
+                    ElasticsearchDistribution bwcDistro = createDistribution(project, "bwc_" + versionString, versionString, false);
+
+                    handleJdkIncompatibleWithOS(version, project, task);
                     task.dependsOn(bwcDistro);
                     registerDistributionInputs(task, bwcDistro);
 
@@ -252,16 +244,63 @@ public Void call(Object... args) {
                         providerFactory.provider(() -> bwcDistro.getExtracted().getSingleFile().getPath())
                     );
 
-                    if (version.getMajor() > 0 && version.before(bwcVersions.getMinimumWireCompatibleVersion())) {
+                    if (version.before(bwcVersions.getMinimumWireCompatibleVersion())) {
                         // If we are upgrade testing older versions we also need to upgrade to 7.last
                         this.call(bwcVersions.getMinimumWireCompatibleVersion());
                     }
                     return null;
                 }
             });
+
+            task.getExtensions().getExtraProperties().set("usesBwcDistributionFromRef", new Closure<Void>(task) {
+                @Override
+                public Void call(Object... args) {
+                    if (args.length != 2 || args[0] instanceof String == false || args[1] instanceof Version == false) {
+                        throw new IllegalArgumentException("Expected arguments (String refSpec, org.elasticsearch.gradle.Version version)");
+                    }
+
+                    String refSpec = (String) args[0];
+                    Version version = (Version) args[1];
+                    boolean isDetachedVersion = true;
+                    String versionString = version.toString();
+
+                    ElasticsearchDistribution bwcDistro = createDistribution(project, "bwc_" + refSpec, versionString, isDetachedVersion);
+                    handleJdkIncompatibleWithOS(version, project, task);
+
+                    task.dependsOn(bwcDistro);
+                    registerDistributionInputs(task, bwcDistro);
+
+                    nonInputSystemProperties.systemProperty(
+                        BWC_SNAPSHOT_DISTRIBUTION_SYSPROP_PREFIX + versionString,
+                        providerFactory.provider(() -> bwcDistro.getExtracted().getSingleFile().getPath())
+                    );
+                    return null;
+                }
+            });
         });
     }
 
+    /**
+     * Older distributions ship with openjdk versions that are not compatible with newer kernels of ubuntu 24.04 and later
+     * Therefore we pass explicitly the runtime java to use the adoptium jdk that is maintained longer and compatible
+     * with newer kernels.
+     * 8.10.4 is the last version shipped with jdk < 21. We configure these cluster to run with jdk 17 adoptium as 17 was
+     * the last LTS release before 21
+     */
+    private static void handleJdkIncompatibleWithOS(Version version, Project project, StandaloneRestIntegTestTask task) {
+        if (jdkIsIncompatibleWithOS(version)) {
+            var toolChainService = project.getExtensions().getByType(JavaToolchainService.class);
+            var fallbackJdk17Launcher = toolChainService.launcherFor(spec -> {
+                spec.getVendor().set(JvmVendorSpec.ADOPTIUM);
+                spec.getLanguageVersion().set(JavaLanguageVersion.of(17));
+            });
+            task.environment(
+                "ES_FALLBACK_JAVA_HOME",
+                fallbackJdk17Launcher.get().getMetadata().getInstallationPath().getAsFile().getPath()
+            );
+        }
+    }
+
     private void copyDependencies(Project project, DependencySet dependencies, Configuration configuration) {
         configuration.getDependencies()
             .stream()
@@ -270,16 +309,23 @@ private void copyDependencies(Project project, DependencySet dependencies, Confi
             .forEach(dependencies::add);
     }
 
-    private ElasticsearchDistribution createDistribution(Project project, String name, String version) {
-        return createDistribution(project, name, version, null);
+    private ElasticsearchDistribution createDistribution(Project project, String name, String version, boolean detachedVersion) {
+        return createDistribution(project, name, version, null, detachedVersion);
     }
 
-    private ElasticsearchDistribution createDistribution(Project project, String name, String version, ElasticsearchDistributionType type) {
+    private ElasticsearchDistribution createDistribution(
+        Project project,
+        String name,
+        String version,
+        ElasticsearchDistributionType type,
+        boolean detachedVersion
+    ) {
         NamedDomainObjectContainer<ElasticsearchDistribution> distributions = DistributionDownloadPlugin.getContainer(project);
         ElasticsearchDistribution maybeDistro = distributions.findByName(name);
         if (maybeDistro == null) {
             return distributions.create(name, distro -> {
                 distro.setVersion(version);
+                distro.setDetachedVersion(detachedVersion);
                 distro.setArchitecture(Architecture.current());
                 if (type != null) {
                     distro.setType(type);

build-tools/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java

@@ -49,6 +49,7 @@ public String toString() {
 
     private final Property<Architecture> architecture;
     private final Property<String> version;
+    private final Property<Boolean> detachedVersion;
     private final Property<ElasticsearchDistributionType> type;
     private final Property<Platform> platform;
     private final Property<Boolean> bundledJdk;
@@ -69,6 +70,7 @@ public String toString() {
         this.configuration = fileConfiguration;
         this.architecture = objectFactory.property(Architecture.class);
         this.version = objectFactory.property(String.class).convention(VersionProperties.getElasticsearch());
+        this.detachedVersion = objectFactory.property(Boolean.class).convention(false);
         this.type = objectFactory.property(ElasticsearchDistributionType.class);
         this.type.convention(ElasticsearchDistributionTypes.ARCHIVE);
         this.platform = objectFactory.property(Platform.class);
@@ -91,6 +93,19 @@ public void setVersion(String version) {
         this.version.set(version);
     }
 
+    /**
+     * Informs if the version is not tied to any Elasticsearch release and is a custom build.
+     * This is true when the distribution is not from HEAD but also not any known released version.
+     * In that case the detached source build needs to be prepared by `usedBwcDistributionFromRef(ref, version)`.
+     */
+    public boolean isDetachedVersion() {
+        return detachedVersion.get();
+    }
+
+    public void setDetachedVersion(boolean detachedVersion) {
+        this.detachedVersion.set(detachedVersion);
+    }
+
     public Platform getPlatform() {
         return platform.getOrNull();
     }

modules/ingest-geoip/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java

@@ -17,7 +17,6 @@
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.cluster.FeatureFlag;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
-import org.elasticsearch.test.cluster.util.Version;
 import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus;
 import org.elasticsearch.upgrades.ParameterizedFullClusterRestartTestCase;
 import org.junit.ClassRule;
@@ -41,7 +40,7 @@ public class FullClusterRestartIT extends ParameterizedFullClusterRestartTestCas
 
     private static final ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .distribution(DistributionType.DEFAULT)
-        .version(Version.fromString(OLD_CLUSTER_VERSION))
+        .version(OLD_CLUSTER_VERSION, isOldClusterDetachedVersion())
         .nodes(2)
         .setting("ingest.geoip.downloader.endpoint", () -> fixture.getAddress(), s -> useFixture)
         .setting("xpack.security.enabled", "false")

qa/full-cluster-restart/build.gradle

@@ -23,10 +23,20 @@ buildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName ->
 }
 
 tasks.register("luceneBwcTest", StandaloneRestIntegTestTask) {
-  // We use a phony version here as the real version is provided via `tests.bwc.main.version` system property
-  usesBwcDistribution(Version.fromString("0.0.0"))
-  systemProperty("tests.old_cluster_version", "0.0.0")
-  onlyIf("tests.bwc.main.version system property exists") { System.getProperty("tests.bwc.main.version") != null }
+  project.gradle.taskGraph.whenReady { allTasks ->
+    if (allTasks.hasTask(getPath())) {
+      if (System.getProperty("tests.bwc.refspec.main") == null) {
+        throw new GradleException("You must set the `tests.bwc.refspec.main` system property to run luceneBwcTest")
+      }
+      if (System.getProperty("tests.bwc.main.version") == null) {
+        throw new GradleException("You must set the `tests.bwc.main.version` system property to run luceneBwcTest")
+      }
+    }
+  }
+  if (System.getProperty("tests.bwc.refspec.main") != null && System.getProperty("tests.bwc.main.version") != null) {
+    usesBwcDistributionFromRef(System.getProperty("tests.bwc.refspec.main"), Version.fromString(System.getProperty("tests.bwc.main.version")))
+    systemProperty("tests.old_cluster_version", System.getProperty("tests.bwc.main.version"))
+  }
 }
 
 tasks.named("bcUpgradeTest").configure {

qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartArchivedSettingsIT.java

@@ -21,7 +21,6 @@
 import org.elasticsearch.test.cluster.FeatureFlag;
 import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
-import org.elasticsearch.test.cluster.util.Version;
 import org.elasticsearch.test.rest.ObjectPath;
 import org.junit.ClassRule;
 import org.junit.rules.RuleChain;
@@ -45,7 +44,7 @@ public class FullClusterRestartArchivedSettingsIT extends ParameterizedFullClust
 
     private static ElasticsearchCluster cluster = ElasticsearchCluster.local()
         .distribution(DistributionType.DEFAULT)
-        .version(Version.fromString(OLD_CLUSTER_VERSION))
+        .version(OLD_CLUSTER_VERSION, isOldClusterDetachedVersion())
         .nodes(2)
         .setting("path.repo", () -> repoDirectory.getRoot().getPath())
         .setting("xpack.security.enabled", "false")

qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartDownsampleIT.java

@@ -50,7 +50,7 @@ private static ElasticsearchCluster buildCluster() {
         Version oldVersion = Version.fromString(OLD_CLUSTER_VERSION);
         var cluster = ElasticsearchCluster.local()
             .distribution(DistributionType.DEFAULT)
-            .version(Version.fromString(OLD_CLUSTER_VERSION))
+            .version(OLD_CLUSTER_VERSION, isOldClusterDetachedVersion())
             .nodes(2)
             .setting("xpack.security.enabled", "false")
             .setting("indices.lifecycle.poll_interval", "5s")