Merge branch 'main' into diskbbq-disk-rescoring

Author: thecoop

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy

@@ -51,7 +51,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
     def "resolves expanded bwc versions from source"() {
         given:
         internalBuild()
-        bwcMajor1ProjectSetup()
+        bwcProjectSetup("major1")
         buildFile << """
             apply plugin: 'elasticsearch.internal-distribution-download'
 
@@ -81,7 +81,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
     def "fails on resolving bwc versions with no bundled jdk"() {
         given:
         internalBuild()
-        bwcMajor1ProjectSetup()
+        bwcProjectSetup("major1")
         buildFile << """
             apply plugin: 'elasticsearch.internal-distribution-download'
 
@@ -105,12 +105,47 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
                 "without a bundled JDK is not supported.")
     }
 
-    private void bwcMajor1ProjectSetup() {
+    def "resolves detached version from source"() {
+        given:
+        internalBuild()
+        bwcProjectSetup("main")
+        buildFile << """
+            apply plugin: 'elasticsearch.internal-distribution-download'
+
+            System.setProperty("tests.bwc.main.version", "9.2.0")
+
+            elasticsearch_distributions {
+              test_distro {
+                  version = "9.2.0"
+                  type = "archive"
+                  platform = "linux"
+                  architecture = Architecture.current();
+                  detachedVersion = true
+              }
+            }
+
+            tasks.register("setupDistro", Sync) {
+                from(elasticsearch_distributions.test_distro)
+                into("build/distro")
+            }
+        """
+
+        when:
+        def result = gradleRunner("setupDistro").build()
+
+        then:
+        result.task(":distribution:bwc:main:buildBwcExpandedTask").outcome == TaskOutcome.SUCCESS
+        result.task(":setupDistro").outcome == TaskOutcome.SUCCESS
+        assertExtractedDistroIsCreated("distribution/bwc/main/build/install/elastic-distro",
+            'bwc-marker.txt')
+    }
+
+    private void bwcProjectSetup(String bwcProjectName) {
         settingsFile << """
-        include ':distribution:bwc:major1'
+        include ':distribution:bwc:$bwcProjectName'
         """
-        def bwcSubProjectFolder = testProjectDir.newFolder("distribution", "bwc", "major1")
-        new File(bwcSubProjectFolder, 'bwc-marker.txt') << "bwc=major1"
+        def bwcSubProjectFolder = testProjectDir.newFolder("distribution", "bwc", bwcProjectName)
+        new File(bwcSubProjectFolder, 'bwc-marker.txt') << "bwc=$bwcProjectName"
         new File(bwcSubProjectFolder, 'build.gradle') << """
             apply plugin:'base'
 

build-tools-internal/src/main/groovy/elasticsearch.bc-upgrade-test.gradle

@@ -11,8 +11,18 @@ import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
 
 tasks.register("bcUpgradeTest", StandaloneRestIntegTestTask) {
-  // We use a phony version here as the real version is provided via `tests.bwc.main.version` system property
-  usesBwcDistribution(Version.fromString("0.0.0"))
-  systemProperty("tests.old_cluster_version", "0.0.0")
-  onlyIf("tests.bwc.main.version system property exists") { System.getProperty("tests.bwc.main.version") != null }
+  project.gradle.taskGraph.whenReady { allTasks ->
+    if (allTasks.hasTask(getPath())) {
+      if (System.getProperty("tests.bwc.refspec.main") == null) {
+        throw new GradleException("You must set the `tests.bwc.refspec.main` system property to run bcUpgradeTest")
+      }
+      if (System.getProperty("tests.bwc.main.version") == null) {
+        throw new GradleException("You must set the `tests.bwc.main.version` system property to run bcUpgradeTest")
+      }
+    }
+  }
+  if (System.getProperty("tests.bwc.refspec.main") != null && System.getProperty("tests.bwc.main.version") != null) {
+    usesBwcDistributionFromRef(System.getProperty("tests.bwc.refspec.main"), Version.fromString(System.getProperty("tests.bwc.main.version")))
+    systemProperty("tests.old_cluster_version", System.getProperty("tests.bwc.main.version"))
+  }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchBuildCompletePlugin.java

@@ -38,8 +38,12 @@
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.io.UncheckedIOException;
+import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -66,7 +70,7 @@ public void apply(Project target) {
             ? System.getenv("BUILD_NUMBER")
             : System.getenv("BUILDKITE_BUILD_NUMBER");
         String performanceTest = System.getenv("BUILD_PERFORMANCE_TEST");
-        if (buildNumber != null && performanceTest == null && GradleUtils.isIncludedBuild(target) == false && OS.current() != OS.WINDOWS) {
+        if (buildNumber != null && performanceTest == null && GradleUtils.isIncludedBuild(target) == false) {
             File targetFile = calculateTargetFile(target, buildNumber);
             File projectDir = target.getProjectDir();
             File gradleWorkersDir = new File(target.getGradle().getGradleUserHomeDir(), "workers/");
@@ -102,6 +106,23 @@ private File calculateTargetFile(Project target, String buildNumber) {
     }
 
     private List<File> resolveProjectLogs(File projectDir) {
+        // HACK: Some tests leave behind symlinks, and gradle throws an exception if it encounters symlinks.
+        // Here we remove them before collecting logs to upload. We could instead build our own path matcher
+        // but that seemed more complex than just deleting the irrelevant files.
+        try {
+            Files.walkFileTree(projectDir.toPath(), new SimpleFileVisitor<>() {
+                @Override
+                public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+                    if (Files.isSymbolicLink(file)) {
+                        Files.delete(file);
+                    }
+                    return FileVisitResult.CONTINUE;
+                }
+            });
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+
         var projectDirFiles = getFileOperations().fileTree(projectDir);
         projectDirFiles.include("**/*.hprof");
         projectDirFiles.include("**/build/reports/configuration-cache/**");
@@ -276,7 +297,12 @@ private static void createBuildArchiveTar(List<File> files, File projectDir, Fil
 
         @NotNull
         private static String calculateArchivePath(Path path, Path projectPath) {
-            return path.startsWith(projectPath) ? projectPath.relativize(path).toString() : path.getFileName().toString();
+            String archivePath = path.startsWith(projectPath) ? projectPath.relativize(path).toString() : path.getFileName().toString();
+            if (OS.current() == OS.WINDOWS) {
+                // tar always uses forward slashes
+                archivePath = archivePath.replace("\\", "/");
+            }
+            return archivePath;
         }
     }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java

@@ -74,7 +74,7 @@ public void apply(Project project) {
      */
     private void registerInternalDistributionResolutions(List<DistributionResolution> resolutions, Provider<BwcVersions> bwcVersions) {
         resolutions.add(new DistributionResolution("local-build", (project, distribution) -> {
-            if (isCurrentVersion(distribution)) {
+            if (isCurrentVersion(distribution) && distribution.isDetachedVersion() == false) {
                 // non-external project, so depend on local build
                 return new ProjectBasedDistributionDependency(
                     config -> projectDependency(project.getDependencies(), distributionProjectPath(distribution), config)
@@ -86,7 +86,7 @@ private void registerInternalDistributionResolutions(List<DistributionResolution
         resolutions.add(new DistributionResolution("bwc", (project, distribution) -> {
             BwcVersions.UnreleasedVersionInfo unreleasedInfo = bwcVersions.get()
                 .unreleasedInfo(Version.fromString(distribution.getVersion()));
-            if (unreleasedInfo != null) {
+            if (unreleasedInfo != null && distribution.isDetachedVersion() == false) {
                 if (distribution.getBundledJdk() == false) {
                     throw new GradleException(
                         "Configuring a snapshot bwc distribution ('"
@@ -103,21 +103,17 @@ private void registerInternalDistributionResolutions(List<DistributionResolution
             return null;
         }));
 
-        // Distribution resolution for "override" versions. This allows for building from source for any version, including the current
+        // Distribution resolution for "detached" versions. This allows for building from source for any version, including the current
         // version of existing released versions from a commit form the main branch. This is done by passing certain system properties, ex:
         //
         // -Dtests.bwc.refspec.main=deadbeef -Dtests.bwc.main.version=9.0.0
         //
-        // The 'test.bwc.main.version' property should map to the version returned by the commit referenced in 'tests.bwc.refspec.main'.
-        resolutions.add(new DistributionResolution("override", (project, distribution) -> {
-            String versionProperty = System.getProperty("tests.bwc.main.version");
-            // We use this phony version as a placeholder for the real version
-            if (distribution.getVersion().equals("0.0.0")) {
-                if (versionProperty == null) {
-                    throw new GradleException("System property 'tests.bwc.main.version' expected for building bwc version.");
-                }
+        // The distribution version set by 'test.bwc.main.version' property should map to the version returned by the commit
+        // referenced in 'tests.bwc.refspec.main'.
+        resolutions.add(new DistributionResolution("detached", (project, distribution) -> {
+            if (distribution.isDetachedVersion()) {
                 BwcVersions.UnreleasedVersionInfo unreleasedVersionInfo = new BwcVersions.UnreleasedVersionInfo(
-                    Version.fromString(versionProperty),
+                    Version.fromString(distribution.getVersion()),
                     "main",
                     ":distribution:bwc:main"
                 );

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestClustersPlugin.java

@@ -11,9 +11,7 @@
 
 import org.elasticsearch.gradle.VersionProperties;
 import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin;
-import org.elasticsearch.gradle.testclusters.ElasticsearchCluster;
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin;
-import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 
@@ -33,17 +31,5 @@ public void apply(Project project) {
             version -> (version.equals(VersionProperties.getElasticsearchVersion()) && buildParams.getSnapshotBuild() == false)
                 || buildParams.getBwcVersions().unreleasedInfo(version) == null
         );
-
-        NamedDomainObjectContainer<ElasticsearchCluster> testClusters = (NamedDomainObjectContainer<ElasticsearchCluster>) project
-            .getExtensions()
-            .getByName(TestClustersPlugin.EXTENSION_NAME);
-        // Limit the number of allocated processors for all nodes to 2 in the cluster by default.
-        // This is to ensure that the tests run consistently across different environments.
-        String processorCount = shouldConfigureTestClustersWithOneProcessor() ? "1" : "2";
-        testClusters.configureEach(elasticsearchCluster -> elasticsearchCluster.setting("node.processors", processorCount));
-    }
-
-    private boolean shouldConfigureTestClustersWithOneProcessor() {
-        return Boolean.parseBoolean(System.getProperty("tests.configure_test_clusters_with_one_processor", "false"));
     }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java

@@ -106,13 +106,15 @@ public void apply(Project project) {
         ElasticsearchDistribution defaultDistro = createDistribution(
             project,
             DEFAULT_REST_INTEG_TEST_DISTRO,
-            VersionProperties.getElasticsearch()
+            VersionProperties.getElasticsearch(),
+            false
         );
         ElasticsearchDistribution integTestDistro = createDistribution(
             project,
             INTEG_TEST_REST_INTEG_TEST_DISTRO,
             VersionProperties.getElasticsearch(),
-            ElasticsearchDistributionTypes.INTEG_TEST_ZIP
+            ElasticsearchDistributionTypes.INTEG_TEST_ZIP,
+            false
         );
 
         // Create configures for module and plugin dependencies
@@ -238,21 +240,11 @@ public Void call(Object... args) {
                     }
 
                     Version version = (Version) args[0];
-                    boolean isReleased = bwcVersions.unreleasedInfo(version) == null && version.toString().equals("0.0.0") == false;
+                    boolean isReleased = bwcVersions.unreleasedInfo(version) == null;
                     String versionString = version.toString();
-                    ElasticsearchDistribution bwcDistro = createDistribution(project, "bwc_" + versionString, versionString);
-
-                    if (jdkIsIncompatibleWithOS(Version.fromString(versionString))) {
-                        var toolChainService = project.getExtensions().getByType(JavaToolchainService.class);
-                        var fallbackJdk17Launcher = toolChainService.launcherFor(spec -> {
-                            spec.getVendor().set(JvmVendorSpec.ADOPTIUM);
-                            spec.getLanguageVersion().set(JavaLanguageVersion.of(17));
-                        });
-                        task.environment(
-                            "ES_FALLBACK_JAVA_HOME",
-                            fallbackJdk17Launcher.get().getMetadata().getInstallationPath().getAsFile().getPath()
-                        );
-                    }
+                    ElasticsearchDistribution bwcDistro = createDistribution(project, "bwc_" + versionString, versionString, false);
+
+                    handleJdkIncompatibleWithOS(version, project, task);
                     task.dependsOn(bwcDistro);
                     registerDistributionInputs(task, bwcDistro);
 
@@ -261,16 +253,63 @@ public Void call(Object... args) {
                         providerFactory.provider(() -> bwcDistro.getExtracted().getSingleFile().getPath())
                     );
 
-                    if (version.getMajor() > 0 && version.before(bwcVersions.getMinimumWireCompatibleVersion())) {
+                    if (version.before(bwcVersions.getMinimumWireCompatibleVersion())) {
                         // If we are upgrade testing older versions we also need to upgrade to 7.last
                         this.call(bwcVersions.getMinimumWireCompatibleVersion());
                     }
                     return null;
                 }
             });
+
+            task.getExtensions().getExtraProperties().set("usesBwcDistributionFromRef", new Closure<Void>(task) {
+                @Override
+                public Void call(Object... args) {
+                    if (args.length != 2 || args[0] instanceof String == false || args[1] instanceof Version == false) {
+                        throw new IllegalArgumentException("Expected arguments (String refSpec, org.elasticsearch.gradle.Version version)");
+                    }
+
+                    String refSpec = (String) args[0];
+                    Version version = (Version) args[1];
+                    boolean isDetachedVersion = true;
+                    String versionString = version.toString();
+
+                    ElasticsearchDistribution bwcDistro = createDistribution(project, "bwc_" + refSpec, versionString, isDetachedVersion);
+                    handleJdkIncompatibleWithOS(version, project, task);
+
+                    task.dependsOn(bwcDistro);
+                    registerDistributionInputs(task, bwcDistro);
+
+                    nonInputSystemProperties.systemProperty(
+                        BWC_SNAPSHOT_DISTRIBUTION_SYSPROP_PREFIX + versionString,
+                        providerFactory.provider(() -> bwcDistro.getExtracted().getSingleFile().getPath())
+                    );
+                    return null;
+                }
+            });
         });
     }
 
+    /**
+     * Older distributions ship with openjdk versions that are not compatible with newer kernels of ubuntu 24.04 and later
+     * Therefore we pass explicitly the runtime java to use the adoptium jdk that is maintained longer and compatible
+     * with newer kernels.
+     * 8.10.4 is the last version shipped with jdk < 21. We configure these cluster to run with jdk 17 adoptium as 17 was
+     * the last LTS release before 21
+     */
+    private static void handleJdkIncompatibleWithOS(Version version, Project project, StandaloneRestIntegTestTask task) {
+        if (jdkIsIncompatibleWithOS(version)) {
+            var toolChainService = project.getExtensions().getByType(JavaToolchainService.class);
+            var fallbackJdk17Launcher = toolChainService.launcherFor(spec -> {
+                spec.getVendor().set(JvmVendorSpec.ADOPTIUM);
+                spec.getLanguageVersion().set(JavaLanguageVersion.of(17));
+            });
+            task.environment(
+                "ES_FALLBACK_JAVA_HOME",
+                fallbackJdk17Launcher.get().getMetadata().getInstallationPath().getAsFile().getPath()
+            );
+        }
+    }
+
     private void copyDependencies(Project project, DependencySet dependencies, Configuration configuration) {
         configuration.getDependencies()
             .stream()
@@ -279,16 +318,23 @@ private void copyDependencies(Project project, DependencySet dependencies, Confi
             .forEach(dependencies::add);
     }
 
-    private ElasticsearchDistribution createDistribution(Project project, String name, String version) {
-        return createDistribution(project, name, version, null);
+    private ElasticsearchDistribution createDistribution(Project project, String name, String version, boolean detachedVersion) {
+        return createDistribution(project, name, version, null, detachedVersion);
     }
 
-    private ElasticsearchDistribution createDistribution(Project project, String name, String version, ElasticsearchDistributionType type) {
+    private ElasticsearchDistribution createDistribution(
+        Project project,
+        String name,
+        String version,
+        ElasticsearchDistributionType type,
+        boolean detachedVersion
+    ) {
         NamedDomainObjectContainer<ElasticsearchDistribution> distributions = DistributionDownloadPlugin.getContainer(project);
         ElasticsearchDistribution maybeDistro = distributions.findByName(name);
         if (maybeDistro == null) {
             return distributions.create(name, distro -> {
                 distro.setVersion(version);
+                distro.setDetachedVersion(detachedVersion);
                 distro.setArchitecture(Architecture.current());
                 if (type != null) {
                     distro.setType(type);

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionDefinition.java

@@ -22,9 +22,11 @@ public static TransportVersionDefinition fromString(Path file, String contents,
 
         String idsLine = null;
         if (contents.isEmpty() == false) {
-            String[] lines = contents.split(System.lineSeparator());
+            // Regardless of whether windows newlines exist (they could be added by git), we split on line feed.
+            // All we care about skipping lines with the comment character, so the remaining \r won't matter
+            String[] lines = contents.split("\n");
             for (String line : lines) {
-                line = line.replaceAll("\\s+", "");
+                line = line.strip();
                 if (line.startsWith("#") == false) {
                     idsLine = line;
                     break;