Fix reproducability of builds against Java EA versions (#132847) (#132975)

* Introduce dedicated toolchain resolver for EA java versions

* Fix reproducability of builds against Java EA versions

This fixes a reproducability issue when using the gradle javaToolChain api.

There is no way to test toolchain candidates reliable against the build number in use.

This meant that ones you e.g. have resolved some version of java 25, gradle toolchain detection
does not detect the difference between certain builds (or even ea vs. released version)
Therefore we fallback to rely on our custom JDKDownloadPlugin for now here.

Syncing with the gradle team, they want to fix this somewhen in 9.x. We will revisit our solution
ones something is available. Ultimately we want to get rid of usages of the JDK download plugin.

* Minor cleanup and fixing non set distribution type in jdk

* Fix ea jdk selection not just based on major version

explicitly requires to add a ea postfix to the runtime sys property

* Fix test reproduce linies when using java ea version

* Apply review feedback

* Use runtime jdk when patching

* More cleanup

(cherry picked from commit 16fe7db)

# Conflicts:
#	BUILDING.md

Author: breskeby

BUILDING.md

@@ -186,6 +186,25 @@ dependencies {
 
 To test an unreleased development version of a third party dependency you have several options.
 
+### How do I test against java early access (ea) versions?
+
+Currently only openjdk EA builds by oracle are supported.
+To test against an early access version java version you can pass the major
+java version appended with `-ea` as a system property (e.g. -Druntime.java=26-ea) to the Gradle build:
+
+```
+./gradlew clean test -Druntime.java=26-ea
+```
+
+This will run the tests using the JDK 26 EA version and pick the latest available build of the matching JDK EA version we expose
+in our custom jdk catalogue at `https://storage.googleapis.com/elasticsearch-jdk-archive/jdks/openjdk/latest.json`.
+
+To run against a specific build number of the EA build you can pass a second system property (e.g. `-Druntime.java.build=6`):
+
+```
+./gradlew clean test -Druntime.java=26-ea -Druntime.java.build=6
+```
+
 #### How to use a Maven based third party dependency via `mavenlocal`?
 
 1. Clone the third party repository locally
@@ -229,11 +248,8 @@ In addition to snapshot builds JitPack supports building Pull Requests. Simply u
 3. Run the Gradle build as needed. Keep in mind the initial resolution might take a bit longer as this needs to be built
 by JitPack in the background before we can resolve the adhoc built dependency.
 
----
-
-**NOTE**
-
-You should only use that approach locally or on a developer branch for production dependencies as we do
+> [!Note]
+> You should only use that approach locally or on a developer branch for production dependencies as we do
 not want to ship unreleased libraries into our releases.
 ---
 
@@ -265,10 +281,8 @@ allprojects {
   ```
 4. Run the Gradle build as needed with `--write-verification-metadata` to ensure the Gradle dependency verification does not fail on your custom dependency.
 
----
-**NOTE**
-
-As Gradle prefers to use modules whose descriptor has been created from real meta-data rather than being generated,
+> [!Note]
+> As Gradle prefers to use modules whose descriptor has been created from real meta-data rather than being generated,
 flat directory repositories cannot be used to override artifacts with real meta-data from other repositories declared in the build.
 For example, if Gradle finds only `jmxri-1.2.1.jar` in a flat directory repository, but `jmxri-1.2.1.pom` in another repository
 that supports meta-data, it will use the second repository to provide the module.

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGitAwareGradleFuncTest.groovy

@@ -10,7 +10,6 @@
 package org.elasticsearch.gradle.fixtures
 
 import org.apache.commons.io.FileUtils
-import org.elasticsearch.gradle.internal.test.InternalAwareGradleRunner
 import org.gradle.testkit.runner.GradleRunner
 import org.junit.Rule
 import org.junit.rules.TemporaryFolder

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/JdkDownloadPluginFuncTest.groovy

@@ -33,13 +33,18 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
     private static final String ADOPT_JDK_VERSION_15 = "15.0.2+7"
     private static final String AZUL_JDK_VERSION_8 = "8u302+b08"
     private static final String AZUL_8_DISTRO_VERSION = "8.56.0.23"
+    private static final String CATALOG_EA_VERSION = "25-ea+30"
     private static final String OPEN_JDK_VERSION = "12.0.1+99@123456789123456789123456789abcde"
     private static final Pattern JDK_HOME_LOGLINE = Pattern.compile("JDK HOME: (.*)")
 
+    def setup() {
+        configurationCacheCompatible = false // JDK class references configurations which break configuration cache
+    }
+
     @Unroll
-    def "jdk #jdkVendor for #platform#suffix are downloaded and extracted"() {
+    def "jdk #distributionVersion #jdkVendor for #platform#suffix are downloaded and extracted"() {
         given:
-        def mockRepoUrl = urlPath(jdkVendor, jdkVersion, platform, arch);
+        def mockRepoUrl = urlPath(jdkVendor, jdkVersion, platform, arch, distributionVersion);
         def mockedContent = filebytes(jdkVendor, platform)
         buildFile.text = """
             plugins {
@@ -77,21 +82,26 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
 
         where:
         platform  | arch      | jdkVendor       | jdkVersion           | distributionVersion   | expectedJavaBin          | suffix
-        "linux"   | "x64"     | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | null                  | "bin/java"               | ""
-        "linux"   | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | null                  | "bin/java"               | ""
-        "linux"   | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | null                  | "bin/java"               | "(old version)"
-        "windows" | "x64"     | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | null                  | "bin/java"               | ""
-        "windows" | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | null                  | "bin/java"               | ""
-        "windows" | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | null                  | "bin/java"               | "(old version)"
-        "darwin"  | "x64"     | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | null                  | "Contents/Home/bin/java" | ""
-        "darwin"  | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | null                  | "Contents/Home/bin/java" | ""
-        "darwin"  | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | null                  | "Contents/Home/bin/java" | "(old version)"
-        "mac"     | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | null                  | "Contents/Home/bin/java" | ""
-        "mac"     | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | null                  | "Contents/Home/bin/java" | "(old version)"
-        "darwin"  | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | null                  | "Contents/Home/bin/java" | ""
-        "linux"   | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | null                  | "bin/java"               | ""
-        "linux"   | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION_11 | null                  | "bin/java"               | "(jdk 11)"
-        "linux"   | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION_15 | null                  | "bin/java"               | "(jdk 15)"
+        "linux"   | "aarch64" | VENDOR_OPENJDK  | CATALOG_EA_VERSION   | "ea"                  | "bin/java"               | ""
+        "linux"   | "x64"     | VENDOR_OPENJDK  | CATALOG_EA_VERSION   | "ea"                  | "bin/java"               | ""
+        "darwin"  | "aarch64" | VENDOR_OPENJDK  | CATALOG_EA_VERSION   | "ea"                  | "Contents/Home/bin/java" | ""
+        "darwin"  | "x64"     | VENDOR_OPENJDK  | CATALOG_EA_VERSION   | "ea"                  | "Contents/Home/bin/java" | ""
+        "windows" | "x64"     | VENDOR_OPENJDK  | CATALOG_EA_VERSION   | "ea"                  | "bin/java"               | ""
+        "linux"   | "x64"     | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | ""                    | "bin/java"               | ""
+        "linux"   | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | ""                    | "bin/java"               | ""
+        "linux"   | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | ""                    | "bin/java"               | "(old version)"
+        "windows" | "x64"     | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | ""                    | "bin/java"               | ""
+        "windows" | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | ""                    | "bin/java"               | ""
+        "windows" | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | ""                    | "bin/java"               | "(old version)"
+        "darwin"  | "x64"     | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | ""                    | "Contents/Home/bin/java" | ""
+        "darwin"  | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | ""                    | "Contents/Home/bin/java" | ""
+        "darwin"  | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | ""                    | "Contents/Home/bin/java" | "(old version)"
+        "mac"     | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | ""                    | "Contents/Home/bin/java" | ""
+        "mac"     | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | ""                    | "Contents/Home/bin/java" | "(old version)"
+        "darwin"  | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | ""                    | "Contents/Home/bin/java" | ""
+        "linux"   | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | ""                    | "bin/java"               | ""
+        "linux"   | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION_11 | ""                    | "bin/java"               | "(jdk 11)"
+        "linux"   | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION_15 | ""                    | "bin/java"               | "(jdk 15)"
         "darwin"  | "aarch64" | VENDOR_ZULU     | AZUL_JDK_VERSION_8   | AZUL_8_DISTRO_VERSION | "Contents/Home/bin/java" | "(jdk 8)"
     }
 
@@ -214,13 +224,19 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
     private static String urlPath(final String vendor,
                                   final String version,
                                   final String platform,
-                                  final String arch = 'x64') {
-        if (vendor.equals(VENDOR_ADOPTIUM)) {
+                                  final String arch = 'x64',
+                                  final String distributedVersion = '') {
+        final boolean isOld = version.equals(OPENJDK_VERSION_OLD);
+
+        if (distributedVersion.equals("ea")) {
+            def effectivePlatform = isMac(platform) ? "macos" : platform;
+            def fileExtension = platform.toLowerCase().equals("windows") ? "zip" : "tar.gz";
+            return "/jdks/openjdk/25/openjdk-${version}/openjdk-${version}_$effectivePlatform-${arch}_bin.$fileExtension";
+        } else if (vendor.equals(VENDOR_ADOPTIUM)) {
             final String module = isMac(platform) ? "mac" : platform;
             return "/jdk-" + version + "/" + module + "/${arch}/jdk/hotspot/normal/adoptium";
         } else if (vendor.equals(VENDOR_OPENJDK)) {
             final String effectivePlatform = isMac(platform) ? "macos" : platform;
-            final boolean isOld = version.equals(OPENJDK_VERSION_OLD);
             final String versionPath = isOld ? "jdk1/99" : "jdk12.0.1/123456789123456789123456789abcde/99";
             final String filename = "openjdk-" + (isOld ? "1" : "12.0.1") + "_" + effectivePlatform + "-x64_bin." + extension(platform);
             return "/java/GA/" + versionPath + "/GPL/" + filename;

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/Jdk.java

@@ -32,6 +32,7 @@ public class Jdk implements Buildable, Iterable<File> {
         "(\\d+)(\\.\\d+\\.\\d+(?:\\.\\d+)?)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?"
     );
     private static final Pattern LEGACY_VERSION_PATTERN = Pattern.compile("(\\d)(u\\d+)\\+(b\\d+?)(@([a-f0-9]{32}))?");
+    private static final Pattern EA_VERSION_PATTERN = Pattern.compile("(\\d+)-ea\\+(\\d+)(@([a-f0-9]{32}))?");
 
     private final String name;
     private final FileCollection configuration;
@@ -78,7 +79,9 @@ public String getVersion() {
     }
 
     public void setVersion(String version) {
-        if (VERSION_PATTERN.matcher(version).matches() == false && LEGACY_VERSION_PATTERN.matcher(version).matches() == false) {
+        if (VERSION_PATTERN.matcher(version).matches() == false
+            && LEGACY_VERSION_PATTERN.matcher(version).matches() == false
+            && EA_VERSION_PATTERN.matcher(version).matches() == false) {
             throw new IllegalArgumentException("malformed version [" + version + "] for jdk [" + name + "]");
         }
         parseVersion(version);
@@ -112,7 +115,7 @@ public void setArchitecture(final String architecture) {
     }
 
     public String getDistributionVersion() {
-        return distributionVersion.get();
+        return distributionVersion.getOrNull();
     }
 
     public void setDistributionVersion(String distributionVersion) {
@@ -218,9 +221,17 @@ private void parseVersion(String version) {
         if (jdkVersionMatcher.matches() == false) {
             // Try again with the pre-Java9 version format
             jdkVersionMatcher = LEGACY_VERSION_PATTERN.matcher(version);
-
             if (jdkVersionMatcher.matches() == false) {
-                throw new IllegalArgumentException("Malformed jdk version [" + version + "]");
+                // Try again with the pre-Java9 version format
+                jdkVersionMatcher = EA_VERSION_PATTERN.matcher(version);
+                if (jdkVersionMatcher.matches() == false) {
+                    throw new IllegalArgumentException("Malformed jdk version [" + version + "]");
+                }
+                baseVersion = version;
+                major = jdkVersionMatcher.group(1);
+                build = jdkVersionMatcher.group(2);
+                hash = null;
+                return;
             }
         }
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/JdkDownloadPlugin.java

@@ -114,6 +114,15 @@ private void setupRepository(Project project, Jdk jdk) {
                     + jdk.getBuild()
                     + "/[module]/[classifier]/jdk/hotspot/normal/adoptium";
             }
+        } else if (jdk.getVendor().equals(VENDOR_OPENJDK) && "ea".equals(jdk.getDistributionVersion())) {
+            repoUrl = "https://builds.es-jdk-archive.com/";
+            // current pattern since 12.0.1
+            artifactPattern = "jdks/openjdk/"
+                + jdk.getMajor()
+                + "/openjdk-"
+                + jdk.getBaseVersion()
+                + "/"
+                + "openjdk-[revision]_[module]-[classifier]_bin.[ext]";
         } else if (jdk.getVendor().equals(VENDOR_OPENJDK)) {
             repoUrl = "https://download.oracle.com";
             if (jdk.getHash() != null) {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java

@@ -12,8 +12,12 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 
 import org.apache.commons.io.IOUtils;
+import org.elasticsearch.gradle.Architecture;
+import org.elasticsearch.gradle.OS;
 import org.elasticsearch.gradle.VersionProperties;
 import org.elasticsearch.gradle.internal.BwcVersions;
+import org.elasticsearch.gradle.internal.Jdk;
+import org.elasticsearch.gradle.internal.JdkDownloadPlugin;
 import org.elasticsearch.gradle.internal.conventions.GitInfoPlugin;
 import org.elasticsearch.gradle.internal.conventions.info.GitInfo;
 import org.elasticsearch.gradle.internal.conventions.info.ParallelDetector;
@@ -22,6 +26,7 @@
 import org.gradle.api.Action;
 import org.gradle.api.GradleException;
 import org.gradle.api.JavaVersion;
+import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.logging.Logger;
@@ -63,6 +68,7 @@
 import javax.inject.Inject;
 
 import static org.elasticsearch.gradle.internal.conventions.GUtils.elvis;
+import static org.elasticsearch.gradle.internal.toolchain.EarlyAccessCatalogJdkToolchainResolver.findLatestEABuildNumber;
 
 public class GlobalBuildInfoPlugin implements Plugin<Project> {
     private static final Logger LOGGER = Logging.getLogger(GlobalBuildInfoPlugin.class);
@@ -92,11 +98,13 @@ public GlobalBuildInfoPlugin(
 
     @Override
     public void apply(Project project) {
+
         if (project != project.getRootProject()) {
             throw new IllegalStateException(this.getClass().getName() + " can only be applied to the root project.");
         }
         this.project = project;
         project.getPlugins().apply(JvmToolchainsPlugin.class);
+        project.getPlugins().apply(JdkDownloadPlugin.class);
         Provider<GitInfo> gitInfo = project.getPlugins().apply(GitInfoPlugin.class).getGitInfo();
 
         toolChainService = project.getExtensions().getByType(JavaToolchainService.class);
@@ -270,7 +278,9 @@ private JavaVersion determineJavaVersion(String description, File javaHome, Java
     private InstallationLocation getJavaInstallation(File javaHome) {
         return getAvailableJavaInstallationLocationSteam().filter(installationLocation -> isSameFile(javaHome, installationLocation))
             .findFirst()
-            .orElseThrow(() -> new GradleException("Could not locate available Java installation in Gradle registry at: " + javaHome));
+            .orElse(
+                InstallationLocation.userDefined(javaHome, "Manually resolved JavaHome (not auto-detected by Gradle toolchain service)")
+            );
     }
 
     private boolean isSameFile(File javaHome, InstallationLocation installationLocation) {
@@ -338,7 +348,14 @@ private Provider<File> findRuntimeJavaHome() {
         String runtimeJavaProperty = System.getProperty("runtime.java");
 
         if (runtimeJavaProperty != null) {
-            return resolveJavaHomeFromToolChainService(runtimeJavaProperty);
+            if (runtimeJavaProperty.toLowerCase().endsWith("-ea")) {
+                // handle EA builds differently due to lack of support in Gradle toolchain service
+                // we resolve them using JdkDownloadPlugin for now.
+                Integer major = Integer.parseInt(runtimeJavaProperty.substring(0, runtimeJavaProperty.length() - 3));
+                return resolveEarlyAccessJavaHome(major);
+            } else {
+                return resolveJavaHomeFromToolChainService(runtimeJavaProperty);
+            }
         }
         if (System.getenv("RUNTIME_JAVA_HOME") != null) {
             return providers.provider(() -> new File(System.getenv("RUNTIME_JAVA_HOME")));
@@ -353,6 +370,23 @@ private Provider<File> findRuntimeJavaHome() {
         });
     }
 
+    private Provider<File> resolveEarlyAccessJavaHome(Integer runtimeJavaProperty) {
+        NamedDomainObjectContainer<Jdk> container = (NamedDomainObjectContainer<Jdk>) project.getExtensions().getByName("jdks");
+        Integer buildNumber = Integer.getInteger("runtime.java.build");
+        if (buildNumber == null) {
+            buildNumber = findLatestEABuildNumber(runtimeJavaProperty);
+        }
+        String eaVersionString = String.format("%d-ea+%d", runtimeJavaProperty, buildNumber);
+        Jdk jdk = container.create("ea_jdk_" + runtimeJavaProperty, j -> {
+            j.setVersion(eaVersionString);
+            j.setVendor("openjdk");
+            j.setPlatform(OS.current().javaOsReference);
+            j.setArchitecture(Architecture.current().javaClassifier);
+            j.setDistributionVersion("ea");
+        });
+        return providers.provider(() -> new File(jdk.getJavaHomePath().toString()));
+    }
+
     @NotNull
     private Provider<File> resolveJavaHomeFromToolChainService(String version) {
         Property<JavaLanguageVersion> value = objectFactory.property(JavaLanguageVersion.class).value(JavaLanguageVersion.of(version));