Fix reproducability of builds against Java EA versions

This fixes a reproducability issue when using the gradle javaToolChain api.

There is no way to test toolchain candidates reliable against the build number in use.

This meant that ones you e.g. have resolved some version of java 25, gradle toolchain detection
does not detect the difference between certain builds (or even ea vs. released version)
Therefore we fallback to rely on our custom JDKDownloadPlugin for now here.

Syncing with the gradle team, they want to fix this somewhen in 9.x. We will revisit our solution
ones something is available. Ultimately we want to get rid of usages of the JDK download plugin.

Author: breskeby

BUILDING.md

@@ -92,7 +92,7 @@ uses the changed dependencies. In most cases, `precommit` or `check` are good ca
 We prefer sha256 checksums as md5 and sha1 are not considered safe anymore these days. The generated entry
 will have the `origin` attribute been set to `Generated by Gradle`.
 
-> [!Tip]  
+> [!Tip]
 > A manual confirmation of the Gradle generated checksums is currently not mandatory.
 > If you want to add a level of verification you can manually confirm the checksum (e.g. by looking it up on the website of the library)
 > Please replace the content of the `origin` attribute by `official site` in that case.
@@ -186,6 +186,25 @@ dependencies {
 
 To test an unreleased development version of a third party dependency you have several options.
 
+### How do I test against java early access (ea) versions?
+
+Currently only openjdk EA builds by oracle are supported.
+To test against an early access version java version you can pass the major java version
+to test against via system property (e.g. -Druntime.java=26):
+
+```
+./gradlew clean test -Druntime.java=26
+```
+
+This will run the tests using the JDK 26 EA version and pick the latest available build of the matching JDK EA version we expose
+in our custom jdk catalogue at `https://storage.googleapis.com/elasticsearch-jdk-archive/jdks/openjdk/latest.json`.
+
+To run against a specific build number of the EA build you can pass a second system property (e.g. `-Druntime.java.build=6`):
+
+```
+./gradlew clean test -Druntime.java=26 -Druntime.java.build=6
+```
+
 #### How to use a Maven based third party dependency via `mavenlocal`?
 
 1. Clone the third party repository locally
@@ -229,7 +248,7 @@ In addition to snapshot builds JitPack supports building Pull Requests. Simply u
 3. Run the Gradle build as needed. Keep in mind the initial resolution might take a bit longer as this needs to be built
 by JitPack in the background before we can resolve the adhoc built dependency.
 
-> [!Note]  
+> [!Note]
 > You should only use that approach locally or on a developer branch for production dependencies as we do
 not want to ship unreleased libraries into our releases.
 
@@ -261,7 +280,7 @@ allprojects {
   ```
 4. Run the Gradle build as needed with `--write-verification-metadata` to ensure the Gradle dependency verification does not fail on your custom dependency.
 
-> [!Note]  
+> [!Note]
 > As Gradle prefers to use modules whose descriptor has been created from real meta-data rather than being generated,
 flat directory repositories cannot be used to override artifacts with real meta-data from other repositories declared in the build.
 > For example, if Gradle finds only `jmxri-1.2.1.jar` in a flat directory repository, but `jmxri-1.2.1.pom` in another repository

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGitAwareGradleFuncTest.groovy

@@ -10,7 +10,6 @@
 package org.elasticsearch.gradle.fixtures
 
 import org.apache.commons.io.FileUtils
-import org.elasticsearch.gradle.internal.test.InternalAwareGradleRunner
 import org.gradle.testkit.runner.GradleRunner
 import org.junit.Rule
 import org.junit.rules.TemporaryFolder

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/JdkDownloadPluginFuncTest.groovy

@@ -33,13 +33,18 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
     private static final String ADOPT_JDK_VERSION_15 = "15.0.2+7"
     private static final String AZUL_JDK_VERSION_8 = "8u302+b08"
     private static final String AZUL_8_DISTRO_VERSION = "8.56.0.23"
+    private static final String CATALOG_EA_VERSION = "25-ea+30"
     private static final String OPEN_JDK_VERSION = "12.0.1+99@123456789123456789123456789abcde"
     private static final Pattern JDK_HOME_LOGLINE = Pattern.compile("JDK HOME: (.*)")
 
+    def setup() {
+        configurationCacheCompatible = false // JDK class references configurations which break configuration cache
+    }
+
     @Unroll
-    def "jdk #jdkVendor for #platform#suffix are downloaded and extracted"() {
+    def "jdk #distributionVersion #jdkVendor for #platform#suffix are downloaded and extracted"() {
         given:
-        def mockRepoUrl = urlPath(jdkVendor, jdkVersion, platform, arch);
+        def mockRepoUrl = urlPath(jdkVendor, jdkVersion, platform, arch, distributionVersion);
         def mockedContent = filebytes(jdkVendor, platform)
         buildFile.text = """
             plugins {
@@ -77,21 +82,26 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
 
         where:
         platform  | arch      | jdkVendor       | jdkVersion           | distributionVersion   | expectedJavaBin          | suffix
-        "linux"   | "x64"     | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | null                  | "bin/java"               | ""
-        "linux"   | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | null                  | "bin/java"               | ""
-        "linux"   | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | null                  | "bin/java"               | "(old version)"
-        "windows" | "x64"     | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | null                  | "bin/java"               | ""
-        "windows" | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | null                  | "bin/java"               | ""
-        "windows" | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | null                  | "bin/java"               | "(old version)"
-        "darwin"  | "x64"     | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | null                  | "Contents/Home/bin/java" | ""
-        "darwin"  | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | null                  | "Contents/Home/bin/java" | ""
-        "darwin"  | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | null                  | "Contents/Home/bin/java" | "(old version)"
-        "mac"     | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | null                  | "Contents/Home/bin/java" | ""
-        "mac"     | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | null                  | "Contents/Home/bin/java" | "(old version)"
-        "darwin"  | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | null                  | "Contents/Home/bin/java" | ""
-        "linux"   | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | null                  | "bin/java"               | ""
-        "linux"   | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION_11 | null                  | "bin/java"               | "(jdk 11)"
-        "linux"   | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION_15 | null                  | "bin/java"               | "(jdk 15)"
+        "linux"   | "aarch64" | VENDOR_OPENJDK  | CATALOG_EA_VERSION   | "ea"                  | "bin/java"               | ""
+        "linux"   | "x64"     | VENDOR_OPENJDK  | CATALOG_EA_VERSION   | "ea"                  | "bin/java"               | ""
+        "darwin"  | "aarch64" | VENDOR_OPENJDK  | CATALOG_EA_VERSION   | "ea"                  | "Contents/Home/bin/java" | ""
+        "darwin"  | "x64"     | VENDOR_OPENJDK  | CATALOG_EA_VERSION   | "ea"                  | "Contents/Home/bin/java" | ""
+        "windows" | "x64"     | VENDOR_OPENJDK  | CATALOG_EA_VERSION   | "ea"                  | "bin/java"               | ""
+        "linux"   | "x64"     | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | ""                    | "bin/java"               | ""
+        "linux"   | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | ""                    | "bin/java"               | ""
+        "linux"   | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | ""                    | "bin/java"               | "(old version)"
+        "windows" | "x64"     | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | ""                    | "bin/java"               | ""
+        "windows" | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | ""                    | "bin/java"               | ""
+        "windows" | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | ""                    | "bin/java"               | "(old version)"
+        "darwin"  | "x64"     | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | ""                    | "Contents/Home/bin/java" | ""
+        "darwin"  | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | ""                    | "Contents/Home/bin/java" | ""
+        "darwin"  | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | ""                    | "Contents/Home/bin/java" | "(old version)"
+        "mac"     | "x64"     | VENDOR_OPENJDK  | OPEN_JDK_VERSION     | ""                    | "Contents/Home/bin/java" | ""
+        "mac"     | "x64"     | VENDOR_OPENJDK  | OPENJDK_VERSION_OLD  | ""                    | "Contents/Home/bin/java" | "(old version)"
+        "darwin"  | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | ""                    | "Contents/Home/bin/java" | ""
+        "linux"   | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION    | ""                    | "bin/java"               | ""
+        "linux"   | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION_11 | ""                    | "bin/java"               | "(jdk 11)"
+        "linux"   | "aarch64" | VENDOR_ADOPTIUM | ADOPT_JDK_VERSION_15 | ""                    | "bin/java"               | "(jdk 15)"
         "darwin"  | "aarch64" | VENDOR_ZULU     | AZUL_JDK_VERSION_8   | AZUL_8_DISTRO_VERSION | "Contents/Home/bin/java" | "(jdk 8)"
     }
 
@@ -214,13 +224,19 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
     private static String urlPath(final String vendor,
                                   final String version,
                                   final String platform,
-                                  final String arch = 'x64') {
-        if (vendor.equals(VENDOR_ADOPTIUM)) {
+                                  final String arch = 'x64',
+                                  final String distributedVersion = '') {
+        final boolean isOld = version.equals(OPENJDK_VERSION_OLD);
+
+        if (distributedVersion.equals("ea")) {
+            def effectivePlatform = isMac(platform) ? "macos" : platform;
+            def fileExtension = platform.toLowerCase().equals("windows") ? "zip" : "tar.gz";
+            return "/jdks/openjdk/25/openjdk-${version}/openjdk-${version}_$effectivePlatform-${arch}_bin.$fileExtension";
+        } else if (vendor.equals(VENDOR_ADOPTIUM)) {
             final String module = isMac(platform) ? "mac" : platform;
             return "/jdk-" + version + "/" + module + "/${arch}/jdk/hotspot/normal/adoptium";
         } else if (vendor.equals(VENDOR_OPENJDK)) {
             final String effectivePlatform = isMac(platform) ? "macos" : platform;
-            final boolean isOld = version.equals(OPENJDK_VERSION_OLD);
             final String versionPath = isOld ? "jdk1/99" : "jdk12.0.1/123456789123456789123456789abcde/99";
             final String filename = "openjdk-" + (isOld ? "1" : "12.0.1") + "_" + effectivePlatform + "-x64_bin." + extension(platform);
             return "/java/GA/" + versionPath + "/GPL/" + filename;

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/Jdk.java

@@ -32,6 +32,7 @@ public class Jdk implements Buildable, Iterable<File> {
         "(\\d+)(\\.\\d+\\.\\d+(?:\\.\\d+)?)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?"
     );
     private static final Pattern LEGACY_VERSION_PATTERN = Pattern.compile("(\\d)(u\\d+)\\+(b\\d+?)(@([a-f0-9]{32}))?");
+    private static final Pattern EA_VERSION_PATTERN = Pattern.compile("(\\d+)-ea\\+(\\d+)(@([a-f0-9]{32}))?");
 
     private final String name;
     private final FileCollection configuration;
@@ -78,7 +79,9 @@ public String getVersion() {
     }
 
     public void setVersion(String version) {
-        if (VERSION_PATTERN.matcher(version).matches() == false && LEGACY_VERSION_PATTERN.matcher(version).matches() == false) {
+        if (VERSION_PATTERN.matcher(version).matches() == false
+            && LEGACY_VERSION_PATTERN.matcher(version).matches() == false
+            && EA_VERSION_PATTERN.matcher(version).matches() == false) {
             throw new IllegalArgumentException("malformed version [" + version + "] for jdk [" + name + "]");
         }
         parseVersion(version);
@@ -218,9 +221,17 @@ private void parseVersion(String version) {
         if (jdkVersionMatcher.matches() == false) {
             // Try again with the pre-Java9 version format
             jdkVersionMatcher = LEGACY_VERSION_PATTERN.matcher(version);
-
             if (jdkVersionMatcher.matches() == false) {
-                throw new IllegalArgumentException("Malformed jdk version [" + version + "]");
+                // Try again with the pre-Java9 version format
+                jdkVersionMatcher = EA_VERSION_PATTERN.matcher(version);
+                if (jdkVersionMatcher.matches() == false) {
+                    throw new IllegalArgumentException("Malformed jdk version [" + version + "]");
+                }
+                baseVersion = version;
+                major = jdkVersionMatcher.group(1);
+                build = jdkVersionMatcher.group(2);
+                hash = null;
+                return;
             }
         }
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/JdkDownloadPlugin.java

@@ -114,6 +114,19 @@ private void setupRepository(Project project, Jdk jdk) {
                     + jdk.getBuild()
                     + "/[module]/[classifier]/jdk/hotspot/normal/adoptium";
             }
+        } else if (jdk.getVendor().equals(VENDOR_OPENJDK) && jdk.getDistributionVersion().equals("ea")) {
+
+            // "https://builds.es-jdk-archive.com/jdks/openjdk/25/openjdk-25-ea+30/openjdk-25-ea+30_linux-x64_bin.tar.gz"],
+            // /jdks/openjdk/25/openjdk-25-ea+30/openjdk-25-ea+30_linux
+            repoUrl = "https://builds.es-jdk-archive.com/";
+
+            // current pattern since 12.0.1
+            artifactPattern = "jdks/openjdk/"
+                + jdk.getMajor()
+                + "/openjdk-"
+                + jdk.getBaseVersion()
+                + "/"
+                + "openjdk-[revision]_[module]-[classifier]_bin.[ext]";
         } else if (jdk.getVendor().equals(VENDOR_OPENJDK)) {
             repoUrl = "https://download.oracle.com";
             if (jdk.getHash() != null) {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java

@@ -12,8 +12,12 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 
 import org.apache.commons.io.IOUtils;
+import org.elasticsearch.gradle.Architecture;
+import org.elasticsearch.gradle.OS;
 import org.elasticsearch.gradle.VersionProperties;
 import org.elasticsearch.gradle.internal.BwcVersions;
+import org.elasticsearch.gradle.internal.Jdk;
+import org.elasticsearch.gradle.internal.JdkDownloadPlugin;
 import org.elasticsearch.gradle.internal.conventions.GitInfoPlugin;
 import org.elasticsearch.gradle.internal.conventions.info.GitInfo;
 import org.elasticsearch.gradle.internal.conventions.info.ParallelDetector;
@@ -22,6 +26,7 @@
 import org.gradle.api.Action;
 import org.gradle.api.GradleException;
 import org.gradle.api.JavaVersion;
+import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.logging.Logger;
@@ -52,12 +57,14 @@
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.UncheckedIOException;
+import java.lang.reflect.Field;
 import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
 import java.util.Random;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -94,6 +101,7 @@ public GlobalBuildInfoPlugin(
 
     @Override
     public void apply(Project project) {
+
         if (project != project.getRootProject()) {
             throw new IllegalStateException(this.getClass().getName() + " can only be applied to the root project.");
         }
@@ -272,7 +280,9 @@ private JavaVersion determineJavaVersion(String description, File javaHome, Java
     private InstallationLocation getJavaInstallation(File javaHome) {
         return getAvailableJavaInstallationLocationSteam().filter(installationLocation -> isSameFile(javaHome, installationLocation))
             .findFirst()
-            .orElseThrow(() -> new GradleException("Could not locate available Java installation in Gradle registry at: " + javaHome));
+            .orElse(
+                InstallationLocation.userDefined(javaHome, "Manually resolved JavaHome (not auto-detected by Gradle toolchain service)")
+            );
     }
 
     private boolean isSameFile(File javaHome, InstallationLocation installationLocation) {
@@ -337,11 +347,15 @@ private static void assertMinimumCompilerVersion(JavaVersion minimumCompilerVers
     }
 
     private Provider<File> findRuntimeJavaHome() {
-        String runtimeJavaProperty = System.getProperty("runtime.java");
-
+        Integer runtimeJavaProperty = Integer.getInteger("runtime.java");
         if (runtimeJavaProperty != null) {
-            System.out.println("GlobalBuildInfoPlugin.findRuntimeJavaHome");
-            return resolveJavaHomeFromToolChainService(runtimeJavaProperty);
+            if (runtimeJavaProperty > Integer.parseInt(VersionProperties.getBundledJdkMajorVersion())) {
+                // handle EA builds differently due to lack of support in Gradle toolchain service
+                // we resolve them using JdkDownloadPlugin for now.
+                return resolveEarlyAccessJavaHome(runtimeJavaProperty);
+            } else {
+                return resolveJavaHomeFromToolChainService(runtimeJavaProperty.toString());
+            }
         }
         if (System.getenv("RUNTIME_JAVA_HOME") != null) {
             return providers.provider(() -> new File(System.getenv("RUNTIME_JAVA_HOME")));
@@ -356,28 +370,31 @@ private Provider<File> findRuntimeJavaHome() {
         });
     }
 
+    private Provider<File> resolveEarlyAccessJavaHome(Integer runtimeJavaProperty) {
+        NamedDomainObjectContainer<Jdk> container = project.getPlugins().apply(JdkDownloadPlugin.class).getContainer(project);
+        Integer buildNumber = Integer.getInteger("runtime.java.build");
+        if (buildNumber == null) {
+            buildNumber = Integer.getInteger("runtime.java." + runtimeJavaProperty + ".build");
+        }
+        if (buildNumber == null) {
+            buildNumber = findLatestEABuildNumber(runtimeJavaProperty);
+        }
+        String eaVersionString = String.format("%d-ea+%d", runtimeJavaProperty, buildNumber);
+        Jdk jdk = container.create("ea_jdk_" + runtimeJavaProperty, j -> {
+            j.setVersion(eaVersionString);
+            j.setVendor("openjdk");
+            j.setPlatform(OS.current().javaOsReference);
+            j.setArchitecture(Architecture.current().javaClassifier);
+            j.setDistributionVersion("ea");
+        });
+        return providers.provider(() -> new File(jdk.getJavaHomePath().toString()));
+    }
+
     @NotNull
     private Provider<File> resolveJavaHomeFromToolChainService(String version) {
-        JavaLanguageVersion languageVersion = JavaLanguageVersion.of(version);
-        Property<JavaLanguageVersion> value = objectFactory.property(JavaLanguageVersion.class).value(languageVersion);
-        return toolChainService.launcherFor(javaToolchainSpec -> {
-            javaToolchainSpec.getLanguageVersion().value(value);
-
-            if (Integer.parseInt(VersionProperties.getBundledJdkMajorVersion()) < Integer.parseInt(version)) {
-                // If the requested version is higher than the bundled JDK, we need trick the toolchain into using our early access catalog
-                // Otherwise, we use the default implementation
-                Integer buildNumber = Integer.getInteger("runtime.java.build");
-
-                if (buildNumber == null) {
-                    buildNumber = Integer.getInteger("runtime.java." + version + ".build");
-                }
-                if (buildNumber == null) {
-                    buildNumber = findLatestEABuildNumber(languageVersion);
-                }
-                javaToolchainSpec.getVendor()
-                    .set(DefaultJvmVendorSpec.of("Oracle[" + languageVersion + "/" + buildNumber + "]"));
-            }
-        }).map(launcher -> launcher.getMetadata().getInstallationPath().getAsFile());
+        Property<JavaLanguageVersion> value = objectFactory.property(JavaLanguageVersion.class).value(JavaLanguageVersion.of(version));
+        return toolChainService.launcherFor(javaToolchainSpec -> javaToolchainSpec.getLanguageVersion().value(value))
+            .map(launcher -> launcher.getMetadata().getInstallationPath().getAsFile());
     }
 
     public static String getResourceContents(String resourcePath) {