Unify resolving pre release java versions like RCs and EAs

This reworks how we deal with pre-release java versions for testing.

Passing `-Druntime.java=25-pre` will pick the lastest build that could be either an EA
or and RC version.

Passing explicitly a build number works by running build via
`-Druntime.java=25-pre -Druntime.java.build=36` which as of now would pick a RC build.

This also tweaks the archive packaging in case of a defined pre release version. This is
used downstream when packaging serverless images including ea / rc versions

Author: breskeby

BUILDING.md

@@ -186,23 +186,23 @@ dependencies {
 
 To test an unreleased development version of a third party dependency you have several options.
 
-### How do I test against java early access (ea) versions?
+### How do I test against java pre-release versions like early access (ea) or release candidates (rc)?
 
 Currently only openjdk EA builds by oracle are supported.
 To test against an early access version java version you can pass the major
-java version appended with `-ea` as a system property (e.g. -Druntime.java=26-ea) to the Gradle build:
+java version appended with `-pre` as a system property (e.g. -Druntime.java=26-pre) to the Gradle build:
 
 ```
-./gradlew clean test -Druntime.java=26-ea
+./gradlew clean test -Druntime.java=26-pre
 ```
 
-This will run the tests using the JDK 26 EA version and pick the latest available build of the matching JDK EA version we expose
-in our custom jdk catalogue at `https://storage.googleapis.com/elasticsearch-jdk-archive/jdks/openjdk/latest.json`.
+This will run the tests using the JDK 26 pre-release version and pick the latest available build of the matching JDK EA version we expose
+in our custom jdk catalogue at `https://builds.es-jdk-archive.com/jdks/openjdk/recent.json`.
 
 To run against a specific build number of the EA build you can pass a second system property (e.g. `-Druntime.java.build=6`):
 
 ```
-./gradlew clean test -Druntime.java=26-ea -Druntime.java.build=6
+./gradlew clean test -Druntime.java=26-pre -Druntime.java.build=6
 ```
 
 #### How to use a Maven based third party dependency via `mavenlocal`?

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/Jdk.java

@@ -32,7 +32,7 @@ public class Jdk implements Buildable, Iterable<File> {
         "(\\d+)(\\.\\d+\\.\\d+(?:\\.\\d+)?)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?"
     );
     private static final Pattern LEGACY_VERSION_PATTERN = Pattern.compile("(\\d)(u\\d+)\\+(b\\d+?)(@([a-f0-9]{32}))?");
-    private static final Pattern EA_VERSION_PATTERN = Pattern.compile("(\\d+)-ea\\+(\\d+)(@([a-f0-9]{32}))?");
+    private static final Pattern EA_VERSION_PATTERN = Pattern.compile("(\\d+)-(?:ea|rc)\\+(\\d+)(@([a-f0-9]{32}))?");
 
     private final String name;
     private final FileCollection configuration;

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/JdkDownloadPlugin.java

@@ -114,33 +114,40 @@ private void setupRepository(Project project, Jdk jdk) {
                     + jdk.getBuild()
                     + "/[module]/[classifier]/jdk/hotspot/normal/adoptium";
             }
-        } else if (jdk.getVendor().equals(VENDOR_OPENJDK) && "ea".equals(jdk.getDistributionVersion())) {
-            repoUrl = "https://builds.es-jdk-archive.com/";
-            // current pattern since 12.0.1
-            artifactPattern = "jdks/openjdk/"
-                + jdk.getMajor()
-                + "/openjdk-"
-                + jdk.getBaseVersion()
-                + "/"
-                + "openjdk-[revision]_[module]-[classifier]_bin.[ext]";
         } else if (jdk.getVendor().equals(VENDOR_OPENJDK)) {
-            repoUrl = "https://download.oracle.com";
-            if (jdk.getHash() != null) {
+            if ("ea".equals(jdk.getDistributionVersion())) {
+                repoUrl = "https://builds.es-jdk-archive.com/";
                 // current pattern since 12.0.1
-                artifactPattern = "java/GA/jdk"
-                    + jdk.getBaseVersion()
-                    + "/"
-                    + jdk.getHash()
-                    + "/"
-                    + jdk.getBuild()
-                    + "/GPL/openjdk-[revision]_[module]-[classifier]_bin.[ext]";
-            } else {
-                // simpler legacy pattern from JDK 9 to JDK 12 that we are advocating to Oracle to bring back
-                artifactPattern = "java/GA/jdk"
+                artifactPattern = "jdks/openjdk/"
                     + jdk.getMajor()
-                    + "/"
-                    + jdk.getBuild()
-                    + "/GPL/openjdk-[revision]_[module]-[classifier]_bin.[ext]";
+                    + "/openjdk-[revision]/openjdk-[revision]_[module]-[classifier]_bin.[ext]";
+            } else if ("rc".equals(jdk.getDistributionVersion())) {
+                repoUrl = "https://builds.es-jdk-archive.com/";
+                // current pattern since 12.0.1
+                artifactPattern = "jdks/openjdk/"
+                    + jdk.getMajor()
+                    + "/openjdk-[revision]/openjdk-"
+                    + jdk.getMajor()
+                    + "_[module]-[classifier]_bin.[ext]";
+            } else {
+                repoUrl = "https://download.oracle.com";
+                if (jdk.getHash() != null) {
+                    // current pattern since 12.0.1
+                    artifactPattern = "java/GA/jdk"
+                        + jdk.getBaseVersion()
+                        + "/"
+                        + jdk.getHash()
+                        + "/"
+                        + jdk.getBuild()
+                        + "/GPL/openjdk-[revision]_[module]-[classifier]_bin.[ext]";
+                } else {
+                    // simpler legacy pattern from JDK 9 to JDK 12 that we are advocating to Oracle to bring back
+                    artifactPattern = "java/GA/jdk"
+                        + jdk.getMajor()
+                        + "/"
+                        + jdk.getBuild()
+                        + "/GPL/openjdk-[revision]_[module]-[classifier]_bin.[ext]";
+                }
             }
         } else if (jdk.getVendor().equals(VENDOR_ZULU)) {
             repoUrl = "https://cdn.azul.com";

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParameterExtension.java

@@ -32,7 +32,7 @@ public interface BuildParameterExtension {
 
     Boolean getIsRuntimeJavaHomeSet();
 
-    Boolean getIsEarlyAccessRuntimeJavaVersion();
+    RuntimeJava getRuntimeJava();
 
     List<JavaHome> getJavaVersions();
 
@@ -46,8 +46,6 @@ public interface BuildParameterExtension {
 
     Provider<? extends Action<JavaToolchainSpec>> getJavaToolChainSpec();
 
-    Provider<String> getRuntimeJavaDetails();
-
     Provider<String> getGitRevision();
 
     Provider<String> getGitOrigin();

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/DefaultBuildParameterExtension.java

@@ -28,8 +28,7 @@
 public abstract class DefaultBuildParameterExtension implements BuildParameterExtension {
     private final Provider<Boolean> inFipsJvm;
     private final Provider<File> runtimeJavaHome;
-    private final Boolean isRuntimeJavaHomeSet;
-    private final Boolean earlyAccessJavaRuntime;
+    private final RuntimeJava runtimeJava;
     private final List<JavaHome> javaVersions;
     private final JavaVersion minimumCompilerVersion;
     private final JavaVersion minimumRuntimeVersion;
@@ -51,12 +50,8 @@ public abstract class DefaultBuildParameterExtension implements BuildParameterEx
 
     public DefaultBuildParameterExtension(
         ProviderFactory providers,
-        Provider<File> runtimeJavaHome,
+        RuntimeJava runtimeJava,
         Provider<? extends Action<JavaToolchainSpec>> javaToolChainSpec,
-        Provider<JavaVersion> runtimeJavaVersion,
-        boolean isRuntimeJavaHomeSet,
-        boolean earlyAccessJavaRuntime,
-        Provider<String> runtimeJavaDetails,
         List<JavaHome> javaVersions,
         JavaVersion minimumCompilerVersion,
         JavaVersion minimumRuntimeVersion,
@@ -70,12 +65,12 @@ public DefaultBuildParameterExtension(
         Provider<BwcVersions> bwcVersions
     ) {
         this.inFipsJvm = providers.systemProperty("tests.fips.enabled").map(DefaultBuildParameterExtension::parseBoolean);
-        this.runtimeJavaHome = cache(providers, runtimeJavaHome);
+        this.runtimeJava = runtimeJava;
+        this.runtimeJavaHome = cache(providers, runtimeJava.getJavahome());
         this.javaToolChainSpec = cache(providers, javaToolChainSpec);
-        this.runtimeJavaVersion = cache(providers, runtimeJavaVersion);
-        this.isRuntimeJavaHomeSet = isRuntimeJavaHomeSet;
-        this.earlyAccessJavaRuntime = earlyAccessJavaRuntime;
-        this.runtimeJavaDetails = cache(providers, runtimeJavaDetails);
+        this.runtimeJavaVersion = cache(providers, runtimeJava.getJavaVersion());
+        // this.runtimeJavaVersion = runtimeJava.getJavaVersion();
+        this.runtimeJavaDetails = cache(providers, runtimeJava.getVendorDetails());
         this.javaVersions = javaVersions;
         this.minimumCompilerVersion = minimumCompilerVersion;
         this.minimumRuntimeVersion = minimumRuntimeVersion;
@@ -119,15 +114,14 @@ public void withFipsEnabledOnly(Task task) {
 
     @Override
     public Boolean getIsRuntimeJavaHomeSet() {
-        return isRuntimeJavaHomeSet;
+        return runtimeJava.isExplicitlySet();
     }
 
     @Override
-    public Boolean getIsEarlyAccessRuntimeJavaVersion() {
-        return earlyAccessJavaRuntime;
+    public RuntimeJava getRuntimeJava() {
+        return runtimeJava;
     }
 
-
     @Override
     public List<JavaHome> getJavaVersions() {
         return javaVersions;
@@ -158,11 +152,6 @@ public Provider<? extends Action<JavaToolchainSpec>> getJavaToolChainSpec() {
         return javaToolChainSpec;
     }
 
-    @Override
-    public Provider<String> getRuntimeJavaDetails() {
-        return runtimeJavaDetails;
-    }
-
     @Override
     public Provider<String> getGitRevision() {
         return gitRevision;

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java

@@ -47,7 +47,6 @@
 import org.gradle.jvm.toolchain.JvmVendorSpec;
 import org.gradle.jvm.toolchain.internal.InstallationLocation;
 import org.gradle.util.GradleVersion;
-import org.jetbrains.annotations.NotNull;
 
 import java.io.BufferedReader;
 import java.io.File;
@@ -68,7 +67,8 @@
 import javax.inject.Inject;
 
 import static org.elasticsearch.gradle.internal.conventions.GUtils.elvis;
-import static org.elasticsearch.gradle.internal.toolchain.EarlyAccessCatalogJdkToolchainResolver.findLatestEABuildNumber;
+import static org.elasticsearch.gradle.internal.toolchain.EarlyAccessCatalogJdkToolchainResolver.findLatestPreReleaseBuild;
+import static org.elasticsearch.gradle.internal.toolchain.EarlyAccessCatalogJdkToolchainResolver.findPreReleaseBuild;
 
 public class GlobalBuildInfoPlugin implements Plugin<Project> {
     private static final Logger LOGGER = Logging.getLogger(GlobalBuildInfoPlugin.class);
@@ -116,37 +116,20 @@ public void apply(Project project) {
         JavaVersion minimumCompilerVersion = JavaVersion.toVersion(getResourceContents("/minimumCompilerVersion"));
         JavaVersion minimumRuntimeVersion = JavaVersion.toVersion(getResourceContents("/minimumRuntimeVersion"));
 
-        RuntimeJavaHome runtimeJavaHome = findRuntimeJavaHome();
-
-        Provider<JvmInstallationMetadata> runtimeJdkMetaData = runtimeJavaHome.getJavahome().map(
-            j -> metadataDetector.getMetadata(getJavaInstallation(j))
-        );
+        RuntimeJava runtimeJavaHome = findRuntimeJavaHome();
         AtomicReference<BwcVersions> cache = new AtomicReference<>();
         Provider<BwcVersions> bwcVersionsProvider = providers.provider(
             () -> cache.updateAndGet(val -> val == null ? resolveBwcVersions() : val)
         );
 
-
         BuildParameterExtension buildParams = project.getExtensions()
             .create(
                 BuildParameterExtension.class,
                 BuildParameterExtension.EXTENSION_NAME,
                 DefaultBuildParameterExtension.class,
                 providers,
-                runtimeJavaHome.getJavahome(),
+                runtimeJavaHome,
                 resolveToolchainSpecFromEnv(),
-                runtimeJavaHome.javahome.map(
-                    javaHome -> determineJavaVersion(
-                        "runtime java.home",
-                        javaHome,
-                        runtimeJavaHome.javahome.isPresent()
-                            ? minimumRuntimeVersion
-                            : JavaVersion.toVersion(VersionProperties.getBundledJdkMajorVersion())
-                    )
-                ),
-                runtimeJavaHome.expclicitlySet,
-                runtimeJavaHome.earlyAccess,
-                runtimeJdkMetaData.map(m -> formatJavaVendorDetails(m)),
                 getAvailableJavaVersions(),
                 minimumCompilerVersion,
                 minimumRuntimeVersion,
@@ -236,7 +219,7 @@ private void logGlobalBuildInfo(BuildParameterExtension buildParams) {
         LOGGER.quiet("Elasticsearch Build Hamster says Hello!");
         LOGGER.quiet("  Gradle Version        : " + GradleVersion.current().getVersion());
         LOGGER.quiet("  OS Info               : " + osName + " " + osVersion + " (" + osArch + ")");
-        if (buildParams.getIsRuntimeJavaHomeSet()) {
+        if (buildParams.getRuntimeJava().isExplicitlySet()) {
             JvmInstallationMetadata runtimeJvm = metadataDetector.getMetadata(getJavaInstallation(buildParams.getRuntimeJavaHome().get()));
             final String runtimeJvmVendorDetails = runtimeJvm.getVendor().getDisplayName();
             final String runtimeJvmImplementationVersion = runtimeJvm.getJvmVersion();
@@ -343,53 +326,74 @@ private static void assertMinimumCompilerVersion(JavaVersion minimumCompilerVers
         }
     }
 
-    private RuntimeJavaHome findRuntimeJavaHome() {
+    private RuntimeJava findRuntimeJavaHome() {
         String runtimeJavaProperty = System.getProperty("runtime.java");
-
         if (runtimeJavaProperty != null) {
-            if (runtimeJavaProperty.toLowerCase().endsWith("-ea")) {
-                // handle EA builds differently due to lack of support in Gradle toolchain service
+            if (runtimeJavaProperty.toLowerCase().endsWith("-pre")) {
+                // handle pre-release builds differently due to lack of support in Gradle toolchain service
                 // we resolve them using JdkDownloadPlugin for now.
-                Integer major = Integer.parseInt(runtimeJavaProperty.substring(0, runtimeJavaProperty.length() - 3));
-                return new RuntimeJavaHome(resolveEarlyAccessJavaHome(major), true, true);
+                return resolvePreReleaseRuntimeJavaHome(runtimeJavaProperty);
             } else {
-                return new RuntimeJavaHome(resolveJavaHomeFromToolChainService(runtimeJavaProperty), true, false);
+                return runtimeJavaHome(resolveJavaHomeFromToolChainService(runtimeJavaProperty), true);
             }
         }
         if (System.getenv("RUNTIME_JAVA_HOME") != null) {
-            return new RuntimeJavaHome(providers.provider(() -> new File(System.getenv("RUNTIME_JAVA_HOME"))), true, false);
+            return runtimeJavaHome(providers.provider(() -> new File(System.getenv("RUNTIME_JAVA_HOME"))), true);
         }
         // fall back to tool chain if set.
         String env = System.getenv("JAVA_TOOLCHAIN_HOME");
         boolean explicitlySet = env != null;
-        Provider<File> javaHome = explicitlySet ?
-            providers.provider(() -> new File(env)) :
-            resolveJavaHomeFromToolChainService(VersionProperties.getBundledJdkMajorVersion());
-        return
-            new RuntimeJavaHome(
+        Provider<File> javaHome = explicitlySet
+            ? providers.provider(() -> new File(env))
+            : resolveJavaHomeFromToolChainService(VersionProperties.getBundledJdkMajorVersion());
+        return runtimeJavaHome(javaHome, explicitlySet);
+    }
+
+    private RuntimeJava runtimeJavaHome(Provider<File> fileProvider, boolean explicitlySet) {
+        return runtimeJavaHome(fileProvider, explicitlySet, null);
+    }
+
+    private RuntimeJava runtimeJavaHome(Provider<File> fileProvider, boolean explicitlySet, String preReleasePostfix) {
+        Provider<JavaVersion> javaVersion = fileProvider.map(
+            javaHome -> determineJavaVersion(
+                "runtime java.home",
                 javaHome,
-                explicitlySet,
-                false);
+                fileProvider.isPresent()
+                    ? JavaVersion.toVersion(getResourceContents("/minimumRuntimeVersion"))
+                    : JavaVersion.toVersion(VersionProperties.getBundledJdkMajorVersion())
+            )
+        );
+
+        Provider<String> vendorDetails = fileProvider.map(j -> metadataDetector.getMetadata(getJavaInstallation(j)))
+            .map(m -> formatJavaVendorDetails(m));
+
+        return new RuntimeJava(fileProvider, javaVersion, vendorDetails, explicitlySet, preReleasePostfix);
     }
 
-    private Provider<File> resolveEarlyAccessJavaHome(Integer runtimeJavaProperty) {
-        NamedDomainObjectContainer<Jdk> container = (NamedDomainObjectContainer<Jdk>) project.getExtensions().getByName("jdks");
+    private RuntimeJava resolvePreReleaseRuntimeJavaHome(String runtimeJavaProperty) {
+        var major = JavaLanguageVersion.of(Integer.parseInt(runtimeJavaProperty.substring(0, runtimeJavaProperty.length() - 4)));
         Integer buildNumber = Integer.getInteger("runtime.java.build");
-        if (buildNumber == null) {
-            buildNumber = findLatestEABuildNumber(runtimeJavaProperty);
-        }
-        String eaVersionString = String.format("%d-ea+%d", runtimeJavaProperty, buildNumber);
-        Jdk jdk = container.create("ea_jdk_" + runtimeJavaProperty, j -> {
-            j.setVersion(eaVersionString);
+        var jdkbuild = buildNumber == null ? findLatestPreReleaseBuild(major) : findPreReleaseBuild(major, buildNumber);
+        String prVersionString = String.format("%d-%s+%d", major.asInt(), jdkbuild.type(), jdkbuild.buildNumber());
+        return resolveJavaHomeFromJdkDownloadPlugin(jdkbuild.type(), major, prVersionString);
+
+    }
+
+    private RuntimeJava resolveJavaHomeFromJdkDownloadPlugin(String preReleaseType, JavaLanguageVersion major, String prVersionString) {
+        NamedDomainObjectContainer<Jdk> container = (NamedDomainObjectContainer<Jdk>) project.getExtensions().getByName("jdks");
+        Jdk jdk = container.create(preReleaseType + "_" + major.asInt(), j -> {
+            j.setVersion(prVersionString);
             j.setVendor("openjdk");
             j.setPlatform(OS.current().javaOsReference);
             j.setArchitecture(Architecture.current().javaClassifier);
-            j.setDistributionVersion("ea");
+            j.setDistributionVersion(preReleaseType);
         });
-        return providers.provider(() -> new File(jdk.getJavaHomePath().toString()));
+        // We on purpose resolve this here eagerly to ensure we resolve the jdk configuration in the context of the root project.
+        // If we keep this lazy we can not guarantee in which project context this is resolved which will fail the build.
+        File file = new File(jdk.getJavaHomePath().toString());
+        return runtimeJavaHome(providers.provider(() -> file), true, preReleaseType);
     }
 
-    @NotNull
     private Provider<File> resolveJavaHomeFromToolChainService(String version) {
         Property<JavaLanguageVersion> value = objectFactory.property(JavaLanguageVersion.class).value(JavaLanguageVersion.of(version));
         return toolChainService.launcherFor(javaToolchainSpec -> javaToolchainSpec.getLanguageVersion().value(value))
@@ -447,24 +451,4 @@ public void execute(JavaToolchainSpec spec) {
         }
     }
 
-    private static class RuntimeJavaHome {
-
-        private final boolean earlyAccess;
-        private final Provider<File> javahome;
-        private final boolean expclicitlySet;
-
-        // RuntimeJavaHome(Provider<File> javahome) {
-        // this(javahome, false);
-        // }
-        //
-        RuntimeJavaHome(Provider<File> javahome, boolean explicitlySet, boolean earlyAccess) {
-            this.javahome = javahome;
-            this.expclicitlySet = explicitlySet;
-            this.earlyAccess = earlyAccess;
-        }
-
-        public Provider<File> getJavahome() {
-            return javahome;
-        }
-    }
 }