Merge branch 'main' into 2215-part-II

Author: yetanothertw

.buildkite/pipelines/periodic.template.yml

@@ -246,7 +246,7 @@ steps:
       image: family/elasticsearch-ubuntu-2404
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
-    if: build.branch == "main" || build.branch == "9.1" || build.branch == "9.0" || build.branch == "8.19" || build.branch == "8.18" || build.branch == "7.17"
+    if: build.branch =~ /^(main|\d+\.\d+|\d+\.x)$/
   - label: check-branch-consistency
     command: .ci/scripts/run-gradle.sh branchConsistency
     timeout_in_minutes: 15

.buildkite/pipelines/periodic.yml

@@ -684,7 +684,7 @@ steps:
       image: family/elasticsearch-ubuntu-2404
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
-    if: build.branch == "main" || build.branch == "9.1" || build.branch == "9.0" || build.branch == "8.19" || build.branch == "8.18" || build.branch == "7.17"
+    if: build.branch =~ /^(main|\d+\.\d+|\d+\.x)$/
   - label: check-branch-consistency
     command: .ci/scripts/run-gradle.sh branchConsistency
     timeout_in_minutes: 15

BUILDING.md

@@ -186,23 +186,23 @@ dependencies {
 
 To test an unreleased development version of a third party dependency you have several options.
 
-### How do I test against java early access (ea) versions?
+### How do I test against java pre-release versions like early access (ea) or release candidates (rc)?
 
 Currently only openjdk EA builds by oracle are supported.
 To test against an early access version java version you can pass the major
-java version appended with `-ea` as a system property (e.g. -Druntime.java=26-ea) to the Gradle build:
+java version appended with `-pre` as a system property (e.g. -Druntime.java=26-pre) to the Gradle build:
 
 ```
-./gradlew clean test -Druntime.java=26-ea
+./gradlew clean test -Druntime.java=26-pre
 ```
 
-This will run the tests using the JDK 26 EA version and pick the latest available build of the matching JDK EA version we expose
-in our custom jdk catalogue at `https://storage.googleapis.com/elasticsearch-jdk-archive/jdks/openjdk/latest.json`.
+This will run the tests using the JDK 26 pre-release version and pick the latest available build of the matching JDK EA version we expose
+in our custom jdk catalogue at `https://builds.es-jdk-archive.com/jdks/openjdk/recent.json`.
 
 To run against a specific build number of the EA build you can pass a second system property (e.g. `-Druntime.java.build=6`):
 
 ```
-./gradlew clean test -Druntime.java=26-ea -Druntime.java.build=6
+./gradlew clean test -Druntime.java=26-pre -Druntime.java.build=6
 ```
 
 #### How to use a Maven based third party dependency via `mavenlocal`?

benchmarks/src/main/java/org/elasticsearch/benchmark/exponentialhistogram/ExponentialHistogramMergeBench.java

@@ -130,7 +130,7 @@ private ExponentialHistogram asCompressedHistogram(ExponentialHistogram histogra
             CompressedExponentialHistogram.writeHistogramBytes(histoBytes, histogram.scale(), negativeBuckets, positiveBuckets);
             CompressedExponentialHistogram result = new CompressedExponentialHistogram();
             BytesRef data = histoBytes.bytes().toBytesRef();
-            result.reset(histogram.zeroBucket().zeroThreshold(), totalCount, histogram.sum(), data);
+            result.reset(histogram.zeroBucket().zeroThreshold(), totalCount, histogram.sum(), histogram.min(), data);
             return result;
         } catch (IOException e) {
             throw new RuntimeException(e);

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/Jdk.java

@@ -32,7 +32,7 @@ public class Jdk implements Buildable, Iterable<File> {
         "(\\d+)(\\.\\d+\\.\\d+(?:\\.\\d+)?)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?"
     );
     private static final Pattern LEGACY_VERSION_PATTERN = Pattern.compile("(\\d)(u\\d+)\\+(b\\d+?)(@([a-f0-9]{32}))?");
-    private static final Pattern EA_VERSION_PATTERN = Pattern.compile("(\\d+)-ea\\+(\\d+)(@([a-f0-9]{32}))?");
+    private static final Pattern EA_VERSION_PATTERN = Pattern.compile("(\\d+)-(?:ea|rc)\\+(\\d+)(@([a-f0-9]{32}))?");
 
     private final String name;
     private final FileCollection configuration;

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/JdkDownloadPlugin.java

@@ -114,33 +114,40 @@ private void setupRepository(Project project, Jdk jdk) {
                     + jdk.getBuild()
                     + "/[module]/[classifier]/jdk/hotspot/normal/adoptium";
             }
-        } else if (jdk.getVendor().equals(VENDOR_OPENJDK) && "ea".equals(jdk.getDistributionVersion())) {
-            repoUrl = "https://builds.es-jdk-archive.com/";
-            // current pattern since 12.0.1
-            artifactPattern = "jdks/openjdk/"
-                + jdk.getMajor()
-                + "/openjdk-"
-                + jdk.getBaseVersion()
-                + "/"
-                + "openjdk-[revision]_[module]-[classifier]_bin.[ext]";
         } else if (jdk.getVendor().equals(VENDOR_OPENJDK)) {
-            repoUrl = "https://download.oracle.com";
-            if (jdk.getHash() != null) {
+            if ("ea".equals(jdk.getDistributionVersion())) {
+                repoUrl = "https://builds.es-jdk-archive.com/";
                 // current pattern since 12.0.1
-                artifactPattern = "java/GA/jdk"
-                    + jdk.getBaseVersion()
-                    + "/"
-                    + jdk.getHash()
-                    + "/"
-                    + jdk.getBuild()
-                    + "/GPL/openjdk-[revision]_[module]-[classifier]_bin.[ext]";
-            } else {
-                // simpler legacy pattern from JDK 9 to JDK 12 that we are advocating to Oracle to bring back
-                artifactPattern = "java/GA/jdk"
+                artifactPattern = "jdks/openjdk/"
                     + jdk.getMajor()
-                    + "/"
-                    + jdk.getBuild()
-                    + "/GPL/openjdk-[revision]_[module]-[classifier]_bin.[ext]";
+                    + "/openjdk-[revision]/openjdk-[revision]_[module]-[classifier]_bin.[ext]";
+            } else if ("rc".equals(jdk.getDistributionVersion())) {
+                repoUrl = "https://builds.es-jdk-archive.com/";
+                // current pattern since 12.0.1
+                artifactPattern = "jdks/openjdk/"
+                    + jdk.getMajor()
+                    + "/openjdk-[revision]/openjdk-"
+                    + jdk.getMajor()
+                    + "_[module]-[classifier]_bin.[ext]";
+            } else {
+                repoUrl = "https://download.oracle.com";
+                if (jdk.getHash() != null) {
+                    // current pattern since 12.0.1
+                    artifactPattern = "java/GA/jdk"
+                        + jdk.getBaseVersion()
+                        + "/"
+                        + jdk.getHash()
+                        + "/"
+                        + jdk.getBuild()
+                        + "/GPL/openjdk-[revision]_[module]-[classifier]_bin.[ext]";
+                } else {
+                    // simpler legacy pattern from JDK 9 to JDK 12 that we are advocating to Oracle to bring back
+                    artifactPattern = "java/GA/jdk"
+                        + jdk.getMajor()
+                        + "/"
+                        + jdk.getBuild()
+                        + "/GPL/openjdk-[revision]_[module]-[classifier]_bin.[ext]";
+                }
             }
         } else if (jdk.getVendor().equals(VENDOR_ZULU)) {
             repoUrl = "https://cdn.azul.com";

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParameterExtension.java

@@ -32,6 +32,8 @@ public interface BuildParameterExtension {
 
     Boolean getIsRuntimeJavaHomeSet();
 
+    RuntimeJava getRuntimeJava();
+
     List<JavaHome> getJavaVersions();
 
     JavaVersion getMinimumCompilerVersion();

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/DefaultBuildParameterExtension.java

@@ -28,7 +28,7 @@
 public abstract class DefaultBuildParameterExtension implements BuildParameterExtension {
     private final Provider<Boolean> inFipsJvm;
     private final Provider<File> runtimeJavaHome;
-    private final Boolean isRuntimeJavaHomeSet;
+    private final RuntimeJava runtimeJava;
     private final List<JavaHome> javaVersions;
     private final JavaVersion minimumCompilerVersion;
     private final JavaVersion minimumRuntimeVersion;
@@ -50,11 +50,8 @@ public abstract class DefaultBuildParameterExtension implements BuildParameterEx
 
     public DefaultBuildParameterExtension(
         ProviderFactory providers,
-        Provider<File> runtimeJavaHome,
+        RuntimeJava runtimeJava,
         Provider<? extends Action<JavaToolchainSpec>> javaToolChainSpec,
-        Provider<JavaVersion> runtimeJavaVersion,
-        boolean isRuntimeJavaHomeSet,
-        Provider<String> runtimeJavaDetails,
         List<JavaHome> javaVersions,
         JavaVersion minimumCompilerVersion,
         JavaVersion minimumRuntimeVersion,
@@ -68,11 +65,11 @@ public DefaultBuildParameterExtension(
         Provider<BwcVersions> bwcVersions
     ) {
         this.inFipsJvm = providers.systemProperty("tests.fips.enabled").map(DefaultBuildParameterExtension::parseBoolean);
-        this.runtimeJavaHome = cache(providers, runtimeJavaHome);
+        this.runtimeJava = runtimeJava;
+        this.runtimeJavaHome = cache(providers, runtimeJava.getJavahome());
         this.javaToolChainSpec = cache(providers, javaToolChainSpec);
-        this.runtimeJavaVersion = cache(providers, runtimeJavaVersion);
-        this.isRuntimeJavaHomeSet = isRuntimeJavaHomeSet;
-        this.runtimeJavaDetails = cache(providers, runtimeJavaDetails);
+        this.runtimeJavaVersion = cache(providers, runtimeJava.getJavaVersion());
+        this.runtimeJavaDetails = cache(providers, runtimeJava.getVendorDetails());
         this.javaVersions = javaVersions;
         this.minimumCompilerVersion = minimumCompilerVersion;
         this.minimumRuntimeVersion = minimumRuntimeVersion;
@@ -116,7 +113,12 @@ public void withFipsEnabledOnly(Task task) {
 
     @Override
     public Boolean getIsRuntimeJavaHomeSet() {
-        return isRuntimeJavaHomeSet;
+        return runtimeJava.isExplicitlySet();
+    }
+
+    @Override
+    public RuntimeJava getRuntimeJava() {
+        return runtimeJava;
     }
 
     @Override

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java

@@ -47,7 +47,6 @@
 import org.gradle.jvm.toolchain.JvmVendorSpec;
 import org.gradle.jvm.toolchain.internal.InstallationLocation;
 import org.gradle.util.GradleVersion;
-import org.jetbrains.annotations.NotNull;
 
 import java.io.BufferedReader;
 import java.io.File;
@@ -68,7 +67,8 @@
 import javax.inject.Inject;
 
 import static org.elasticsearch.gradle.internal.conventions.GUtils.elvis;
-import static org.elasticsearch.gradle.internal.toolchain.EarlyAccessCatalogJdkToolchainResolver.findLatestEABuildNumber;
+import static org.elasticsearch.gradle.internal.toolchain.EarlyAccessCatalogJdkToolchainResolver.findLatestPreReleaseBuild;
+import static org.elasticsearch.gradle.internal.toolchain.EarlyAccessCatalogJdkToolchainResolver.findPreReleaseBuild;
 
 public class GlobalBuildInfoPlugin implements Plugin<Project> {
     private static final Logger LOGGER = Logging.getLogger(GlobalBuildInfoPlugin.class);
@@ -116,38 +116,20 @@ public void apply(Project project) {
         JavaVersion minimumCompilerVersion = JavaVersion.toVersion(getResourceContents("/minimumCompilerVersion"));
         JavaVersion minimumRuntimeVersion = JavaVersion.toVersion(getResourceContents("/minimumRuntimeVersion"));
 
-        Provider<File> explicitRuntimeJavaHome = findRuntimeJavaHome();
-        boolean isRuntimeJavaHomeExplicitlySet = explicitRuntimeJavaHome.isPresent();
-        Provider<File> actualRuntimeJavaHome = isRuntimeJavaHomeExplicitlySet
-            ? explicitRuntimeJavaHome
-            : resolveJavaHomeFromToolChainService(VersionProperties.getBundledJdkMajorVersion());
-
-        Provider<JvmInstallationMetadata> runtimeJdkMetaData = actualRuntimeJavaHome.map(
-            runtimeJavaHome -> metadataDetector.getMetadata(getJavaInstallation(runtimeJavaHome))
-        );
+        RuntimeJava runtimeJavaHome = findRuntimeJavaHome();
         AtomicReference<BwcVersions> cache = new AtomicReference<>();
         Provider<BwcVersions> bwcVersionsProvider = providers.provider(
             () -> cache.updateAndGet(val -> val == null ? resolveBwcVersions() : val)
         );
+
         BuildParameterExtension buildParams = project.getExtensions()
             .create(
                 BuildParameterExtension.class,
                 BuildParameterExtension.EXTENSION_NAME,
                 DefaultBuildParameterExtension.class,
                 providers,
-                actualRuntimeJavaHome,
+                runtimeJavaHome,
                 resolveToolchainSpecFromEnv(),
-                actualRuntimeJavaHome.map(
-                    javaHome -> determineJavaVersion(
-                        "runtime java.home",
-                        javaHome,
-                        isRuntimeJavaHomeExplicitlySet
-                            ? minimumRuntimeVersion
-                            : JavaVersion.toVersion(VersionProperties.getBundledJdkMajorVersion())
-                    )
-                ),
-                isRuntimeJavaHomeExplicitlySet,
-                runtimeJdkMetaData.map(m -> formatJavaVendorDetails(m)),
                 getAvailableJavaVersions(),
                 minimumCompilerVersion,
                 minimumRuntimeVersion,
@@ -237,7 +219,7 @@ private void logGlobalBuildInfo(BuildParameterExtension buildParams) {
         LOGGER.quiet("Elasticsearch Build Hamster says Hello!");
         LOGGER.quiet("  Gradle Version        : " + GradleVersion.current().getVersion());
         LOGGER.quiet("  OS Info               : " + osName + " " + osVersion + " (" + osArch + ")");
-        if (buildParams.getIsRuntimeJavaHomeSet()) {
+        if (buildParams.getRuntimeJava().isExplicitlySet()) {
             JvmInstallationMetadata runtimeJvm = metadataDetector.getMetadata(getJavaInstallation(buildParams.getRuntimeJavaHome().get()));
             final String runtimeJvmVendorDetails = runtimeJvm.getVendor().getDisplayName();
             final String runtimeJvmImplementationVersion = runtimeJvm.getJvmVersion();
@@ -344,50 +326,74 @@ private static void assertMinimumCompilerVersion(JavaVersion minimumCompilerVers
         }
     }
 
-    private Provider<File> findRuntimeJavaHome() {
+    private RuntimeJava findRuntimeJavaHome() {
         String runtimeJavaProperty = System.getProperty("runtime.java");
-
         if (runtimeJavaProperty != null) {
-            if (runtimeJavaProperty.toLowerCase().endsWith("-ea")) {
-                // handle EA builds differently due to lack of support in Gradle toolchain service
+            if (runtimeJavaProperty.toLowerCase().endsWith("-pre")) {
+                // handle pre-release builds differently due to lack of support in Gradle toolchain service
                 // we resolve them using JdkDownloadPlugin for now.
-                Integer major = Integer.parseInt(runtimeJavaProperty.substring(0, runtimeJavaProperty.length() - 3));
-                return resolveEarlyAccessJavaHome(major);
+                return resolvePreReleaseRuntimeJavaHome(runtimeJavaProperty);
             } else {
-                return resolveJavaHomeFromToolChainService(runtimeJavaProperty);
+                return runtimeJavaHome(resolveJavaHomeFromToolChainService(runtimeJavaProperty), true);
             }
         }
         if (System.getenv("RUNTIME_JAVA_HOME") != null) {
-            return providers.provider(() -> new File(System.getenv("RUNTIME_JAVA_HOME")));
+            return runtimeJavaHome(providers.provider(() -> new File(System.getenv("RUNTIME_JAVA_HOME"))), true);
         }
         // fall back to tool chain if set.
         String env = System.getenv("JAVA_TOOLCHAIN_HOME");
-        return providers.provider(() -> {
-            if (env == null) {
-                return null;
-            }
-            return new File(env);
-        });
+        boolean explicitlySet = env != null;
+        Provider<File> javaHome = explicitlySet
+            ? providers.provider(() -> new File(env))
+            : resolveJavaHomeFromToolChainService(VersionProperties.getBundledJdkMajorVersion());
+        return runtimeJavaHome(javaHome, explicitlySet);
     }
 
-    private Provider<File> resolveEarlyAccessJavaHome(Integer runtimeJavaProperty) {
-        NamedDomainObjectContainer<Jdk> container = (NamedDomainObjectContainer<Jdk>) project.getExtensions().getByName("jdks");
+    private RuntimeJava runtimeJavaHome(Provider<File> fileProvider, boolean explicitlySet) {
+        return runtimeJavaHome(fileProvider, explicitlySet, null);
+    }
+
+    private RuntimeJava runtimeJavaHome(Provider<File> fileProvider, boolean explicitlySet, String preReleasePostfix) {
+        Provider<JavaVersion> javaVersion = fileProvider.map(
+            javaHome -> determineJavaVersion(
+                "runtime java.home",
+                javaHome,
+                fileProvider.isPresent()
+                    ? JavaVersion.toVersion(getResourceContents("/minimumRuntimeVersion"))
+                    : JavaVersion.toVersion(VersionProperties.getBundledJdkMajorVersion())
+            )
+        );
+
+        Provider<String> vendorDetails = fileProvider.map(j -> metadataDetector.getMetadata(getJavaInstallation(j)))
+            .map(m -> formatJavaVendorDetails(m));
+
+        return new RuntimeJava(fileProvider, javaVersion, vendorDetails, explicitlySet, preReleasePostfix);
+    }
+
+    private RuntimeJava resolvePreReleaseRuntimeJavaHome(String runtimeJavaProperty) {
+        var major = JavaLanguageVersion.of(Integer.parseInt(runtimeJavaProperty.substring(0, runtimeJavaProperty.length() - 4)));
         Integer buildNumber = Integer.getInteger("runtime.java.build");
-        if (buildNumber == null) {
-            buildNumber = findLatestEABuildNumber(runtimeJavaProperty);
-        }
-        String eaVersionString = String.format("%d-ea+%d", runtimeJavaProperty, buildNumber);
-        Jdk jdk = container.create("ea_jdk_" + runtimeJavaProperty, j -> {
-            j.setVersion(eaVersionString);
+        var jdkbuild = buildNumber == null ? findLatestPreReleaseBuild(major) : findPreReleaseBuild(major, buildNumber);
+        String prVersionString = String.format("%d-%s+%d", major.asInt(), jdkbuild.type(), jdkbuild.buildNumber());
+        return resolveJavaHomeFromJdkDownloadPlugin(jdkbuild.type(), major, prVersionString);
+
+    }
+
+    private RuntimeJava resolveJavaHomeFromJdkDownloadPlugin(String preReleaseType, JavaLanguageVersion major, String prVersionString) {
+        NamedDomainObjectContainer<Jdk> container = (NamedDomainObjectContainer<Jdk>) project.getExtensions().getByName("jdks");
+        Jdk jdk = container.create(preReleaseType + "_" + major.asInt(), j -> {
+            j.setVersion(prVersionString);
             j.setVendor("openjdk");
             j.setPlatform(OS.current().javaOsReference);
             j.setArchitecture(Architecture.current().javaClassifier);
-            j.setDistributionVersion("ea");
+            j.setDistributionVersion(preReleaseType);
         });
-        return providers.provider(() -> new File(jdk.getJavaHomePath().toString()));
+        // We on purpose resolve this here eagerly to ensure we resolve the jdk configuration in the context of the root project.
+        // If we keep this lazy we can not guarantee in which project context this is resolved which will fail the build.
+        File file = new File(jdk.getJavaHomePath().toString());
+        return runtimeJavaHome(providers.provider(() -> file), true, preReleaseType);
     }
 
-    @NotNull
     private Provider<File> resolveJavaHomeFromToolChainService(String version) {
         Property<JavaLanguageVersion> value = objectFactory.property(JavaLanguageVersion.class).value(JavaLanguageVersion.of(version));
         return toolChainService.launcherFor(javaToolchainSpec -> javaToolchainSpec.getLanguageVersion().value(value))
@@ -444,4 +450,5 @@ public void execute(JavaToolchainSpec spec) {
             spec.getLanguageVersion().set(expectedJavaLanguageVersion);
         }
     }
+
 }