Merge remote-tracking branch 'elastic/main' into exp-histo-hashcode-equals

# Conflicts:
#	libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogram.java
#	libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramUtils.java
#	x-pack/plugin/mapper-exponential-histogram/src/main/java/org/elasticsearch/xpack/exponentialhistogram/CompressedExponentialHistogram.java

Author: JonasKunz

.buildkite/pipelines/intake.yml

@@ -65,7 +65,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.18.6", "8.19.3", "9.0.6", "9.1.3", "9.2.0"]
+            BWC_VERSION: ["8.18.7", "8.19.4", "9.0.7", "9.1.4", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404

.buildkite/pipelines/periodic-packaging.yml

@@ -318,8 +318,8 @@ steps:
         env:
           BWC_VERSION: 8.17.10
 
-      - label: "{{matrix.image}} / 8.18.6 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.6
+      - label: "{{matrix.image}} / 8.18.7 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.7
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -332,10 +332,10 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.18.6
+          BWC_VERSION: 8.18.7
 
-      - label: "{{matrix.image}} / 8.19.3 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.3
+      - label: "{{matrix.image}} / 8.19.4 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.4
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -348,10 +348,10 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.19.3
+          BWC_VERSION: 8.19.4
 
-      - label: "{{matrix.image}} / 9.0.6 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.6
+      - label: "{{matrix.image}} / 9.0.7 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.7
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -364,10 +364,10 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 9.0.6
+          BWC_VERSION: 9.0.7
 
-      - label: "{{matrix.image}} / 9.1.3 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.1.3
+      - label: "{{matrix.image}} / 9.1.4 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.1.4
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -380,7 +380,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 9.1.3
+          BWC_VERSION: 9.1.4
 
       - label: "{{matrix.image}} / 9.2.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.2.0

.buildkite/pipelines/periodic.template.yml

@@ -246,7 +246,7 @@ steps:
       image: family/elasticsearch-ubuntu-2404
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
-    if: build.branch == "main" || build.branch == "9.1" || build.branch == "9.0" || build.branch == "8.19" || build.branch == "8.18" || build.branch == "7.17"
+    if: build.branch =~ /^(main|\d+\.\d+|\d+\.x)$/
   - label: check-branch-consistency
     command: .ci/scripts/run-gradle.sh branchConsistency
     timeout_in_minutes: 15

.buildkite/pipelines/periodic.yml

@@ -344,8 +344,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.18.6 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.6#bwcTest
+      - label: 8.18.7 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.7#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -354,7 +354,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.18.6
+          BWC_VERSION: 8.18.7
         retry:
           automatic:
             - exit_status: "-1"
@@ -363,8 +363,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.19.3 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.19.3#bwcTest
+      - label: 8.19.4 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.19.4#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -373,7 +373,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.19.3
+          BWC_VERSION: 8.19.4
         retry:
           automatic:
             - exit_status: "-1"
@@ -382,8 +382,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 9.0.6 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.6#bwcTest
+      - label: 9.0.7 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.7#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -392,7 +392,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 9.0.6
+          BWC_VERSION: 9.0.7
         retry:
           automatic:
             - exit_status: "-1"
@@ -401,8 +401,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 9.1.3 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.1.3#bwcTest
+      - label: 9.1.4 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.1.4#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -411,7 +411,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 9.1.3
+          BWC_VERSION: 9.1.4
         retry:
           automatic:
             - exit_status: "-1"
@@ -506,7 +506,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.18.6", "8.19.3", "9.0.6", "9.1.3", "9.2.0"]
+            BWC_VERSION: ["8.18.7", "8.19.4", "9.0.7", "9.1.4", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404
@@ -550,7 +550,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.18.6", "8.19.3", "9.0.6", "9.1.3", "9.2.0"]
+            BWC_VERSION: ["8.18.7", "8.19.4", "9.0.7", "9.1.4", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404
@@ -684,7 +684,7 @@ steps:
       image: family/elasticsearch-ubuntu-2404
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
-    if: build.branch == "main" || build.branch == "9.1" || build.branch == "9.0" || build.branch == "8.19" || build.branch == "8.18" || build.branch == "7.17"
+    if: build.branch =~ /^(main|\d+\.\d+|\d+\.x)$/
   - label: check-branch-consistency
     command: .ci/scripts/run-gradle.sh branchConsistency
     timeout_in_minutes: 15

.ci/bwcVersions

@@ -17,8 +17,8 @@ BWC_VERSION:
   - "8.15.5"
   - "8.16.6"
   - "8.17.10"
-  - "8.18.6"
-  - "8.19.3"
-  - "9.0.6"
-  - "9.1.3"
+  - "8.18.7"
+  - "8.19.4"
+  - "9.0.7"
+  - "9.1.4"
   - "9.2.0"

.ci/snapshotBwcVersions

@@ -1,6 +1,6 @@
 BWC_VERSION:
-  - "8.18.6"
-  - "8.19.3"
-  - "9.0.6"
-  - "9.1.3"
+  - "8.18.7"
+  - "8.19.4"
+  - "9.0.7"
+  - "9.1.4"
   - "9.2.0"

BUILDING.md

@@ -186,23 +186,23 @@ dependencies {
 
 To test an unreleased development version of a third party dependency you have several options.
 
-### How do I test against java early access (ea) versions?
+### How do I test against java pre-release versions like early access (ea) or release candidates (rc)?
 
 Currently only openjdk EA builds by oracle are supported.
 To test against an early access version java version you can pass the major
-java version appended with `-ea` as a system property (e.g. -Druntime.java=26-ea) to the Gradle build:
+java version appended with `-pre` as a system property (e.g. -Druntime.java=26-pre) to the Gradle build:
 
 ```
-./gradlew clean test -Druntime.java=26-ea
+./gradlew clean test -Druntime.java=26-pre
 ```
 
-This will run the tests using the JDK 26 EA version and pick the latest available build of the matching JDK EA version we expose
-in our custom jdk catalogue at `https://storage.googleapis.com/elasticsearch-jdk-archive/jdks/openjdk/latest.json`.
+This will run the tests using the JDK 26 pre-release version and pick the latest available build of the matching JDK EA version we expose
+in our custom jdk catalogue at `https://builds.es-jdk-archive.com/jdks/openjdk/recent.json`.
 
 To run against a specific build number of the EA build you can pass a second system property (e.g. `-Druntime.java.build=6`):
 
 ```
-./gradlew clean test -Druntime.java=26-ea -Druntime.java.build=6
+./gradlew clean test -Druntime.java=26-pre -Druntime.java.build=6
 ```
 
 #### How to use a Maven based third party dependency via `mavenlocal`?

benchmarks/src/main/java/org/elasticsearch/benchmark/exponentialhistogram/ExponentialHistogramMergeBench.java

@@ -130,7 +130,7 @@ private ExponentialHistogram asCompressedHistogram(ExponentialHistogram histogra
             CompressedExponentialHistogram.writeHistogramBytes(histoBytes, histogram.scale(), negativeBuckets, positiveBuckets);
             CompressedExponentialHistogram result = new CompressedExponentialHistogram();
             BytesRef data = histoBytes.bytes().toBytesRef();
-            result.reset(histogram.zeroBucket().zeroThreshold(), totalCount, histogram.sum(), data);
+            result.reset(histogram.zeroBucket().zeroThreshold(), totalCount, histogram.sum(), histogram.min(), data);
             return result;
         } catch (IOException e) {
             throw new RuntimeException(e);

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionValidationFuncTest.groovy

@@ -223,10 +223,33 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes
 
     def "unreferable definitions can have primary ids that are patches"() {
         given:
-        unreferableTransportVersion("initial_10.0.1", "10000001")
+        unreferableTransportVersion("initial_7.0.1", "7000001")
         when:
         def result = gradleRunner(":myserver:validateTransportVersionResources").build()
         then:
         result.task(":myserver:validateTransportVersionResources").outcome == TaskOutcome.SUCCESS
     }
+
+    def "highest id in an referable definition should exist in an upper bounds file"() {
+        given:
+        referableAndReferencedTransportVersion("some_tv", "10000000")
+        when:
+        def result = validateResourcesFails()
+        then:
+        assertValidateResourcesFailure(result, "Transport version definition file " +
+            "[myserver/src/main/resources/transport/definitions/referable/some_tv.csv] " +
+            "has the highest transport version id [10000000] but is not present in any upper bounds files")
+    }
+
+    def "highest id in an unreferable definition should exist in an upper bounds file"() {
+        given:
+        unreferableTransportVersion("initial_10.0.0", "10000000")
+        when:
+        def result = validateResourcesFails()
+        then:
+        // TODO: this should be _unreferable_ in the error message, but will require some rework
+        assertValidateResourcesFailure(result, "Transport version definition file " +
+            "[myserver/src/main/resources/transport/definitions/referable/initial_10.0.0.csv] " +
+            "has the highest transport version id [10000000] but is not present in any upper bounds files")
+    }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

@@ -330,24 +330,22 @@ private static void configureEntitlements(Project project) {
             .matching(test -> TEST_TASKS_WITH_ENTITLEMENTS.contains(test.getName()))
             .configureEach(test -> {
                 // See also SystemJvmOptions.maybeAttachEntitlementAgent.
+                SystemPropertyCommandLineArgumentProvider nonInputSystemProperties = test.getExtensions()
+                    .getByType(SystemPropertyCommandLineArgumentProvider.class);
 
                 // Agent
-                if (agentFiles.isEmpty() == false) {
-                    test.getInputs().files(agentFiles);
-                    test.systemProperty("es.entitlement.agentJar", agentFiles.getAsPath());
-                    test.systemProperty("jdk.attach.allowAttachSelf", true);
-                }
+                test.getInputs().files(agentFiles).optional(true);
+                nonInputSystemProperties.systemProperty("es.entitlement.agentJar", agentFiles::getAsPath);
+                nonInputSystemProperties.systemProperty("jdk.attach.allowAttachSelf", () -> agentFiles.isEmpty() ? "false" : "true");
 
                 // Bridge
-                if (bridgeFiles.isEmpty() == false) {
-                    String modulesContainingEntitlementInstrumentation = "java.logging,java.net.http,java.naming,jdk.net";
-                    test.getInputs().files(bridgeFiles);
-                    // Tests may not be modular, but the JDK still is
-                    test.jvmArgs(
-                        "--add-exports=java.base/org.elasticsearch.entitlement.bridge=ALL-UNNAMED,"
-                            + modulesContainingEntitlementInstrumentation
-                    );
-                }
+                String modulesContainingEntitlementInstrumentation = "java.logging,java.net.http,java.naming,jdk.net";
+                test.getInputs().files(bridgeFiles).optional(true);
+                // Tests may not be modular, but the JDK still is
+                test.jvmArgs(
+                    "--add-exports=java.base/org.elasticsearch.entitlement.bridge=ALL-UNNAMED,"
+                        + modulesContainingEntitlementInstrumentation
+                );
             });
     }