Merge branch 'main' into kderusso/remove-300-word-limit-on-sentence-boundary-chunking

Author: kderusso

.buildkite/pipelines/intake.yml

@@ -65,7 +65,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.18.6", "8.19.4", "9.0.6", "9.1.3", "9.2.0"]
+            BWC_VERSION: ["8.18.7", "8.19.4", "9.0.7", "9.1.4", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404

.buildkite/pipelines/periodic-packaging.yml

@@ -318,8 +318,8 @@ steps:
         env:
           BWC_VERSION: 8.17.10
 
-      - label: "{{matrix.image}} / 8.18.6 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.6
+      - label: "{{matrix.image}} / 8.18.7 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.7
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -332,7 +332,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.18.6
+          BWC_VERSION: 8.18.7
 
       - label: "{{matrix.image}} / 8.19.4 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.4
@@ -350,8 +350,8 @@ steps:
         env:
           BWC_VERSION: 8.19.4
 
-      - label: "{{matrix.image}} / 9.0.6 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.6
+      - label: "{{matrix.image}} / 9.0.7 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.7
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -364,10 +364,10 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 9.0.6
+          BWC_VERSION: 9.0.7
 
-      - label: "{{matrix.image}} / 9.1.3 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.1.3
+      - label: "{{matrix.image}} / 9.1.4 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.1.4
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -380,7 +380,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 9.1.3
+          BWC_VERSION: 9.1.4
 
       - label: "{{matrix.image}} / 9.2.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.2.0

.buildkite/pipelines/periodic.yml

@@ -344,8 +344,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.18.6 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.6#bwcTest
+      - label: 8.18.7 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.7#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -354,7 +354,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.18.6
+          BWC_VERSION: 8.18.7
         retry:
           automatic:
             - exit_status: "-1"
@@ -382,8 +382,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 9.0.6 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.6#bwcTest
+      - label: 9.0.7 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.7#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -392,7 +392,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 9.0.6
+          BWC_VERSION: 9.0.7
         retry:
           automatic:
             - exit_status: "-1"
@@ -401,8 +401,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 9.1.3 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.1.3#bwcTest
+      - label: 9.1.4 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.1.4#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -411,7 +411,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 9.1.3
+          BWC_VERSION: 9.1.4
         retry:
           automatic:
             - exit_status: "-1"
@@ -506,7 +506,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.18.6", "8.19.4", "9.0.6", "9.1.3", "9.2.0"]
+            BWC_VERSION: ["8.18.7", "8.19.4", "9.0.7", "9.1.4", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404
@@ -550,7 +550,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.18.6", "8.19.4", "9.0.6", "9.1.3", "9.2.0"]
+            BWC_VERSION: ["8.18.7", "8.19.4", "9.0.7", "9.1.4", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404

.ci/bwcVersions

@@ -17,8 +17,8 @@ BWC_VERSION:
   - "8.15.5"
   - "8.16.6"
   - "8.17.10"
-  - "8.18.6"
+  - "8.18.7"
   - "8.19.4"
-  - "9.0.6"
-  - "9.1.3"
+  - "9.0.7"
+  - "9.1.4"
   - "9.2.0"

.ci/snapshotBwcVersions

@@ -1,6 +1,6 @@
 BWC_VERSION:
-  - "8.18.6"
+  - "8.18.7"
   - "8.19.4"
-  - "9.0.6"
-  - "9.1.3"
+  - "9.0.7"
+  - "9.1.4"
   - "9.2.0"

BUILDING.md

@@ -186,23 +186,23 @@ dependencies {
 
 To test an unreleased development version of a third party dependency you have several options.
 
-### How do I test against java early access (ea) versions?
+### How do I test against java pre-release versions like early access (ea) or release candidates (rc)?
 
 Currently only openjdk EA builds by oracle are supported.
 To test against an early access version java version you can pass the major
-java version appended with `-ea` as a system property (e.g. -Druntime.java=26-ea) to the Gradle build:
+java version appended with `-pre` as a system property (e.g. -Druntime.java=26-pre) to the Gradle build:
 
 ```
-./gradlew clean test -Druntime.java=26-ea
+./gradlew clean test -Druntime.java=26-pre
 ```
 
-This will run the tests using the JDK 26 EA version and pick the latest available build of the matching JDK EA version we expose
-in our custom jdk catalogue at `https://storage.googleapis.com/elasticsearch-jdk-archive/jdks/openjdk/latest.json`.
+This will run the tests using the JDK 26 pre-release version and pick the latest available build of the matching JDK EA version we expose
+in our custom jdk catalogue at `https://builds.es-jdk-archive.com/jdks/openjdk/recent.json`.
 
 To run against a specific build number of the EA build you can pass a second system property (e.g. `-Druntime.java.build=6`):
 
 ```
-./gradlew clean test -Druntime.java=26-ea -Druntime.java.build=6
+./gradlew clean test -Druntime.java=26-pre -Druntime.java.build=6
 ```
 
 #### How to use a Maven based third party dependency via `mavenlocal`?

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionValidationFuncTest.groovy

@@ -223,10 +223,33 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes
 
     def "unreferable definitions can have primary ids that are patches"() {
         given:
-        unreferableTransportVersion("initial_10.0.1", "10000001")
+        unreferableTransportVersion("initial_7.0.1", "7000001")
         when:
         def result = gradleRunner(":myserver:validateTransportVersionResources").build()
         then:
         result.task(":myserver:validateTransportVersionResources").outcome == TaskOutcome.SUCCESS
     }
+
+    def "highest id in an referable definition should exist in an upper bounds file"() {
+        given:
+        referableAndReferencedTransportVersion("some_tv", "10000000")
+        when:
+        def result = validateResourcesFails()
+        then:
+        assertValidateResourcesFailure(result, "Transport version definition file " +
+            "[myserver/src/main/resources/transport/definitions/referable/some_tv.csv] " +
+            "has the highest transport version id [10000000] but is not present in any upper bounds files")
+    }
+
+    def "highest id in an unreferable definition should exist in an upper bounds file"() {
+        given:
+        unreferableTransportVersion("initial_10.0.0", "10000000")
+        when:
+        def result = validateResourcesFails()
+        then:
+        // TODO: this should be _unreferable_ in the error message, but will require some rework
+        assertValidateResourcesFailure(result, "Transport version definition file " +
+            "[myserver/src/main/resources/transport/definitions/referable/initial_10.0.0.csv] " +
+            "has the highest transport version id [10000000] but is not present in any upper bounds files")
+    }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/Jdk.java

@@ -32,7 +32,7 @@ public class Jdk implements Buildable, Iterable<File> {
         "(\\d+)(\\.\\d+\\.\\d+(?:\\.\\d+)?)?\\+(\\d+(?:\\.\\d+)?)(@([a-f0-9]{32}))?"
     );
     private static final Pattern LEGACY_VERSION_PATTERN = Pattern.compile("(\\d)(u\\d+)\\+(b\\d+?)(@([a-f0-9]{32}))?");
-    private static final Pattern EA_VERSION_PATTERN = Pattern.compile("(\\d+)-ea\\+(\\d+)(@([a-f0-9]{32}))?");
+    private static final Pattern EA_VERSION_PATTERN = Pattern.compile("(\\d+)-(?:ea|rc)\\+(\\d+)(@([a-f0-9]{32}))?");
 
     private final String name;
     private final FileCollection configuration;

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/JdkDownloadPlugin.java

@@ -114,33 +114,40 @@ private void setupRepository(Project project, Jdk jdk) {
                     + jdk.getBuild()
                     + "/[module]/[classifier]/jdk/hotspot/normal/adoptium";
             }
-        } else if (jdk.getVendor().equals(VENDOR_OPENJDK) && "ea".equals(jdk.getDistributionVersion())) {
-            repoUrl = "https://builds.es-jdk-archive.com/";
-            // current pattern since 12.0.1
-            artifactPattern = "jdks/openjdk/"
-                + jdk.getMajor()
-                + "/openjdk-"
-                + jdk.getBaseVersion()
-                + "/"
-                + "openjdk-[revision]_[module]-[classifier]_bin.[ext]";
         } else if (jdk.getVendor().equals(VENDOR_OPENJDK)) {
-            repoUrl = "https://download.oracle.com";
-            if (jdk.getHash() != null) {
+            if ("ea".equals(jdk.getDistributionVersion())) {
+                repoUrl = "https://builds.es-jdk-archive.com/";
                 // current pattern since 12.0.1
-                artifactPattern = "java/GA/jdk"
-                    + jdk.getBaseVersion()
-                    + "/"
-                    + jdk.getHash()
-                    + "/"
-                    + jdk.getBuild()
-                    + "/GPL/openjdk-[revision]_[module]-[classifier]_bin.[ext]";
-            } else {
-                // simpler legacy pattern from JDK 9 to JDK 12 that we are advocating to Oracle to bring back
-                artifactPattern = "java/GA/jdk"
+                artifactPattern = "jdks/openjdk/"
                     + jdk.getMajor()
-                    + "/"
-                    + jdk.getBuild()
-                    + "/GPL/openjdk-[revision]_[module]-[classifier]_bin.[ext]";
+                    + "/openjdk-[revision]/openjdk-[revision]_[module]-[classifier]_bin.[ext]";
+            } else if ("rc".equals(jdk.getDistributionVersion())) {
+                repoUrl = "https://builds.es-jdk-archive.com/";
+                // current pattern since 12.0.1
+                artifactPattern = "jdks/openjdk/"
+                    + jdk.getMajor()
+                    + "/openjdk-[revision]/openjdk-"
+                    + jdk.getMajor()
+                    + "_[module]-[classifier]_bin.[ext]";
+            } else {
+                repoUrl = "https://download.oracle.com";
+                if (jdk.getHash() != null) {
+                    // current pattern since 12.0.1
+                    artifactPattern = "java/GA/jdk"
+                        + jdk.getBaseVersion()
+                        + "/"
+                        + jdk.getHash()
+                        + "/"
+                        + jdk.getBuild()
+                        + "/GPL/openjdk-[revision]_[module]-[classifier]_bin.[ext]";
+                } else {
+                    // simpler legacy pattern from JDK 9 to JDK 12 that we are advocating to Oracle to bring back
+                    artifactPattern = "java/GA/jdk"
+                        + jdk.getMajor()
+                        + "/"
+                        + jdk.getBuild()
+                        + "/GPL/openjdk-[revision]_[module]-[classifier]_bin.[ext]";
+                }
             }
         } else if (jdk.getVendor().equals(VENDOR_ZULU)) {
             repoUrl = "https://cdn.azul.com";

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParameterExtension.java

@@ -32,6 +32,8 @@ public interface BuildParameterExtension {
 
     Boolean getIsRuntimeJavaHomeSet();
 
+    RuntimeJava getRuntimeJava();
+
     List<JavaHome> getJavaVersions();
 
     JavaVersion getMinimumCompilerVersion();