Merge branch '9.0' into backport/9.0/pr-133526

Author: breskeby

.backportrc.json

@@ -3,7 +3,7 @@
   "targetBranchChoices" : [ "main", "9.0", "8.18", "8.17", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
   "targetPRLabels" : [ "backport" ],
   "branchLabelMapping" : {
-    "^v9.0.6$" : "main",
+    "^v9.0.7$" : "main",
     "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
   }
 }

.buildkite/pipelines/intake.yml

@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.17.10", "8.18.6", "9.0.6"]
+            BWC_VERSION: ["8.18.7", "9.0.7"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404

.buildkite/pipelines/periodic-fwc.yml

@@ -10,6 +10,6 @@ steps:
       buildDirectory: /dev/shm/bk
     matrix:
       setup:
-        FWC_VERSION: ["9.0.0", "9.0.1", "9.0.2", "9.0.3", "9.0.4", "9.0.5"]
+        FWC_VERSION: ["9.0.0", "9.0.1", "9.0.2", "9.0.3", "9.0.4", "9.0.5", "9.0.6"]
     env:
       FWC_VERSION: $FWC_VERSION

.buildkite/pipelines/periodic-packaging.yml

@@ -318,8 +318,8 @@ steps:
         env:
           BWC_VERSION: 8.17.10
 
-      - label: "{{matrix.image}} / 8.18.6 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.6
+      - label: "{{matrix.image}} / 8.18.7 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.7
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -332,10 +332,10 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.18.6
+          BWC_VERSION: 8.18.7
 
-      - label: "{{matrix.image}} / 9.0.6 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.6
+      - label: "{{matrix.image}} / 9.0.7 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.7
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -348,7 +348,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 9.0.6
+          BWC_VERSION: 9.0.7
 
   - group: packaging-tests-windows
     steps:

.buildkite/pipelines/periodic.template.yml

@@ -232,7 +232,7 @@ steps:
       image: family/elasticsearch-ubuntu-2404
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
-    if: build.branch == "main" || build.branch == "9.1" || build.branch == "9.0" || build.branch == "8.19" || build.branch == "8.18" || build.branch == "7.17"
+    if: build.branch =~ /^(main|\d+\.\d+|\d+\.x)$/
   - label: check-branch-consistency
     command: .ci/scripts/run-gradle.sh branchConsistency
     timeout_in_minutes: 15

.buildkite/pipelines/periodic.yml

@@ -344,8 +344,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.18.6 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.6#bwcTest
+      - label: 8.18.7 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.7#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -354,7 +354,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.18.6
+          BWC_VERSION: 8.18.7
         retry:
           automatic:
             - exit_status: "-1"
@@ -363,8 +363,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 9.0.6 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.6#bwcTest
+      - label: 9.0.7 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.7#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -373,7 +373,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 9.0.6
+          BWC_VERSION: 9.0.7
         retry:
           automatic:
             - exit_status: "-1"
@@ -448,7 +448,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.17.10", "8.18.6", "9.0.6"]
+            BWC_VERSION: ["8.18.7", "9.0.7"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404
@@ -490,7 +490,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.17.10", "8.18.6", "9.0.6"]
+            BWC_VERSION: ["8.18.7", "9.0.7"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404
@@ -613,7 +613,7 @@ steps:
       image: family/elasticsearch-ubuntu-2404
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
-    if: build.branch == "main" || build.branch == "9.1" || build.branch == "9.0" || build.branch == "8.19" || build.branch == "8.18" || build.branch == "7.17"
+    if: build.branch =~ /^(main|\d+\.\d+|\d+\.x)$/
   - label: check-branch-consistency
     command: .ci/scripts/run-gradle.sh branchConsistency
     timeout_in_minutes: 15

.ci/bwcVersions

@@ -17,5 +17,5 @@ BWC_VERSION:
   - "8.15.5"
   - "8.16.6"
   - "8.17.10"
-  - "8.18.6"
-  - "9.0.6"
+  - "8.18.7"
+  - "9.0.7"

.ci/snapshotBwcVersions

@@ -1,4 +1,3 @@
 BWC_VERSION:
-  - "8.17.10"
-  - "8.18.6"
-  - "9.0.6"
+  - "8.18.7"
+  - "9.0.7"

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/TransportVersionValidationFuncTest.groovy

@@ -223,10 +223,33 @@ class TransportVersionValidationFuncTest extends AbstractTransportVersionFuncTes
 
     def "unreferable definitions can have primary ids that are patches"() {
         given:
-        unreferableTransportVersion("initial_10.0.1", "10000001")
+        unreferableTransportVersion("initial_7.0.1", "7000001")
         when:
         def result = gradleRunner(":myserver:validateTransportVersionResources").build()
         then:
         result.task(":myserver:validateTransportVersionResources").outcome == TaskOutcome.SUCCESS
     }
+
+    def "highest id in an referable definition should exist in an upper bounds file"() {
+        given:
+        referableAndReferencedTransportVersion("some_tv", "10000000")
+        when:
+        def result = validateResourcesFails()
+        then:
+        assertValidateResourcesFailure(result, "Transport version definition file " +
+            "[myserver/src/main/resources/transport/definitions/referable/some_tv.csv] " +
+            "has the highest transport version id [10000000] but is not present in any upper bounds files")
+    }
+
+    def "highest id in an unreferable definition should exist in an upper bounds file"() {
+        given:
+        unreferableTransportVersion("initial_10.0.0", "10000000")
+        when:
+        def result = validateResourcesFails()
+        then:
+        // TODO: this should be _unreferable_ in the error message, but will require some rework
+        assertValidateResourcesFailure(result, "Transport version definition file " +
+            "[myserver/src/main/resources/transport/definitions/referable/initial_10.0.0.csv] " +
+            "has the highest transport version id [10000000] but is not present in any upper bounds files")
+    }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ProjectSubscribeBuildService.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal;
+
+import org.gradle.api.Project;
+import org.gradle.api.provider.Provider;
+import org.gradle.api.provider.ProviderFactory;
+import org.gradle.api.services.BuildService;
+import org.gradle.api.services.BuildServiceParameters;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.inject.Inject;
+
+public abstract class ProjectSubscribeBuildService implements BuildService<BuildServiceParameters.None> {
+
+    private final ProviderFactory providerFactory;
+
+    /**
+     * The filling of this map depends on the order of #registerProjectForTopic being called.
+     * This is usually done during configuration phase, but we do not enforce yet the time of this method call.
+     * The values are LinkedHashSet to preserve the order of registration mostly to provide a predicatable order
+     * when running consecutive builds.
+     * */
+    private final Map<String, Collection<String>> versionsByTopic = new HashMap<>();
+
+    @Inject
+    public ProjectSubscribeBuildService(ProviderFactory providerFactory) {
+        this.providerFactory = providerFactory;
+    }
+
+    /**
+     * Returning a provider so the evaluation of the map value is deferred to when the provider is queried.
+     * */
+    public Provider<Collection<String>> getProjectsByTopic(String topic) {
+        return providerFactory.provider(() -> versionsByTopic.computeIfAbsent(topic, k -> new java.util.LinkedHashSet<>()));
+    }
+
+    public void registerProjectForTopic(String topic, Project project) {
+        versionsByTopic.computeIfAbsent(topic, k -> new java.util.LinkedHashSet<>()).add(project.getPath());
+    }
+}