Merge branch 'main' into lucene_snapshot

Author: thecoop

.backportrc.json

@@ -1,9 +1,9 @@
 {
   "upstream" : "elastic/elasticsearch",
-  "targetBranchChoices" : [ "main", "9.1", "9.0", "8.19", "8.18", "8.17", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
+  "targetBranchChoices" : [ "main", "9.2", "9.1", "9.0", "8.19", "8.18", "8.17", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
   "targetPRLabels" : [ "backport" ],
   "branchLabelMapping" : {
-    "^v9.2.0$" : "main",
+    "^v9.3.0$" : "main",
     "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
   }
 }

.buildkite/pipelines/periodic-fwc.yml

@@ -10,6 +10,6 @@ steps:
       buildDirectory: /dev/shm/bk
     matrix:
       setup:
-        FWC_VERSION: []
+        FWC_VERSION: ["9.3.0"]
     env:
       FWC_VERSION: "{{matrix.FWC_VERSION}}"

.buildkite/pipelines/periodic-java-ea.yml

@@ -442,6 +442,25 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
+      - label: 9.3.0 / bwc
+        command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true v9.3.0#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2404
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+          preemptible: true
+        env:
+          BWC_VERSION: 9.3.0
+        retry:
+          automatic:
+            - exit_status: "-1"
+              limit: 3
+              signal_reason: none
+            - signal_reason: agent_stop
+              limit: 3
+
   - label: concurrent-search-tests
     command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true -Dtests.jvm.argline=-Des.concurrent_search=true -Des.concurrent_search=true functionalTests
     timeout_in_minutes: 420

.buildkite/pipelines/periodic-packaging.yml

@@ -398,6 +398,22 @@ steps:
         env:
           BWC_VERSION: 9.2.0
 
+      - label: "{{matrix.image}} / 9.3.0 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.3.0
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2404
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 9.3.0
+
   - group: packaging-tests-windows
     steps:
       - label: "{{matrix.image}} / packaging-tests-windows"

.buildkite/pipelines/periodic.yml

@@ -439,6 +439,25 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
+      - label: 9.3.0 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.3.0#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2404
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+          preemptible: true
+        env:
+          BWC_VERSION: 9.3.0
+        retry:
+          automatic:
+            - exit_status: "-1"
+              limit: 3
+              signal_reason: none
+            - signal_reason: agent_stop
+              limit: 3
+
   - label: concurrent-search-tests
     command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.jvm.argline=-Des.concurrent_search=true -Des.concurrent_search=true functionalTests
     timeout_in_minutes: 420

.ci/bwcVersions

@@ -22,3 +22,4 @@ BWC_VERSION:
   - "9.0.8"
   - "9.1.5"
   - "9.2.0"
+  - "9.3.0"

branches.json

@@ -3,6 +3,10 @@
   "branches" : [
     {
       "branch" : "main",
+      "version" : "9.3.0"
+    },
+    {
+      "branch" : "9.2",
       "version" : "9.2.0"
     },
     {

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy

@@ -115,9 +115,6 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest {
             include ':myserver'
             include ':myplugin'
         """
-        propertiesFile << """
-            org.elasticsearch.transport.upstreamRef=main
-        """
         versionPropertiesFile.text = versionPropertiesFile.text.replace("9.1.0", "9.2.0")
 
         file("myserver/build.gradle") << """

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateInitialTransportVersionTask.java

@@ -37,31 +37,31 @@ public void run() throws IOException {
         Version releaseVersion = Version.fromString(getReleaseVersion().get());
         String upperBoundName = getUpperBoundName(releaseVersion);
         TransportVersionResourcesService resources = getResourceService().get();
-        TransportVersionUpperBound upstreamUpperBound = resources.getUpperBoundFromUpstream(upperBoundName);
+        TransportVersionUpperBound baseUpperBound = resources.getUpperBoundFromGitBase(upperBoundName);
         String initialDefinitionName = "initial_" + releaseVersion;
-        TransportVersionDefinition existingDefinition = resources.getUnreferableDefinitionFromUpstream(initialDefinitionName);
+        TransportVersionDefinition existingDefinition = resources.getUnreferableDefinitionFromGitBase(initialDefinitionName);
 
-        if (existingDefinition != null) {
-            // this initial version has already been created upstream
-            return;
-        }
+        // This task runs on main and release branches. In release branches we will generate the exact same
+        // upper bound result because we always look at the base branch (ie upstream/main).
+        if (existingDefinition == null) {
+            if (baseUpperBound == null) {
+                throw new RuntimeException("Missing upper bound " + upperBoundName + " for release version " + releaseVersion);
+            }
 
-        if (upstreamUpperBound == null) {
-            throw new RuntimeException("Missing upper bound " + upperBoundName + " for release version " + releaseVersion);
-        }
-        // minors increment by 1000 to create a unique base, patches increment by 1 as other patches do
-        int increment = releaseVersion.getRevision() == 0 ? 1000 : 1;
-        var id = TransportVersionId.fromInt(upstreamUpperBound.definitionId().complete() + increment);
-        var definition = new TransportVersionDefinition(initialDefinitionName, List.of(id), false);
-        resources.writeDefinition(definition);
-        var newUpperBound = new TransportVersionUpperBound(upperBoundName, initialDefinitionName, id);
-        resources.writeUpperBound(newUpperBound, false);
+            // minors increment by 1000 to create a unique base, patches increment by 1 as other patches do
+            int increment = releaseVersion.getRevision() == 0 ? 1000 : 1;
+            var id = TransportVersionId.fromInt(baseUpperBound.definitionId().complete() + increment);
+            var definition = new TransportVersionDefinition(initialDefinitionName, List.of(id), false);
+            resources.writeDefinition(definition);
+            var newUpperBound = new TransportVersionUpperBound(upperBoundName, initialDefinitionName, id);
+            resources.writeUpperBound(newUpperBound, false);
 
-        if (releaseVersion.getRevision() == 0) {
-            Version currentVersion = getCurrentVersion().get();
-            String currentUpperBoundName = getUpperBoundName(currentVersion);
-            var currentUpperBound = new TransportVersionUpperBound(currentUpperBoundName, initialDefinitionName, id);
-            resources.writeUpperBound(currentUpperBound, false);
+            if (releaseVersion.getRevision() == 0) {
+                Version currentVersion = getCurrentVersion().get();
+                String currentUpperBoundName = getUpperBoundName(currentVersion);
+                var currentUpperBound = new TransportVersionUpperBound(currentUpperBoundName, initialDefinitionName, id);
+                resources.writeUpperBound(currentUpperBound, false);
+            }
         }
     }
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java

@@ -109,7 +109,7 @@ public void run() throws IOException {
             resetAllUpperBounds(resources, idsByBase);
         } else {
             getLogger().lifecycle("Generating transport version name: " + targetDefinitionName);
-            List<TransportVersionUpperBound> upstreamUpperBounds = resources.getUpperBoundsFromUpstream();
+            List<TransportVersionUpperBound> upstreamUpperBounds = resources.getUpperBoundsFromGitBase();
             Set<String> targetUpperBoundNames = getTargetUpperBoundNames(resources, upstreamUpperBounds, targetDefinitionName);
 
             List<TransportVersionId> ids = updateUpperBounds(
@@ -143,7 +143,7 @@ private List<TransportVersionId> updateUpperBounds(
         List<TransportVersionId> ids = new ArrayList<>();
         boolean stageInGit = getResolveConflict().getOrElse(false);
 
-        TransportVersionDefinition existingDefinition = resources.getReferableDefinitionFromUpstream(definitionName);
+        TransportVersionDefinition existingDefinition = resources.getReferableDefinitionFromGitBase(definitionName);
         for (TransportVersionUpperBound existingUpperBound : existingUpperBounds) {
             String upperBoundName = existingUpperBound.name();
 
@@ -263,7 +263,7 @@ private Set<String> getUpperBoundNamesFromDefinition(
     private void resetAllUpperBounds(TransportVersionResourcesService resources, Map<Integer, List<IdAndDefinition>> idsByBase)
         throws IOException {
         for (String upperBoundName : resources.getChangedUpperBoundNames()) {
-            TransportVersionUpperBound upstreamUpperBound = resources.getUpperBoundFromUpstream(upperBoundName);
+            TransportVersionUpperBound upstreamUpperBound = resources.getUpperBoundFromGitBase(upperBoundName);
             resetUpperBound(resources, upstreamUpperBound, idsByBase, null);
         }
     }