Merge branch 'main' into cardagg_no_vectors

Author: john-wagster

.backportrc.json

@@ -1,9 +1,9 @@
 {
   "upstream" : "elastic/elasticsearch",
-  "targetBranchChoices" : [ "main", "9.1", "9.0", "8.19", "8.18", "8.17", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
+  "targetBranchChoices" : [ "main", "9.2", "9.1", "9.0", "8.19", "8.18", "8.17", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
   "targetPRLabels" : [ "backport" ],
   "branchLabelMapping" : {
-    "^v9.2.0$" : "main",
+    "^v9.3.0$" : "main",
     "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
   }
 }

.buildkite/pipelines/periodic-fwc.yml

@@ -10,6 +10,6 @@ steps:
       buildDirectory: /dev/shm/bk
     matrix:
       setup:
-        FWC_VERSION: []
+        FWC_VERSION: ["9.3.0"]
     env:
       FWC_VERSION: "{{matrix.FWC_VERSION}}"

.buildkite/pipelines/periodic-java-ea.yml

@@ -442,6 +442,25 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
+      - label: 9.3.0 / bwc
+        command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true v9.3.0#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2404
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+          preemptible: true
+        env:
+          BWC_VERSION: 9.3.0
+        retry:
+          automatic:
+            - exit_status: "-1"
+              limit: 3
+              signal_reason: none
+            - signal_reason: agent_stop
+              limit: 3
+
   - label: concurrent-search-tests
     command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true -Dtests.jvm.argline=-Des.concurrent_search=true -Des.concurrent_search=true functionalTests
     timeout_in_minutes: 420

.buildkite/pipelines/periodic-packaging.yml

@@ -398,6 +398,22 @@ steps:
         env:
           BWC_VERSION: 9.2.0
 
+      - label: "{{matrix.image}} / 9.3.0 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.3.0
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2404
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 9.3.0
+
   - group: packaging-tests-windows
     steps:
       - label: "{{matrix.image}} / packaging-tests-windows"

.buildkite/pipelines/periodic.yml

@@ -439,6 +439,25 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
+      - label: 9.3.0 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.3.0#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2404
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+          preemptible: true
+        env:
+          BWC_VERSION: 9.3.0
+        retry:
+          automatic:
+            - exit_status: "-1"
+              limit: 3
+              signal_reason: none
+            - signal_reason: agent_stop
+              limit: 3
+
   - label: concurrent-search-tests
     command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.jvm.argline=-Des.concurrent_search=true -Des.concurrent_search=true functionalTests
     timeout_in_minutes: 420

.ci/bwcVersions

@@ -22,3 +22,4 @@ BWC_VERSION:
   - "9.0.8"
   - "9.1.5"
   - "9.2.0"
+  - "9.3.0"

branches.json

@@ -3,6 +3,10 @@
   "branches" : [
     {
       "branch" : "main",
+      "version" : "9.3.0"
+    },
+    {
+      "branch" : "9.2",
       "version" : "9.2.0"
     },
     {

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy

@@ -51,7 +51,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
     def "resolves expanded bwc versions from source"() {
         given:
         internalBuild()
-        bwcMajor1ProjectSetup()
+        bwcProjectSetup("major1")
         buildFile << """
             apply plugin: 'elasticsearch.internal-distribution-download'
 
@@ -81,7 +81,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
     def "fails on resolving bwc versions with no bundled jdk"() {
         given:
         internalBuild()
-        bwcMajor1ProjectSetup()
+        bwcProjectSetup("major1")
         buildFile << """
             apply plugin: 'elasticsearch.internal-distribution-download'
 
@@ -105,12 +105,47 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
                 "without a bundled JDK is not supported.")
     }
 
-    private void bwcMajor1ProjectSetup() {
+    def "resolves detached version from source"() {
+        given:
+        internalBuild()
+        bwcProjectSetup("main")
+        buildFile << """
+            apply plugin: 'elasticsearch.internal-distribution-download'
+
+            System.setProperty("tests.bwc.main.version", "9.2.0")
+
+            elasticsearch_distributions {
+              test_distro {
+                  version = "9.2.0"
+                  type = "archive"
+                  platform = "linux"
+                  architecture = Architecture.current();
+                  detachedVersion = true
+              }
+            }
+
+            tasks.register("setupDistro", Sync) {
+                from(elasticsearch_distributions.test_distro)
+                into("build/distro")
+            }
+        """
+
+        when:
+        def result = gradleRunner("setupDistro").build()
+
+        then:
+        result.task(":distribution:bwc:main:buildBwcExpandedTask").outcome == TaskOutcome.SUCCESS
+        result.task(":setupDistro").outcome == TaskOutcome.SUCCESS
+        assertExtractedDistroIsCreated("distribution/bwc/main/build/install/elastic-distro",
+            'bwc-marker.txt')
+    }
+
+    private void bwcProjectSetup(String bwcProjectName) {
         settingsFile << """
-        include ':distribution:bwc:major1'
+        include ':distribution:bwc:$bwcProjectName'
         """
-        def bwcSubProjectFolder = testProjectDir.newFolder("distribution", "bwc", "major1")
-        new File(bwcSubProjectFolder, 'bwc-marker.txt') << "bwc=major1"
+        def bwcSubProjectFolder = testProjectDir.newFolder("distribution", "bwc", bwcProjectName)
+        new File(bwcSubProjectFolder, 'bwc-marker.txt') << "bwc=$bwcProjectName"
         new File(bwcSubProjectFolder, 'build.gradle') << """
             apply plugin:'base'
 

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy

@@ -115,9 +115,6 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest {
             include ':myserver'
             include ':myplugin'
         """
-        propertiesFile << """
-            org.elasticsearch.transports.upstreamRef=main
-        """
         versionPropertiesFile.text = versionPropertiesFile.text.replace("9.1.0", "9.2.0")
 
         file("myserver/build.gradle") << """

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/ResolveTransportVersionConflictFuncTest.groovy

@@ -96,4 +96,35 @@ class ResolveTransportVersionConflictFuncTest extends AbstractTransportVersionFu
         assertResolveAndValidateSuccess(result)
         assertUpperBound("9.2", "existing_92,8123000")
     }
+
+    def "upstream changes don't affect merge"() {
+        given:
+        // setup main with 2 commits, but we will only merge in the first one
+        execute("git checkout main")
+        referableAndReferencedTransportVersion("upstream_new_tv1", "8124000")
+        transportVersionUpperBound("9.2", "upstream_new_tv1", "8124000")
+        execute("git add .")
+        execute("git commit -m update1")
+        String toMerge = execute("git rev-parse HEAD")
+        referableAndReferencedTransportVersion("upstream_new_tv2", "8125000")
+        transportVersionUpperBound("9.2", "upstream_new_tv2", "8125000")
+        execute("git add .")
+        execute("git commit -m update2")
+        execute("git checkout test")
+        // now commit a conflict on the test branch, a new TV
+        referableAndReferencedTransportVersion("branch_new_tv", "8124000")
+        transportVersionUpperBound("9.2", "branch_new_tv", "8124000")
+        execute("git add .")
+        execute("git commit -m branch")
+        // and finally initiate the merge
+        System.out.println("Merging commit " + toMerge);
+        execute("git merge " + toMerge, testProjectDir.root, true);
+
+        when:
+        def result = runResolveAndValidateTask().build()
+
+        then:
+        assertResolveAndValidateSuccess(result)
+        assertUpperBound("9.2", "branch_new_tv,8125000")
+    }
 }