Another try on fixing pipeline dynamic pipeline setup

Author: breskeby

.buildkite/scripts/run-bc-upgrade-tests.sh

@@ -56,31 +56,17 @@ echo "Running BC upgrade tests on $BUILDKITE_BRANCH [$BC_VERSION] using BC (or s
 
 cat <<EOF | buildkite-agent pipeline upload
 steps:
-    - group: "bc-upgrade $BC_BUILD_ID -> $BUILDKITE_BRANCH"
-      steps:
-        - label: "bc-upgrade-tests-part{{matrix.part}}"
-          command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=${BC_VERSION} -Dtests.bwc.refspec.main=${BC_COMMIT_HASH} bcUpgradeTestPart{{matrix.part}} -Dtests.jvm.argline="-Des.serverless_transport=true"
-          timeout_in_minutes: 300
-          agents:
-            provider: gcp
-            image: family/elasticsearch-ubuntu-2004
-            machineType: n1-standard-32
-            buildDirectory: /dev/shm/bk
-            preemptible: true
-          retry:
-            automatic:
-              - exit_status: "-1"
-                limit: 3
-                signal_reason: none
-              - signal_reason: agent_stop
-                limit: 3
-          matrix:
-            setup: matrix
-            variations:
-              - part: 1
-              - part: 2
-              - part: 3
-              - part: 4
-              - part: 5
-              - part: 6
+  - group: "bc-upgrade $BC_BUILD_ID -> $BUILDKITE_BRANCH"
+    steps:
+      - label: "bc-upgrade-tests-part{{matrix.PART}}"
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=${BC_VERSION} -Dtests.bwc.refspec.main=${BC_COMMIT_HASH} bcUpgradeTestPart{{matrix.PART}} -Dtests.jvm.argline="-Des.serverless_transport=true"
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        matrix:
+          setup:
+            PART: ["1", "2", "3", "4", "5", "6"]
 EOF

.buildkite/scripts/run-pr-upgrade-tests.sh

@@ -24,33 +24,19 @@ VERSION=$(sed -n 's/^elasticsearch[[:space:]]*=[[:space:]]*\(.*\)/\1/p' build-to
 
 echo "Running PR upgrade tests from $BUILDKITE_PULL_REQUEST_BASE_BRANCH [$BASE_COMMIT] to $BUILDKITE_BRANCH [$BUILDKITE_COMMIT]."
 
-cat <<EOF | buildkite-agent pipeline upload
+cat << EOF | buildkite-agent pipeline upload
 steps:
     - group: "pr-upgrade $BC_BUILD_ID -> $BUILDKITE_BRANCH"
       steps:
-          - label: "pr-upgrade-part-{{matrix.part}}"
-            command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=${VERSION}-SNAPSHOT -Dtests.bwc.refspec.main=${BASE_COMMIT} bcUpgradeTestPart{{matrix.part}} -Dtests.jvm.argline="-Des.serverless_transport=true"
-            timeout_in_minutes: 300
-            agents:
-              provider: gcp
-              image: family/elasticsearch-ubuntu-2004
-              machineType: n1-standard-32
-              buildDirectory: /dev/shm/bk
-              preemptible: true
-            retry:
-              automatic:
-                - exit_status: "-1"
-                  limit: 3
-                  signal_reason: none
-                - signal_reason: agent_stop
-                  limit: 3
-            matrix:
-              setup: matrix
-              variations:
-                - part: 1
-                - part: 2
-                - part: 3
-                - part: 4
-                - part: 5
-                - part: 6
+        - label: "pr-upgrade-part-{{matrix.PART}}"
+          command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints -Dtests.bwc.main.version=${VERSION}-SNAPSHOT -Dtests.bwc.refspec.main=${BASE_COMMIT} bcUpgradeTestPart{{matrix.PART}} -Dtests.jvm.argline="-Des.serverless_transport=true"
+          timeout_in_minutes: 300
+          agents:
+            provider: gcp
+            image: family/elasticsearch-ubuntu-2004
+            machineType: n1-standard-32
+            buildDirectory: /dev/shm/bk
+          matrix:
+            setup:
+              PART: ["1", "2", "3", "4", "5", "6"]
 EOF