From 7781b1327512e469a76cb2bebf945cc385d29c54 Mon Sep 17 00:00:00 2001
From: Ed Savage <ed.savage@elastic.co>
Date: Fri, 13 Mar 2026 12:54:40 +1300
Subject: [PATCH] [ML] Split ES integration tests into parallel steps
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Split each per-architecture Elasticsearch integration test step into
two independent steps that run in parallel:

- Multi-Node Tests (javaRestTest) — ~27 min on x86_64, ~37 min on aarch64
- YAML REST Tests (yamlRestTest) — ~12 min on x86_64, ~15 min on aarch64

Previously these ran sequentially in a single step, making the total
wall-clock time ~40 min (x86_64) / ~53 min (aarch64). Running them
in parallel reduces the critical path to the duration of the slower
suite, saving ~11-14 minutes per PR build.

The split also improves failure attribution (immediately visible which
suite failed) and enables selective retry of just the failed suite.

A new ES_TEST_SUITE environment variable controls which Gradle command
to run. When unset, both suites run sequentially for backward
compatibility with local developer use.

Made-with: Cursor
---
 .../pipelines/run_es_tests_aarch64.yml.sh     | 29 +++++++++++++++++--
 .../pipelines/run_es_tests_x86_64.yml.sh      | 29 +++++++++++++++++--
 dev-tools/run_es_tests.sh                     | 19 ++++++++++--
 3 files changed, 69 insertions(+), 8 deletions(-)

diff --git a/.buildkite/pipelines/run_es_tests_aarch64.yml.sh b/.buildkite/pipelines/run_es_tests_aarch64.yml.sh
index d0dddebbd..ef96ad2b1 100755
--- a/.buildkite/pipelines/run_es_tests_aarch64.yml.sh
+++ b/.buildkite/pipelines/run_es_tests_aarch64.yml.sh
@@ -10,8 +10,8 @@
 
 cat <<EOL
 steps:
-  - label: "Java :java: Integration Tests for aarch64 :hammer:"
-    key: "java_integration_tests_aarch64"
+  - label: "Java :java: Multi-Node Tests for aarch64 :hammer:"
+    key: "java_multinode_tests_aarch64"
     command:
       - 'sudo rpm --import https://yum.corretto.aws/corretto.key'
       - 'sudo curl -L -o /etc/yum.repos.d/corretto.repo https://yum.corretto.aws/corretto.repo'
@@ -28,7 +28,30 @@ steps:
     env:
       IVY_REPO: "../ivy"
       GRADLE_JVM_OPTS: "-Dorg.gradle.jvmargs=-Xmx16g"
+      ES_TEST_SUITE: "javaRestTest"
     notify:
       - github_commit_status:
-          context: "Java Integration Tests for aarch64"
+          context: "Java Multi-Node Tests for aarch64"
+  - label: "Java :java: YAML REST Tests for aarch64 :hammer:"
+    key: "java_yaml_rest_tests_aarch64"
+    command:
+      - 'sudo rpm --import https://yum.corretto.aws/corretto.key'
+      - 'sudo curl -L -o /etc/yum.repos.d/corretto.repo https://yum.corretto.aws/corretto.repo'
+      - 'sudo dnf install -y java-21-amazon-corretto-devel'
+      - 'buildkite-agent artifact download "build/*" . --step build_test_linux-aarch64-RelWithDebInfo'
+      - '.buildkite/scripts/steps/run_es_tests.sh || (cd ../elasticsearch && find x-pack -name logs | xargs tar cvzf logs.tgz && buildkite-agent artifact upload logs.tgz && false)'
+    depends_on: "build_test_linux-aarch64-RelWithDebInfo"
+    agents:
+      provider: aws
+      instanceType: m6g.2xlarge
+      imagePrefix: core-almalinux-8-aarch64
+      diskSizeGb: 100
+      diskName: '/dev/xvda'
+    env:
+      IVY_REPO: "../ivy"
+      GRADLE_JVM_OPTS: "-Dorg.gradle.jvmargs=-Xmx16g"
+      ES_TEST_SUITE: "yamlRestTest"
+    notify:
+      - github_commit_status:
+          context: "Java YAML REST Tests for aarch64"
 EOL
diff --git a/.buildkite/pipelines/run_es_tests_x86_64.yml.sh b/.buildkite/pipelines/run_es_tests_x86_64.yml.sh
index 1542dbe2e..4d6facff9 100755
--- a/.buildkite/pipelines/run_es_tests_x86_64.yml.sh
+++ b/.buildkite/pipelines/run_es_tests_x86_64.yml.sh
@@ -10,8 +10,8 @@
 
 cat <<EOL
 steps:
-  - label: "Java :java: Integration Tests for x86_64 :hammer:"
-    key: "java_integration_tests_x86_64"
+  - label: "Java :java: Multi-Node Tests for x86_64 :hammer:"
+    key: "java_multinode_tests_x86_64"
     command:
       - 'sudo rpm --import https://yum.corretto.aws/corretto.key'
       - 'sudo curl -L -o /etc/yum.repos.d/corretto.repo https://yum.corretto.aws/corretto.repo'
@@ -28,7 +28,30 @@ steps:
     env:
       IVY_REPO: "../ivy"
       GRADLE_JVM_OPTS: "-Dorg.gradle.jvmargs=-Xmx16g"
+      ES_TEST_SUITE: "javaRestTest"
     notify:
       - github_commit_status:
-          context: "Java Integration Tests for x86_64"
+          context: "Java Multi-Node Tests for x86_64"
+  - label: "Java :java: YAML REST Tests for x86_64 :hammer:"
+    key: "java_yaml_rest_tests_x86_64"
+    command:
+      - 'sudo rpm --import https://yum.corretto.aws/corretto.key'
+      - 'sudo curl -L -o /etc/yum.repos.d/corretto.repo https://yum.corretto.aws/corretto.repo'
+      - 'sudo dnf install -y java-21-amazon-corretto-devel'
+      - 'buildkite-agent artifact download "build/*" . --step build_test_linux-x86_64-RelWithDebInfo'
+      - '.buildkite/scripts/steps/run_es_tests.sh || (cd ../elasticsearch && find x-pack -name logs | xargs tar cvzf logs.tgz && buildkite-agent artifact upload logs.tgz && false)'
+    depends_on: "build_test_linux-x86_64-RelWithDebInfo"
+    agents:
+      provider: aws
+      instanceType: m6i.2xlarge
+      imagePrefix: core-amazonlinux-2023
+      diskSizeGb: 100
+      diskName: '/dev/xvda'
+    env:
+      IVY_REPO: "../ivy"
+      GRADLE_JVM_OPTS: "-Dorg.gradle.jvmargs=-Xmx16g"
+      ES_TEST_SUITE: "yamlRestTest"
+    notify:
+      - github_commit_status:
+          context: "Java YAML REST Tests for x86_64"
 EOL
diff --git a/dev-tools/run_es_tests.sh b/dev-tools/run_es_tests.sh
index 953408357..13d2a5e34 100755
--- a/dev-tools/run_es_tests.sh
+++ b/dev-tools/run_es_tests.sh
@@ -113,6 +113,21 @@ export GIT_COMMIT="$(git rev-parse HEAD)"
 export GIT_PREVIOUS_COMMIT="$GIT_COMMIT"
 
 IVY_REPO_URL="file://$2"
-./gradlew $GRADLE_JVM_OPTS -Dbuild.ml_cpp.repo="$IVY_REPO_URL" :x-pack:plugin:ml:qa:native-multi-node-tests:javaRestTest $EXTRA_TEST_OPTS
-./gradlew $GRADLE_JVM_OPTS -Dbuild.ml_cpp.repo="$IVY_REPO_URL" :x-pack:plugin:yamlRestTest --tests "org.elasticsearch.xpack.test.rest.XPackRestIT.test {p0=ml/*}" $EXTRA_TEST_OPTS
+
+# ES_TEST_SUITE selects which test suite to run:
+#   javaRestTest  - native multi-node integration tests only
+#   yamlRestTest  - ML YAML REST tests only
+#   (unset/empty) - both suites sequentially (backward compatible)
+case "${ES_TEST_SUITE:-}" in
+  javaRestTest)
+    ./gradlew $GRADLE_JVM_OPTS -Dbuild.ml_cpp.repo="$IVY_REPO_URL" :x-pack:plugin:ml:qa:native-multi-node-tests:javaRestTest $EXTRA_TEST_OPTS
+    ;;
+  yamlRestTest)
+    ./gradlew $GRADLE_JVM_OPTS -Dbuild.ml_cpp.repo="$IVY_REPO_URL" :x-pack:plugin:yamlRestTest --tests "org.elasticsearch.xpack.test.rest.XPackRestIT.test {p0=ml/*}" $EXTRA_TEST_OPTS
+    ;;
+  *)
+    ./gradlew $GRADLE_JVM_OPTS -Dbuild.ml_cpp.repo="$IVY_REPO_URL" :x-pack:plugin:ml:qa:native-multi-node-tests:javaRestTest $EXTRA_TEST_OPTS
+    ./gradlew $GRADLE_JVM_OPTS -Dbuild.ml_cpp.repo="$IVY_REPO_URL" :x-pack:plugin:yamlRestTest --tests "org.elasticsearch.xpack.test.rest.XPackRestIT.test {p0=ml/*}" $EXTRA_TEST_OPTS
+    ;;
+esac