[ML] Split ES integration tests into parallel steps (#2990)

Split each per-architecture Elasticsearch integration test step into
two independent steps that run in parallel:

- Multi-Node Tests (javaRestTest) — ~27 min on x86_64, ~37 min on aarch64
- YAML REST Tests (yamlRestTest) — ~12 min on x86_64, ~15 min on aarch64

Previously these ran sequentially in a single step, making the total
wall-clock time ~40 min (x86_64) / ~53 min (aarch64). Running them
in parallel reduces the critical path to the duration of the slower
suite, saving ~11-14 minutes per PR build.

The split also improves failure attribution (immediately visible which
suite failed) and enables selective retry of just the failed suite.

A new ES_TEST_SUITE environment variable controls which Gradle command
to run. When unset, both suites run sequentially for backward
compatibility with local developer use.

Made-with: Cursor

Author: edsavage

.buildkite/pipelines/run_es_tests_aarch64.yml.sh

@@ -10,8 +10,8 @@
 
 cat <<EOL
 steps:
-  - label: "Java :java: Integration Tests for aarch64 :hammer:"
-    key: "java_integration_tests_aarch64"
+  - label: "Java :java: Multi-Node Tests for aarch64 :hammer:"
+    key: "java_multinode_tests_aarch64"
     command:
       - 'sudo rpm --import https://yum.corretto.aws/corretto.key'
       - 'sudo curl -L -o /etc/yum.repos.d/corretto.repo https://yum.corretto.aws/corretto.repo'
@@ -28,7 +28,30 @@ steps:
     env:
       IVY_REPO: "../ivy"
       GRADLE_JVM_OPTS: "-Dorg.gradle.jvmargs=-Xmx16g"
+      ES_TEST_SUITE: "javaRestTest"
     notify:
       - github_commit_status:
-          context: "Java Integration Tests for aarch64"
+          context: "Java Multi-Node Tests for aarch64"
+  - label: "Java :java: YAML REST Tests for aarch64 :hammer:"
+    key: "java_yaml_rest_tests_aarch64"
+    command:
+      - 'sudo rpm --import https://yum.corretto.aws/corretto.key'
+      - 'sudo curl -L -o /etc/yum.repos.d/corretto.repo https://yum.corretto.aws/corretto.repo'
+      - 'sudo dnf install -y java-21-amazon-corretto-devel'
+      - 'buildkite-agent artifact download "build/*" . --step build_test_linux-aarch64-RelWithDebInfo'
+      - '.buildkite/scripts/steps/run_es_tests.sh || (cd ../elasticsearch && find x-pack -name logs | xargs tar cvzf logs.tgz && buildkite-agent artifact upload logs.tgz && false)'
+    depends_on: "build_test_linux-aarch64-RelWithDebInfo"
+    agents:
+      provider: aws
+      instanceType: m6g.2xlarge
+      imagePrefix: core-almalinux-8-aarch64
+      diskSizeGb: 100
+      diskName: '/dev/xvda'
+    env:
+      IVY_REPO: "../ivy"
+      GRADLE_JVM_OPTS: "-Dorg.gradle.jvmargs=-Xmx16g"
+      ES_TEST_SUITE: "yamlRestTest"
+    notify:
+      - github_commit_status:
+          context: "Java YAML REST Tests for aarch64"
 EOL

.buildkite/pipelines/run_es_tests_x86_64.yml.sh

@@ -10,8 +10,8 @@
 
 cat <<EOL
 steps:
-  - label: "Java :java: Integration Tests for x86_64 :hammer:"
-    key: "java_integration_tests_x86_64"
+  - label: "Java :java: Multi-Node Tests for x86_64 :hammer:"
+    key: "java_multinode_tests_x86_64"
     command:
       - 'sudo rpm --import https://yum.corretto.aws/corretto.key'
       - 'sudo curl -L -o /etc/yum.repos.d/corretto.repo https://yum.corretto.aws/corretto.repo'
@@ -28,7 +28,30 @@ steps:
     env:
       IVY_REPO: "../ivy"
       GRADLE_JVM_OPTS: "-Dorg.gradle.jvmargs=-Xmx16g"
+      ES_TEST_SUITE: "javaRestTest"
     notify:
       - github_commit_status:
-          context: "Java Integration Tests for x86_64"
+          context: "Java Multi-Node Tests for x86_64"
+  - label: "Java :java: YAML REST Tests for x86_64 :hammer:"
+    key: "java_yaml_rest_tests_x86_64"
+    command:
+      - 'sudo rpm --import https://yum.corretto.aws/corretto.key'
+      - 'sudo curl -L -o /etc/yum.repos.d/corretto.repo https://yum.corretto.aws/corretto.repo'
+      - 'sudo dnf install -y java-21-amazon-corretto-devel'
+      - 'buildkite-agent artifact download "build/*" . --step build_test_linux-x86_64-RelWithDebInfo'
+      - '.buildkite/scripts/steps/run_es_tests.sh || (cd ../elasticsearch && find x-pack -name logs | xargs tar cvzf logs.tgz && buildkite-agent artifact upload logs.tgz && false)'
+    depends_on: "build_test_linux-x86_64-RelWithDebInfo"
+    agents:
+      provider: aws
+      instanceType: m6i.2xlarge
+      imagePrefix: core-amazonlinux-2023
+      diskSizeGb: 100
+      diskName: '/dev/xvda'
+    env:
+      IVY_REPO: "../ivy"
+      GRADLE_JVM_OPTS: "-Dorg.gradle.jvmargs=-Xmx16g"
+      ES_TEST_SUITE: "yamlRestTest"
+    notify:
+      - github_commit_status:
+          context: "Java YAML REST Tests for x86_64"
 EOL

dev-tools/run_es_tests.sh

@@ -131,8 +131,6 @@ GCS_CACHE_PATH=""
 if [ -n "${GRADLE_BUILD_CACHE_GCS_BUCKET:-}" ] && [ -n "${GOOGLE_APPLICATION_CREDENTIALS:-}" ]; then
     GCS_CACHE_PATH="gs://${GRADLE_BUILD_CACHE_GCS_BUCKET}/${CACHE_KEY}.tar.gz"
     if command -v gsutil &>/dev/null; then
-        # The gcloud SDK gsutil needs explicit service account activation;
-        # GOOGLE_APPLICATION_CREDENTIALS alone is not sufficient.
         if command -v gcloud &>/dev/null; then
             gcloud auth activate-service-account --key-file="$GOOGLE_APPLICATION_CREDENTIALS" 2>/dev/null || true
         fi
@@ -152,8 +150,22 @@ if [ -n "${GRADLE_BUILD_CACHE_GCS_BUCKET:-}" ] && [ -n "${GOOGLE_APPLICATION_CRE
     fi
 fi
 
-./gradlew $GRADLE_JVM_OPTS $CACHE_ARGS -Dbuild.ml_cpp.repo="$IVY_REPO_URL" :x-pack:plugin:ml:qa:native-multi-node-tests:javaRestTest $EXTRA_TEST_OPTS
-./gradlew $GRADLE_JVM_OPTS $CACHE_ARGS -Dbuild.ml_cpp.repo="$IVY_REPO_URL" :x-pack:plugin:yamlRestTest --tests "org.elasticsearch.xpack.test.rest.XPackRestIT.test {p0=ml/*}" $EXTRA_TEST_OPTS
+# ES_TEST_SUITE selects which test suite to run:
+#   javaRestTest  - native multi-node integration tests only
+#   yamlRestTest  - ML YAML REST tests only
+#   (unset/empty) - both suites sequentially (backward compatible)
+case "${ES_TEST_SUITE:-}" in
+  javaRestTest)
+    ./gradlew $GRADLE_JVM_OPTS $CACHE_ARGS -Dbuild.ml_cpp.repo="$IVY_REPO_URL" :x-pack:plugin:ml:qa:native-multi-node-tests:javaRestTest $EXTRA_TEST_OPTS
+    ;;
+  yamlRestTest)
+    ./gradlew $GRADLE_JVM_OPTS $CACHE_ARGS -Dbuild.ml_cpp.repo="$IVY_REPO_URL" :x-pack:plugin:yamlRestTest --tests "org.elasticsearch.xpack.test.rest.XPackRestIT.test {p0=ml/*}" $EXTRA_TEST_OPTS
+    ;;
+  *)
+    ./gradlew $GRADLE_JVM_OPTS $CACHE_ARGS -Dbuild.ml_cpp.repo="$IVY_REPO_URL" :x-pack:plugin:ml:qa:native-multi-node-tests:javaRestTest $EXTRA_TEST_OPTS
+    ./gradlew $GRADLE_JVM_OPTS $CACHE_ARGS -Dbuild.ml_cpp.repo="$IVY_REPO_URL" :x-pack:plugin:yamlRestTest --tests "org.elasticsearch.xpack.test.rest.XPackRestIT.test {p0=ml/*}" $EXTRA_TEST_OPTS
+    ;;
+esac
 
 # Upload Gradle build cache to GCS for future builds.
 if [ -n "$GCS_CACHE_PATH" ] && [ -d "$GRADLE_CACHE_DIR" ] && command -v gsutil &>/dev/null; then
@@ -169,4 +181,3 @@ if [ -n "$GCS_CACHE_PATH" ] && [ -d "$GRADLE_CACHE_DIR" ] && command -v gsutil &
         echo "Skipping cache upload (size=${CACHE_SIZE:-0}M, expected 1-4095M)"
     fi
 fi
-