[ML] Add Gradle build cache for Java integration tests

Enable Gradle's local build cache for the ES integration test builds
and persist it to GCS between CI runs. On cache-warm builds (same ES
commit), compilation drops from ~15-20 min to ~1-2 min, saving
significant CI wall-clock time.

- Add gradle-build-cache-init.gradle init script
- Pass --build-cache --init-script to both Gradle invocations
- Download/upload cache tarball from GCS bucket
- Extend post-checkout hook to inject GCS credentials for Java IT steps
- Install gsutil on ES test agents when needed

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: edsavage

.buildkite/hooks/post-checkout

@@ -31,10 +31,10 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == ml-cpp* ]]; then
     export BUILDKITE_API_READ_TOKEN=$(vault read -field=token secret/ci/elastic-ml-cpp/buildkite/api_read_token 2>/dev/null || echo "")
   fi
 
-  # sccache with GCS backend — inject credentials for all build_test steps.
-  # The GCS service account key is stored in Vault. Build scripts detect
-  # SCCACHE_GCS_BUCKET and configure sccache automatically.
-  if [[ "$BUILDKITE_STEP_KEY" == build_test_* ]]; then
+  # GCS service account — inject credentials for build and Java IT steps.
+  # Build steps use it for sccache; Java IT steps use it for the Gradle
+  # build cache.  The key is stored in Vault.
+  if [[ "$BUILDKITE_STEP_KEY" == build_test_* || "$BUILDKITE_STEP_KEY" == java_integration_tests_* ]]; then
     SCCACHE_GCS_KEY_JSON=$(vault read -field=key secret/ci/elastic-ml-cpp/sccache/gcs_service_account 2>/dev/null || echo "")
     if [ -n "$SCCACHE_GCS_KEY_JSON" ]; then
       export SCCACHE_GCS_BUCKET="elastic-ml-cpp-sccache"

.buildkite/scripts/steps/run_es_tests.sh

@@ -24,6 +24,18 @@ export PR_AUTHOR=$(expr "$BUILDKITE_BRANCH" : '\(.*\):.*')
 export PR_SOURCE_BRANCH=$(expr "$BUILDKITE_BRANCH" : '.*:\(.*\)')
 export PR_TARGET_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
 
+# Set up GCS credentials for Gradle build cache persistence (if available).
+# The post-checkout hook writes the GCS service account key for sccache;
+# reuse the same credentials for the Gradle cache bucket.
+if [ -n "${SCCACHE_GCS_BUCKET:-}" ] && [ -n "${GOOGLE_APPLICATION_CREDENTIALS:-}" ]; then
+    export GRADLE_BUILD_CACHE_GCS_BUCKET="${SCCACHE_GCS_BUCKET}"
+    # Install gsutil if not already present
+    if ! command -v gsutil &>/dev/null; then
+        echo "--- Installing gsutil"
+        pip3 install --quiet gsutil 2>/dev/null || pip install --quiet gsutil 2>/dev/null || echo "Warning: failed to install gsutil"
+    fi
+fi
+
 mkdir -p "${IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/$VERSION"
 cp "build/distributions/ml-cpp-$VERSION-linux-$HARDWARE_ARCH.zip" "${IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/$VERSION/ml-cpp-$VERSION.zip"
 # Since this is all local, for simplicity, cheat with the dependencies/no-dependencies split

dev-tools/gradle-build-cache-init.gradle

@@ -0,0 +1,17 @@
+/*
+ * Gradle init script to enable the local build cache for ES integration test
+ * builds.  Injected via --init-script so that we don't need to modify the
+ * cloned Elasticsearch repository.
+ *
+ * The local build cache stores task outputs keyed on their inputs.  When the
+ * cache directory is persisted between CI runs (e.g. via GCS), subsequent
+ * builds with the same ES commit get near-instant compilation.
+ */
+
+settingsEvaluated { settings ->
+    settings.buildCache {
+        local {
+            enabled = true
+        }
+    }
+}

dev-tools/run_es_tests.sh

@@ -24,6 +24,9 @@
 
 set -e
 
+# Resolve the ml-cpp repo root before we cd away.
+ML_CPP_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+
 function isCloneTargetValid {
     FORK_TO_CHECK="$1"
     BRANCH_TO_CHECK="$2"
@@ -113,6 +116,52 @@ export GIT_COMMIT="$(git rev-parse HEAD)"
 export GIT_PREVIOUS_COMMIT="$GIT_COMMIT"
 
 IVY_REPO_URL="file://$2"
-./gradlew $GRADLE_JVM_OPTS -Dbuild.ml_cpp.repo="$IVY_REPO_URL" :x-pack:plugin:ml:qa:native-multi-node-tests:javaRestTest $EXTRA_TEST_OPTS
-./gradlew $GRADLE_JVM_OPTS -Dbuild.ml_cpp.repo="$IVY_REPO_URL" :x-pack:plugin:yamlRestTest --tests "org.elasticsearch.xpack.test.rest.XPackRestIT.test {p0=ml/*}" $EXTRA_TEST_OPTS
+
+INIT_SCRIPT="$ML_CPP_ROOT/dev-tools/gradle-build-cache-init.gradle"
+GRADLE_CACHE_DIR="$HOME/.gradle/caches/build-cache-1"
+CACHE_ARGS=""
+if [ -f "$INIT_SCRIPT" ]; then
+    CACHE_ARGS="--build-cache --init-script $INIT_SCRIPT"
+fi
+
+# Restore Gradle build cache from GCS if credentials are available.
+# This lets ephemeral CI agents reuse compilation outputs from prior builds.
+CACHE_KEY="gradle-build-cache-$(uname -m)"
+GCS_CACHE_PATH=""
+if [ -n "${GRADLE_BUILD_CACHE_GCS_BUCKET:-}" ] && [ -n "${GOOGLE_APPLICATION_CREDENTIALS:-}" ]; then
+    GCS_CACHE_PATH="gs://${GRADLE_BUILD_CACHE_GCS_BUCKET}/${CACHE_KEY}.tar.gz"
+    if command -v gsutil &>/dev/null; then
+        echo "--- Restoring Gradle build cache from $GCS_CACHE_PATH"
+        mkdir -p "$GRADLE_CACHE_DIR"
+        if gsutil -q stat "$GCS_CACHE_PATH" 2>/dev/null; then
+            gsutil cp "$GCS_CACHE_PATH" /tmp/gradle-cache.tar.gz \
+                && tar xzf /tmp/gradle-cache.tar.gz -C "$HOME/.gradle/caches/" \
+                && rm -f /tmp/gradle-cache.tar.gz \
+                && echo "Gradle build cache restored ($(du -sh "$GRADLE_CACHE_DIR" 2>/dev/null | cut -f1))" \
+                || echo "Warning: failed to restore Gradle build cache, continuing without it"
+        else
+            echo "No cached Gradle build cache found, will build from scratch"
+        fi
+    else
+        echo "gsutil not found, skipping Gradle build cache restore"
+    fi
+fi
+
+./gradlew $GRADLE_JVM_OPTS $CACHE_ARGS -Dbuild.ml_cpp.repo="$IVY_REPO_URL" :x-pack:plugin:ml:qa:native-multi-node-tests:javaRestTest $EXTRA_TEST_OPTS
+./gradlew $GRADLE_JVM_OPTS $CACHE_ARGS -Dbuild.ml_cpp.repo="$IVY_REPO_URL" :x-pack:plugin:yamlRestTest --tests "org.elasticsearch.xpack.test.rest.XPackRestIT.test {p0=ml/*}" $EXTRA_TEST_OPTS
+
+# Upload Gradle build cache to GCS for future builds.
+if [ -n "$GCS_CACHE_PATH" ] && [ -d "$GRADLE_CACHE_DIR" ] && command -v gsutil &>/dev/null; then
+    echo "--- Uploading Gradle build cache to $GCS_CACHE_PATH"
+    CACHE_SIZE=$(du -sm "$GRADLE_CACHE_DIR" 2>/dev/null | cut -f1)
+    if [ "${CACHE_SIZE:-0}" -gt 0 ] && [ "${CACHE_SIZE:-0}" -lt 4096 ]; then
+        tar czf /tmp/gradle-cache.tar.gz -C "$HOME/.gradle/caches/" build-cache-1 \
+            && gsutil -o "GSUtil:parallel_composite_upload_threshold=50M" cp /tmp/gradle-cache.tar.gz "$GCS_CACHE_PATH" \
+            && rm -f /tmp/gradle-cache.tar.gz \
+            && echo "Gradle build cache uploaded (${CACHE_SIZE}M)" \
+            || echo "Warning: failed to upload Gradle build cache"
+    else
+        echo "Skipping cache upload (size=${CACHE_SIZE:-0}M, expected 1-4095M)"
+    fi
+fi