[ML] Replace shell test runner with portable CMake/CTest infrastructure

.buildkite/branch.json.py

@@ -40,6 +40,10 @@ def main():
         build_linux = pipeline_steps.generate_step_template("Linux", "build", config.build_aarch64, config.build_x86_64)
         pipeline_steps.append(build_linux)
 
+    # Analyse build timings after all build+test steps complete
+    pipeline_steps.append(pipeline_steps.generate_step("Analyse build timings",
+                                                       ".buildkite/pipelines/analyze_build_timings.yml.sh"))
+
     # Build the DRA artifacts and upload to S3 and GCS
     pipeline_steps.append(pipeline_steps.generate_step("Create daily releasable artifacts",
                                                        ".buildkite/pipelines/create_dra.yml.sh"))

.buildkite/hooks/post-checkout

@@ -27,6 +27,24 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == ml-cpp* ]]; then
     export BUILDKITE_ANALYTICS_TOKEN=$(vault read secret/ci/elastic-ml-cpp/buildkite/test_analytics/windows_x86_64 | awk '/^token/ {print $2;}')
   fi
 
+  if [[ "$BUILDKITE_STEP_KEY" == "analyze_build_timings" ]]; then
+    export BUILDKITE_API_READ_TOKEN=$(vault read -field=token secret/ci/elastic-ml-cpp/buildkite/api_read_token 2>/dev/null || echo "")
+  fi
+
+  # GCS service account — inject credentials for build and Java IT steps.
+  # Build steps use it for sccache; Java IT steps use it for the Gradle
+  # build cache.  The key is stored in Vault.
+  if [[ "$BUILDKITE_STEP_KEY" == build_test_* || "$BUILDKITE_STEP_KEY" == java_integration_tests_* ]]; then
+    SCCACHE_GCS_KEY_JSON=$(vault read -field=key secret/ci/elastic-ml-cpp/sccache/gcs_service_account 2>/dev/null || echo "")
+    if [ -n "$SCCACHE_GCS_KEY_JSON" ]; then
+      export SCCACHE_GCS_BUCKET="elastic-ml-cpp-sccache"
+      export SCCACHE_GCS_KEY_FILE=$(mktemp)
+      echo "$SCCACHE_GCS_KEY_JSON" > "$SCCACHE_GCS_KEY_FILE"
+      export GOOGLE_APPLICATION_CREDENTIALS="$SCCACHE_GCS_KEY_FILE"
+      export SCCACHE_GCS_KEY_PATH="$SCCACHE_GCS_KEY_FILE"
+    fi
+  fi
+
   if [[ "$BUILDKITE_STEP_KEY" == "build_pytorch_docker_image" ]]; then
     export DOCKER_REGISTRY_USERNAME=$(vault read --field=username  secret/ci/elastic-ml-cpp/prod_docker_registry_credentials)
     export DOCKER_REGISTRY_PASSWORD=$(vault read --field=password  secret/ci/elastic-ml-cpp/prod_docker_registry_credentials)

.buildkite/pipelines/analyze_build_timings.yml.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the following additional limitation. Functionality enabled by the
+# files subject to the Elastic License 2.0 may only be used in production when
+# invoked by an Elasticsearch process with a license key installed that permits
+# use of machine learning features. You may not use this file except in
+# compliance with the Elastic License 2.0 and the foregoing additional
+# limitation.
+
+cat <<EOL
+steps:
+  - label: "Analyse build timings :chart_with_upwards_trend:"
+    key: "analyze_build_timings"
+    command:
+        - "python3 .buildkite/scripts/steps/analyze_build_timings.py"
+    depends_on:
+        - "build_test_linux-aarch64-RelWithDebInfo"
+        - "build_test_linux-x86_64-RelWithDebInfo"
+        - "build_test_macos-aarch64-RelWithDebInfo"
+        - "build_test_Windows-x86_64-RelWithDebInfo"
+    allow_dependency_failure: true
+    soft_fail: true
+    agents:
+      image: "python:3-slim"
+EOL

.buildkite/pipelines/build_linux.json.py

@@ -93,7 +93,7 @@ def main(args):
                     "key": build_key,
                     "env": {
                       **common_env,
-                      "CMAKE_FLAGS": f"-DCMAKE_TOOLCHAIN_FILE=cmake/linux-{arch}.cmake",
+                      "CMAKE_FLAGS": f"-DCMAKE_TOOLCHAIN_FILE=cmake/linux-{arch}.cmake -DCMAKE_UNITY_BUILD=ON -DML_PCH=ON",
                       "RUN_TESTS": "false",
                     },
                     "notify": [
@@ -118,7 +118,7 @@ def main(args):
                     "env": {
                       **common_env,
                       "BUILD_STEP_KEY": build_key,
-                      "CMAKE_FLAGS": f"-DCMAKE_TOOLCHAIN_FILE=cmake/linux-{arch}.cmake",
+                      "CMAKE_FLAGS": f"-DCMAKE_TOOLCHAIN_FILE=cmake/linux-{arch}.cmake -DCMAKE_UNITY_BUILD=ON -DML_PCH=ON",
                       "BOOST_TEST_OUTPUT_FORMAT_FLAGS": "--logger=JUNIT,error,boost_test_results.junit",
                     },
                     "plugins": {
@@ -151,7 +151,7 @@ def main(args):
                     "key": build_key,
                     "env": {
                       **common_env,
-                      "CMAKE_FLAGS": f"-DCMAKE_TOOLCHAIN_FILE=cmake/linux-{arch}.cmake",
+                      "CMAKE_FLAGS": f"-DCMAKE_TOOLCHAIN_FILE=cmake/linux-{arch}.cmake -DCMAKE_UNITY_BUILD=ON -DML_PCH=ON",
                       "RUN_TESTS": "false",
                     },
                     "notify": [
@@ -176,7 +176,7 @@ def main(args):
                     "env": {
                       **common_env,
                       "BUILD_STEP_KEY": build_key,
-                      "CMAKE_FLAGS": f"-DCMAKE_TOOLCHAIN_FILE=cmake/linux-{arch}.cmake",
+                      "CMAKE_FLAGS": f"-DCMAKE_TOOLCHAIN_FILE=cmake/linux-{arch}.cmake -DCMAKE_UNITY_BUILD=ON -DML_PCH=ON",
                       "BOOST_TEST_OUTPUT_FORMAT_FLAGS": "--logger=JUNIT,error,boost_test_results.junit",
                     },
                     "plugins": {
@@ -212,7 +212,7 @@ def main(args):
             "env": {
               **common_env,
               "ML_DEBUG": "1",
-              "CMAKE_FLAGS": "-DCMAKE_TOOLCHAIN_FILE=cmake/linux-x86_64.cmake",
+              "CMAKE_FLAGS": "-DCMAKE_TOOLCHAIN_FILE=cmake/linux-x86_64.cmake -DCMAKE_UNITY_BUILD=ON -DML_PCH=ON",
               "RUN_TESTS": "false",
               "SKIP_ARTIFACT_UPLOAD": "true",
             },
@@ -239,7 +239,7 @@ def main(args):
               **common_env,
               "BUILD_STEP_KEY": debug_build_key,
               "ML_DEBUG": "1",
-              "CMAKE_FLAGS": "-DCMAKE_TOOLCHAIN_FILE=cmake/linux-x86_64.cmake",
+              "CMAKE_FLAGS": "-DCMAKE_TOOLCHAIN_FILE=cmake/linux-x86_64.cmake -DCMAKE_UNITY_BUILD=ON -DML_PCH=ON",
               "BOOST_TEST_OUTPUT_FORMAT_FLAGS": "--logger=JUNIT,error,boost_test_results.junit",
             },
             "plugins": {

.buildkite/pipelines/build_macos.json.py

@@ -43,7 +43,7 @@
       "PATH": "/opt/homebrew/bin:$PATH",
       "ML_DEBUG": "0",
       "CPP_CROSS_COMPILE": "",
-      "CMAKE_FLAGS": "-DCMAKE_TOOLCHAIN_FILE=cmake/darwin-aarch64.cmake",
+      "CMAKE_FLAGS": "-DCMAKE_TOOLCHAIN_FILE=cmake/darwin-aarch64.cmake -DCMAKE_UNITY_BUILD=ON -DML_PCH=ON",
       "RUN_TESTS": "true",
       "BOOST_TEST_OUTPUT_FORMAT_FLAGS": "--logger=JUNIT,error,boost_test_results.junit",
     }

.buildkite/pipelines/build_windows.json.py

@@ -41,7 +41,8 @@
 common_env = {
     "ML_DEBUG": "0",
     "CPP_CROSS_COMPILE": "",
-    "CMAKE_FLAGS": "-DCMAKE_TOOLCHAIN_FILE=cmake/windows-x86_64.cmake",
+    "CMAKE_GENERATOR": "Ninja Multi-Config",
+    "CMAKE_FLAGS": "-DCMAKE_TOOLCHAIN_FILE=cmake/windows-x86_64.cmake -DCMAKE_UNITY_BUILD=ON -DML_PCH=ON",
 }
 
 def main(args):

.buildkite/scripts/steps/analyze_build_timings.py

@@ -0,0 +1,185 @@
+#!/usr/bin/env python3
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the following additional limitation. Functionality enabled by the
+# files subject to the Elastic License 2.0 may only be used in production when
+# invoked by an Elasticsearch process with a license key installed that permits
+# use of machine learning features. You may not use this file except in
+# compliance with the Elastic License 2.0 and the foregoing additional
+# limitation.
+
+"""
+Analyse build+test timings for the current snapshot build and compare
+against recent history.  Produces a Buildkite annotation with a summary
+table and flags any regressions.
+"""
+
+import json
+import math
+import os
+import subprocess
+import sys
+import urllib.request
+import urllib.error
+
+PIPELINE_SLUG = "ml-cpp-snapshot-builds"
+ORG_SLUG = "elastic"
+API_BASE = f"https://api.buildkite.com/v2/organizations/{ORG_SLUG}/pipelines/{PIPELINE_SLUG}"
+HISTORY_COUNT = 14
+
+PLATFORM_MAP = {
+    "Windows": "windows_x86_64",
+    "MacOS": "macos_aarch64",
+    "linux-x86_64": "linux_x86_64",
+    "linux-aarch64": "linux_aarch64",
+}
+
+
+def api_get(path, token):
+    url = f"{API_BASE}{path}"
+    req = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}"})
+    try:
+        with urllib.request.urlopen(req, timeout=30) as resp:
+            return json.loads(resp.read())
+    except urllib.error.HTTPError as e:
+        print(f"API error {e.code} for {url}: {e.read().decode()}", file=sys.stderr)
+        sys.exit(1)
+
+
+def extract_timings(build_data):
+    """Extract per-platform build+test timings from a build's jobs."""
+    timings = {}
+    for job in build_data.get("jobs", []):
+        name = job.get("name") or ""
+        if "Build & test" not in name:
+            continue
+        if "debug" in name.lower():
+            continue
+        started = job.get("started_at")
+        finished = job.get("finished_at")
+        if not started or not finished:
+            continue
+
+        for pattern, key in PLATFORM_MAP.items():
+            if pattern in name:
+                from datetime import datetime, timezone
+                fmt = "%Y-%m-%dT%H:%M:%S.%fZ"
+                t_start = datetime.strptime(started, fmt).replace(tzinfo=timezone.utc)
+                t_end = datetime.strptime(finished, fmt).replace(tzinfo=timezone.utc)
+                mins = (t_end - t_start).total_seconds() / 60.0
+                timings[key] = round(mins, 1)
+                break
+    return timings
+
+
+def mean_stddev(values):
+    if not values:
+        return 0.0, 0.0
+    n = len(values)
+    m = sum(values) / n
+    if n < 2:
+        return m, 0.0
+    variance = sum((x - m) ** 2 for x in values) / (n - 1)
+    return m, math.sqrt(variance)
+
+
+def annotate(markdown, style="info"):
+    """Create a Buildkite annotation."""
+    cmd = ["buildkite-agent", "annotate", "--style", style, "--context", "build-timings"]
+    proc = subprocess.run(cmd, input=markdown.encode(), capture_output=True)
+    if proc.returncode != 0:
+        print(f"buildkite-agent annotate failed: {proc.stderr.decode()}", file=sys.stderr)
+
+
+def main():
+    token = os.environ.get("BUILDKITE_API_READ_TOKEN", "")
+    if not token:
+        print("BUILDKITE_API_READ_TOKEN not set, skipping timing analysis", file=sys.stderr)
+        sys.exit(0)
+
+    build_number = os.environ.get("BUILDKITE_BUILD_NUMBER", "")
+    branch = os.environ.get("BUILDKITE_BRANCH", "main")
+
+    # Fetch current build
+    current = api_get(f"/builds/{build_number}", token)
+    current_timings = extract_timings(current)
+    current_date = current.get("created_at", "")[:10]
+
+    if not current_timings:
+        print("No build+test timings found for current build")
+        sys.exit(0)
+
+    # Fetch historical builds for the same branch
+    history_data = api_get(
+        f"/builds?branch={branch}&state=passed&per_page={HISTORY_COUNT + 1}", token
+    )
+
+    # Exclude the current build from history
+    history_builds = [
+        b for b in history_data if str(b.get("number")) != str(build_number)
+    ][:HISTORY_COUNT]
+
+    # Collect historical timings per platform
+    history = {key: [] for key in PLATFORM_MAP.values()}
+    for build in history_builds:
+        full_build = api_get(f"/builds/{build['number']}", token)
+        timings = extract_timings(full_build)
+        for key, val in timings.items():
+            history[key].append(val)
+
+    # Build the summary table
+    platforms = ["linux_x86_64", "linux_aarch64", "macos_aarch64", "windows_x86_64"]
+    platform_labels = {
+        "linux_x86_64": "Linux x86_64",
+        "linux_aarch64": "Linux aarch64",
+        "macos_aarch64": "macOS aarch64",
+        "windows_x86_64": "Windows x86_64",
+    }
+
+    lines = []
+    lines.append(f"### Build Timing Analysis — {current_date} (build #{build_number})")
+    lines.append("")
+    lines.append("| Platform | Current (min) | Avg (min) | Std Dev | Delta | Status |")
+    lines.append("|----------|:------------:|:---------:|:-------:|:-----:|:------:|")
+
+    has_regression = False
+    for plat in platforms:
+        cur = current_timings.get(plat)
+        hist = history.get(plat, [])
+        avg, sd = mean_stddev(hist)
+
+        if cur is None:
+            lines.append(f"| {platform_labels[plat]} | — | {avg:.1f} | {sd:.1f} | — | — |")
+            continue
+
+        delta = cur - avg
+        delta_pct = (delta / avg * 100) if avg > 0 else 0
+        sign = "+" if delta >= 0 else ""
+
+        if avg > 0 and sd > 0 and cur > avg + 2 * sd:
+            status = ":rotating_light: Regression"
+            has_regression = True
+        elif avg > 0 and cur < avg - sd:
+            status = ":rocket: Faster"
+        else:
+            status = ":white_check_mark: Normal"
+
+        lines.append(
+            f"| {platform_labels[plat]} | **{cur:.1f}** | {avg:.1f} | {sd:.1f} "
+            f"| {sign}{delta:.1f} ({sign}{delta_pct:.0f}%) | {status} |"
+        )
+
+    n_hist = len(history_builds)
+    lines.append("")
+    lines.append(f"_Compared against {n_hist} recent `{branch}` builds._")
+
+    markdown = "\n".join(lines)
+    print(markdown)
+
+    style = "warning" if has_regression else "info"
+    annotate(markdown, style)
+
+
+if __name__ == "__main__":
+    main()

.buildkite/scripts/steps/build_and_test.ps1

@@ -53,6 +53,37 @@ if (Test-Path Env:ML_DEBUG) {
     $DebugOption=""
 }
 
+# Ensure Ninja is available (required for Ninja Multi-Config generator)
+$ninjaCmd = Get-Command ninja -ErrorAction SilentlyContinue
+if ($ninjaCmd) {
+    Write-Output "ninja found: $($ninjaCmd.Source)"
+} else {
+    $ninjaVersion = "1.12.1"
+    $ninjaDir = "$Env:LOCALAPPDATA\ninja"
+    $ninjaExe = "$ninjaDir\ninja.exe"
+    if (Test-Path $ninjaExe) {
+        Write-Output "ninja already downloaded: $ninjaExe"
+    } else {
+        Write-Output "Downloading ninja v${ninjaVersion}..."
+        $url = "https://github.com/ninja-build/ninja/releases/download/v${ninjaVersion}/ninja-win.zip"
+        $zipPath = "$Env:TEMP\ninja-win.zip"
+        if (-not (Test-Path $ninjaDir)) { New-Item -ItemType Directory -Path $ninjaDir | Out-Null }
+        (New-Object Net.WebClient).DownloadFile($url, $zipPath)
+        Expand-Archive -Path $zipPath -DestinationPath $ninjaDir -Force
+        Remove-Item $zipPath -ErrorAction SilentlyContinue
+        Write-Output "ninja installed: $ninjaExe"
+    }
+    if ($Env:PATH -notlike "*$ninjaDir*") {
+        $Env:PATH = "$ninjaDir;$Env:PATH"
+    }
+}
+& ninja --version
+
+# Set up sccache with GCS backend if the bucket env var has been injected
+if (Test-Path Env:SCCACHE_GCS_BUCKET) {
+    . "$PSScriptRoot\..\..\..\dev-tools\setup_sccache.ps1"
+}
+
 # The exit code of the gradlew commands is checked explicitly, and their
 # stderr is treated as an error by PowerShell without this
 $ErrorActionPreference="Continue"
@@ -69,4 +100,10 @@ if ($ExitCode -ne 0) {
     Exit $ExitCode
 }
 
+# Print sccache stats if it was used
+if (Test-Path Env:SCCACHE_PATH) {
+    & $Env:SCCACHE_PATH --show-stats 2>$null
+    & $Env:SCCACHE_PATH --stop-server 2>$null
+}
+
 buildkite-agent artifact upload "build/distributions/*"

.buildkite/scripts/steps/run_es_tests.sh

@@ -24,6 +24,18 @@ export PR_AUTHOR=$(expr "$BUILDKITE_BRANCH" : '\(.*\):.*')
 export PR_SOURCE_BRANCH=$(expr "$BUILDKITE_BRANCH" : '.*:\(.*\)')
 export PR_TARGET_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
 
+# Set up GCS credentials for Gradle build cache persistence (if available).
+# The post-checkout hook writes the GCS service account key for sccache;
+# reuse the same credentials for the Gradle cache bucket.
+if [ -n "${SCCACHE_GCS_BUCKET:-}" ] && [ -n "${GOOGLE_APPLICATION_CREDENTIALS:-}" ]; then
+    export GRADLE_BUILD_CACHE_GCS_BUCKET="${SCCACHE_GCS_BUCKET}"
+    # Install gsutil if not already present
+    if ! command -v gsutil &>/dev/null; then
+        echo "--- Installing gsutil"
+        pip3 install --quiet gsutil 2>/dev/null || pip install --quiet gsutil 2>/dev/null || echo "Warning: failed to install gsutil"
+    fi
+fi
+
 mkdir -p "${IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/$VERSION"
 cp "build/distributions/ml-cpp-$VERSION-linux-$HARDWARE_ARCH.zip" "${IVY_REPO}/maven/org/elasticsearch/ml/ml-cpp/$VERSION/ml-cpp-$VERSION.zip"
 # Since this is all local, for simplicity, cheat with the dependencies/no-dependencies split