Add Buildkite pipeline to check flagship AMIP performance (#2438)

Author: petebachant

.buildkite/perf/ClimaCoupler.jl

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+SYPD_FILE=output/gpu_amip_progedmf_1M_land_he16/artifacts/sypd.txt
+
+if [[ ! -f "$SYPD_FILE" ]]; then
+    echo "❌ SYPD file not found: $SYPD_FILE"
+    exit 1
+fi
+
+SYPD=$(cat "$SYPD_FILE" | tr -d '[:space:]')
+
+if [[ -z "$SYPD" ]]; then
+    echo "❌ SYPD file is empty"
+    exit 1
+fi
+
+echo "SYPD: $SYPD"
+
+PERCENT_CHANGE=$(echo "scale=2; (($SYPD - $BASELINE_SYPD) / $BASELINE_SYPD) * 100" | bc)
+
+if (( $(echo "$PERCENT_CHANGE <= $MIN_PERCENT_CHANGE" | bc -l) )); then
+    echo "❌ SYPD changed by $PERCENT_CHANGE% (threshold: $MIN_PERCENT_CHANGE%)"
+    exit 1
+else
+    echo "✅ SYPD change ($PERCENT_CHANGE%) is okay (threshold: $MIN_PERCENT_CHANGE%)"
+fi

.buildkite/perf/check-sypd.sh

@@ -0,0 +1,69 @@
+# A pipeline for downstream performance checks
+agents:
+  queue: clima
+  slurm_time: 14:00:00
+  modules: climacommon/2025_05_15
+
+env:
+  JULIA_DEPOT_PATH: "${BUILDKITE_BUILD_PATH}/${BUILDKITE_PIPELINE_SLUG}/depot/default"
+  JULIA_MAX_NUM_PRECOMPILE_FILES: 100
+  GKSwstype: 100
+  SLURM_KILL_BAD_EXIT: 1
+  COUPLER_PATH: ".buildkite/perf/ClimaCoupler.jl"
+  CONFIG_PATH: "$COUPLER_PATH/config/nightly_configs"
+  BENCHMARK_CONFIG_PATH: "$COUPLER_PATH/config/benchmark_configs"
+
+timeout_in_minutes: 840
+
+steps:
+  - label: "Check if pipeline should run :mag:"
+    key: "check_changes"
+    soft_fail: true
+    command:
+      - git submodule update --init --recursive .buildkite/perf/ClimaCoupler.jl
+      - |
+        # Check if files in src, ext, or .buildkite/perf have changed
+        if git diff --quiet origin/main...HEAD -- src ext .buildkite/perf; then
+          echo "⏭️ No relevant changes detected, skipping downstream checks"
+          exit 1
+        else
+          echo "✅ Relevant changes detected, running downstream checks"
+          exit 0
+        fi
+
+  - label: "init :GPU:"
+    key: "init_gpu_env"
+    depends_on: "check_changes"
+    command:
+      - "echo $$JULIA_DEPOT_PATH"
+      - echo "--- Instantiate AMIP env"
+      # Instantiate and dev install ClimaCore
+      - "julia --project=$COUPLER_PATH/experiments/ClimaEarth/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'"
+      - 'julia --project=$COUPLER_PATH/experiments/ClimaEarth/ -e ''using Pkg; Pkg.develop(path=".")'''
+      - 'julia --project=$COUPLER_PATH/experiments/ClimaEarth/ -e ''using Pkg; Pkg.add("MPI")'''
+      - "julia --project=$COUPLER_PATH/experiments/ClimaEarth/ -e 'using Pkg; Pkg.precompile()'"
+      - "julia --project=$COUPLER_PATH/experiments/ClimaEarth/ -e 'using Pkg; Pkg.status()'"
+    agents:
+      slurm_gpus: 1
+      slurm_cpus_per_task: 8
+    env:
+      JULIA_NUM_PRECOMPILE_TASKS: 8
+      JULIA_MAX_NUM_PRECOMPILE_FILES: 50
+
+  - label: "Flagship AMIP GPU with prognostic EDMF + 1M + integrated land (16 helems)"
+    key: "gpu_amip_progedmf_1M_land_he16"
+    depends_on: "init_gpu_env"
+    command:
+      - "srun julia --threads=3 --color=yes --project=$COUPLER_PATH/experiments/ClimaEarth/ $COUPLER_PATH/experiments/ClimaEarth/run_amip.jl --config_file $BENCHMARK_CONFIG_PATH/amip_progedmf_1m_land_he16.yml --job_id gpu_amip_progedmf_1M_land_he16"
+      - echo "--- Checking SYPD from artifacts"
+      - bash .buildkite/perf/check-sypd.sh
+    artifact_paths: "output/gpu_amip_progedmf_1M_land_he16/artifacts/*"
+    env:
+      CLIMACOMMS_DEVICE: "CUDA"
+      BASELINE_SYPD: "0.081"
+      MIN_PERCENT_CHANGE: "-1"
+    agents:
+      slurm_gpus_per_task: 1
+      slurm_cpus_per_task: 4
+      slurm_ntasks: 1
+      slurm_mem: 16GB

.buildkite/perf/pipeline.yml

@@ -0,0 +1,3 @@
+[submodule ".buildkite/perf/ClimaCoupler.jl"]
+	path = .buildkite/perf/ClimaCoupler.jl
+	url = https://github.com/CliMA/ClimaCoupler.jl.git