CliMA · petebachant · Oct 6, 2025 · Oct 6, 2025 · Oct 7, 2025 · Oct 14, 2025
diff --git a/.buildkite/Manifest-v1.11.toml b/.buildkite/Manifest-v1.11.toml
@@ -406,7 +406,9 @@ weakdeps = ["CUDA", "MPI"]
 
 [[deps.ClimaCore]]
 deps = ["Adapt", "BandedMatrices", "BlockArrays", "ClimaComms", "CubedSphere", "DataStructures", "ForwardDiff", "GaussQuadrature", "GilbertCurves", "HDF5", "InteractiveUtils", "IntervalSets", "KrylovKit", "LazyBroadcast", "LinearAlgebra", "MultiBroadcastFusion", "NVTX", "PkgVersion", "RecursiveArrayTools", "RootSolvers", "SparseArrays", "StaticArrays", "Statistics", "UnrolledUtilities"]
-git-tree-sha1 = "4fdb73dfcf8dc1c9655c81c605989da89b18fdd1"
+git-tree-sha1 = "6f1589a275831b575f103e82ceb2a4adee077b3f"
+repo-rev = "cf830bf6263da935a3d34f4081f0a74a2f8378d5"
+repo-url = "https://github.com/CliMA/ClimaCore.jl"
 uuid = "d414da3d-4745-48bb-8d80-42e94e092884"
 version = "0.14.40"
 weakdeps = ["CUDA", "Krylov"]

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -20,7 +20,7 @@ steps:
   - label: "init :computer:"
     key: "init_cpu_env"
     concurrency: 1
-    concurrency_group: 'depot/climaatmos-ci'
+    concurrency_group: "depot/climaatmos-ci"
     command:
       - "echo $$JULIA_DEPOT_PATH"
 
@@ -41,13 +41,11 @@ steps:
 
   - group: "Reproducibility infrastructure"
     steps:
-
       - label: ":computer: Test reproducibility infrastructure"
         command: "julia --color=yes --project=.buildkite test/unit_reproducibility_infra.jl"
 
   - group: "Radiation"
     steps:
-
       - label: ":computer: single column radiative equilibrium gray"
         command: >
           julia --color=yes --project=.buildkite .buildkite/ci_driver.jl
@@ -98,7 +96,6 @@ steps:
 
   - group: "Gravity wave"
     steps:
-
       - label: ":computer: non-orographic gravity wave parameterization unit test 3d"
         command: "julia --color=yes --project=.buildkite test/parameterized_tendencies/gravity_wave/non_orographic_gravity_wave/nogw_test_3d.jl"
         artifact_paths: "nonorographic_gravity_wave_test_3d/*"
@@ -132,7 +129,6 @@ steps:
 
   - group: "Column Examples"
     steps:
-
       - label: ":computer: single column hydrostatic balance float64"
         command: >
           julia --color=yes --project=.buildkite .buildkite/ci_driver.jl
@@ -149,7 +145,6 @@ steps:
 
   - group: "Box Examples"
     steps:
-
       - label: ":computer: Box hydrostatic balance"
         command: >
           julia --color=yes --project=.buildkite .buildkite/ci_driver.jl
@@ -188,7 +183,6 @@ steps:
 
   - group: "Plane Examples"
     steps:
-
       - label: ":computer: Density current experiment"
         command: >
           julia --color=yes --project=.buildkite .buildkite/ci_driver.jl
@@ -279,7 +273,6 @@ steps:
 
   - group: "Conservation check"
     steps:
-
       - label: ":computer: baroclinic wave check conservation"
         command: >
           julia --color=yes --project=.buildkite .buildkite/ci_driver.jl --config_file $CONFIG_PATH/baroclinic_wave_conservation.yml
@@ -305,7 +298,7 @@ steps:
         artifact_paths: "baroclinic_wave_equil_conservation_source/output_active/*"
         agents:
           slurm_mem: 16GB
-      
+
       - label: ":computer: baroclinic wave nonequil moist check conservation with sources"
         command: >
           julia --color=yes --project=.buildkite .buildkite/ci_driver.jl --config_file $CONFIG_PATH/baroclinic_wave_nonequil_conservation_source.yml
@@ -316,7 +309,6 @@ steps:
 
   - group: "Sphere Examples (Dycore)"
     steps:
-
       - label: ":computer: hydrostatic balance float64"
         command: >
           julia --color=yes --project=.buildkite .buildkite/ci_driver.jl
@@ -374,7 +366,6 @@ steps:
 
   - group: "Sphere Examples (Aquaplanet)"
     steps:
-
       - label: ":umbrella: aquaplanet nonequil allsky monin_obukhov varying insol gravity wave (gfdl_restart) high top 1-moment"
         command: >
           julia --color=yes --project=.buildkite .buildkite/ci_driver.jl
@@ -442,7 +433,6 @@ steps:
 
   - group: "Sphere Examples (Topography)"
     steps:
-
       - label: ":computer: baroclinic wave topography (dcmip)"
         command: >
           julia --color=yes --project=.buildkite .buildkite/ci_driver.jl
@@ -469,7 +459,6 @@ steps:
 
   - group: "Restarting"
     steps:
-
       - label: ":computer: test restart"
         command: >
           julia --color=yes --project=.buildkite test/restart.jl
@@ -534,7 +523,6 @@ steps:
 
   - group: "MPI Examples"
     steps:
-
       - label: ":computer: Prep restart for MPI"
         key: "mpi_baro_wave_make_restart"
         command: >
@@ -604,7 +592,6 @@ steps:
 
   - group: "EDOnlyEDMFX"
     steps:
-
       - label: ":man_in_business_suit_levitating: EDOnly EDMFX aquaplanet"
         command: >
           julia --color=yes --project=.buildkite .buildkite/ci_driver.jl
@@ -627,7 +614,6 @@ steps:
 
   - group: "Diagnostic EDMFX"
     steps:
-
       - label: ":genie: Diagnostic EDMFX test in a box"
         command: >
           julia --color=yes --project=.buildkite .buildkite/ci_driver.jl
@@ -744,14 +730,12 @@ steps:
 
   - group: "Prognostic EDMFX"
     steps:
-
       - label: ":genie: Prognostic EDMFX advection test in a column"
         command: >
           julia --color=yes --project=.buildkite .buildkite/ci_driver.jl
           --config_file $CONFIG_PATH/prognostic_edmfx_adv_test_column.yml
           --job_id prognostic_edmfx_adv_test_column
         artifact_paths: "prognostic_edmfx_adv_test_column/output_active/*"
-
         agents:
           slurm_mem: 20GB
 
@@ -853,7 +837,7 @@ steps:
         artifact_paths: "prognostic_edmfx_rico_column/output_active/*"
         agents:
           slurm_mem: 20GB
-      
+
       - label: ":umbrella: Prognostic EDMFX Rico in a column (implicit)"
         command: >
           julia --color=yes --project=.buildkite .buildkite/ci_driver.jl
@@ -946,7 +930,6 @@ steps:
 
   - group: "Autodiff"
     steps:
-
       - label: "baroclinic wave moist check conservation float64 sparse autodiff"
         command: >
           julia --color=yes --project=.buildkite .buildkite/ci_driver.jl --config_file $CONFIG_PATH/baroclinic_wave_equil_conservation_ft64_sparse_autodiff.yml
@@ -989,7 +972,7 @@ steps:
         artifact_paths: "prognostic_edmfx_bomex_column_sparse_autodiff/output_active/*"
         agents:
           slurm_mem: 20GB
-      
+
       - label: "Prognostic EDMFX Dycoms RF02 in a column sparse autodiff"
         command: >
           julia --color=yes --project=.buildkite .buildkite/ci_driver.jl
@@ -999,7 +982,7 @@ steps:
         agents:
           slurm_mem: 20GB
         soft_fail: true
-      
+
       - label: "Prognostic EDMFX TRMM in a column sparse autodiff"
         command: >
           julia --color=yes --project=.buildkite .buildkite/ci_driver.jl
@@ -1029,7 +1012,7 @@ steps:
         artifact_paths: "prognostic_edmfx_aquaplanet_sparse_autodiff/output_active/*"
         agents:
           slurm_mem: 20GB
-      
+
       - label: "AMIP Target EDOnly nonequil sparse autodiff"
         command: >
           julia --color=yes --project=.buildkite .buildkite/ci_driver.jl
@@ -1042,7 +1025,6 @@ steps:
 
   - group: "GPU"
     steps:
-
       - label: "GPU: Gravity waves"
         command: >
           julia --color=yes --project=.buildkite .buildkite/ci_driver.jl
@@ -1111,7 +1093,6 @@ steps:
           - "baroclinic_wave"
           - "baroclinic_wave_gpu"
 
-
       - label: "GPU: baroclinic wave - 2 gpus"
         key: "baroclinic_wave_2gpu"
         command:
@@ -1231,7 +1212,6 @@ steps:
 
   - group: "Benchmarks"
     steps:
-
       - label: ":computer: Benchmark: CPU baroclinic wave moist"
         command: >
           julia --color=yes --project=.buildkite perf/benchmark_step.jl
@@ -1249,6 +1229,7 @@ steps:
         artifact_paths: "bm_baroclinic_wave_moist_gpu/output_active/*"
         env:
           CLIMACOMMS_DEVICE: "CUDA"
+          CLIMA_NAME_CUDA_KERNELS_FROM_STACK_TRACE: "true"
         agents:
           slurm_mem: 16G
           slurm_gpus: 1
@@ -1300,7 +1281,6 @@ steps:
 
   - group: "Flame graphs"
     steps:
-
       - label: ":fire: Flame graph: gpu job"
         command: >
           julia --color=yes --project=.buildkite perf/flame.jl
@@ -1418,7 +1398,6 @@ steps:
 
   - group: "Checkbounds/Inference/Invalidations"
     steps:
-
       # TODO: we should somehow decouple this unit test from the perf env / scripts
       # Checkbounds
       - label: ":computer: checkbounds"

diff --git a/perf/benchmark_step.jl b/perf/benchmark_step.jl
@@ -30,13 +30,64 @@ Y₀ = deepcopy(integrator.u);
 CA.benchmark_step!(integrator, Y₀); # compile first
 
 @info "Running benchmark_step!..."
-n_steps = 10
 comms_ctx = ClimaComms.context(integrator.u.c)
 device = ClimaComms.device(comms_ctx)
-local e
-s = CA.@timed_str begin
-    e = ClimaComms.elapsed(device) do
-        CA.benchmark_step!(integrator, Y₀, n_steps) # run
+
+# Robustly parse boolean-like environment variables
+function getenv_bool(var::AbstractString; default::Bool = false)
+    raw = get(ENV, var, nothing)
+    raw === nothing && return default
+    s = lowercase(strip(String(raw)))
+    if s in ("1", "true", "t", "yes", "y", "on")
+        return true
+    elseif s in ("0", "false", "f", "no", "n", "off")
+        return false
+    else
+        # fall back to parse as integer (non-zero -> true)
+        try
+            return parse(Int, s) != 0
+        catch
+            @warn "Unrecognized boolean env var value; using default" var = var val = raw default =
+                default
+            return default
+        end
+    end
+end
+
+# If we're running on CUDA, use CUDA's profiler
+if ENV["CLIMACOMMS_DEVICE"] == "CUDA" && device isa ClimaComms.CUDADevice
+    import CUDA
+    if getenv_bool("CLIMA_NAME_CUDA_KERNELS_FROM_STACK_TRACE", default = false)
+        import ClimaCore
+        ClimaCore.DebugOnly.name_kernels_from_stack_trace() = true
+    end
+    e = 0.0
+    n_steps = 5
+    use_external_profiler = CUDA.Profile.detect_cupti()
+    if use_external_profiler
+        @info "Using external CUDA profiler"
+        CUDA.@profile external = true begin
+            e = CUDA.@elapsed begin
+                CA.benchmark_step!(integrator, Y₀, n_steps)
+            end
+        end
+    else
+        @info "Using internal CUDA profiler"
+        CUDA.@profile external = false begin
+            e = CUDA.@elapsed begin
+                CA.benchmark_step!(integrator, Y₀, n_steps)
+            end
+        end
+    end
+    @info "Ran step! with CUDA $n_steps times in $e s, ($(CA.prettytime(e/n_steps*1e9)) per step)"
+else
+    # Profile with Julia's built-in profiler
+    n_steps = 10
+    local e
+    s = CA.@timed_str begin
+        e = ClimaComms.elapsed(device) do
+            CA.benchmark_step!(integrator, Y₀, n_steps) # run
+        end
     end
+    @info "Ran step! $n_steps times in $s, ($(CA.prettytime(e/n_steps*1e9)) per step)"
 end
-@info "Ran step! $n_steps times in $s, ($(CA.prettytime(e/n_steps*1e9)) per step)"