diff --git a/toolchain/bootstrap/modules.sh b/toolchain/bootstrap/modules.sh index 2a0245fc8..fbe28a959 100644 --- a/toolchain/bootstrap/modules.sh +++ b/toolchain/bootstrap/modules.sh @@ -39,12 +39,13 @@ done if [ -v $u_c ]; then log "Select a system:" log "$G""ORNL$W: Ascent (a) | Frontier (f) | Summit (s) | Wombat (w)" + log "$B""LLNL $W: Tuolumne (t)" log "$C""ACCESS$W: Bridges2 (b) | Expanse (e) | Delta (d) | DeltaAI (dai)" log "$Y""Gatech$W: Phoenix (p)" log "$R""Caltech$W: Richardson (r)" - log "$BR""Brown$W: Oscar (o)" + log "$BR""Brown$W: Oscar (o)" log "$B""DoD$W: Carpenter Cray (cc) | Carpenter GNU (c) | Nautilus (n)" - log_n "($G""a$W/$G""f$W/$G""s$W/$G""w$W/$C""b$W/$C""e$CR/$C""d/$C""dai$CR/$Y""p$CR/$R""r$CR/$B""cc$CR/$B""c$CR/$B""n$CR/$BR""o"$CR"): " + log_n "($G""a$W/$G""f$W/$G""s$W/$G""w$W/$B""t$W/$C""b$W/$C""e$CR/$C""d/$C""dai$CR/$Y""p$CR/$R""r$CR/$B""cc$CR/$B""c$CR/$B""n$CR/$BR""o"$CR"): " read u_c log fi diff --git a/toolchain/modules b/toolchain/modules index 65686085b..365a7dfde 100644 --- a/toolchain/modules +++ b/toolchain/modules @@ -45,6 +45,12 @@ f-all cpe/25.03 rocm/6.3.1 f-all cray-fftw cray-hdf5 python cmake f-gpu python craype-accel-amd-gfx90a rocprofiler-compute/3.0.0 +t OLCF Tuolumne +t-all cpe/25.03 rocm/6.3.1 +t-all cray-fftw cray-hdf5 cray-python cmake +t-gpu craype-accel-amd-gfx942 +t-gpu HSA_XNACK=1 + d NCSA Delta d-all python/3.11.6 d-cpu gcc/11.4.0 openmpi diff --git a/toolchain/templates/tuo.mako b/toolchain/templates/tuo.mako new file mode 100644 index 000000000..6821b3af1 --- /dev/null +++ b/toolchain/templates/tuo.mako @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +<%namespace name="helpers" file="helpers.mako"/> + +% if engine == 'batch': +# flux: -N ${nodes} +# flux: -n ${tasks_per_node*nodes} +# flux: --job-name="${name}" +# flux: --output="${name}.out" +# flux: --error="${name}.err" +# flux: --time=${walltime} +# flux: --exclusive +# flux:--setattr=thp=always +# flux: --coral2-hugepages=512GB +% if account: +# flux: --bank=${account} +% endif +% if partition: +# flux: --queue=${partition} +% endif +% if unified: +# flux:--setattr=thp=always +# flux: --coral2-hugepages=512GB +% endif +% endif + +${helpers.template_prologue()} + +ok ":) Loading modules:\n" +cd "${MFC_ROOT_DIR}" +% if engine == 'batch': +. ./mfc.sh load -c t -m ${'g' if gpu else 'c'} +% endif +cd - > /dev/null +echo + +% if gpu: + export MPICH_GPU_SUPPORT_ENABLED=1 +% else: + export MPICH_GPU_SUPPORT_ENABLED=0 +% endif + +% for target in targets: + ${helpers.run_prologue(target)} + + % if not mpi: + (set -x; ${profiler} "${target.get_install_binpath(case)}") + % else: + (set -x; flux run \ + --nodes=${nodes} --ntasks=${tasks_per_node * nodes} \ + --exclusive \ + % if gpu: + --gpus-per-task 1 \ + % endif + ${profiler} "${target.get_install_binpath(case)}") + % endif + + ${helpers.run_epilogue(target)} + + echo +% endfor + +${helpers.template_epilogue()}