diff --git a/toolchain/modules b/toolchain/modules index 65686085b..69757c3c0 100644 --- a/toolchain/modules +++ b/toolchain/modules @@ -54,7 +54,7 @@ d-gpu MFC_CUDA_CC=80,86 o Brown Oscar o-all python -o-cpu hpcx-mpi +o-cpu hpcx-mpi python/3.13.10s o-gpu nvhpc cuda/12.3.0 cmake/3.26.3 o-gpu CC=nvc CXX=nvc++ FC=nvfortran diff --git a/toolchain/templates/oscar.mako b/toolchain/templates/oscar.mako index af54ca718..223c0e6ae 100644 --- a/toolchain/templates/oscar.mako +++ b/toolchain/templates/oscar.mako @@ -4,7 +4,7 @@ % if engine == 'batch': #SBATCH --nodes=${nodes} -#SBATCH --ntasks-per-node=${tasks_per_node} +#SBATCH --tasks-per-node=${tasks_per_node} #SBATCH --cpus-per-task=1 #SBATCH --job-name="${name}" #SBATCH --time=${walltime} @@ -15,9 +15,8 @@ #SBATCH --account="${account}" % endif % if gpu_enabled: -#SBATCH --gpus-per-node=${tasks_per_node} -#SBATCH --mem=64G -#SBATCH --gpu-bind=closest +#SBATCH --gpu-bind=verbose,closest +#SBATCH --gres=gpu:v100-16:${tasks_per_node} % endif #SBATCH --output="${name}.out" #SBATCH --error="${name}.err" @@ -31,7 +30,7 @@ ${helpers.template_prologue()} ok ":) Loading modules:\n" -cd "${MFC_ROOTDIR}" +cd "${MFC_ROOT_DIR}" . ./mfc.sh load -c o -m ${'g' if gpu_enabled else 'c'} cd - > /dev/null echo @@ -42,9 +41,8 @@ echo % if not mpi: (set -x; ${profiler} "${target.get_install_binpath(case)}") % else: - (set -x; ${profiler} \ - mpirun -np ${nodes*tasks_per_node} \ - ${' '.join([f"'{x}'" for x in ARG('--') ])} \ + (set -x; ${profiler} \ + mpirun -np ${nodes*tasks_per_node} \ "${target.get_install_binpath(case)}") % endif