From 105ae274025a1218d2068faecf7032979ed9cbdd Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Wed, 16 Apr 2025 13:05:41 -0400 Subject: [PATCH 01/12] Update m_bubbles.fpp --- src/simulation/m_bubbles.fpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/simulation/m_bubbles.fpp b/src/simulation/m_bubbles.fpp index b7b249139d..54fcd825a8 100644 --- a/src/simulation/m_bubbles.fpp +++ b/src/simulation/m_bubbles.fpp @@ -7,7 +7,7 @@ !> @brief This module contains the procedures shared by the ensemble-averaged and volume-averaged bubble models. module m_bubbles - use m_derived_types !< Definitions of the derived types + use m_derived_types !< Definitions of the derived types use m_global_parameters !< Definitions of the global parameters From 99fd4c224287e68f4df472271e997c712c0a08e8 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Wed, 16 Apr 2025 15:40:08 -0400 Subject: [PATCH 02/12] Update bench.sh --- .github/workflows/phoenix/bench.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/phoenix/bench.sh b/.github/workflows/phoenix/bench.sh index 4fc99cc1fa..bb6b1ad1da 100644 --- a/.github/workflows/phoenix/bench.sh +++ b/.github/workflows/phoenix/bench.sh @@ -2,6 +2,8 @@ n_ranks=12 +export TMPDIR=/storage/home/hcoda1/6/sbryngelson3/scratch/runner-tmp + if [ "$job_device" == "gpu" ]; then n_ranks=$(nvidia-smi -L | wc -l) # number of GPUs on node gpu_ids=$(seq -s ' ' 0 $(($n_ranks-1))) # 0,1,2,...,gpu_count-1 From fcee17f21fa98ee598a28feced2f61c38e1c5948 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Wed, 16 Apr 2025 16:17:23 -0400 Subject: [PATCH 03/12] Update bench.sh --- .github/workflows/phoenix/bench.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/phoenix/bench.sh b/.github/workflows/phoenix/bench.sh index bb6b1ad1da..6c8076bbed 100644 --- a/.github/workflows/phoenix/bench.sh +++ b/.github/workflows/phoenix/bench.sh @@ -2,7 +2,14 @@ n_ranks=12 -export TMPDIR=/storage/home/hcoda1/6/sbryngelson3/scratch/runner-tmp +rand="" +for i in {1..10}; do + rand="${rand}$((RANDOM % 10))" +done +TMPDIR=/tmp/$rand +mkdir $TMPDIR +chmod 777 $TMPDIR +export TMPDIR=$TMPDIR if [ "$job_device" == "gpu" ]; then n_ranks=$(nvidia-smi -L | wc -l) # number of GPUs on node From 2faaadc31a682cc1b6e239eb51133cad8198ad57 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Wed, 16 Apr 2025 16:17:34 -0400 Subject: [PATCH 04/12] Discard changes to src/simulation/m_bubbles.fpp --- src/simulation/m_bubbles.fpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/simulation/m_bubbles.fpp b/src/simulation/m_bubbles.fpp index 54fcd825a8..b7b249139d 100644 --- a/src/simulation/m_bubbles.fpp +++ b/src/simulation/m_bubbles.fpp @@ -7,7 +7,7 @@ !> @brief This module contains the procedures shared by the ensemble-averaged and volume-averaged bubble models. module m_bubbles - use m_derived_types !< Definitions of the derived types + use m_derived_types !< Definitions of the derived types use m_global_parameters !< Definitions of the global parameters From eda654543255eade71d48737a93f727ecedc9ab3 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Thu, 17 Apr 2025 09:22:50 -0400 Subject: [PATCH 05/12] Discard changes to .github/workflows/phoenix/bench.sh --- .github/workflows/phoenix/bench.sh | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/phoenix/bench.sh b/.github/workflows/phoenix/bench.sh index 6c8076bbed..4fc99cc1fa 100644 --- a/.github/workflows/phoenix/bench.sh +++ b/.github/workflows/phoenix/bench.sh @@ -2,15 +2,6 @@ n_ranks=12 -rand="" -for i in {1..10}; do - rand="${rand}$((RANDOM % 10))" -done -TMPDIR=/tmp/$rand -mkdir $TMPDIR -chmod 777 $TMPDIR -export TMPDIR=$TMPDIR - if [ "$job_device" == "gpu" ]; then n_ranks=$(nvidia-smi -L | wc -l) # number of GPUs on node gpu_ids=$(seq -s ' ' 0 $(($n_ranks-1))) # 0,1,2,...,gpu_count-1 From d3232e9fa9f4f984b8a221a877056a07b08eef9a Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Thu, 17 Apr 2025 09:23:48 -0400 Subject: [PATCH 06/12] Update phoenix.mako --- toolchain/templates/phoenix.mako | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/toolchain/templates/phoenix.mako b/toolchain/templates/phoenix.mako index e5e139905f..d67a8717e9 100644 --- a/toolchain/templates/phoenix.mako +++ b/toolchain/templates/phoenix.mako @@ -42,8 +42,7 @@ echo (set -x; ${profiler} "${target.get_install_binpath(case)}") % else: (set -x; ${profiler} \ - mpirun -np ${nodes*tasks_per_node} \ - --bind-to none \ + srun -n ${nodes*tasks_per_node} \ "${target.get_install_binpath(case)}") % endif From 51dcd5d101b5774076d71219ff9b822d12a817d1 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Thu, 17 Apr 2025 09:25:10 -0400 Subject: [PATCH 07/12] Update bench.sh From 27a6a160413e004a56cc7da6e13c072701f6d365 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Thu, 17 Apr 2025 09:33:26 -0400 Subject: [PATCH 08/12] Update m_bubbles.fpp --- src/simulation/m_bubbles.fpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/simulation/m_bubbles.fpp b/src/simulation/m_bubbles.fpp index b7b249139d..07fd608e6d 100644 --- a/src/simulation/m_bubbles.fpp +++ b/src/simulation/m_bubbles.fpp @@ -1,6 +1,6 @@ !> !! @file m_bubbles.f90 -!! @brief Contains module m_bubbles +!! @brief Contains module m_bubbles #:include 'macros.fpp' From b3af841858bc654e2821adf59349f59d6d2b0d5c Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Thu, 17 Apr 2025 13:20:11 -0400 Subject: [PATCH 09/12] Update bench.sh --- .github/workflows/phoenix/bench.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/phoenix/bench.sh b/.github/workflows/phoenix/bench.sh index 4fc99cc1fa..0bcab20b88 100644 --- a/.github/workflows/phoenix/bench.sh +++ b/.github/workflows/phoenix/bench.sh @@ -13,3 +13,5 @@ if ["$job_device" == "gpu"]; then else ./mfc.sh bench --mem 1 -j $(nproc) -o "$job_slug.yaml" -- -c phoenix $device_opts -n $n_ranks fi + +sleep 10 From 724ff7a9184af6c1617114d82a5782b0585d63dd Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Thu, 17 Apr 2025 13:20:39 -0400 Subject: [PATCH 10/12] Update bench.yaml --- toolchain/bench.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/toolchain/bench.yaml b/toolchain/bench.yaml index a578ec22c4..2c3cfbb7d3 100644 --- a/toolchain/bench.yaml +++ b/toolchain/bench.yaml @@ -35,3 +35,6 @@ path: benchmarks/hypo_hll/case.py args: [] +- slug: ibm + path: benchmarks/ibm/case.py + args: [] From 39d0294a1c29eb3a86fb67624e3836bee99579e3 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Thu, 24 Apr 2025 08:49:04 -0400 Subject: [PATCH 11/12] Discard changes to toolchain/bench.yaml --- toolchain/bench.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/toolchain/bench.yaml b/toolchain/bench.yaml index 2c3cfbb7d3..a578ec22c4 100644 --- a/toolchain/bench.yaml +++ b/toolchain/bench.yaml @@ -35,6 +35,3 @@ path: benchmarks/hypo_hll/case.py args: [] -- slug: ibm - path: benchmarks/ibm/case.py - args: [] From f85115f6b82b2fd27a88825bef6b5889e3583b56 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Thu, 24 Apr 2025 08:59:07 -0400 Subject: [PATCH 12/12] Update phoenix.mako --- toolchain/templates/phoenix.mako | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/toolchain/templates/phoenix.mako b/toolchain/templates/phoenix.mako index 891ce8a21f..2705305111 100644 --- a/toolchain/templates/phoenix.mako +++ b/toolchain/templates/phoenix.mako @@ -40,9 +40,15 @@ echo % if not mpi: (set -x; ${profiler} "${target.get_install_binpath(case)}") % else: - (set -x; ${profiler} \ - srun -n ${nodes*tasks_per_node} \ - "${target.get_install_binpath(case)}") + (set -x; srun \ + % if engine == 'interactive': + --nodes ${nodes} --ntasks-per-node ${tasks_per_node} \ + --cpus-per-task 1 \ + % if gpu: + --gpus-per-task 1 --gpu-bind closest \ + % endif + % endif + ${profiler} "${target.get_install_binpath(case)}") % endif ${helpers.run_epilogue(target)}