Frontier Benchmarking (#453)

.github/workflows/bench.yml

@@ -18,7 +18,7 @@ jobs:
         with: 
           filters: ".github/file-filter.yml"
 
-  self:
+  phoenix:
     name: Georgia Tech | Phoenix (NVHPC)
     if: github.repository == 'MFlowCode/MFC' && needs.file-changes.outputs.checkall == 'true'
     needs: file-changes
@@ -66,3 +66,58 @@ jobs:
             pr/build/benchmarks/*
             master/bench-${{ matrix.device }}.*
             master/build/benchmarks/*
+
+  frontier:
+      name: Oak Ridge | Frontier (AMD ROCm)
+      if: github.repository == 'MFlowCode/MFC' && needs.file-changes.outputs.checkall == 'true'
+      needs: file-changes
+      strategy:
+        matrix:
+          device: ['cpu', 'gpu']
+      runs-on:
+        group:  frontier
+        labels: olcf
+      timeout-minutes: 1400
+      env:
+        ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16
+        ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+      steps:
+        - name: Clone - PR
+          uses: actions/checkout@v4
+          with:
+            path: pr
+
+        - name: Clone - Master
+          uses: actions/checkout@v4
+          with:
+            repository: MFlowCode/MFC
+            ref: master
+            path: master
+
+        - name: Setup & Build
+          run: | 
+            (cd pr     && bash .github/workflows/frontier/build.sh ${{ matrix.device }}) &
+            (cd master && bash .github/workflows/frontier/build.sh ${{ matrix.device }}) &
+            wait %1 && wait %2
+
+        - name: Bench (Master v. PR)
+          run: |
+            (cd pr     && bash .github/workflows/frontier/submit-bench.sh .github/workflows/frontier/bench.sh ${{ matrix.device }}) &
+            (cd master && bash .github/workflows/frontier/submit-bench.sh .github/workflows/frontier/bench.sh ${{ matrix.device }}) &
+            wait %1 && wait %2
+
+        - name: Generate & Post Comment
+          run: |
+            (cd pr && . ./mfc.sh load -c p -m g)
+            (cd pr && ./mfc.sh bench_diff ../master/bench-${{ matrix.device }}.yaml ../pr/bench-${{ matrix.device }}.yaml)
+
+        - name: Archive Logs
+          uses: actions/upload-artifact@v4
+          if:   always()
+          with:
+            name: logs-frontier-${{ matrix.device }}
+            path: |
+              pr/bench-${{ matrix.device }}.*
+              pr/build/benchmarks/*
+              master/bench-${{ matrix.device }}.*
+              master/build/benchmarks/*

.github/workflows/frontier/bench.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+n_ranks=12
+
+if [ "$job_device" == "gpu" ]; then
+    gpus=$(rocm-smi --showid | awk '{print $1}' | grep -Eo '[0-9]+' | uniq | tr '\n' ' ')
+    n_ranks=$(echo "$gpus" | wc -w)         # number of GPUs on node
+    gpu_ids=$(echo "$gpus" | tr ' ' '\n' | tr '\n' ' ' | sed 's/ $//')  # GPU IDs from rocm-smi
+    device_opts="--gpu -g $gpu_ids"
+fi
+
+mkdir -p /storage/scratch1/6/sbryngelson3/mytmp_build
+export TMPDIR=/storage/scratch1/6/sbryngelson3/mytmp_build
+
+if [ "$job_device" == "gpu" ]; then
+    ./mfc.sh bench --mem 12 -j $n_ranks -o "$job_slug.yaml" -- -c frontier-bench $device_opts -n $n_ranks
+else
+    ./mfc.sh bench --mem 1 -j $(nproc) -o "$job_slug.yaml" -- -c frontier-bench $device_opts -n $n_ranks
+fi
+
+unset TMPDIR

.github/workflows/frontier/submit-bench.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+set -e
+
+usage() {
+    echo "Usage: $0 [script.sh] [cpu|gpu]"
+}
+
+if [ ! -z "$1" ]; then
+    sbatch_script_contents=`cat $1`
+else
+    usage
+    exit 1
+fi
+
+if [ "$2" == "cpu" ]; then
+    sbatch_device_opts="\
+#SBATCH -n 32                       # Number of cores required"
+elif [ "$2" == "gpu" ]; then
+    sbatch_device_opts="\
+#SBATCH -n 8                       # Number of cores required"
+else
+    usage
+    exit 1
+fi
+
+
+job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2"
+
+sbatch <<EOT
+#!/bin/bash
+#SBATCH -JMFC-$job_slug            # Job name
+#SBATCH -A CFD154                  # charge account
+#SBATCH -N 1                       # Number of nodes required
+$sbatch_device_opts
+#SBATCH -t 01:59:00                # Duration of the job (Ex: 15 mins)
+#SBATCH -o$job_slug.out            # Combined output and error messages file
+#SBATCH -p extended                # Extended partition for shorter queues
+#SBATCH -q debug                   # Use debug QOS - only one job per user allowed in queue!
+#SBATCH -W                         # Do not exit until the submitted job terminates.
+
+set -e
+set -x
+
+cd "\$SLURM_SUBMIT_DIR"
+echo "Running in $(pwd):"
+
+job_slug="$job_slug"
+job_device="$2"
+
+. ./mfc.sh load -c f -m g
+
+$sbatch_script_contents
+
+EOT
+