fix

Author: sbryngelson

.github/workflows/bench.yml

@@ -79,11 +79,19 @@ jobs:
           wait %1 && wait %2
 
       - name: Bench (Master v. PR)
+        if: matrix.cluster == 'frontier'
         run: |
           (cd pr     && bash .github/workflows/${{ matrix.cluster }}/submit-bench.sh .github/workflows/${{ matrix.cluster }}/bench.sh ${{ matrix.device }}) &
           (cd master && bash .github/workflows/${{ matrix.cluster }}/submit-bench.sh .github/workflows/${{ matrix.cluster }}/bench.sh ${{ matrix.device }}) &
           wait %1 && wait %2
 
+      - name: Bench (Master v. PR)
+        if: matrix.cluster == 'phoenix'
+        run: |
+          (cd pr     && bash .github/workflows/${{ matrix.cluster }}/submit-bench.sh ${{ matrix.device }}) &
+          (cd master && bash .github/workflows/${{ matrix.cluster }}/submit-bench.sh ${{ matrix.device }}) &
+          wait %1 && wait %2
+
       - name: Generate & Post Comment
         run: |
           (cd pr && . ./mfc.sh load -c ${{ matrix.flag }} -m g)

.github/workflows/phoenix/bench.sh

@@ -2,16 +2,14 @@
 set -euo pipefail
 
 usage() {
-    echo "Usage: $0 [script.sh] [cpu|gpu]"
+    echo "Usage: $0 [cpu|gpu]"
     exit 1
 }
 
-[[ $# -eq 2 ]] || usage
+[[ $# -eq 1 ]] || usage
 
-sbatch_script="$1"
-
-device="$2"
-job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2"
+device="$1"
+job_slug="bench-$1"
 
 # read the body of the user script
 sbatch_body=$(<"$sbatch_script")
@@ -65,7 +63,39 @@ JOBID=$(sbatch <<-EOT | awk '{print $4}'
 	. ./mfc.sh load -c p -m $device
 
 	# user script contents
-	${sbatch_body}
+    n_ranks=12
+
+    echo "My benchmarking device is:" $device
+    if [ "$device" = "gpu" ]; then
+        echo "Set device opts for GPU cases."
+        n_ranks=$(nvidia-smi -L | wc -l)        # number of GPUs on node
+        gpu_ids=$(seq -s ' ' 0 $(($n_ranks-1))) # 0,1,2,...,gpu_count-1
+        device_opts="--gpu -g $gpu_ids"
+    fi
+
+    tmpbuild=/storage/scratch1/6/sbryngelson3/mytmp_build
+    currentdir=$tmpbuild/run-$(( RANDOM % 900 ))
+    mkdir -p $tmpbuild
+    mkdir -p $currentdir
+
+    export TMPDIR=$currentdir
+
+    if [ "$device" = "gpu" ]; then
+        echo "running GPU benchmarks"
+        ./mfc.sh bench --mem 12 -j $(nproc) -o "$job_slug.yaml" -- -c phoenix-bench $device_opts -n $n_ranks
+    elif [ "$device" = "cpu" ]; then
+        echo "running CPU benchmarks"
+        ./mfc.sh bench --mem 1 -j $(nproc) -o "$job_slug.yaml" -- -c phoenix-bench $device_opts -n $n_ranks
+    else
+        echo "didn't find a device"
+        echo "device is" $device
+        exit 1
+    fi
+
+    sleep 10
+    rm -rf "$currentdir" || true
+
+    unset TMPDIR
 EOT
 )
 

.github/workflows/phoenix/submit-bench.sh

@@ -2,16 +2,14 @@
 set -euo pipefail
 
 usage() {
-    echo "Usage: $0 [script.sh] [cpu|gpu]"
+    echo "Usage: $0 [cpu|gpu]"
     exit 1
 }
 
-[[ $# -eq 2 ]] || usage
+[[ $# -eq 1 ]] || usage
 
-sbatch_script="$1"
-device="$2"
-
-job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2"
+device="$1"
+job_slug="test-$1"
 
 # read the body of the user script
 sbatch_body=$(<"$sbatch_script")
@@ -49,6 +47,12 @@ JOBID=$(sbatch <<-EOT | awk '{print $4}'
 	${sbatch_common_opts}
 	${sbatch_device_opts}
 
+	export job_slug="${job_slug}"
+	export device="${device}"
+
+	echo "Job slug is:" $job_slug
+ 	echo "Device is:" $device
+
 	set -e -x
 
 	cd "\$SLURM_SUBMIT_DIR"
@@ -58,7 +62,45 @@ JOBID=$(sbatch <<-EOT | awk '{print $4}'
 	. ./mfc.sh load -c p -m $device
 
 	# user script contents
-	${sbatch_body}
+    tmpbuild=/storage/scratch1/6/sbryngelson3/mytmp_build
+    currentdir=$tmpbuild/run-$(( RANDOM % 900 ))
+    mkdir -p $tmpbuild
+    mkdir -p $currentdir
+    export TMPDIR=$currentdir
+
+    n_test_threads=8
+
+    build_opts=""
+    if [ "$device" = "gpu" ]; then
+        build_opts="--gpu"
+    fi
+    echo "build_opts =" $build_opts
+
+    if [[ "$device" == "cpu" ]]; then
+        echo "CPU BUILD"
+    elif [[ "$device" == "gpu" ]]; then
+        echo "GPU BUILD"
+    else
+      exit 1
+    fi
+
+    exit 1
+
+    ./mfc.sh test --dry-run -j $n_test_threads $build_opts
+
+    if [ "$device" = "gpu" ]; then
+        gpu_count=$(nvidia-smi -L | wc -l)        # number of GPUs on node
+        gpu_ids=$(seq -s ' ' 0 $(($gpu_count-1))) # 0,1,2,...,gpu_count-1
+        device_opts="-g $gpu_ids"
+        n_test_threads=`expr $gpu_count \* 2`
+    fi
+
+    ./mfc.sh test --max-attempts 3 -a -j $n_test_threads $device_opts -- -c phoenix
+
+    sleep 10
+    rm -rf "$currentdir" || true
+
+    unset TMPDIR
 EOT
 )
 

.github/workflows/phoenix/submit.sh

@@ -111,7 +111,7 @@ jobs:
 
       - name: Build & Test
         if:   matrix.lbl == 'gt'
-        run:  bash .github/workflows/phoenix/submit.sh .github/workflows/phoenix/test.sh ${{ matrix.device }}
+        run:  bash .github/workflows/phoenix/submit.sh ${{ matrix.device }}
 
       - name: Build
         if:   matrix.lbl == 'frontier'