Skip to content

Commit ced589a

Browse files
prathi-windprathi-wind
committed
OpenMP CI added
1 parent 8db9452 commit ced589a

File tree

12 files changed

+98
-28
lines changed

12 files changed

+98
-28
lines changed

‎.github/workflows/bench.yml‎

Lines changed: 28 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -36,21 +36,40 @@ jobs:
3636
labels: gt
3737
flag: p
3838
device: cpu
39+
interface: none
3940
build_script: ""
4041
- cluster: phoenix
4142
name: Georgia Tech | Phoenix (NVHPC)
4243
group: phoenix
4344
labels: gt
4445
flag: p
4546
device: gpu
47+
interface: acc
4648
build_script: ""
49+
- cluster: phoenix
50+
name: Georgia Tech | Phoenix (NVHPC)
51+
group: phoenix
52+
labels: gt
53+
flag: p
54+
device: gpu
55+
interface: omp
56+
build_script: ""
57+
- cluster: frontier
58+
name: Oak Ridge | Frontier (CCE)
59+
group: phoenix
60+
labels: frontier
61+
flag: f
62+
device: gpu
63+
interface: acc
64+
build_script: "bash .github/workflows/frontier/build.sh gpu acc bench"
4765
- cluster: frontier
4866
name: Oak Ridge | Frontier (CCE)
4967
group: phoenix
5068
labels: frontier
5169
flag: f
5270
device: gpu
53-
build_script: "bash .github/workflows/frontier/build.sh gpu bench"
71+
interface: omp
72+
build_script: "bash .github/workflows/frontier/build.sh gpu omp bench"
5473
runs-on:
5574
group: ${{ matrix.group }}
5675
labels: ${{ matrix.labels }}
@@ -80,29 +99,29 @@ jobs:
8099
81100
- name: Bench (Master v. PR)
82101
run: |
83-
(cd pr && bash .github/workflows/${{ matrix.cluster }}/submit-bench.sh .github/workflows/${{ matrix.cluster }}/bench.sh ${{ matrix.device }}) &
84-
(cd master && bash .github/workflows/${{ matrix.cluster }}/submit-bench.sh .github/workflows/${{ matrix.cluster }}/bench.sh ${{ matrix.device }}) &
102+
(cd pr && bash .github/workflows/${{ matrix.cluster }}/submit-bench.sh .github/workflows/${{ matrix.cluster }}/bench.sh ${{ matrix.device }} ${{ matrix.interface }}) &
103+
(cd master && bash .github/workflows/${{ matrix.cluster }}/submit-bench.sh .github/workflows/${{ matrix.cluster }}/bench.sh ${{ matrix.device }} ${{ matrix.interface }}) &
85104
wait %1 && wait %2
86105
87106
- name: Generate & Post Comment
88107
run: |
89108
(cd pr && . ./mfc.sh load -c ${{ matrix.flag }} -m g)
90-
(cd pr && ./mfc.sh bench_diff ../master/bench-${{ matrix.device }}.yaml ../pr/bench-${{ matrix.device }}.yaml)
109+
(cd pr && ./mfc.sh bench_diff ../master/bench-${{ matrix.device }}-${{ matrix.interface }}.yaml ../pr/bench-${{ matrix.device }}-${{ matrix.interface }}.yaml)
91110
92111
- name: Print Logs
93112
if: always()
94113
run: |
95-
cat pr/bench-${{ matrix.device }}.* 2>/dev/null || true
96-
cat master/bench-${{ matrix.device }}.* 2>/dev/null || true
114+
cat pr/bench-${{ matrix.device }}-${{ matrix.interface }}.* 2>/dev/null || true
115+
cat master/bench-${{ matrix.device }}-${{ matrix.interface }}.* 2>/dev/null || true
97116
98117
# All other runners (non-Phoenix) just run without special env
99118
- name: Archive Logs (Frontier)
100119
if: always() && matrix.cluster != 'phoenix'
101120
uses: actions/upload-artifact@v4
102121
with:
103-
name: ${{ matrix.cluster }}-${{ matrix.device }}
122+
name: ${{ matrix.cluster }}-${{ matrix.device }}-${{ matrix.interface }}
104123
path: |
105-
pr/bench-${{ matrix.device }}.*
124+
pr/bench-${{ matrix.device }}-${{ matrix.interface }}.*
106125
pr/build/benchmarks/*
107-
master/bench-${{ matrix.device }}.*
126+
master/bench-${{ matrix.device }}-${{ matrix.interface }}.*
108127
master/build/benchmarks/*

‎.github/workflows/frontier/bench.sh‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,18 @@
11
#!/bin/bash
22

33
n_ranks=12
4-
4+
device_opts=""
55
if [ "$job_device" = "gpu" ]; then
66
gpus=$(rocm-smi --showid | awk '{print $1}' | grep -Eo '[0-9]+' | uniq | tr '\n' ' ')
77
n_ranks=$(echo "$gpus" | wc -w) # number of GPUs on node
88
gpu_ids=$(echo "$gpus" | tr ' ' '\n' | tr '\n' ' ' | sed 's/ $//') # GPU IDs from rocm-smi
9-
device_opts="--gpu -g $gpu_ids"
9+
device_opts+="--gpu"
10+
if [ "$job_interface" = "acc" ]; then
11+
device_opts+=" acc"
12+
elif [ "$job_interface" = "omp" ]; then
13+
device_opts+=" mp"
14+
fi
15+
device_opts+=" -g $gpu_ids"
1016
fi
1117

1218
if [ "$job_device" = "gpu" ]; then

‎.github/workflows/frontier/build.sh‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,21 @@
11
#!/bin/bash
22

3+
job_device=$1
4+
job_interface=$2
5+
run_bench=$3
36
build_opts=""
4-
if [ "$1" = "gpu" ]; then
5-
build_opts="--gpu"
7+
if [ "$job_device" = "gpu" ]; then
8+
build_opts+="--gpu"
9+
if [ "$job_interface" = "acc" ]; then
10+
build_opts+=" acc"
11+
elif [ "$job_interface" = "omp" ]; then
12+
build_opts+=" mp"
13+
fi
614
fi
715

816
. ./mfc.sh load -c f -m g
917

10-
if [ "$2" == "bench" ]; then
18+
if [ "$run_bench" == "bench" ]; then
1119
for dir in benchmarks/*/; do
1220
dirname=$(basename "$dir")
1321
./mfc.sh run "$dir/case.py" --case-optimization -j 8 --dry-run $build_opts

‎.github/workflows/frontier/submit-bench.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ echo "Running in $(pwd):"
4545
4646
job_slug="$job_slug"
4747
job_device="$2"
48+
job_interface="$3"
4849
4950
. ./mfc.sh load -c f -m g
5051

‎.github/workflows/frontier/submit.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ echo "Running in $(pwd):"
4646
4747
job_slug="$job_slug"
4848
job_device="$2"
49+
job_interface="$3"
4950
5051
. ./mfc.sh load -c f -m g
5152

‎.github/workflows/frontier/test.sh‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,21 @@
33
gpus=`rocm-smi --showid | awk '{print $1}' | grep -Eo '[0-9]+' | uniq | tr '\n' ' '`
44
ngpus=`echo "$gpus" | tr -d '[:space:]' | wc -c`
55

6+
device_opts=""
67
if [ "$job_device" = "gpu" ]; then
7-
./mfc.sh test -a --rdma-mpi --max-attempts 3 -j $ngpus -- -c frontier
8+
n_ranks=$(echo "$gpus" | wc -w) # number of GPUs on node
9+
gpu_ids=$(echo "$gpus" | tr ' ' '\n' | tr '\n' ' ' | sed 's/ $//') # GPU IDs from rocm-smi
10+
device_opts+="--gpu"
11+
if [ "$job_interface" = "acc" ]; then
12+
device_opts+=" acc"
13+
elif [ "$job_interface" = "omp" ]; then
14+
device_opts+=" mp"
15+
fi
16+
device_opts+=" -g $gpu_ids"
17+
fi
18+
19+
if [ "$job_device" = "gpu" ]; then
20+
./mfc.sh test -a --rdma-mpi --max-attempts 3 -j $ngpus $job_device -- -c frontier
821
else
922
./mfc.sh test -a --rdma-mpi --max-attempts 3 -j 32 -- -c frontier
1023
fi

‎.github/workflows/phoenix/bench.sh‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,18 @@
22

33
n_ranks=12
44

5+
echo "My interface is:" $job_interface
6+
device_opts=""
57
if [ "$job_device" = "gpu" ]; then
68
n_ranks=$(nvidia-smi -L | wc -l) # number of GPUs on node
79
gpu_ids=$(seq -s ' ' 0 $(($n_ranks-1))) # 0,1,2,...,gpu_count-1
8-
device_opts="--gpu -g $gpu_ids"
10+
device_opts+="--gpu"
11+
if [ "$job_interface" = "acc" ]; then
12+
device_opts+=" acc"
13+
elif [ "$job_interface" = "omp" ]; then
14+
device_opts+=" mp"
15+
fi
16+
device_opts+=" -g $gpu_ids"
917
fi
1018

1119
tmpbuild=/storage/scratch1/6/sbryngelson3/mytmp_build

‎.github/workflows/phoenix/submit-bench.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
set -e
44

55
usage() {
6-
echo "Usage: $0 [script.sh] [cpu|gpu]"
6+
echo "Usage: $0 [script.sh] [cpu|gpu] [none|acc|omp]"
77
}
88

99
if [ ! -z "$1" ]; then
@@ -55,6 +55,7 @@ echo "Running in $(pwd):"
5555
5656
job_slug="$job_slug"
5757
job_device="$2"
58+
job_interface="$3"
5859
5960
. ./mfc.sh load -c p -m $2
6061

‎.github/workflows/phoenix/submit.sh‎

100644100755
Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
set -e
44

55
usage() {
6-
echo "Usage: $0 [script.sh] [cpu|gpu]"
6+
echo "Usage: $0 [script.sh] [cpu|gpu] [none|acc|omp]"
77
}
88

99
if [ ! -z "$1" ]; then
@@ -55,10 +55,10 @@ echo "Running in $(pwd):"
5555
5656
job_slug="$job_slug"
5757
job_device="$2"
58+
job_interface="$3"
5859
5960
. ./mfc.sh load -c p -m $2
6061
6162
$sbatch_script_contents
6263
63-
EOT
64-
64+
EOT

‎.github/workflows/phoenix/test.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,11 @@
33
build_opts=""
44
if [ "$job_device" = "gpu" ]; then
55
build_opts="--gpu"
6+
if [ "$job_interface" = "omp" ]; then
7+
build_opts+=" mp"
8+
elif [ "$job_interface" = "acc" ]; then
9+
build_opts+=" acc"
10+
fi
611
fi
712

813
./mfc.sh test --dry-run -j 8 $build_opts

0 commit comments

Comments
 (0)