Skip to content

Commit 531a46f

Browse files
prathi-windTanush PrathiAnand RadhakrishnanwilfonbaTanush Prathi
authored
Add OpenMP support for Nvidia hardware (#999)
Co-authored-by: Tanush Prathi <[email protected]> Co-authored-by: Anand Radhakrishnan <[email protected]> Co-authored-by: Ben Wilfong <[email protected]> Co-authored-by: Tanush Prathi <[email protected]> Co-authored-by: Tanush Prathi <[email protected]> Co-authored-by: Anand Radhakrishnan <[email protected]> Co-authored-by: Tanush Prathi <[email protected]> Co-authored-by: Tanush Prathi <[email protected]> Co-authored-by: Tanush Prathi <[email protected]> Co-authored-by: Tanush Prathi <[email protected]>
1 parent 2da0daf commit 531a46f

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

70 files changed

+12578
-11502
lines changed

‎.github/workflows/bench.yml‎

Lines changed: 20 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -36,21 +36,32 @@ jobs:
3636
labels: gt
3737
flag: p
3838
device: cpu
39+
interface: none
3940
build_script: ""
4041
- cluster: phoenix
4142
name: Georgia Tech | Phoenix (NVHPC)
4243
group: phoenix
4344
labels: gt
4445
flag: p
4546
device: gpu
47+
interface: acc
48+
build_script: ""
49+
- cluster: phoenix
50+
name: Georgia Tech | Phoenix (NVHPC)
51+
group: phoenix
52+
labels: gt
53+
flag: p
54+
device: gpu
55+
interface: omp
4656
build_script: ""
4757
- cluster: frontier
4858
name: Oak Ridge | Frontier (CCE)
4959
group: phoenix
5060
labels: frontier
5161
flag: f
5262
device: gpu
53-
build_script: "bash .github/workflows/frontier/build.sh gpu bench"
63+
interface: acc
64+
build_script: "bash .github/workflows/frontier/build.sh gpu acc bench"
5465
runs-on:
5566
group: ${{ matrix.group }}
5667
labels: ${{ matrix.labels }}
@@ -80,29 +91,29 @@ jobs:
8091
8192
- name: Bench (Master v. PR)
8293
run: |
83-
(cd pr && bash .github/workflows/${{ matrix.cluster }}/submit-bench.sh .github/workflows/${{ matrix.cluster }}/bench.sh ${{ matrix.device }}) &
84-
(cd master && bash .github/workflows/${{ matrix.cluster }}/submit-bench.sh .github/workflows/${{ matrix.cluster }}/bench.sh ${{ matrix.device }}) &
94+
(cd pr && bash .github/workflows/${{ matrix.cluster }}/submit-bench.sh .github/workflows/${{ matrix.cluster }}/bench.sh ${{ matrix.device }} ${{ matrix.interface }}) &
95+
(cd master && bash .github/workflows/${{ matrix.cluster }}/submit-bench.sh .github/workflows/${{ matrix.cluster }}/bench.sh ${{ matrix.device }} ${{ matrix.interface }}) &
8596
wait %1 && wait %2
8697
8798
- name: Generate & Post Comment
8899
run: |
89100
(cd pr && . ./mfc.sh load -c ${{ matrix.flag }} -m g)
90-
(cd pr && ./mfc.sh bench_diff ../master/bench-${{ matrix.device }}.yaml ../pr/bench-${{ matrix.device }}.yaml)
101+
(cd pr && ./mfc.sh bench_diff ../master/bench-${{ matrix.device }}-${{ matrix.interface }}.yaml ../pr/bench-${{ matrix.device }}-${{ matrix.interface }}.yaml)
91102
92103
- name: Print Logs
93104
if: always()
94105
run: |
95-
cat pr/bench-${{ matrix.device }}.* 2>/dev/null || true
96-
cat master/bench-${{ matrix.device }}.* 2>/dev/null || true
106+
cat pr/bench-${{ matrix.device }}-${{ matrix.interface }}.* 2>/dev/null || true
107+
cat master/bench-${{ matrix.device }}-${{ matrix.interface }}.* 2>/dev/null || true
97108
98109
# All other runners (non-Phoenix) just run without special env
99110
- name: Archive Logs (Frontier)
100111
if: always() && matrix.cluster != 'phoenix'
101112
uses: actions/upload-artifact@v4
102113
with:
103-
name: ${{ matrix.cluster }}-${{ matrix.device }}
114+
name: ${{ matrix.cluster }}-${{ matrix.device }}-${{ matrix.interface }}
104115
path: |
105-
pr/bench-${{ matrix.device }}.*
116+
pr/bench-${{ matrix.device }}-${{ matrix.interface }}.*
106117
pr/build/benchmarks/*
107-
master/bench-${{ matrix.device }}.*
118+
master/bench-${{ matrix.device }}-${{ matrix.interface }}.*
108119
master/build/benchmarks/*

‎.github/workflows/frontier/bench.sh‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,18 @@
11
#!/bin/bash
22

33
n_ranks=12
4-
4+
device_opts=""
55
if [ "$job_device" = "gpu" ]; then
66
gpus=$(rocm-smi --showid | awk '{print $1}' | grep -Eo '[0-9]+' | uniq | tr '\n' ' ')
77
n_ranks=$(echo "$gpus" | wc -w) # number of GPUs on node
88
gpu_ids=$(echo "$gpus" | tr ' ' '\n' | tr '\n' ' ' | sed 's/ $//') # GPU IDs from rocm-smi
9-
device_opts="--gpu -g $gpu_ids"
9+
device_opts+="--gpu"
10+
if [ "$job_interface" = "acc" ]; then
11+
device_opts+=" acc"
12+
elif [ "$job_interface" = "omp" ]; then
13+
device_opts+=" mp"
14+
fi
15+
device_opts+=" -g $gpu_ids"
1016
fi
1117

1218
if [ "$job_device" = "gpu" ]; then

‎.github/workflows/frontier/build.sh‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,21 @@
11
#!/bin/bash
22

3+
job_device=$1
4+
job_interface=$2
5+
run_bench=$3
36
build_opts=""
4-
if [ "$1" = "gpu" ]; then
5-
build_opts="--gpu"
7+
if [ "$job_device" = "gpu" ]; then
8+
build_opts+="--gpu"
9+
if [ "$job_interface" = "acc" ]; then
10+
build_opts+=" acc"
11+
elif [ "$job_interface" = "omp" ]; then
12+
build_opts+=" mp"
13+
fi
614
fi
715

816
. ./mfc.sh load -c f -m g
917

10-
if [ "$2" == "bench" ]; then
18+
if [ "$run_bench" == "bench" ]; then
1119
for dir in benchmarks/*/; do
1220
dirname=$(basename "$dir")
1321
./mfc.sh run "$dir/case.py" --case-optimization -j 8 --dry-run $build_opts

‎.github/workflows/frontier/submit-bench.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ else
2424
fi
2525

2626

27-
job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2"
27+
job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2-$3"
2828

2929
sbatch <<EOT
3030
#!/bin/bash
@@ -45,6 +45,7 @@ echo "Running in $(pwd):"
4545
4646
job_slug="$job_slug"
4747
job_device="$2"
48+
job_interface="$3"
4849
4950
. ./mfc.sh load -c f -m g
5051

‎.github/workflows/frontier/submit.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ else
2525
fi
2626

2727

28-
job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2"
28+
job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2-$3"
2929

3030
sbatch <<EOT
3131
#!/bin/bash
@@ -46,6 +46,7 @@ echo "Running in $(pwd):"
4646
4747
job_slug="$job_slug"
4848
job_device="$2"
49+
job_interface="$3"
4950
5051
. ./mfc.sh load -c f -m g
5152

‎.github/workflows/frontier/test.sh‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,18 @@
33
gpus=`rocm-smi --showid | awk '{print $1}' | grep -Eo '[0-9]+' | uniq | tr '\n' ' '`
44
ngpus=`echo "$gpus" | tr -d '[:space:]' | wc -c`
55

6+
device_opts=""
67
if [ "$job_device" = "gpu" ]; then
7-
./mfc.sh test -a --rdma-mpi --max-attempts 3 -j $ngpus -- -c frontier
8+
device_opts+="--gpu"
9+
if [ "$job_interface" = "acc" ]; then
10+
device_opts+=" acc"
11+
elif [ "$job_interface" = "omp" ]; then
12+
device_opts+=" mp"
13+
fi
14+
fi
15+
16+
if [ "$job_device" = "gpu" ]; then
17+
./mfc.sh test -a --rdma-mpi --max-attempts 3 -j $ngpus $device_opts -- -c frontier
818
else
919
./mfc.sh test -a --max-attempts 3 -j 32 -- -c frontier
1020
fi

‎.github/workflows/lint-source.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ jobs:
3030

3131
- name: Looking for raw directives
3232
run: |
33-
! grep -iR '!\$acc\|!\$omp' --exclude="parallel_macros.fpp" --exclude="syscheck.fpp" ./src/*
33+
! grep -iR '!\$acc\|!\$omp' --exclude="parallel_macros.fpp" --exclude="acc_macros.fpp" --exclude="omp_macros.fpp" --exclude="shared_parallel_macros.fpp" --exclude="syscheck.fpp" ./src/*
3434
3535
- name: No double precision intrinsics
3636
run: |

‎.github/workflows/phoenix/bench.sh‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,18 @@
22

33
n_ranks=12
44

5+
echo "My interface is:" $job_interface
6+
device_opts=""
57
if [ "$job_device" = "gpu" ]; then
68
n_ranks=$(nvidia-smi -L | wc -l) # number of GPUs on node
79
gpu_ids=$(seq -s ' ' 0 $(($n_ranks-1))) # 0,1,2,...,gpu_count-1
8-
device_opts="--gpu -g $gpu_ids"
10+
device_opts+="--gpu"
11+
if [ "$job_interface" = "acc" ]; then
12+
device_opts+=" acc"
13+
elif [ "$job_interface" = "omp" ]; then
14+
device_opts+=" mp"
15+
fi
16+
device_opts+=" -g $gpu_ids"
917
fi
1018

1119
tmpbuild=/storage/scratch1/6/sbryngelson3/mytmp_build

‎.github/workflows/phoenix/submit-bench.sh‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
set -e
44

55
usage() {
6-
echo "Usage: $0 [script.sh] [cpu|gpu]"
6+
echo "Usage: $0 [script.sh] [cpu|gpu] [none|acc|omp]"
77
}
88

99
if [ ! -z "$1" ]; then
@@ -34,7 +34,7 @@ else
3434
exit 1
3535
fi
3636

37-
job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2"
37+
job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2-$3"
3838

3939
sbatch <<EOT
4040
#!/bin/bash
@@ -55,6 +55,7 @@ echo "Running in $(pwd):"
5555
5656
job_slug="$job_slug"
5757
job_device="$2"
58+
job_interface="$3"
5859
5960
. ./mfc.sh load -c p -m $2
6061

‎.github/workflows/phoenix/submit.sh‎

100644100755
Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
set -e
44

55
usage() {
6-
echo "Usage: $0 [script.sh] [cpu|gpu]"
6+
echo "Usage: $0 [script.sh] [cpu|gpu] [none|acc|omp]"
77
}
88

99
if [ ! -z "$1" ]; then
@@ -34,7 +34,7 @@ else
3434
exit 1
3535
fi
3636

37-
job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2"
37+
job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2-$3"
3838

3939
sbatch <<EOT
4040
#!/bin/bash
@@ -55,10 +55,10 @@ echo "Running in $(pwd):"
5555
5656
job_slug="$job_slug"
5757
job_device="$2"
58+
job_interface="$3"
5859
5960
. ./mfc.sh load -c p -m $2
6061
6162
$sbatch_script_contents
6263
63-
EOT
64-
64+
EOT

0 commit comments

Comments
 (0)