Skip to content

Commit 2b3d35d

Browse files
anandrdbzhenryleberreabbottsAnand RadhakrishnanAnand Radhakrishnan
authored
OpenACC + Cray CCE + AMD MI200+ (#368)
Co-authored-by: Henry Le Berre <[email protected]> Co-authored-by: Steve Abbott <[email protected]> Co-authored-by: Anand Radhakrishnan <[email protected]> Co-authored-by: Anand Radhakrishnan <[email protected]> Co-authored-by: Anand Radhakrishnan <[email protected]> Co-authored-by: Anand Radhakrishnan <[email protected]> Co-authored-by: wilfonba <[email protected]> Co-authored-by: Anand Radhakrishnan <[email protected]> Co-authored-by: Anand Radhakrishnan <[email protected]> Co-authored-by: Anand Radhakrishnan <[email protected]> Co-authored-by: Steve Abbott <[email protected]> Co-authored-by: Abbott, Stephen R <[email protected]> Co-authored-by: Spencer Bryngelson <[email protected]> Co-authored-by: Anand Radhakrishnan <[email protected]> Co-authored-by: Ben Wilfong <[email protected]> Co-authored-by: Anand Radhakrishnan <[email protected]> Co-authored-by: Anand Radhakrishnan <[email protected]> Co-authored-by: Anand Radhakrishnan <[email protected]> Co-authored-by: Anand Radhakrishnan <[email protected]> Co-authored-by: Anand Radhakrishnan <[email protected]> Co-authored-by: Anand Radhakrishnan <[email protected]> Co-authored-by: Anand Radhakrishnan <[email protected]> Co-authored-by: Anand Radhakrishnan <[email protected]> Co-authored-by: Anand Radhakrishnan <[email protected]> Co-authored-by: Anand Radhakrishnan <[email protected]> Co-authored-by: Anand Radhakrishnan <[email protected]> Co-authored-by: Anand Radhakrishnan <[email protected]> Co-authored-by: Anand <[email protected]> Co-authored-by: Anand Radhakrishnan <[email protected]> Co-authored-by: Anand <[email protected]> Co-authored-by: Spencer Bryngelson <[email protected]>
1 parent ddb3edd commit 2b3d35d

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

53 files changed

+2064
-1087
lines changed

‎.github/workflows/frontier/build.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#!/bin/bash
2+
3+
. ./mfc.sh load -c f -m g
4+
./mfc.sh build -j 8 --gpu

‎.github/workflows/frontier/submit.sh‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
usage() {
6+
echo "Usage: $0 [script.sh] [cpu|gpu]"
7+
}
8+
9+
if [ ! -z "$1" ]; then
10+
sbatch_script_contents=`cat $1`
11+
else
12+
usage
13+
exit 1
14+
fi
15+
16+
job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2"
17+
18+
sbatch <<EOT
19+
#!/bin/bash
20+
#SBATCH -JMFC-$job_slug # Job name
21+
#SBATCH -A CFD154 # charge account
22+
#SBATCH -N 1 # Number of nodes required
23+
#SBATCH -n 8 # Number of cores required
24+
#SBATCH -t 02:00:00 # Duration of the job (Ex: 15 mins)
25+
#SBATCH -q debug # QOS Name
26+
#SBATCH -o$job_slug.out # Combined output and error messages file
27+
#SBATCH -W # Do not exit until the submitted job terminates.
28+
29+
set -e
30+
set -x
31+
32+
cd "\$SLURM_SUBMIT_DIR"
33+
echo "Running in $(pwd):"
34+
35+
job_slug="$job_slug"
36+
job_device="$2"
37+
38+
. ./mfc.sh load -c f -m g
39+
40+
$sbatch_script_contents
41+
42+
EOT
43+

‎.github/workflows/frontier/test.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/bash
2+
3+
./mfc.sh test -j 4 -a -- -c frontier

‎.github/workflows/test.yml‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -99,22 +99,35 @@ jobs:
9999
run: sudo ./mfc.sh docker ./mfc.sh test -j $(nproc) -a
100100

101101
self:
102-
name: Georgia Tech | Phoenix (NVHPC)
102+
name: Self Hosted
103103
if: github.repository == 'MFlowCode/MFC'
104104
continue-on-error: true
105105
strategy:
106106
matrix:
107107
device: ['cpu', 'gpu']
108+
lbl: ['gt', 'frontier']
109+
exclude:
110+
- device: cpu
111+
lbl: frontier
108112
runs-on:
109113
group: phoenix
110-
labels: gt
114+
labels: ${{ matrix.lbl }}
111115
steps:
112116
- name: Clone
113117
uses: actions/checkout@v3
114118

115119
- name: Build & Test
120+
if: matrix.lbl == 'gt'
116121
run: bash .github/workflows/phoenix/submit.sh .github/workflows/phoenix/test.sh ${{ matrix.device }}
117122

123+
- name: Build
124+
if: matrix.lbl == 'frontier'
125+
run: bash .github/workflows/frontier/build.sh
126+
127+
- name: Test
128+
if: matrix.lbl == 'frontier'
129+
run: bash .github/workflows/frontier/submit.sh .github/workflows/frontier/test.sh ${{matrix.device}}
130+
118131
- name: Print Logs
119132
if: always()
120133
run: cat test-${{ matrix.device }}.out

‎.typos.toml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ AttributeIDSupressMenu = "AttributeIDSupressMenu"
1010
INOUT = "INOUT"
1111
WRONLY = "WRONLY"
1212
nd = "nd"
13+
iy = "iy"
14+
HPE = "HPE"
1315

1416
[files]
15-
extend-exclude = ["docs/documentation/references*", "tests/"]
17+
extend-exclude = ["docs/documentation/references*", "tests/", "toolchain/cce_simulation_workgroup_256.sh"]

‎CMakeLists.txt‎

Lines changed: 48 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,7 @@ cmake_minimum_required(VERSION 3.20)
1212
# We include C as a language because - for some reason -
1313
# FIND_LIBRARY_USE_LIB64_PATHS is otherwise ignored.
1414

15-
project(MFC LANGUAGES C Fortran)
16-
15+
project(MFC LANGUAGES C CXX Fortran)
1716

1817
# Build options exposed to users and their default values.
1918

@@ -109,7 +108,7 @@ if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
109108

110109
if (CMAKE_BUILD_TYPE STREQUAL "Debug")
111110
add_compile_options(
112-
-fcheck=all,no-array-temps
111+
-fcheck=all,no-array-temps
113112
-fbacktrace
114113
-fimplicit-none
115114
#-ffpe-trap=invalid,zero,denormal,overflow
@@ -125,11 +124,22 @@ if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
125124
elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")
126125
add_compile_options(
127126
"SHELL:-h nomessage=296:878:1391:1069"
128-
"SHELL:-h msgs" "SHELL:-h static" "SHELL:-h keepfiles"
127+
"SHELL:-h static" "SHELL:-h keepfiles"
128+
"SHELL:-h acc_model=auto_async_none"
129+
"SHELL: -h acc_model=no_fast_addr"
130+
"SHELL: -h list=adm" "-DCRAY_ACC_SIMPLIFY" "-DCRAY_ACC_WAR"
129131
)
130132

133+
add_link_options("SHELL:-hkeepfiles")
134+
131135
if (CMAKE_BUILD_TYPE STREQUAL "Debug")
132-
add_compile_options(-e D "SHELL:-h func_trace")
136+
add_compile_options(
137+
"SHELL:-h acc_model=auto_async_none"
138+
"SHELL: -h acc_model=no_fast_addr"
139+
"SHELL: -K trap=fp" "SHELL: -G2"
140+
141+
)
142+
add_link_options("SHELL: -K trap=fp" "SHELL: -G2")
133143
endif()
134144
elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Flang")
135145
add_compile_options(
@@ -197,13 +207,14 @@ if (CMAKE_BUILD_TYPE STREQUAL "Debug")
197207
endif()
198208

199209

210+
200211
# HANDLE_SOURCES: Given a target (herein <target>):
201212
#
202213
# * Locate all source files for <target> of the type
203214
#
204215
# src/[<target>,common]/[.,include]/*.[f90,fpp].
205-
#
206-
# * For each .fpp file found with filepath <dirpath>/<filename>.fpp, using a
216+
#
217+
# * For each .fpp file found with filepath <dirpath>/<filename>.fpp, using a
207218
# custom command, instruct CMake how to generate a file with path
208219
#
209220
# src/<target>/fypp/<filename>.f90
@@ -224,7 +235,7 @@ endif()
224235
# file is modified, but also when any file with filepath of the form
225236
#
226237
# src/[<target>,common]/include/*.fpp
227-
#
238+
#
228239
# is modified. This is a reasonable compromise as modifications to .fpp files
229240
# in the include directories will be rare - by design. Other approaches would
230241
# have required a more complex CMakeLists.txt file (perhaps parsing the .fpp
@@ -333,11 +344,11 @@ function(MFC_SETUP_TARGET)
333344

334345
set_target_properties(${ARGS_TARGET} PROPERTIES Fortran_PREPROCESS ON)
335346

336-
target_include_directories(${ARGS_TARGET} PRIVATE
347+
target_include_directories(${ARGS_TARGET} PRIVATE
337348
"${CMAKE_SOURCE_DIR}/src/common"
338349
"${CMAKE_SOURCE_DIR}/src/common/include"
339350
"${CMAKE_SOURCE_DIR}/src/${ARGS_TARGET}")
340-
351+
341352
if (EXISTS "${CMAKE_SOURCE_DIR}/src/${ARGS_TARGET}/include")
342353
target_include_directories(${ARGS_TARGET} PRIVATE
343354
"${CMAKE_SOURCE_DIR}/src/${ARGS_TARGET}/include")
@@ -367,9 +378,14 @@ function(MFC_SETUP_TARGET)
367378
endif()
368379

369380
if (ARGS_FFTW)
370-
if (MFC_OpenACC AND ARGS_OpenACC AND (CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC" OR CMAKE_Fortran_COMPILER_ID STREQUAL "PGI"))
371-
find_package(CUDAToolkit REQUIRED)
372-
target_link_libraries(${ARGS_TARGET} PRIVATE CUDA::cudart CUDA::cufft)
381+
if (MFC_OpenACC AND ARGS_OpenACC)
382+
if (CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC" OR CMAKE_Fortran_COMPILER_ID STREQUAL "PGI")
383+
find_package(CUDAToolkit REQUIRED)
384+
target_link_libraries(${ARGS_TARGET} PRIVATE CUDA::cudart CUDA::cufft)
385+
else()
386+
find_package(hipfort COMPONENTS hipfft CONFIG REQUIRED)
387+
target_link_libraries(${ARGS_TARGET} PRIVATE hipfort::hipfft)
388+
endif()
373389
else()
374390
find_package(FFTW REQUIRED)
375391
target_link_libraries(${ARGS_TARGET} PRIVATE FFTW::FFTW)
@@ -420,6 +436,9 @@ function(MFC_SETUP_TARGET)
420436
PRIVATE -gpu=autocompare,debug
421437
)
422438
endif()
439+
elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")
440+
find_package(hipfort COMPONENTS hip CONFIG REQUIRED)
441+
target_link_libraries(${ARGS_TARGET} PRIVATE hipfort::hip hipfort::hipfort-amdgcn)
423442
endif()
424443
elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")
425444
target_compile_options(${ARGS_TARGET} PRIVATE "SHELL:-h noacc" "SHELL:-x acc")
@@ -438,19 +457,31 @@ if (MFC_PRE_PROCESS)
438457
MFC_SETUP_TARGET(TARGET pre_process
439458
SOURCES "${pre_process_SRCs}"
440459
MPI)
460+
if(CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")
461+
target_compile_options(pre_process PRIVATE -hfp0)
462+
endif()
441463
endif()
442464

443465
if (MFC_SIMULATION)
444466
MFC_SETUP_TARGET(TARGET simulation
445467
SOURCES "${simulation_SRCs}"
446468
MPI OpenACC FFTW)
469+
470+
if (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray" AND MFC_OpenACC)
471+
add_custom_command(TARGET simulation POST_BUILD
472+
COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/toolchain/cce_simulation_workgroup_256.sh"
473+
"${CMAKE_CURRENT_BINARY_DIR}"
474+
WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
475+
COMMENT "Patching & Rebuilding with Cray hacks"
476+
)
477+
endif()
447478
endif()
448479

449480
if (MFC_POST_PROCESS)
450481
MFC_SETUP_TARGET(TARGET post_process
451482
SOURCES "${post_process_SRCs}"
452483
MPI SILO HDF5 FFTW)
453-
484+
454485
# -O0 is in response to https://github.com/MFlowCode/MFC-develop/issues/95
455486
target_compile_options(post_process PRIVATE -O0)
456487
endif()
@@ -468,7 +499,7 @@ if (MFC_DOCUMENTATION)
468499
add_custom_command(
469500
OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/docs/documentation/examples.md"
470501
DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/docs/examples.sh;${examples_DOCs}"
471-
COMMAND "bash" "${CMAKE_CURRENT_SOURCE_DIR}/docs/examples.sh"
502+
COMMAND "bash" "${CMAKE_CURRENT_SOURCE_DIR}/docs/examples.sh"
472503
"${CMAKE_CURRENT_SOURCE_DIR}"
473504
COMMENT "Generating examples.md"
474505
VERBATIM
@@ -500,7 +531,7 @@ if (MFC_DOCUMENTATION)
500531
set(DOXYGEN_IMAGE_PATH "\"${CMAKE_CURRENT_SOURCE_DIR}/docs/res\"\
501532
\"${CMAKE_CURRENT_SOURCE_DIR}/docs/${target}\"")
502533

503-
file(MAKE_DIRECTORY "${DOXYGEN_OUTPUT_DIRECTORY}")
534+
file(MAKE_DIRECTORY "${DOXYGEN_OUTPUT_DIRECTORY}")
504535

505536
configure_file(
506537
"${CMAKE_CURRENT_SOURCE_DIR}/docs/Doxyfile.in"
@@ -537,7 +568,7 @@ if (MFC_DOCUMENTATION)
537568
endmacro()
538569

539570
add_custom_target(documentation)
540-
571+
541572
find_package(Doxygen REQUIRED dot REQUIRED)
542573

543574
# > Fetch CSS Theme

‎docs/Doxyfile.in‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -548,7 +548,7 @@ INTERNAL_DOCS = YES
548548
# names in lower-case letters. If set to YES, upper-case letters are also
549549
# allowed. This is useful if you have classes or files whose names only differ
550550
# in case and if your file system supports case sensitive file names. Windows
551-
# (including Cygwin) ands Mac users are advised to set this option to NO.
551+
# (including Cygwin) and Mac users are advised to set this option to NO.
552552
# The default value is: system dependent.
553553

554554
CASE_SENSE_NAMES = YES

‎examples/3D_weak_scaling/analyze.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# This script is ran from the 3D_weak_scaling case directory after running
2+
# MFC with --omni -n <name>. To analyze, run chmod u+x ./analyze.sh followed
3+
# by ./analyze.sh <name>
4+
5+
omniperf analyze -p workloads/$1/mi200 --metric 0 7.1.5 7.1.6 7.1.7 7.1.8 7.1.9 16.3.1 16.3.2 16.3.7 17.3.2 17.3.3 17.3.8

‎misc/profiling_amdgpu.txt‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
Profile MFC using omniperf
2+
3+
0) Start an interactive session with the desired number of nodes and total tasks using `salloc -A [account] -J interactive -t 2:00:00 -p batch -N [nnodes] -n [total tasks]`
4+
5+
1) Generate MFC input files by running `./mfc.sh run [path to casefile] -N [nnodes] -n [total tasks] --gpu -t pre_process simulation --case-optimization`
6+
7+
2) Move to the simulation directory using `cd [path to casefile]`
8+
9+
3) - `module load` the following modules:
10+
- rocm/5.5.1
11+
- cray-python
12+
- omniperf
13+
These must be loaded in the order that they are listed.
14+
15+
4) Run `omniperf profile -n [profile name] -- [path to MFC beginning with /]/build/install/bin/simulation`
16+
17+
5) Run `omniperf analyze --gui -p [path to casefile]/workloads/[profile name]/mi200`
18+
19+
6) Determine what login node you're on, call it [node name]
20+
21+
7) Open a new terminal window and log into Frontier using `ssh -L8050:localhost:8050 username@[node name].frontier.olcf.ornl.gov`
22+
23+
8) Open a web browser and navigate to `http://localhost:8050/`

‎src/common/include/inline_conversions.fpp‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
#:def s_compute_speed_of_sound()
22
subroutine s_compute_speed_of_sound(pres, rho, gamma, pi_inf, H, adv, vel_sum, c)
3-
3+
#ifdef CRAY_ACC_WAR
4+
!DIR$ INLINEALWAYS s_compute_speed_of_sound
5+
#else
6+
!$acc routine seq
7+
#endif
48
real(kind(0d0)), intent(IN) :: pres
59
real(kind(0d0)), intent(IN) :: rho, gamma, pi_inf
610
real(kind(0d0)), intent(IN) :: H
@@ -39,7 +43,6 @@
3943
(pres + pi_inf/(gamma + 1d0))/ &
4044
(rho*(1d0 - adv(num_fluids)))
4145
end if
42-
4346
else
4447
c = ((H - 5d-1*vel_sum)/gamma)
4548
end if
@@ -49,7 +52,6 @@
4952
else
5053
c = sqrt(c)
5154
end if
52-
5355
end subroutine s_compute_speed_of_sound
5456
#:enddef
5557

0 commit comments

Comments
 (0)