Skip to content

Commit 7169711

Browse files
anandrdbzanandrdbz
authored
Merge pull request #1035 from prathi-wind/openmp_cce
Openmp cce
2 parents 76b6620 + 5fa68fa commit 7169711

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

67 files changed

+4798
-3312
lines changed

‎.github/workflows/bench.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,14 @@ jobs:
6262
device: gpu
6363
interface: acc
6464
build_script: "bash .github/workflows/frontier/build.sh gpu acc bench"
65+
- cluster: frontier
66+
name: Oak Ridge | Frontier (CCE)
67+
group: phoenix
68+
labels: frontier
69+
flag: f
70+
device: gpu
71+
interface: omp
72+
build_script: "bash .github/workflows/frontier/build.sh gpu omp bench"
6573
runs-on:
6674
group: ${{ matrix.group }}
6775
labels: ${{ matrix.labels }}

‎.github/workflows/test.yml‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -111,10 +111,6 @@ jobs:
111111
- device: 'cpu'
112112
interface: 'none'
113113
lbl: 'frontier'
114-
exclude:
115-
- device: 'gpu'
116-
interface: 'omp'
117-
lbl: 'frontier'
118114
runs-on:
119115
group: phoenix
120116
labels: ${{ matrix.lbl }}

‎CMakeLists.txt‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ option(MFC_SYSCHECK "Build syscheck" OFF
2929
option(MFC_DOCUMENTATION "Build documentation" OFF)
3030
option(MFC_ALL "Build everything" OFF)
3131
option(MFC_SINGLE_PRECISION "Build single precision" OFF)
32+
option(MFC_MIXED_PRECISION "Build mixed precision" OFF)
3233

3334
if (MFC_ALL)
3435
set(MFC_PRE_PROCESS ON FORCE)
@@ -43,6 +44,10 @@ else()
4344
add_compile_definitions(MFC_DOUBLE_PRECISION)
4445
endif()
4546

47+
if (MFC_MIXED_PRECISION)
48+
add_compile_definitions(MFC_MIXED_PRECISION)
49+
endif()
50+
4651

4752
# CMake Library Imports
4853

@@ -196,7 +201,7 @@ elseif ((CMAKE_Fortran_COMPILER_ID STREQUAL "NVHPC") OR (CMAKE_Fortran_COMPILER_
196201
add_compile_options(
197202
$<$<COMPILE_LANGUAGE:Fortran>:-Mfreeform>
198203
$<$<COMPILE_LANGUAGE:Fortran>:-cpp>
199-
$<$<COMPILE_LANGUAGE:Fortran>:-Minfo=inline>
204+
$<$<COMPILE_LANGUAGE:Fortran>:-Minfo=inline>
200205
$<$<COMPILE_LANGUAGE:Fortran>:-Minfo=accel>
201206
)
202207

@@ -474,10 +479,10 @@ function(MFC_SETUP_TARGET)
474479
endif()
475480
endif()
476481

477-
if (ARGS_LAPACK)
478-
find_package(LAPACK REQUIRED)
479-
target_link_libraries(${a_target} PRIVATE LAPACK::LAPACK)
480-
endif()
482+
if (ARGS_LAPACK)
483+
find_package(LAPACK REQUIRED)
484+
target_link_libraries(${a_target} PRIVATE LAPACK::LAPACK)
485+
endif()
481486

482487
if ((MFC_OpenACC AND ARGS_OpenACC) OR (MFC_OpenMP AND ARGS_OpenMP))
483488
if ((MFC_OpenACC AND ARGS_OpenACC))
@@ -573,7 +578,13 @@ function(MFC_SETUP_TARGET)
573578
find_package(hipfort COMPONENTS hip CONFIG REQUIRED)
574579
target_link_libraries(${a_target} PRIVATE hipfort::hip hipfort::hipfort-amdgcn)
575580
elseif(CMAKE_Fortran_COMPILER_ID STREQUAL "LLVMFlang")
576-
find_package(hipfort COMPONENTS hip CONFIG REQUIRED)
581+
582+
if (MFC_Unified)
583+
target_compile_options(${ARGS_TARGET}
584+
PRIVATE -DFRONTIER_UNIFIED)
585+
endif()
586+
587+
find_package(hipfort COMPONENTS hip CONFIG REQUIRED)
577588
target_link_libraries(${a_target} PRIVATE hipfort::hip hipfort::hipfort-amdgcn flang_rt.hostdevice)
578589
endif()
579590
elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")

‎examples/3D_IGR_33jet/case.py‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
#!/usr/bin/env python3
2+
import math
3+
import json
4+
5+
# Domain parameters
6+
D = 2.5 # Jet diameter
7+
Nd = 711 # Cells per jet diameter
8+
9+
x0 = 0 # x_beg coordinate
10+
x1 = 19 * D # x_end coordinate
11+
y0 = -22 * D / 2 # y_beg coordinate
12+
y1 = 22 * D / 2 # y_end coordinate
13+
z0 = -22 * D / 2 # y_beg coordinate
14+
z1 = 22 * D / 2 # y_end coordinate
15+
Nx = int(Nd * (x1 - x0) / D) - 1
16+
Ny = int(Nd * (y1 - y0) / D) - 1
17+
Nz = int(Nd * (z1 - z0) / D) - 1
18+
19+
time_end = 5
20+
igrIters = 5
21+
22+
dx = D / Nd
23+
dt = dx / 36
24+
25+
Nt = int(time_end / dt)
26+
27+
# Configuring case dictionary
28+
print(
29+
json.dumps(
30+
{
31+
# Logistics
32+
"run_time_info": "T",
33+
# Computational Domain Parameters
34+
"x_domain%beg": x0,
35+
"x_domain%end": x1,
36+
"y_domain%beg": y0,
37+
"y_domain%end": y1,
38+
"z_domain%beg": z0,
39+
"z_domain%end": z1,
40+
"m": int(Nx),
41+
"n": int(Ny),
42+
"p": int(Nz),
43+
"dt": dt,
44+
"t_step_start": 0,
45+
"t_step_stop": 10, # Nt,
46+
"t_step_save": 10, # int(Nt/50),
47+
# Simulation Algorithm Parameters
48+
"num_patches": 1,
49+
"num_bc_patches": 0,
50+
"model_eqns": 2,
51+
"alt_soundspeed": "F",
52+
"num_fluids": 1,
53+
"mpp_lim": "F",
54+
"time_stepper": 3,
55+
"igr": "T",
56+
"igr_order": 3,
57+
"igr_pres_lim": "T",
58+
"igr_iter_solver": 1,
59+
"num_igr_iters": igrIters,
60+
"num_igr_warm_start_iters": 10 * igrIters,
61+
"alf_factor": 10,
62+
"bc_x%beg": -17,
63+
"bc_x%end": -3,
64+
"bc_y%beg": -3,
65+
"bc_y%end": -3,
66+
"bc_z%beg": -3,
67+
"bc_z%end": -3,
68+
# Formatted Database Files Structure Parameters
69+
"format": 1,
70+
"precision": 1,
71+
"prim_vars_wrt": "T",
72+
"file_per_process": "T",
73+
"parallel_io": "T",
74+
"down_sample": "T",
75+
# Patch
76+
"patch_icpp(1)%geometry": 9,
77+
"patch_icpp(1)%x_centroid": (x1 + x0) / 2,
78+
"patch_icpp(1)%y_centroid": (y1 + y0) / 2,
79+
"patch_icpp(1)%z_centroid": (z1 + z0) / 2,
80+
"patch_icpp(1)%length_x": 2 * (x1 - x0),
81+
"patch_icpp(1)%length_y": 2 * (y1 - y0),
82+
"patch_icpp(1)%length_z": 2 * (z1 - z0),
83+
"patch_icpp(1)%hcid": 303,
84+
"patch_icpp(1)%vel(1)": 0.0e00,
85+
"patch_icpp(1)%vel(2)": 0.0e00,
86+
"patch_icpp(1)%vel(3)": 0.0e00,
87+
"patch_icpp(1)%pres": 1.0e00,
88+
"patch_icpp(1)%alpha_rho(1)": 1.0,
89+
"patch_icpp(1)%alpha(1)": 1.0,
90+
# Perturbation of velocity field
91+
"simplex_perturb": "T",
92+
"simplex_params%perturb_vel(1)": "T",
93+
"simplex_params%perturb_vel_freq(1)": 3,
94+
"simplex_params%perturb_vel_scale(1)": 0.02,
95+
"simplex_params%perturb_vel_offset(1,1)": 12.3,
96+
"simplex_params%perturb_vel_offset(1,2)": -11.3,
97+
"simplex_params%perturb_vel_offset(1,3)": 34.6,
98+
"simplex_params%perturb_vel(2)": "T",
99+
"simplex_params%perturb_vel_freq(2)": 2,
100+
"simplex_params%perturb_vel_scale(2)": 0.02,
101+
"simplex_params%perturb_vel_offset(2,1)": -70.3,
102+
"simplex_params%perturb_vel_offset(2,2)": 33.4,
103+
"simplex_params%perturb_vel_offset(2,3)": -34.6,
104+
"simplex_params%perturb_vel(3)": "T",
105+
"simplex_params%perturb_vel_freq(3)": 2,
106+
"simplex_params%perturb_vel_scale(3)": 0.02,
107+
"simplex_params%perturb_vel_offset(3,1)": 123.3,
108+
"simplex_params%perturb_vel_offset(3,2)": -654.3,
109+
"simplex_params%perturb_vel_offset(3,3)": -64.5,
110+
# Fluids Physical Parameters
111+
"fluid_pp(1)%gamma": 1.0e00 / (1.4e00 - 1.0e00),
112+
"fluid_pp(1)%pi_inf": 0.0,
113+
"viscous": "T",
114+
"fluid_pp(1)%Re(1)": 5e5,
115+
},
116+
indent=4,
117+
)
118+
)

‎examples/3D_IGR_33jet/jets.csv‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
3.000000,0.000000,0.001
2+
-1.500000,2.598076,0.001
3+
-1.500000,-2.598076,0.001
4+
9.000000,0.000000,0.001
5+
7.281153,5.290067,0.001
6+
2.781153,8.559509,0.001
7+
-2.781153,8.559509,0.001
8+
-7.281153,5.290067,0.001
9+
-9.000000,0.000000,0.001
10+
-7.281153,-5.290067,0.001
11+
-2.781153,-8.559509,0.001
12+
2.781153,-8.559509,0.001
13+
7.281153,-5.290067,0.001
14+
15.000000,0.000000,0.001
15+
14.265848,4.635255,0.001
16+
12.135255,8.816779,0.001
17+
8.816779,12.135255,0.001
18+
4.635255,14.265848,0.001
19+
0.000000,15.000000,0.001
20+
-4.635255,14.265848,0.001
21+
-8.816779,12.135255,0.001
22+
-12.135255,8.816779,0.001
23+
-14.265848,4.635255,0.001
24+
-15.000000,0.000000,0.001
25+
-14.265848,-4.635255,0.001
26+
-12.135255,-8.816779,0.001
27+
-8.816779,-12.135255,0.001
28+
-4.635255,-14.265848,0.001
29+
-0.000000,-15.000000,0.001
30+
4.635255,-14.265848,0.001
31+
8.816779,-12.135255,0.001
32+
12.135255,-8.816779,0.001
33+
14.265848,-4.635255,0.001

‎examples/3D_IGR_33jet/njet.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
33

‎examples/3D_IGR_jet_1fluid/case.py‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
#!/usr/bin/env python3
2+
import json
3+
4+
# Domain parameters
5+
alfFactor = 10
6+
igrIters = 3
7+
8+
NPx = 2
9+
NPy = 2
10+
NPz = 2
11+
12+
x0 = 0
13+
x1 = 2 * NPx
14+
y0 = -1.0 * NPy
15+
y1 = 1.0 * NPy
16+
z0 = -1.0 * NPz
17+
z1 = 1.0 * NPz
18+
19+
N = 1383
20+
21+
Nx = N * NPx - 1
22+
Ny = N * NPy - 1
23+
Nz = N * NPz - 1
24+
25+
dx = (x1 - x0) / Nx
26+
dt = dx / 20000
27+
28+
# Configuring case dictionary
29+
print(
30+
json.dumps(
31+
{
32+
# Logistics
33+
"run_time_info": "F",
34+
# Computational Domain Parameters
35+
"x_domain%beg": x0,
36+
"x_domain%end": x1,
37+
"y_domain%beg": y0,
38+
"y_domain%end": y1,
39+
"z_domain%beg": z0,
40+
"z_domain%end": z1,
41+
"m": int(Nx),
42+
"n": int(Ny),
43+
"p": int(Nz),
44+
"dt": dt,
45+
"t_step_start": 0,
46+
"t_step_stop": 10, # Nt,
47+
"t_step_save": 10, # int(Nt / 20),
48+
# Simulation Algorithm Parameters
49+
"num_patches": 1,
50+
"num_bc_patches": 0,
51+
"model_eqns": 2,
52+
"alt_soundspeed": "F",
53+
"num_fluids": 1,
54+
"mpp_lim": "F",
55+
"time_stepper": 3,
56+
"igr": "T",
57+
"igr_order": 5,
58+
"igr_pres_lim": "T",
59+
"igr_iter_solver": 1,
60+
"num_igr_iters": igrIters,
61+
"num_igr_warm_start_iters": igrIters,
62+
"alf_factor": 10,
63+
"bc_x%beg": -17,
64+
"bc_x%end": -3,
65+
"bc_y%beg": -3,
66+
"bc_y%end": -3,
67+
"bc_z%beg": -3,
68+
"bc_z%end": -3,
69+
# Formatted Database Files Structure Parameters
70+
"format": 1,
71+
"precision": 2,
72+
"prim_vars_wrt": "T",
73+
"c_wrt": "F",
74+
"parallel_io": "T",
75+
"file_per_process": "T",
76+
"down_sample": "F",
77+
# Background
78+
"patch_icpp(1)%geometry": 9,
79+
"patch_icpp(1)%x_centroid": (x1 + x0) / 2,
80+
"patch_icpp(1)%y_centroid": (y1 + y0) / 2,
81+
"patch_icpp(1)%z_centroid": (z1 + z0) / 2,
82+
"patch_icpp(1)%length_x": (x1 - x0),
83+
"patch_icpp(1)%length_y": (y1 - y0),
84+
"patch_icpp(1)%length_z": (z1 - z0),
85+
"patch_icpp(1)%hcid": 302,
86+
"patch_icpp(1)%vel(1)": 1.0,
87+
"patch_icpp(1)%vel(2)": 0.0e00,
88+
"patch_icpp(1)%vel(3)": 0.0e00,
89+
"patch_icpp(1)%pres": 1.0,
90+
"patch_icpp(1)%alpha_rho(1)": 1.0e00,
91+
"patch_icpp(1)%alpha(1)": 1.0e00,
92+
# Fluids Physical Parameters
93+
"fluid_pp(1)%gamma": 1.0e00 / (1.4e00 - 1.0e00),
94+
"fluid_pp(1)%pi_inf": 0.0,
95+
"viscous": "T",
96+
"fluid_pp(1)%Re(1)": 5e4,
97+
},
98+
indent=4,
99+
)
100+
)

‎load_amd.sh‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
module use /ccs/home/bcornille/afar-drops/modulefiles/Core/
2+
module load rocm-afar-drop mpich cray-python
3+
module load cmake
4+
#export OMPX_APU_MAPS=0
5+
#export HSA_XNACK=0
6+
#export LIBOMPTARGET_INFO=0
7+
#export AMD_LOG_LEVEL=1

0 commit comments

Comments
 (0)