Skip to content

Commit 6ac97d7

Browse files
wilfonbawilfonba
committed
Merge remote-tracking branch 'upstream/master' into meta-directive
2 parents 6bc96a8 + 1ab0f5e commit 6ac97d7

File tree

5 files changed

+163
-107
lines changed

5 files changed

+163
-107
lines changed

‎.github/workflows/bench.yml‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
name: 'Benchmark'
22

3-
on:
3+
on:
4+
pull_request:
45
pull_request_review:
56
types: [submitted]
67
workflow_dispatch:
@@ -23,7 +24,10 @@ jobs:
2324

2425
self:
2526
name: Georgia Tech | Phoenix (NVHPC)
26-
if: github.repository == 'MFlowCode/MFC' && needs.file-changes.outputs.checkall == 'true' && ${{ github.event.review.state == 'approved' }}
27+
if: ${{ github.repository == 'MFlowCode/MFC' && needs.file-changes.outputs.checkall == 'true' && (
28+
(github.event_name == 'pull_request_review' && github.event.review.state == 'approved') ||
29+
(github.event_name == 'pull_request' && github.event.pull_request.user.login == 'sbryngelson')
30+
) }}
2731
needs: file-changes
2832
strategy:
2933
matrix:

‎src/simulation/m_sim_helpers.fpp‎

Lines changed: 97 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,65 @@ module m_sim_helpers
1616

1717
contains
1818

19+
!> Computes the modified dtheta for Fourier filtering in azimuthal direction
20+
!! @param k y coordinate index
21+
!! @param l z coordinate index
22+
!! @return fltr_dtheta Modified dtheta value for cylindrical coordinates
23+
pure function f_compute_filtered_dtheta(k, l) result(fltr_dtheta)
24+
!$acc routine seq
25+
integer, intent(in) :: k, l
26+
real(wp) :: fltr_dtheta
27+
integer :: Nfq
28+
29+
if (grid_geometry == 3) then
30+
if (k == 0) then
31+
fltr_dtheta = 2._wp*pi*y_cb(0)/3._wp
32+
elseif (k <= fourier_rings) then
33+
Nfq = min(floor(2._wp*real(k, wp)*pi), (p + 1)/2 + 1)
34+
fltr_dtheta = 2._wp*pi*y_cb(k - 1)/real(Nfq, wp)
35+
else
36+
fltr_dtheta = y_cb(k - 1)*dz(l)
37+
end if
38+
else
39+
fltr_dtheta = 0._wp
40+
end if
41+
end function f_compute_filtered_dtheta
42+
43+
!> Computes inviscid CFL terms for multi-dimensional cases (2D/3D only)
44+
!! @param vel directional velocities
45+
!! @param c mixture speed of sound
46+
!! @param j x coordinate index
47+
!! @param k y coordinate index
48+
!! @param l z coordinate index
49+
!! @return cfl_terms computed CFL terms for 2D/3D cases
50+
pure function f_compute_multidim_cfl_terms(vel, c, j, k, l) result(cfl_terms)
51+
!$acc routine seq
52+
real(wp), dimension(num_vels), intent(in) :: vel
53+
real(wp), intent(in) :: c
54+
integer, intent(in) :: j, k, l
55+
real(wp) :: cfl_terms
56+
real(wp) :: fltr_dtheta
57+
58+
fltr_dtheta = f_compute_filtered_dtheta(k, l)
59+
60+
if (p > 0) then
61+
!3D
62+
if (grid_geometry == 3) then
63+
cfl_terms = min(dx(j)/(abs(vel(1)) + c), &
64+
dy(k)/(abs(vel(2)) + c), &
65+
fltr_dtheta/(abs(vel(3)) + c))
66+
else
67+
cfl_terms = min(dx(j)/(abs(vel(1)) + c), &
68+
dy(k)/(abs(vel(2)) + c), &
69+
dz(l)/(abs(vel(3)) + c))
70+
end if
71+
else
72+
!2D
73+
cfl_terms = min(dx(j)/(abs(vel(1)) + c), &
74+
dy(k)/(abs(vel(2)) + c))
75+
end if
76+
end function f_compute_multidim_cfl_terms
77+
1978
!> Computes enthalpy
2079
!! @param q_prim_vf cell centered primitive variables
2180
!! @param pres mixture pressure
@@ -76,7 +135,7 @@ contains
76135

77136
E = gamma*pres + pi_inf + 5.e-1_wp*rho*vel_sum + qv
78137

79-
! ENERGY ADJUSTMENTS FOR HYPERELASTIC ENERGY
138+
! Adjust energy for hyperelasticity
80139
if (hyperelasticity) then
81140
E = E + G*q_prim_vf(xiend + 1)%sf(j, k, l)
82141
end if
@@ -92,8 +151,8 @@ contains
92151
!! @param j x index
93152
!! @param k y index
94153
!! @param l z index
95-
!! @param icfl_sf cell centered inviscid cfl number
96-
!! @param vcfl_sf (optional) cell centered viscous cfl number
154+
!! @param icfl_sf cell-centered inviscid cfl number
155+
!! @param vcfl_sf (optional) cell-centered viscous CFL number
97156
!! @param Rc_sf (optional) cell centered Rc
98157
pure subroutine s_compute_stability_from_dt(vel, c, rho, Re_l, j, k, l, icfl_sf, vcfl_sf, Rc_sf)
99158
$:GPU_ROUTINE(parallelism='[seq]')
@@ -104,82 +163,48 @@ contains
104163
real(wp), dimension(2), intent(in) :: Re_l
105164
integer, intent(in) :: j, k, l
106165

107-
real(wp) :: fltr_dtheta !<
108-
!! Modified dtheta accounting for Fourier filtering in azimuthal direction.
109-
integer :: Nfq
166+
real(wp) :: fltr_dtheta
110167

111-
if (grid_geometry == 3) then
112-
if (k == 0) then
113-
fltr_dtheta = 2._wp*pi*y_cb(0)/3._wp
114-
elseif (k <= fourier_rings) then
115-
Nfq = min(floor(2._wp*real(k, wp)*pi), (p + 1)/2 + 1)
116-
fltr_dtheta = 2._wp*pi*y_cb(k - 1)/real(Nfq, wp)
117-
else
118-
fltr_dtheta = y_cb(k - 1)*dz(l)
119-
end if
168+
! Inviscid CFL calculation
169+
if (p > 0 .or. n > 0) then
170+
! 2D/3D
171+
icfl_sf(j, k, l) = dt/f_compute_multidim_cfl_terms(vel, c, j, k, l)
172+
else
173+
! 1D
174+
icfl_sf(j, k, l) = (dt/dx(j))*(abs(vel(1)) + c)
120175
end if
121176

122-
if (p > 0) then
123-
!3D
124-
if (grid_geometry == 3) then
125-
icfl_sf(j, k, l) = dt/min(dx(j)/(abs(vel(1)) + c), &
126-
dy(k)/(abs(vel(2)) + c), &
127-
fltr_dtheta/(abs(vel(3)) + c))
128-
else
129-
icfl_sf(j, k, l) = dt/min(dx(j)/(abs(vel(1)) + c), &
130-
dy(k)/(abs(vel(2)) + c), &
131-
dz(l)/(abs(vel(3)) + c))
132-
end if
133-
134-
if (viscous) then
135-
177+
! Viscous calculations
178+
if (viscous) then
179+
if (p > 0) then
180+
!3D
136181
if (grid_geometry == 3) then
182+
fltr_dtheta = f_compute_filtered_dtheta(k, l)
137183
vcfl_sf(j, k, l) = maxval(dt/Re_l/rho) &
138184
/min(dx(j), dy(k), fltr_dtheta)**2._wp
139-
140185
Rc_sf(j, k, l) = min(dx(j)*(abs(vel(1)) + c), &
141186
dy(k)*(abs(vel(2)) + c), &
142187
fltr_dtheta*(abs(vel(3)) + c)) &
143188
/maxval(1._wp/Re_l)
144189
else
145190
vcfl_sf(j, k, l) = maxval(dt/Re_l/rho) &
146191
/min(dx(j), dy(k), dz(l))**2._wp
147-
148192
Rc_sf(j, k, l) = min(dx(j)*(abs(vel(1)) + c), &
149193
dy(k)*(abs(vel(2)) + c), &
150194
dz(l)*(abs(vel(3)) + c)) &
151195
/maxval(1._wp/Re_l)
152196
end if
153-
154-
end if
155-
156-
elseif (n > 0) then
157-
!2D
158-
icfl_sf(j, k, l) = dt/min(dx(j)/(abs(vel(1)) + c), &
159-
dy(k)/(abs(vel(2)) + c))
160-
161-
if (viscous) then
162-
197+
elseif (n > 0) then
198+
!2D
163199
vcfl_sf(j, k, l) = maxval(dt/Re_l/rho)/min(dx(j), dy(k))**2._wp
164-
165200
Rc_sf(j, k, l) = min(dx(j)*(abs(vel(1)) + c), &
166201
dy(k)*(abs(vel(2)) + c)) &
167202
/maxval(1._wp/Re_l)
168-
169-
end if
170-
171-
else
172-
!1D
173-
icfl_sf(j, k, l) = (dt/dx(j))*(abs(vel(1)) + c)
174-
175-
if (viscous) then
176-
203+
else
204+
!1D
177205
vcfl_sf(j, k, l) = maxval(dt/Re_l/rho)/dx(j)**2._wp
178-
179206
Rc_sf(j, k, l) = dx(j)*(abs(vel(1)) + c)/maxval(1._wp/Re_l)
180-
181207
end if
182-
183208
end if
184209

185210
end subroutine s_compute_stability_from_dt
@@ -201,64 +226,39 @@ contains
201226
integer, intent(in) :: j, k, l
202227

203228
real(wp) :: icfl_dt, vcfl_dt
204-
real(wp) :: fltr_dtheta !<
205-
!! Modified dtheta accounting for Fourier filtering in azimuthal direction.
229+
real(wp) :: fltr_dtheta
206230

207-
integer :: Nfq
208-
209-
if (grid_geometry == 3) then
210-
if (k == 0) then
211-
fltr_dtheta = 2._wp*pi*y_cb(0)/3._wp
212-
elseif (k <= fourier_rings) then
213-
Nfq = min(floor(2._wp*real(k, wp)*pi), (p + 1)/2 + 1)
214-
fltr_dtheta = 2._wp*pi*y_cb(k - 1)/real(Nfq, wp)
215-
else
216-
fltr_dtheta = y_cb(k - 1)*dz(l)
217-
end if
231+
! Inviscid CFL calculation
232+
if (p > 0 .or. n > 0) then
233+
! 2D/3D cases
234+
icfl_dt = cfl_target*f_compute_multidim_cfl_terms(vel, c, j, k, l)
235+
else
236+
! 1D case
237+
icfl_dt = cfl_target*(dx(j)/(abs(vel(1)) + c))
218238
end if
219239

220-
if (p > 0) then
221-
!3D
222-
if (grid_geometry == 3) then
223-
icfl_dt = cfl_target*min(dx(j)/(abs(vel(1)) + c), &
224-
dy(k)/(abs(vel(2)) + c), &
225-
fltr_dtheta/(abs(vel(3)) + c))
226-
else
227-
icfl_dt = cfl_target*min(dx(j)/(abs(vel(1)) + c), &
228-
dy(k)/(abs(vel(2)) + c), &
229-
dz(l)/(abs(vel(3)) + c))
230-
end if
231-
232-
if (viscous) then
240+
! Viscous calculations
241+
if (viscous) then
242+
if (p > 0) then
243+
!3D
233244
if (grid_geometry == 3) then
245+
fltr_dtheta = f_compute_filtered_dtheta(k, l)
234246
vcfl_dt = cfl_target*(min(dx(j), dy(k), fltr_dtheta)**2._wp) &
235247
/minval(1/(rho*Re_l))
236248
else
237249
vcfl_dt = cfl_target*(min(dx(j), dy(k), dz(l))**2._wp) &
238250
/minval(1/(rho*Re_l))
239251
end if
240-
end if
241-
242-
elseif (n > 0) then
243-
!2D
244-
icfl_dt = cfl_target*min(dx(j)/(abs(vel(1)) + c), &
245-
dy(k)/(abs(vel(2)) + c))
246-
247-
if (viscous) then
252+
elseif (n > 0) then
253+
!2D
248254
vcfl_dt = cfl_target*(min(dx(j), dy(k))**2._wp)/maxval((1/Re_l)/rho)
249-
end if
250-
251-
else
252-
!1D
253-
icfl_dt = cfl_target*(dx(j)/(abs(vel(1)) + c))
254-
255-
if (viscous) then
255+
else
256+
!1D
256257
vcfl_dt = cfl_target*(dx(j)**2._wp)/minval(1/(rho*Re_l))
257258
end if
258-
259259
end if
260260

261-
if (any(re_size > 0)) then
261+
if (any(Re_size > 0)) then
262262
max_dt(j, k, l) = min(icfl_dt, vcfl_dt)
263263
else
264264
max_dt(j, k, l) = icfl_dt

‎toolchain/bootstrap/modules.sh‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,8 @@ if [ -v $u_c ]; then
2424
log "$Y""Gatech$W: Phoenix (p)"
2525
log "$R""Caltech$W: Richardson (r)"
2626
log "$BR""Brown$W: Oscar (o)"
27-
log "$B""DoD$W: Carpenter (c) | Nautilus (n)"
28-
log_n "($G""a$W/$G""f$W/$G""s$W/$G""w$W/$C""b$W/$C""e$CR/$C""d/$C""dai$CR/$Y""p$CR/$R""r$CR/$B""c$CR/$B""n$CR/$BR""o"$CR"): "
27+
log "$B""DoD$W: Carpenter Cray (cc) | Carpenter GNU (c) | Nautilus (n)"
28+
log_n "($G""a$W/$G""f$W/$G""s$W/$G""w$W/$C""b$W/$C""e$CR/$C""d/$C""dai$CR/$Y""p$CR/$R""r$CR/$B""cc$CR/$B""c$CR/$B""n$CR/$BR""o"$CR"): "
2929
read u_c
3030
log
3131
fi
@@ -68,7 +68,7 @@ fi
6868
log "Loading modules (& env variables) for $M$COMPUTER$CR on $M$CG$CR"'s:'
6969

7070
# Reset modules to default system configuration (unless Carpenter)
71-
if [ "$u_c" != 'c' ]; then
71+
if [ "$u_c" '!=' 'cc' ] && [ "$u_c" '!=' 'c' ]; then
7272
module reset > /dev/null 2>&1
7373
code="$?"
7474

@@ -97,7 +97,7 @@ if [ $(echo "$VARIABLES" | grep = | wc -c) -gt 0 ]; then
9797
fi
9898

9999
# Don't check for Cray paths on Carpenter, otherwise do check if they exist
100-
if [ ! -z ${CRAY_LD_LIBRARY_PATH+x} ] && [ "$u_c" != 'c' ]; then
100+
if [ ! -z ${CRAY_LD_LIBRARY_PATH+x} ] && [ "$u_c" '!=' 'c' ]; then
101101
ok "Found $M\$CRAY_LD_LIBRARY_PATH$CR. Prepending to $M\$LD_LIBRARY_PATH$CR."
102102
export LD_LIBRARY_PATH="$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH"
103103
fi

‎toolchain/modules‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,10 @@ s-gpu nvhpc/22.11 cuda/nvhpc
1414
s-gpu CC=nvc CXX=nvc++ FC=nvfortran
1515

1616
b PSC Bridges2
17-
b-all python/3.8.6
17+
b-all python/3.8.6 hdf5 anaconda3
1818
b-cpu allocations/1.0 gcc/10.2.0 openmpi/4.0.5-gcc10.2.0
19-
b-gpu openmpi/4.0.5-nvhpc22.9 nvhpc/22.9 cuda
20-
b-gpu CC=nvc CXX=nvc++ FC=nvfortran
19+
b-gpu nvhpc/22.9 cuda/11.7 openmpi/4.0.5-nvhpc22.9
20+
b-gpu MFC_CUDA_CC=70,75,80 NVHPC_CUDA_HOME=$CUDA_HOME CC=nvc CXX=nvc++ FC=nvfortran
2121

2222
a OLCF Ascent
2323
a-all python cmake/3.22.2
@@ -70,11 +70,15 @@ dai-all python cmake nvhpc-openmpi3/24.3 cuda
7070
dai-all CC=nvc CXX=nvc++ FC=nvfortran
7171
dai-gpu MFC_CUDA_CC=89,90
7272

73-
c DoD Carpenter
73+
c DoD Carpenter (GNU)
7474
c-all python/3.12.1
7575
c-cpu compiler-rt/2024.2.0 ifort/2024.2.0 icc/2023.1.0 mpi/latest cmake/3.28.1-intel-2023.0.0
7676
c-cpu CC=gcc CXX=g++ FC=gfortran
7777

78+
cc DoD Carpenter (Cray)
79+
cc-all cray-python/3.11.7 craype-x86-rome
80+
cc-cpu PrgEnv-cray/8.5.0 cray-fftw/3.3.10.8 cray-hdf5/1.14.3.1 cray-pals/1.4.0 cray-libpals/1.4.0
81+
7882
n DoD Nautilus
7983
n-all slurm
8084
n-cpu penguin/openmpi/4.1.5/gcc-8.5.0

‎toolchain/templates/carpenter-cray.mako‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
#!/usr/bin/env bash
2+
3+
<%namespace name="helpers" file="helpers.mako"/>
4+
5+
% if engine == 'batch':
6+
#PBS -l select=${nodes}:ncpus=192:mpiprocs=${tasks_per_node}
7+
#PBS -N "${name}"
8+
#PBS -l walltime=${walltime}
9+
% if partition:
10+
#PBS -q ${partition}
11+
% endif
12+
% if account:
13+
#PBS -A ${account}
14+
% endif
15+
% if email:
16+
#PBS -M ${email}
17+
#PBS -m abe
18+
% endif
19+
#PBS -o "${name}.out"
20+
#PBS -e "${name}.err"
21+
#PBS -V
22+
% endif
23+
24+
${helpers.template_prologue()}
25+
26+
ok ":) Loading modules:\n"
27+
cd "${MFC_ROOT_DIR}"
28+
. ./mfc.sh load -c cc -m ${'g' if gpu else 'c'}
29+
cd - > /dev/null
30+
echo
31+
32+
% for target in targets:
33+
${helpers.run_prologue(target)}
34+
35+
% if not mpi:
36+
(set -x; ${profiler} "${target.get_install_binpath(case)}")
37+
% else:
38+
(set -x; ${profiler} \
39+
mpirun -np ${nodes*tasks_per_node} \
40+
"${target.get_install_binpath(case)}")
41+
% endif
42+
43+
${helpers.run_epilogue(target)}
44+
45+
echo
46+
% endfor
47+
48+
${helpers.template_epilogue()}

0 commit comments

Comments
 (0)