Skip to content

Commit 7f53861

Browse files
wilfonbawilfonba
authored
Fix Benchmarking (#423)
Co-authored-by: Ben Wilfong <[email protected]>
1 parent c208d59 commit 7f53861

File tree

5 files changed

+145
-126
lines changed

5 files changed

+145
-126
lines changed

‎benchmarks/hypo_hll/case.py‎

Lines changed: 105 additions & 112 deletions
Original file line numberDiff line numberDiff line change
@@ -26,119 +26,112 @@
2626
s = math.floor((ncells / 2.0) ** (1/3))
2727
Nx, Ny, Nz = 2*s, s, s
2828

29-
Mu = 1.84E-05
30-
gam_a = 1.4
31-
32-
D = 0.1
3329

3430
# Configuring case dictionary
3531
print(json.dumps({
36-
# Logistics ================================================================
37-
'run_time_info' : 'T',
38-
# ==========================================================================
39-
40-
# Computational Domain Parameters ==========================================
41-
# x direction
42-
'x_domain%beg' : -5*D,
43-
'x_domain%end' : 5.0*D,
44-
# y direction
45-
'y_domain%beg' : -2.5*D,
46-
'y_domain%end' : 2.5*D,
47-
# z direction
48-
'z_domain%beg' : -2.5*D,
49-
'z_domain%end' : 2.5*D,
50-
51-
'cyl_coord' : 'F',
52-
'm' : Nx,
53-
'n' : Ny,
54-
'p' : Nz,
55-
'dt' : 1.0E-7,
56-
't_step_start' : 0,
57-
't_step_stop' : int(20*(45*size + 5)),
58-
't_step_save' : int(4*(45*size + 5)),
59-
# ==========================================================================
60-
61-
# Simulation Algorithm Parameters ==========================================
62-
# Only one patches are necessary, the air tube
63-
'num_patches' : 1,
64-
# Use the 5 equation model
65-
'model_eqns' : 2,
66-
# 6 equations model does not need the K \div(u) term
67-
'alt_soundspeed' : 'F',
68-
# One fluids: air
69-
'num_fluids' : 1,
70-
# Advect both volume fractions
71-
'adv_alphan' : 'T',
72-
# No need to ensure the volume fractions sum to unity at the end of each
73-
# time step
74-
'mpp_lim' : 'F',
75-
# Correct errors when computing speed of sound
76-
'mixture_err' : 'T',
77-
# Use TVD RK3 for time marching
78-
'time_stepper' : 3,
79-
# Reconstruct the primitive variables to minimize spurious
80-
# Use WENO5
81-
'weno_order' : 5,
82-
'weno_eps' : 1.E-16,
83-
'weno_Re_flux' : 'T',
84-
'weno_avg' : 'T',
85-
'avg_state' : 2,
86-
'mapped_weno' : 'T',
87-
'null_weights' : 'F',
88-
'mp_weno' : 'T',
89-
'riemann_solver' : 2,
90-
'wave_speeds' : 1,
91-
# We use ghost-cell extrapolation
92-
'bc_x%beg' : -3,
93-
'bc_x%end' : -3,
94-
'bc_y%beg' : -3,
95-
'bc_y%end' : -3,
96-
'bc_z%beg' : -3,
97-
'bc_z%end' : -3,
98-
# Set IB to True and add 1 patch
99-
'ib' : 'T',
100-
'num_ibs' : 1,
101-
# ==========================================================================
102-
103-
# Formatted Database Files Structure Parameters ============================
104-
'format' : 1,
105-
'precision' : 2,
106-
'prim_vars_wrt' :'T',
107-
'E_wrt' :'T',
108-
'parallel_io' :'T',
109-
# ==========================================================================
110-
111-
# Patch: Constant Tube filled with air =====================================
112-
# Specify the cylindrical air tube grid geometry
113-
'patch_icpp(1)%geometry' : 9,
114-
'patch_icpp(1)%x_centroid' : 0.0,
115-
# Uniform medium density, centroid is at the center of the domain
116-
'patch_icpp(1)%y_centroid' : 0.0,
117-
'patch_icpp(1)%z_centroid' : 0.0,
118-
'patch_icpp(1)%length_x' : 10*D,
119-
'patch_icpp(1)%length_y' : 5*D,
120-
'patch_icpp(1)%length_z' : 5*D,
121-
# Specify the patch primitive variables
122-
'patch_icpp(1)%vel(1)' : 527.2E+00,
123-
'patch_icpp(1)%vel(2)' : 0.0E+00,
124-
'patch_icpp(1)%vel(3)' : 0.0E+00,
125-
'patch_icpp(1)%pres' : 10918.2549,
126-
'patch_icpp(1)%alpha_rho(1)' : 0.2199,
127-
'patch_icpp(1)%alpha(1)' : 1.E+00,
128-
# # ========================================================================
129-
130-
# Patch: Sphere Immersed Boundary ========================================
131-
'patch_ib(1)%geometry' : 8,
132-
'patch_ib(1)%x_centroid' : -3.0E-3,
133-
'patch_ib(1)%y_centroid' : 0.0,
134-
'patch_ib(1)%z_centroid' : 0.0,
135-
'patch_ib(1)%radius' : D/2,
136-
'patch_ib(1)%slip' : 'T',
137-
# ==========================================================================
138-
139-
# Fluids Physical Parameters ===============================================
140-
'fluid_pp(1)%gamma' : 1.E+00/(gam_a-1.E+00), # 2.50(Not 1.40)
141-
'fluid_pp(1)%pi_inf' : 0,
142-
'fluid_pp(1)%Re(1)' : 7535533.2,
143-
# ==========================================================================
32+
# Logistics ================================================
33+
'run_time_info' : 'T',
34+
# ==========================================================
35+
36+
# Computational Domain Parameters ==========================
37+
'x_domain%beg' : 0.E+00,
38+
'x_domain%end' : 1.E+00,
39+
'y_domain%beg' : 0.E+00,
40+
'y_domain%end' : 2.E+00,
41+
'z_domain%beg' : 0.E+00,
42+
'z_domain%end' : 1.E+00,
43+
'm' : Nx,
44+
'n' : Ny,
45+
'p' : Nz,
46+
'dt' : 1e-8,
47+
't_step_start' : 0,
48+
't_step_stop' : int(60*(95*size + 5)),
49+
't_step_save' : int(12*(95*size + 5)),
50+
# ==========================================================
51+
52+
# Simulation Algorithm Parameters ==========================
53+
'num_patches' : 2,
54+
'model_eqns' : 2,
55+
'alt_soundspeed' : 'F',
56+
'num_fluids' : 2,
57+
'adv_alphan' : 'T',
58+
'mpp_lim' : 'F',
59+
'mixture_err' : 'F',
60+
'time_stepper' : 3,
61+
'weno_order' : 3,
62+
'weno_eps' : 1.E-16,
63+
'weno_Re_flux' : 'F',
64+
'weno_avg' : 'F',
65+
'mapped_weno' : 'F',
66+
'null_weights' : 'F',
67+
'mp_weno' : 'F',
68+
'riemann_solver' : 1,
69+
'wave_speeds' : 1,
70+
'avg_state' : 2,
71+
'bc_x%beg' : -3,
72+
'bc_x%end' : -3,
73+
'bc_y%beg' : -3,
74+
'bc_y%end' : -3,
75+
'bc_z%beg' : -3,
76+
'bc_z%end' : -3,
77+
# ==========================================================
78+
79+
# Turning on Hypoelasticity ================================
80+
'hypoelasticity' : 'T',
81+
# ==========================================================
82+
83+
# Formatted Database Files Structure Parameters ============
84+
'format' : 1,
85+
'precision' : 2,
86+
'prim_vars_wrt' :'T',
87+
'parallel_io' :'F',
88+
# ==========================================================
89+
90+
# Patch 1 L ================================================
91+
'patch_icpp(1)%geometry' : 9,
92+
'patch_icpp(1)%x_centroid' : 0.35,
93+
'patch_icpp(1)%y_centroid' : 1.0,
94+
'patch_icpp(1)%z_centroid' : 0.5,
95+
'patch_icpp(1)%length_x' : 0.7,
96+
'patch_icpp(1)%length_y' : 2,
97+
'patch_icpp(1)%length_z' : 1,
98+
'patch_icpp(1)%vel(1)' : 0.0,
99+
'patch_icpp(1)%vel(2)' : 0.0,
100+
'patch_icpp(1)%vel(3)' : 0.0,
101+
'patch_icpp(1)%pres' : 1E+09,
102+
'patch_icpp(1)%alpha_rho(1)' : 1000,
103+
'patch_icpp(1)%alpha_rho(2)' : 0.,
104+
'patch_icpp(1)%alpha(1)' : 1,
105+
'patch_icpp(1)%alpha(2)' : 0.,
106+
'patch_icpp(1)%tau_e(1)' : 0.0,
107+
# ==========================================================
108+
109+
# Patch 2 R ================================================
110+
'patch_icpp(2)%geometry' : 9,
111+
'patch_icpp(2)%x_centroid' : 0.85,
112+
'patch_icpp(2)%y_centroid' : 1.0,
113+
'patch_icpp(2)%z_centroid' : 0.5,
114+
'patch_icpp(2)%length_x' : 0.3,
115+
'patch_icpp(2)%length_y' : 2.0,
116+
'patch_icpp(2)%length_z' : 1.0,
117+
'patch_icpp(2)%vel(1)' : 0,
118+
'patch_icpp(2)%vel(2)' : 0,
119+
'patch_icpp(2)%vel(3)' : 0,
120+
'patch_icpp(2)%pres' : 1E+05,
121+
'patch_icpp(2)%alpha_rho(1)' : 0.,
122+
'patch_icpp(2)%alpha_rho(2)' : 50.,
123+
'patch_icpp(2)%alpha(1)' : 0,
124+
'patch_icpp(2)%alpha(2)' : 1,
125+
'patch_icpp(2)%tau_e(1)' : 0.0,
126+
# ==========================================================
127+
128+
# Fluids Physical Parameters ===============================
129+
'fluid_pp(1)%gamma' : 1.E+00/(4.4E+00-1.E+00),
130+
'fluid_pp(1)%pi_inf' : 4.4E+00*6.E+08/(4.4E+00 - 1.E+00),
131+
'fluid_pp(1)%G' : 1.E+09,
132+
'fluid_pp(2)%gamma' : 1.E+00/(1.4E+00-1.E+00),
133+
'fluid_pp(2)%pi_inf' : 0.,
134+
'fluid_pp(2)%G' : 0.,
135+
# ==========================================================
144136
}))
137+
# ==============================================================================

‎src/simulation/m_mpi_proxy.fpp‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -143,11 +143,11 @@ contains
143143
144144
#:for VAR in ['t_step_old', 'm', 'n', 'p', 'm_glb', 'n_glb', 'p_glb', &
145145
& 't_step_start','t_step_stop','t_step_save','t_step_print', &
146-
& 'model_eqns','num_fluids','time_stepper', 'riemann_solver', &
146+
& 'model_eqns','time_stepper', 'riemann_solver', &
147147
& 'wave_speeds', 'avg_state', 'precision', 'bc_x%beg', 'bc_x%end', &
148148
& 'bc_y%beg', 'bc_y%end', 'bc_z%beg', 'bc_z%end', 'fd_order', &
149149
& 'num_probes', 'num_integrals', 'bubble_model', 'thermal', &
150-
& 'R0_type', 'num_mono', 'relax_model']
150+
& 'R0_type', 'num_mono', 'relax_model', 'num_ibs']
151151
call MPI_BCAST(${VAR}$, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr)
152152
#:endfor
153153
@@ -157,7 +157,7 @@ contains
157157
& 'parallel_io', 'hypoelasticity', 'bubbles', 'polytropic', &
158158
& 'polydisperse', 'qbmm', 'monopole', 'probe_wrt', 'integral_wrt', &
159159
& 'prim_vars_wrt', 'weno_avg', 'file_per_process', 'relax', &
160-
& 'adv_n', 'adap_dt', 'ib', 'num_ibs' ]
160+
& 'adv_n', 'adap_dt', 'ib' ]
161161
call MPI_BCAST(${VAR}$, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, ierr)
162162
#:endfor
163163
@@ -174,6 +174,7 @@ contains
174174
#:if not MFC_CASE_OPTIMIZATION
175175
call MPI_BCAST(weno_order, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr)
176176
call MPI_BCAST(nb, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr)
177+
call MPI_BCAST(num_fluids, 1, MPI_INTEGER, 0, MPI_COMM_WORLD, ierr)
177178
#:endif
178179
179180
do i = 1, num_fluids_max

‎src/simulation/m_start_up.fpp‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -191,8 +191,8 @@ contains
191191
if (CRAY_ACC_MODULE == "") then
192192
call s_mpi_abort("CRAY_ACC_MODULE is not set. Exiting...")
193193
end if
194-
#endif
195-
#endif
194+
#endif
195+
#endif
196196

197197
end subroutine s_read_input_file ! -------------------------------------
198198

@@ -481,7 +481,7 @@ contains
481481
else
482482
call s_mpi_abort('File '//trim(file_loc)//' is missing. Exiting...')
483483
end if
484-
484+
485485
! Assigning local cell boundary locations
486486
x_cb(-1:m) = x_cb_glb((start_idx(1) - 1):(start_idx(1) + m))
487487
! Computing the cell width distribution
@@ -1247,7 +1247,7 @@ contains
12471247
call acc_present_dump()
12481248
#endif
12491249

1250-
if (hypoelasticity) call s_initialize_hypoelastic_module()
1250+
if (hypoelasticity) call s_initialize_hypoelastic_module()
12511251
if (relax) call s_initialize_phasechange_module()
12521252
call s_initialize_data_output_module()
12531253
call s_initialize_derived_variables_module()
@@ -1355,7 +1355,9 @@ contains
13551355
! carried out if the simulation is in fact not truly executed in parallel.
13561356

13571357
call s_mpi_bcast_user_inputs()
1358+
13581359
call s_initialize_parallel_io()
1360+
13591361
call s_mpi_decompose_computational_domain()
13601362

13611363
end subroutine s_initialize_mpi_domain
@@ -1371,7 +1373,7 @@ contains
13711373
end if
13721374
!$acc update device(nb, R0ref, Ca, Web, Re_inv, weight, R0, V0, bubbles, polytropic, polydisperse, qbmm, R0_type, ptil, bubble_model, thermal, poly_sigma, adv_n, adap_dt, n_idx, pi_fac)
13731375
!$acc update device(R_n, R_v, phi_vn, phi_nv, Pe_c, Tw, pv, M_n, M_v, k_n, k_v, pb0, mass_n0, mass_v0, Pe_T, Re_trans_T, Re_trans_c, Im_trans_T, Im_trans_c, omegaN , mul0, ss, gamma_v, mu_v, gamma_m, gamma_n, mu_n, gam)
1374-
!$acc update device(dx, dy, dz, x_cb, x_cc, y_cb, y_cc, z_cb, z_cc)
1376+
!$acc update device(dx, dy, dz, x_cb, x_cc, y_cb, y_cc, z_cb, z_cc)
13751377
!$acc update device(bc_x%vb1, bc_x%vb2, bc_x%vb3, bc_x%ve1, bc_x%ve2, bc_x%ve3)
13761378
!$acc update device(bc_y%vb1, bc_y%vb2, bc_y%vb3, bc_y%ve1, bc_y%ve2, bc_y%ve3)
13771379
!$acc update device(bc_z%vb1, bc_z%vb2, bc_z%vb3, bc_z%ve1, bc_z%ve2, bc_z%ve3)
@@ -1400,7 +1402,7 @@ contains
14001402
if (grid_geometry == 3) call s_finalize_fftw_module
14011403
call s_finalize_mpi_proxy_module()
14021404
call s_finalize_global_parameters_module()
1403-
if (relax) call s_finalize_relaxation_solver_module()
1405+
if (relax) call s_finalize_relaxation_solver_module()
14041406
if (any(Re_size > 0)) then
14051407
call s_finalize_viscous_module()
14061408
end if

‎toolchain/mfc/bench.py‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import os, sys, uuid, subprocess, dataclasses, typing
1+
import os, sys, uuid, subprocess, dataclasses, typing, math
22

33
import rich.table
44

@@ -7,6 +7,7 @@
77
from .build import get_targets, DEFAULT_TARGETS
88
from .common import system, MFC_BENCH_FILEPATH, MFC_SUBDIR, format_list_to_string
99
from .common import file_load_yaml, file_dump_yaml, create_directory
10+
from .common import MFCException
1011

1112

1213
@dataclasses.dataclass
@@ -106,18 +107,40 @@ def _lock_to_str(lock):
106107
table.add_column("[bold]Simulation[/bold]", justify="right")
107108
table.add_column("[bold]Post Process[/bold]", justify="right")
108109

110+
err = 0
111+
109112
for slug in slugs:
110113
lhs_summary = lhs["cases"][slug]["output_summary"]
111114
rhs_summary = rhs["cases"][slug]["output_summary"]
112115

113116
speedups = ['N/A', 'N/A', 'N/A']
114117

115118
for i, target in enumerate(sorted(DEFAULT_TARGETS, key=lambda t: t.runOrder)):
116-
if target.name not in lhs_summary or target.name not in rhs_summary:
119+
if (target.name not in lhs_summary) or (target.name not in rhs_summary):
120+
121+
err = 1
122+
123+
if target.name not in lhs_summary:
124+
cons.print(f"{target.name} not present in lhs_summary - Case: {slug}")
125+
126+
if target.name not in rhs_summary:
127+
cons.print(f"{target.name} not present in rhs_summary - Case: {slug}")
128+
117129
continue
118130

131+
if (float(f"{lhs_summary[target.name]}") <= 0.0) or math.isnan(float(f"{lhs_summary[target.name]}")):
132+
err = 1
133+
cons.print(f"lhs_summary reports non-positive or NaN runtime for {target.name} - Case: {slug}")
134+
135+
if (float(f"{rhs_summary[target.name]}") <= 0.0) or math.isnan(float(f"{rhs_summary[target.name]}")):
136+
err = 1
137+
cons.print(f"rhs_summary reports non-positive or NaN runtime for {target.name} - Case: {slug}")
138+
119139
speedups[i] = f"{lhs_summary[target.name] / rhs_summary[target.name]:.2f}x"
120140

121141
table.add_row(f"[magenta]{slug}[/magenta]", *speedups)
122142

123143
cons.raw.print(table)
144+
145+
if err != 0:
146+
raise MFCException("Benchmarking failed")

‎toolchain/templates/include/helpers.mako‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -64,13 +64,13 @@ fi
6464
6565
cd "${os.path.dirname(input)}"
6666
67-
t_${target.name}_start=$(date +%s)
67+
t_${target.name}_start=$(perl -MTime::HiRes=time -e 'printf "%.9f\n", time')
6868
</%def>
6969
7070
<%def name="run_epilogue(target)">
7171
code=$?
7272
73-
t_${target.name}_stop=$(date +%s)
73+
t_${target.name}_stop=$(perl -MTime::HiRes=time -e 'printf "%.9f\n", time')
7474
7575
if [ $code -ne 0 ]; then
7676
echo
@@ -86,7 +86,7 @@ unset CRAY_ACC_MODULE
8686
cd "${MFC_ROOTDIR}"
8787
8888
cat >>"${output_summary}" <<EOL
89-
${target.name}: $(expr $t_${target.name}_stop - $t_${target.name}_start)
89+
${target.name}: $(echo "$t_${target.name}_stop - $t_${target.name}_start" | bc -l)
9090
EOL
9191
9292
cd - > /dev/null

0 commit comments

Comments
 (0)