Skip to content

Commit 2e16041

Browse files
max-Hawkinsmax-Hawkins
committed
Standardize timing ranges
1 parent 0400700 commit 2e16041

File tree

5 files changed

+66
-50
lines changed

5 files changed

+66
-50
lines changed

‎src/common/m_mpi_common.fpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ module m_mpi_common
1616
use m_derived_types !< Definitions of the derived types
1717
1818
use m_global_parameters !< Definitions of the global parameters
19+
20+
use m_nvtx
1921
! ==========================================================================
2022
2123
implicit none
@@ -239,6 +241,7 @@ contains
239241
#ifdef MFC_SIMULATION
240242
#ifdef MFC_MPI
241243
244+
call nvtxStartRange("TSTEP-RUNTIME-INFO-REDUCE-CRIT")
242245
! Reducing local extrema of ICFL, VCFL, CCFL and Rc numbers to their
243246
! global extrema and bookkeeping the results on the rank 0 processor
244247
call MPI_REDUCE(icfl_max_loc, icfl_max_glb, 1, &
@@ -253,6 +256,7 @@ contains
253256
MPI_DOUBLE_PRECISION, MPI_MIN, 0, &
254257
MPI_COMM_WORLD, ierr)
255258
end if
259+
call nvtxEndRange
256260
257261
#else
258262

‎src/simulation/m_data_output.fpp‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ module m_data_output
3535

3636
implicit none
3737

38-
private;
38+
private;
3939
public :: s_initialize_data_output_module, &
4040
s_open_run_time_information_file, &
4141
s_open_probe_files, &
@@ -236,8 +236,10 @@ contains
236236
real(kind(0d0)), dimension(2) :: Re !< Cell-avg. Reynolds numbers
237237
integer :: j, k, l
238238

239+
call nvtxStartRange("TSTEP-RUNTIME-INFO")
239240
! Computing Stability Criteria at Current Time-step ================
240-
!$acc parallel loop collapse(3) gang vector default(present) private(vel, alpha, Re)
241+
call nvtxStartRange("TSTEP-RUNTIME-INFO-COMPUTE-LOCAL")
242+
!$acc parallel loop collapse(3) gang vector default(present) private(alpha_rho, vel, alpha, Re, fltr_dtheta, Nfq)
241243
do l = 0, p
242244
do k = 0, n
243245
do j = 0, m
@@ -253,6 +255,7 @@ contains
253255
end do
254256
end do
255257
end do
258+
256259
! END: Computing Stability Criteria at Current Time-step ===========
257260

258261
! Determining local stability criteria extrema at current time-step
@@ -282,7 +285,7 @@ contains
282285
!$acc end kernels
283286
end if
284287
#endif
285-
288+
call nvtxEndRange ! RUNTIME-INFO-COMPUTE-LOCAL
286289
! Determining global stability criteria extrema at current time-step
287290
if (num_procs > 1) then
288291
call s_mpi_reduce_stability_criteria_extrema(icfl_max_loc, &
@@ -337,6 +340,7 @@ contains
337340
end if
338341

339342
call s_mpi_barrier()
343+
call nvtxEndRange ! RUNTIME-INFO
340344

341345
end subroutine s_write_run_time_information
342346

‎src/simulation/m_mpi_proxy.fpp‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -878,6 +878,7 @@ contains
878878
879879
#ifdef MFC_MPI
880880
881+
call nvtxStartRange("RHS-COMM-PACKBUF")
881882
!$acc update device(v_size)
882883
883884
if (qbmm .and. .not. polytropic) then
@@ -1070,6 +1071,7 @@ contains
10701071
#:endif
10711072
end if
10721073
#:endfor
1074+
call nvtxEndRange ! Packbuf
10731075
10741076
! Send/Recv
10751077
#:for rdma_mpi in [False, True]
@@ -1079,28 +1081,34 @@ contains
10791081
#:if rdma_mpi
10801082
!$acc data attach(p_send, p_recv)
10811083
!$acc host_data use_device(p_send, p_recv)
1084+
call nvtxStartRange("RHS-COMM-SENDRECV-RDMA")
10821085
#:else
1086+
call nvtxStartRange("RHS-COMM-DEV2HOST")
10831087
!$acc update host(q_cons_buff_send, ib_buff_send)
1088+
call nvtxEndRange
1089+
call nvtxStartRange("RHS-COMM-SENDRECV-NO-RMDA")
10841090
#:endif
10851091
1086-
call nvtxStartRange("RHS-MPI-SENDRECV")
10871092
call MPI_SENDRECV( &
10881093
p_send, buffer_count, MPI_DOUBLE_PRECISION, dst_proc, send_tag, &
10891094
p_recv, buffer_count, MPI_DOUBLE_PRECISION, src_proc, recv_tag, &
10901095
MPI_COMM_WORLD, MPI_STATUS_IGNORE, ierr)
1091-
call nvtxEndRange
1096+
call nvtxEndRange ! RHS-MPI-SENDRECV-(NO)-RDMA
10921097
10931098
#:if rdma_mpi
10941099
!$acc end host_data
10951100
!$acc end data
10961101
!$acc wait
10971102
#:else
1103+
call nvtxStartRange("RHS-COMM-HOST2DEV")
10981104
!$acc update device(q_cons_buff_recv)
1105+
call nvtxEndRange
10991106
#:endif
11001107
end if
11011108
#:endfor
11021109
11031110
! Unpack Received Buffer
1111+
call nvtxStartRange("RHS-COMM-UNPACKBUF")
11041112
#:for mpi_dir in [1, 2, 3]
11051113
if (mpi_dir == ${mpi_dir}$) then
11061114
#:if mpi_dir == 1
@@ -1269,6 +1277,7 @@ contains
12691277
#:endif
12701278
end if
12711279
#:endfor
1280+
call nvtxEndRange
12721281
12731282
#endif
12741283

‎src/simulation/m_rhs.fpp‎

Lines changed: 34 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -703,7 +703,7 @@ contains
703703
real(kind(0d0)) :: t_start, t_finish
704704
integer :: i, j, k, l, id !< Generic loop iterators
705705

706-
call nvtxStartRange("Compute_RHS")
706+
call nvtxStartRange("COMPUTE-RHS")
707707

708708
call cpu_time(t_start)
709709
! Association/Population of Working Variables ======================
@@ -750,7 +750,7 @@ contains
750750
gm_alpha_qp%vf)
751751
call nvtxEndRange
752752

753-
call nvtxStartRange("RHS-MPI+BufPack")
753+
call nvtxStartRange("RHS-MPI+BUFPACK")
754754
call s_populate_variables_buffers(q_prim_qp%vf, pb, mv)
755755
call nvtxEndRange
756756

@@ -764,7 +764,7 @@ contains
764764

765765
if (qbmm) call s_mom_inv(q_cons_qp%vf, q_prim_qp%vf, mom_sp, mom_3d, pb, rhs_pb, mv, rhs_mv, idwbuff(1), idwbuff(2), idwbuff(3), nbub)
766766

767-
call nvtxStartRange("Viscous")
767+
call nvtxStartRange("RHS-VISCOUS")
768768
if (viscous) call s_get_viscous(qL_rsx_vf, qL_rsy_vf, qL_rsz_vf, &
769769
dqL_prim_dx_n, dqL_prim_dy_n, dqL_prim_dz_n, &
770770
qL_prim, &
@@ -776,7 +776,7 @@ contains
776776
idwbuff(1), idwbuff(2), idwbuff(3))
777777
call nvtxEndRange
778778

779-
call nvtxStartRange("Surface_Tension")
779+
call nvtxStartRange("RHS-SURFACE-TENSION")
780780
if (surface_tension) call s_get_capilary(q_prim_qp%vf)
781781
call nvtxEndRange
782782
! Dimensional Splitting Loop =======================================
@@ -858,7 +858,7 @@ contains
858858
irx%end = m; iry%end = n; irz%end = p
859859
! ===============================================================
860860

861-
call nvtxStartRange("RHS_riemann_solver")
861+
call nvtxStartRange("RHS-RIEMANN-SOLVER")
862862

863863
! Computing Riemann Solver Flux and Source Flux =================
864864

@@ -882,7 +882,7 @@ contains
882882
! Additional physics and source terms ==============================
883883

884884
! RHS addition for advection source
885-
call nvtxStartRange("RHS_advection_source")
885+
call nvtxStartRange("RHS-ADVECTION-SRC")
886886
call s_compute_advection_source_term(id, &
887887
rhs_vf, &
888888
q_cons_qp, &
@@ -891,14 +891,14 @@ contains
891891
call nvtxEndRange
892892

893893
! RHS additions for hypoelasticity
894-
call nvtxStartRange("RHS_Hypoelasticity")
894+
call nvtxStartRange("RHS-HYPOELASTICITY")
895895
if (hypoelasticity) call s_compute_hypoelastic_rhs(id, &
896896
q_prim_qp%vf, &
897897
rhs_vf)
898898
call nvtxEndRange
899899

900900
! RHS additions for viscosity
901-
call nvtxStartRange("RHS_add_phys")
901+
call nvtxStartRange("RHS-ADD-PHYSICS")
902902
if (viscous .or. surface_tension) then
903903
call s_compute_additional_physics_rhs(id, &
904904
q_prim_qp%vf, &
@@ -911,14 +911,16 @@ contains
911911
call nvtxEndRange
912912

913913
! RHS additions for sub-grid bubbles
914-
call nvtxStartRange("RHS_bubbles")
915-
if (bubbles) call s_compute_bubbles_rhs(id, &
916-
q_prim_qp%vf)
917-
call nvtxEndRange
914+
if (bubbles) then
915+
call nvtxStartRange("RHS-BUBBLES-COMPUTE")
916+
call s_compute_bubbles_rhs(id, q_prim_qp%vf)
917+
call nvtxEndRange
918+
end if
918919

919920
! RHS additions for qbmm bubbles
920-
call nvtxStartRange("RHS_qbmm")
921-
if (qbmm) call s_compute_qbmm_rhs(id, &
921+
if (qbmm) then
922+
call nvtxStartRange("RHS-QBMM")
923+
call s_compute_qbmm_rhs(id, &
922924
q_cons_qp%vf, &
923925
q_prim_qp%vf, &
924926
rhs_vf, &
@@ -927,14 +929,15 @@ contains
927929
rhs_pb, &
928930
mv, &
929931
rhs_mv)
930-
call nvtxEndRange
932+
call nvtxEndRange
933+
end if
931934
! END: Additional physics and source terms =========================
932935

933936
end do
934937
! END: Dimensional Splitting Loop =================================
935938

936939
if (chemistry) then
937-
call nvtxStartRange("RHS_Chem_Advection")
940+
call nvtxStartRange("RHS-CHEM-ADVECTION")
938941
call s_compute_chemistry_advection_flux(flux_n, rhs_vf)
939942
call nvtxEndRange
940943
end if
@@ -956,25 +959,29 @@ contains
956959

957960
! Additional Physics and Source Temrs ==================================
958961
! Additions for acoustic_source
959-
call nvtxStartRange("RHS_acoustic_src")
960-
if (acoustic_source) call s_acoustic_src_calculations(q_cons_qp%vf(1:sys_size), &
962+
if (acoustic_source) then
963+
call nvtxStartRange("RHS-ACOUSTIC-SRC")
964+
call s_acoustic_src_calculations(q_cons_qp%vf(1:sys_size), &
961965
q_prim_qp%vf(1:sys_size), &
962966
t_step, &
963967
rhs_vf)
964-
call nvtxEndRange
968+
call nvtxEndRange
969+
end if
965970

966971
! Add bubles source term
967-
call nvtxStartRange("RHS_bubbles")
968-
if (bubbles .and. (.not. adap_dt) .and. (.not. qbmm)) call s_compute_bubble_source( &
969-
q_cons_qp%vf(1:sys_size), &
970-
q_prim_qp%vf(1:sys_size), &
971-
t_step, &
972-
rhs_vf)
973-
call nvtxEndRange
972+
if (bubbles .and. (.not. adap_dt) .and. (.not. qbmm)) then
973+
call nvtxStartRange("RHS-BUBBLES-SOURCE")
974+
call s_compute_bubble_source( &
975+
q_cons_qp%vf(1:sys_size), &
976+
q_prim_qp%vf(1:sys_size), &
977+
t_step, &
978+
rhs_vf)
979+
call nvtxEndRange
980+
end if
974981

975982
if (chemistry) then
976983
if (chem_params%reactions) then
977-
call nvtxStartRange("RHS_Chem_Reactions")
984+
call nvtxStartRange("RHS-CHEM-REACTIONS")
978985
call s_compute_chemistry_reaction_flux(rhs_vf, q_cons_qp%vf, q_prim_qp%vf)
979986
call nvtxEndRange
980987
end if

0 commit comments

Comments
 (0)