Skip to content

Increase granularity of halo-exchange timing info #639

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Nov 12, 2024
Merged

Increase granularity of halo-exchange timing info #639

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
2f3be63
Increase granularity of halo-exchange timing info
max-Hawkins Oct 1, 2024
2b7d551
Standardize timing ranges
max-Hawkins Oct 17, 2024
8cfeee9
Include NVTX range in conditionals
max-Hawkins Nov 11, 2024
26af09c
Add global simulation nvtx range
max-Hawkins Nov 5, 2024
e00bc64
Change NVTX names and add subranges
max-Hawkins Nov 11, 2024
ce5c8af
Improve NVTX range tracking
max-Hawkins Nov 12, 2024
095d890
Add NVTX ranges for initialization and finalization
max-Hawkins Nov 12, 2024
29f0b7b
Fix formatting
max-Hawkins Nov 12, 2024
022f593
Remove chemistry straggler
max-Hawkins Nov 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions src/simulation/m_mpi_proxy.fpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ module m_mpi_proxy

use m_mpi_common

use m_nvtx

use ieee_arithmetic
! ==========================================================================

Expand Down Expand Up @@ -865,6 +867,7 @@ contains

#ifdef MFC_MPI

call nvtxStartRange("RHS-COMM-PACKBUF")
!$acc update device(v_size)

if (qbmm .and. .not. polytropic) then
Expand Down Expand Up @@ -1057,6 +1060,7 @@ contains
#:endif
end if
#:endfor
call nvtxEndRange ! Packbuf

! Send/Recv
#:for rdma_mpi in [False, True]
Expand All @@ -1066,26 +1070,34 @@ contains
#:if rdma_mpi
!$acc data attach(p_send, p_recv)
!$acc host_data use_device(p_send, p_recv)
call nvtxStartRange("RHS-COMM-SENDRECV-RDMA")
#:else
call nvtxStartRange("RHS-COMM-DEV2HOST")
!$acc update host(q_cons_buff_send, ib_buff_send)
call nvtxEndRange
call nvtxStartRange("RHS-COMM-SENDRECV-NO-RMDA")
#:endif

call MPI_SENDRECV( &
p_send, buffer_count, MPI_DOUBLE_PRECISION, dst_proc, send_tag, &
p_recv, buffer_count, MPI_DOUBLE_PRECISION, src_proc, recv_tag, &
MPI_COMM_WORLD, MPI_STATUS_IGNORE, ierr)
call nvtxEndRange ! RHS-MPI-SENDRECV-(NO)-RDMA

#:if rdma_mpi
!$acc end host_data
!$acc end data
!$acc wait
#:else
call nvtxStartRange("RHS-COMM-HOST2DEV")
!$acc update device(q_cons_buff_recv)
call nvtxEndRange
#:endif
end if
#:endfor

! Unpack Received Buffer
call nvtxStartRange("RHS-COMM-UNPACKBUF")
#:for mpi_dir in [1, 2, 3]
if (mpi_dir == ${mpi_dir}$) then
#:if mpi_dir == 1
Expand Down Expand Up @@ -1254,6 +1266,7 @@ contains
#:endif
end if
#:endfor
call nvtxEndRange

#endif

Expand Down
97 changes: 65 additions & 32 deletions src/simulation/m_rhs.fpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -622,7 +622,7 @@ contains
real(kind(0d0)) :: t_start, t_finish
integer :: i, j, k, l, id !< Generic loop iterators

call nvtxStartRange("Compute_RHS")
call nvtxStartRange("COMPUTE-RHS")

call cpu_time(t_start)
! Association/Population of Working Variables ======================
Expand Down Expand Up @@ -669,7 +669,7 @@ contains
gm_alpha_qp%vf)
call nvtxEndRange

call nvtxStartRange("RHS-MPI")
call nvtxStartRange("RHS-COMMUNICATION")
call s_populate_variables_buffers(q_prim_qp%vf, pb, mv)
call nvtxEndRange

Expand All @@ -683,8 +683,9 @@ contains

if (qbmm) call s_mom_inv(q_cons_qp%vf, q_prim_qp%vf, mom_sp, mom_3d, pb, rhs_pb, mv, rhs_mv, idwbuff(1), idwbuff(2), idwbuff(3), nbub)

call nvtxStartRange("Viscous")
if (viscous) call s_get_viscous(qL_rsx_vf, qL_rsy_vf, qL_rsz_vf, &
if (viscous) then
call nvtxStartRange("RHS-VISCOUS")
call s_get_viscous(qL_rsx_vf, qL_rsy_vf, qL_rsz_vf, &
dqL_prim_dx_n, dqL_prim_dy_n, dqL_prim_dz_n, &
qL_prim, &
qR_rsx_vf, qR_rsy_vf, qR_rsz_vf, &
Expand All @@ -693,11 +694,15 @@ contains
q_prim_qp, &
dq_prim_dx_qp, dq_prim_dy_qp, dq_prim_dz_qp, &
idwbuff(1), idwbuff(2), idwbuff(3))
call nvtxEndRange
call nvtxEndRange
end if

call nvtxStartRange("Surface_Tension")
if (surface_tension) call s_get_capilary(q_prim_qp%vf)
call nvtxEndRange

if (surface_tension) then
call nvtxStartRange("RHS-SURFACE-TENSION")
call s_get_capilary(q_prim_qp%vf)
call nvtxEndRange
end if
! Dimensional Splitting Loop =======================================

do id = 1, num_dims
Expand Down Expand Up @@ -777,7 +782,7 @@ contains
irx%end = m; iry%end = n; irz%end = p
! ===============================================================

call nvtxStartRange("RHS_riemann_solver")
call nvtxStartRange("RHS-RIEMANN-SOLVER")

! Computing Riemann Solver Flux and Source Flux =================

Expand All @@ -801,7 +806,7 @@ contains
! Additional physics and source terms ==============================

! RHS addition for advection source
call nvtxStartRange("RHS_advection_source")
call nvtxStartRange("RHS-ADVECTION-SRC")
call s_compute_advection_source_term(id, &
rhs_vf, &
q_cons_qp, &
Expand All @@ -810,34 +815,37 @@ contains
call nvtxEndRange

! RHS additions for hypoelasticity
call nvtxStartRange("RHS_Hypoelasticity")
call nvtxStartRange("RHS-HYPOELASTICITY")
if (hypoelasticity) call s_compute_hypoelastic_rhs(id, &
q_prim_qp%vf, &
rhs_vf)
call nvtxEndRange

! RHS additions for viscosity
call nvtxStartRange("RHS_add_phys")
if (viscous .or. surface_tension) then
call nvtxStartRange("RHS-ADD-PHYSICS")
call s_compute_additional_physics_rhs(id, &
q_prim_qp%vf, &
rhs_vf, &
flux_src_n(id)%vf, &
dq_prim_dx_qp(1)%vf, &
dq_prim_dy_qp(1)%vf, &
dq_prim_dz_qp(1)%vf)
call nvtxEndRange
end if
call nvtxEndRange

! RHS additions for sub-grid bubbles
call nvtxStartRange("RHS_bubbles")
if (bubbles) call s_compute_bubbles_rhs(id, &
q_prim_qp%vf)
call nvtxEndRange
if (bubbles) then
call nvtxStartRange("RHS-BUBBLES-COMPUTE")
call s_compute_bubbles_rhs(id, q_prim_qp%vf)
call nvtxEndRange
end if

! RHS additions for qbmm bubbles
call nvtxStartRange("RHS_qbmm")
if (qbmm) call s_compute_qbmm_rhs(id, &

if (qbmm) then
call nvtxStartRange("RHS-QBMM")
call s_compute_qbmm_rhs(id, &
q_cons_qp%vf, &
q_prim_qp%vf, &
rhs_vf, &
Expand All @@ -846,14 +854,35 @@ contains
rhs_pb, &
mv, &
rhs_mv)
call nvtxEndRange
call nvtxEndRange
end if
! END: Additional physics and source terms =========================

#:if chemistry
if (chem_params%advection) then
call nvtxStartRange("RHS-CHEM-ADVECTION")

#:for NORM_DIR, XYZ in [(1, 'x'), (2, 'y'), (3, 'z')]

if (id == ${NORM_DIR}$) then
call s_compute_chemistry_rhs_${XYZ}$ ( &
flux_n, &
rhs_vf, &
flux_src_n(${NORM_DIR}$)%vf, &
q_prim_vf)
end if

#:endfor

call nvtxEndRange
end if
#:endif

end do
! END: Dimensional Splitting Loop =================================

if (chemistry) then
call nvtxStartRange("RHS_Chem_Advection")
call nvtxStartRange("RHS-CHEM-ADVECTION")
call s_compute_chemistry_advection_flux(flux_n, rhs_vf)
call nvtxEndRange
end if
Expand All @@ -875,25 +904,29 @@ contains

! Additional Physics and Source Temrs ==================================
! Additions for acoustic_source
call nvtxStartRange("RHS_acoustic_src")
if (acoustic_source) call s_acoustic_src_calculations(q_cons_qp%vf(1:sys_size), &
if (acoustic_source) then
call nvtxStartRange("RHS-ACOUSTIC-SRC")
call s_acoustic_src_calculations(q_cons_qp%vf(1:sys_size), &
q_prim_qp%vf(1:sys_size), &
t_step, &
rhs_vf)
call nvtxEndRange
call nvtxEndRange
end if

! Add bubles source term
call nvtxStartRange("RHS_bubbles")
if (bubbles .and. (.not. adap_dt) .and. (.not. qbmm)) call s_compute_bubble_source( &
q_cons_qp%vf(1:sys_size), &
q_prim_qp%vf(1:sys_size), &
t_step, &
rhs_vf)
call nvtxEndRange
if (bubbles .and. (.not. adap_dt) .and. (.not. qbmm)) then
call nvtxStartRange("RHS-BUBBLES-SRC")
call s_compute_bubble_source( &
q_cons_qp%vf(1:sys_size), &
q_prim_qp%vf(1:sys_size), &
t_step, &
rhs_vf)
call nvtxEndRange
end if

if (chemistry) then
if (chem_params%reactions) then
call nvtxStartRange("RHS_Chem_Reactions")
call nvtxStartRange("RHS-CHEM-REACTIONS")
call s_compute_chemistry_reaction_flux(rhs_vf, q_cons_qp%vf, q_prim_qp%vf)
call nvtxEndRange
end if
Expand Down
4 changes: 2 additions & 2 deletions src/simulation/m_start_up.fpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1239,7 +1239,7 @@ contains
integer :: save_count

call cpu_time(start)
! call nvtxStartRange("I/O")
call nvtxStartRange("SAVE-DATA")
do i = 1, sys_size
!$acc update host(q_cons_ts(1)%vf(i)%sf)
do l = 0, p
Expand Down Expand Up @@ -1267,7 +1267,7 @@ contains

call s_write_data_files(q_cons_ts(1)%vf, q_prim_vf, save_count)

! call nvtxEndRange
call nvtxEndRange
call cpu_time(finish)
if (cfl_dt) then
nt = mytime/t_save
Expand Down
24 changes: 7 additions & 17 deletions src/simulation/m_time_steppers.fpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -279,7 +279,7 @@ contains
integer :: i, j, k, l, q !< Generic loop iterator

! Stage 1 of 1 =====================================================
call nvtxStartRange("Time_Step")
call nvtxStartRange("TIMESTEP")

call s_compute_rhs(q_cons_ts(1)%vf, q_prim_vf, rhs_vf, pb_ts(1)%sf, rhs_pb, mv_ts(1)%sf, rhs_mv, t_step, time_avg)

Expand Down Expand Up @@ -353,7 +353,6 @@ contains
end do
end if

call nvtxStartRange("body_forces")
if (bodyForces) call s_apply_bodyforces(q_cons_ts(1)%vf, q_prim_vf, rhs_vf, dt)
call nvtxEndRange

Expand All @@ -371,8 +370,6 @@ contains
end if
end if

call nvtxEndRange

! ==================================================================

end subroutine s_1st_order_tvd_rk
Expand All @@ -391,7 +388,7 @@ contains

call cpu_time(start)

call nvtxStartRange("Time_Step")
call nvtxStartRange("TIMESTEP")

call s_compute_rhs(q_cons_ts(1)%vf, q_prim_vf, rhs_vf, pb_ts(1)%sf, rhs_pb, mv_ts(1)%sf, rhs_mv, t_step, time_avg)

Expand Down Expand Up @@ -457,9 +454,7 @@ contains
end do
end if

call nvtxStartRange("body_forces")
if (bodyForces) call s_apply_bodyforces(q_cons_ts(2)%vf, q_prim_vf, rhs_vf, dt)
call nvtxEndRange

if (grid_geometry == 3) call s_apply_fourier_filter(q_cons_ts(2)%vf)

Expand Down Expand Up @@ -532,9 +527,7 @@ contains
end do
end if

call nvtxStartRange("body_forces")
if (bodyForces) call s_apply_bodyforces(q_cons_ts(1)%vf, q_prim_vf, rhs_vf, 2d0*dt/3d0)
call nvtxEndRange

if (grid_geometry == 3) call s_apply_fourier_filter(q_cons_ts(1)%vf)

Expand Down Expand Up @@ -573,7 +566,7 @@ contains

if (.not. adap_dt) then
call cpu_time(start)
call nvtxStartRange("Time_Step")
call nvtxStartRange("TIMESTEP")
end if

call s_compute_rhs(q_cons_ts(1)%vf, q_prim_vf, rhs_vf, pb_ts(1)%sf, rhs_pb, mv_ts(1)%sf, rhs_mv, t_step, time_avg)
Expand Down Expand Up @@ -640,9 +633,7 @@ contains
end do
end if

call nvtxStartRange("body_forces")
if (bodyForces) call s_apply_bodyforces(q_cons_ts(2)%vf, q_prim_vf, rhs_vf, dt)
call nvtxEndRange

if (grid_geometry == 3) call s_apply_fourier_filter(q_cons_ts(2)%vf)

Expand Down Expand Up @@ -715,9 +706,7 @@ contains
end do
end if

call nvtxStartRange("body_forces")
if (bodyForces) call s_apply_bodyforces(q_cons_ts(2)%vf, q_prim_vf, rhs_vf, dt/4d0)
call nvtxEndRange

if (grid_geometry == 3) call s_apply_fourier_filter(q_cons_ts(2)%vf)

Expand Down Expand Up @@ -789,9 +778,7 @@ contains
end do
end if

call nvtxStartRange("body_forces")
if (bodyForces) call s_apply_bodyforces(q_cons_ts(1)%vf, q_prim_vf, rhs_vf, 2d0*dt/3d0)
call nvtxEndRange

if (grid_geometry == 3) call s_apply_fourier_filter(q_cons_ts(1)%vf)

Expand Down Expand Up @@ -832,7 +819,7 @@ contains

call cpu_time(start)

call nvtxStartRange("Time_Step")
call nvtxStartRange("TIMESTEP")

! Stage 1 of 3 =====================================================
call s_adaptive_dt_bubble(t_step)
Expand Down Expand Up @@ -935,6 +922,7 @@ contains

integer :: i, j, k, l

call nvtxStartRange("RHS-BODYFORCES")
call s_compute_body_forces_rhs(q_prim_vf, q_cons_vf, rhs_vf)

!$acc parallel loop collapse(4) gang vector default(present)
Expand All @@ -949,6 +937,8 @@ contains
end do
end do

call nvtxEndRange

end subroutine s_apply_bodyforces

!> This subroutine saves the temporary q_prim_vf vector
Expand Down
Loading
Loading