Skip to content

Commit a79e2e0

Browse files
max-Hawkinsmax-Hawkins
authored
Increase granularity of halo-exchange timing info (#639)
1 parent f1e53d4 commit a79e2e0

File tree

5 files changed

+102
-70
lines changed

5 files changed

+102
-70
lines changed

‎src/simulation/m_mpi_proxy.fpp‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ module m_mpi_proxy
2727
2828
use m_mpi_common
2929
30+
use m_nvtx
31+
3032
use ieee_arithmetic
3133
! ==========================================================================
3234
@@ -865,6 +867,7 @@ contains
865867
866868
#ifdef MFC_MPI
867869
870+
call nvtxStartRange("RHS-COMM-PACKBUF")
868871
!$acc update device(v_size)
869872
870873
if (qbmm .and. .not. polytropic) then
@@ -1057,6 +1060,7 @@ contains
10571060
#:endif
10581061
end if
10591062
#:endfor
1063+
call nvtxEndRange ! Packbuf
10601064
10611065
! Send/Recv
10621066
#:for rdma_mpi in [False, True]
@@ -1066,26 +1070,34 @@ contains
10661070
#:if rdma_mpi
10671071
!$acc data attach(p_send, p_recv)
10681072
!$acc host_data use_device(p_send, p_recv)
1073+
call nvtxStartRange("RHS-COMM-SENDRECV-RDMA")
10691074
#:else
1075+
call nvtxStartRange("RHS-COMM-DEV2HOST")
10701076
!$acc update host(q_cons_buff_send, ib_buff_send)
1077+
call nvtxEndRange
1078+
call nvtxStartRange("RHS-COMM-SENDRECV-NO-RMDA")
10711079
#:endif
10721080
10731081
call MPI_SENDRECV( &
10741082
p_send, buffer_count, MPI_DOUBLE_PRECISION, dst_proc, send_tag, &
10751083
p_recv, buffer_count, MPI_DOUBLE_PRECISION, src_proc, recv_tag, &
10761084
MPI_COMM_WORLD, MPI_STATUS_IGNORE, ierr)
1085+
call nvtxEndRange ! RHS-MPI-SENDRECV-(NO)-RDMA
10771086
10781087
#:if rdma_mpi
10791088
!$acc end host_data
10801089
!$acc end data
10811090
!$acc wait
10821091
#:else
1092+
call nvtxStartRange("RHS-COMM-HOST2DEV")
10831093
!$acc update device(q_cons_buff_recv)
1094+
call nvtxEndRange
10841095
#:endif
10851096
end if
10861097
#:endfor
10871098
10881099
! Unpack Received Buffer
1100+
call nvtxStartRange("RHS-COMM-UNPACKBUF")
10891101
#:for mpi_dir in [1, 2, 3]
10901102
if (mpi_dir == ${mpi_dir}$) then
10911103
#:if mpi_dir == 1
@@ -1254,6 +1266,7 @@ contains
12541266
#:endif
12551267
end if
12561268
#:endfor
1269+
call nvtxEndRange
12571270
12581271
#endif
12591272

‎src/simulation/m_rhs.fpp‎

Lines changed: 63 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -622,7 +622,7 @@ contains
622622
real(kind(0d0)) :: t_start, t_finish
623623
integer :: i, j, k, l, id !< Generic loop iterators
624624

625-
call nvtxStartRange("Compute_RHS")
625+
call nvtxStartRange("COMPUTE-RHS")
626626

627627
call cpu_time(t_start)
628628
! Association/Population of Working Variables ======================
@@ -669,7 +669,7 @@ contains
669669
gm_alpha_qp%vf)
670670
call nvtxEndRange
671671

672-
call nvtxStartRange("RHS-MPI")
672+
call nvtxStartRange("RHS-COMMUNICATION")
673673
call s_populate_variables_buffers(q_prim_qp%vf, pb, mv)
674674
call nvtxEndRange
675675

@@ -683,21 +683,25 @@ contains
683683

684684
if (qbmm) call s_mom_inv(q_cons_qp%vf, q_prim_qp%vf, mom_sp, mom_3d, pb, rhs_pb, mv, rhs_mv, idwbuff(1), idwbuff(2), idwbuff(3), nbub)
685685

686-
call nvtxStartRange("Viscous")
687-
if (viscous) call s_get_viscous(qL_rsx_vf, qL_rsy_vf, qL_rsz_vf, &
688-
dqL_prim_dx_n, dqL_prim_dy_n, dqL_prim_dz_n, &
689-
qL_prim, &
690-
qR_rsx_vf, qR_rsy_vf, qR_rsz_vf, &
691-
dqR_prim_dx_n, dqR_prim_dy_n, dqR_prim_dz_n, &
692-
qR_prim, &
693-
q_prim_qp, &
694-
dq_prim_dx_qp, dq_prim_dy_qp, dq_prim_dz_qp, &
695-
idwbuff(1), idwbuff(2), idwbuff(3))
696-
call nvtxEndRange
686+
if (viscous) then
687+
call nvtxStartRange("RHS-VISCOUS")
688+
call s_get_viscous(qL_rsx_vf, qL_rsy_vf, qL_rsz_vf, &
689+
dqL_prim_dx_n, dqL_prim_dy_n, dqL_prim_dz_n, &
690+
qL_prim, &
691+
qR_rsx_vf, qR_rsy_vf, qR_rsz_vf, &
692+
dqR_prim_dx_n, dqR_prim_dy_n, dqR_prim_dz_n, &
693+
qR_prim, &
694+
q_prim_qp, &
695+
dq_prim_dx_qp, dq_prim_dy_qp, dq_prim_dz_qp, &
696+
idwbuff(1), idwbuff(2), idwbuff(3))
697+
call nvtxEndRange
698+
end if
697699

698-
call nvtxStartRange("Surface_Tension")
699-
if (surface_tension) call s_get_capilary(q_prim_qp%vf)
700-
call nvtxEndRange
700+
if (surface_tension) then
701+
call nvtxStartRange("RHS-SURFACE-TENSION")
702+
call s_get_capilary(q_prim_qp%vf)
703+
call nvtxEndRange
704+
end if
701705
! Dimensional Splitting Loop =======================================
702706

703707
do id = 1, num_dims
@@ -777,7 +781,7 @@ contains
777781
irx%end = m; iry%end = n; irz%end = p
778782
! ===============================================================
779783

780-
call nvtxStartRange("RHS_riemann_solver")
784+
call nvtxStartRange("RHS-RIEMANN-SOLVER")
781785

782786
! Computing Riemann Solver Flux and Source Flux =================
783787

@@ -801,7 +805,7 @@ contains
801805
! Additional physics and source terms ==============================
802806

803807
! RHS addition for advection source
804-
call nvtxStartRange("RHS_advection_source")
808+
call nvtxStartRange("RHS-ADVECTION-SRC")
805809
call s_compute_advection_source_term(id, &
806810
rhs_vf, &
807811
q_cons_qp, &
@@ -810,50 +814,54 @@ contains
810814
call nvtxEndRange
811815

812816
! RHS additions for hypoelasticity
813-
call nvtxStartRange("RHS_Hypoelasticity")
817+
call nvtxStartRange("RHS-HYPOELASTICITY")
814818
if (hypoelasticity) call s_compute_hypoelastic_rhs(id, &
815819
q_prim_qp%vf, &
816820
rhs_vf)
817821
call nvtxEndRange
818822

819823
! RHS additions for viscosity
820-
call nvtxStartRange("RHS_add_phys")
821824
if (viscous .or. surface_tension) then
825+
call nvtxStartRange("RHS-ADD-PHYSICS")
822826
call s_compute_additional_physics_rhs(id, &
823827
q_prim_qp%vf, &
824828
rhs_vf, &
825829
flux_src_n(id)%vf, &
826830
dq_prim_dx_qp(1)%vf, &
827831
dq_prim_dy_qp(1)%vf, &
828832
dq_prim_dz_qp(1)%vf)
833+
call nvtxEndRange
829834
end if
830-
call nvtxEndRange
831835

832836
! RHS additions for sub-grid bubbles
833-
call nvtxStartRange("RHS_bubbles")
834-
if (bubbles) call s_compute_bubbles_rhs(id, &
835-
q_prim_qp%vf)
836-
call nvtxEndRange
837+
if (bubbles) then
838+
call nvtxStartRange("RHS-BUBBLES-COMPUTE")
839+
call s_compute_bubbles_rhs(id, q_prim_qp%vf)
840+
call nvtxEndRange
841+
end if
837842

838843
! RHS additions for qbmm bubbles
839-
call nvtxStartRange("RHS_qbmm")
840-
if (qbmm) call s_compute_qbmm_rhs(id, &
841-
q_cons_qp%vf, &
842-
q_prim_qp%vf, &
843-
rhs_vf, &
844-
flux_n(id)%vf, &
845-
pb, &
846-
rhs_pb, &
847-
mv, &
848-
rhs_mv)
849-
call nvtxEndRange
844+
845+
if (qbmm) then
846+
call nvtxStartRange("RHS-QBMM")
847+
call s_compute_qbmm_rhs(id, &
848+
q_cons_qp%vf, &
849+
q_prim_qp%vf, &
850+
rhs_vf, &
851+
flux_n(id)%vf, &
852+
pb, &
853+
rhs_pb, &
854+
mv, &
855+
rhs_mv)
856+
call nvtxEndRange
857+
end if
850858
! END: Additional physics and source terms =========================
851859

852860
end do
853861
! END: Dimensional Splitting Loop =================================
854862

855863
if (chemistry) then
856-
call nvtxStartRange("RHS_Chem_Advection")
864+
call nvtxStartRange("RHS-CHEM-ADVECTION")
857865
call s_compute_chemistry_advection_flux(flux_n, rhs_vf)
858866
call nvtxEndRange
859867
end if
@@ -875,25 +883,29 @@ contains
875883

876884
! Additional Physics and Source Temrs ==================================
877885
! Additions for acoustic_source
878-
call nvtxStartRange("RHS_acoustic_src")
879-
if (acoustic_source) call s_acoustic_src_calculations(q_cons_qp%vf(1:sys_size), &
880-
q_prim_qp%vf(1:sys_size), &
881-
t_step, &
882-
rhs_vf)
883-
call nvtxEndRange
886+
if (acoustic_source) then
887+
call nvtxStartRange("RHS-ACOUSTIC-SRC")
888+
call s_acoustic_src_calculations(q_cons_qp%vf(1:sys_size), &
889+
q_prim_qp%vf(1:sys_size), &
890+
t_step, &
891+
rhs_vf)
892+
call nvtxEndRange
893+
end if
884894

885895
! Add bubles source term
886-
call nvtxStartRange("RHS_bubbles")
887-
if (bubbles .and. (.not. adap_dt) .and. (.not. qbmm)) call s_compute_bubble_source( &
888-
q_cons_qp%vf(1:sys_size), &
889-
q_prim_qp%vf(1:sys_size), &
890-
t_step, &
891-
rhs_vf)
892-
call nvtxEndRange
896+
if (bubbles .and. (.not. adap_dt) .and. (.not. qbmm)) then
897+
call nvtxStartRange("RHS-BUBBLES-SRC")
898+
call s_compute_bubble_source( &
899+
q_cons_qp%vf(1:sys_size), &
900+
q_prim_qp%vf(1:sys_size), &
901+
t_step, &
902+
rhs_vf)
903+
call nvtxEndRange
904+
end if
893905

894906
if (chemistry) then
895907
if (chem_params%reactions) then
896-
call nvtxStartRange("RHS_Chem_Reactions")
908+
call nvtxStartRange("RHS-CHEM-REACTIONS")
897909
call s_compute_chemistry_reaction_flux(rhs_vf, q_cons_qp%vf, q_prim_qp%vf)
898910
call nvtxEndRange
899911
end if

‎src/simulation/m_start_up.fpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1352,7 +1352,7 @@ contains
13521352
integer :: save_count
13531353

13541354
call cpu_time(start)
1355-
! call nvtxStartRange("I/O")
1355+
call nvtxStartRange("SAVE-DATA")
13561356
do i = 1, sys_size
13571357
!$acc update host(q_cons_ts(1)%vf(i)%sf)
13581358
do l = 0, p
@@ -1380,7 +1380,7 @@ contains
13801380

13811381
call s_write_data_files(q_cons_ts(1)%vf, q_prim_vf, save_count)
13821382

1383-
! call nvtxEndRange
1383+
call nvtxEndRange
13841384
call cpu_time(finish)
13851385
if (cfl_dt) then
13861386
nt = mytime/t_save

‎src/simulation/m_time_steppers.fpp‎

Lines changed: 7 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -279,7 +279,7 @@ contains
279279
integer :: i, j, k, l, q !< Generic loop iterator
280280

281281
! Stage 1 of 1 =====================================================
282-
call nvtxStartRange("Time_Step")
282+
call nvtxStartRange("TIMESTEP")
283283

284284
call s_compute_rhs(q_cons_ts(1)%vf, q_prim_vf, rhs_vf, pb_ts(1)%sf, rhs_pb, mv_ts(1)%sf, rhs_mv, t_step, time_avg)
285285

@@ -353,7 +353,6 @@ contains
353353
end do
354354
end if
355355

356-
call nvtxStartRange("body_forces")
357356
if (bodyForces) call s_apply_bodyforces(q_cons_ts(1)%vf, q_prim_vf, rhs_vf, dt)
358357
call nvtxEndRange
359358

@@ -371,8 +370,6 @@ contains
371370
end if
372371
end if
373372

374-
call nvtxEndRange
375-
376373
! ==================================================================
377374

378375
end subroutine s_1st_order_tvd_rk
@@ -391,7 +388,7 @@ contains
391388

392389
call cpu_time(start)
393390

394-
call nvtxStartRange("Time_Step")
391+
call nvtxStartRange("TIMESTEP")
395392

396393
call s_compute_rhs(q_cons_ts(1)%vf, q_prim_vf, rhs_vf, pb_ts(1)%sf, rhs_pb, mv_ts(1)%sf, rhs_mv, t_step, time_avg)
397394

@@ -457,9 +454,7 @@ contains
457454
end do
458455
end if
459456

460-
call nvtxStartRange("body_forces")
461457
if (bodyForces) call s_apply_bodyforces(q_cons_ts(2)%vf, q_prim_vf, rhs_vf, dt)
462-
call nvtxEndRange
463458

464459
if (grid_geometry == 3) call s_apply_fourier_filter(q_cons_ts(2)%vf)
465460

@@ -532,9 +527,7 @@ contains
532527
end do
533528
end if
534529

535-
call nvtxStartRange("body_forces")
536530
if (bodyForces) call s_apply_bodyforces(q_cons_ts(1)%vf, q_prim_vf, rhs_vf, 2d0*dt/3d0)
537-
call nvtxEndRange
538531

539532
if (grid_geometry == 3) call s_apply_fourier_filter(q_cons_ts(1)%vf)
540533

@@ -573,7 +566,7 @@ contains
573566

574567
if (.not. adap_dt) then
575568
call cpu_time(start)
576-
call nvtxStartRange("Time_Step")
569+
call nvtxStartRange("TIMESTEP")
577570
end if
578571

579572
call s_compute_rhs(q_cons_ts(1)%vf, q_prim_vf, rhs_vf, pb_ts(1)%sf, rhs_pb, mv_ts(1)%sf, rhs_mv, t_step, time_avg)
@@ -640,9 +633,7 @@ contains
640633
end do
641634
end if
642635

643-
call nvtxStartRange("body_forces")
644636
if (bodyForces) call s_apply_bodyforces(q_cons_ts(2)%vf, q_prim_vf, rhs_vf, dt)
645-
call nvtxEndRange
646637

647638
if (grid_geometry == 3) call s_apply_fourier_filter(q_cons_ts(2)%vf)
648639

@@ -715,9 +706,7 @@ contains
715706
end do
716707
end if
717708

718-
call nvtxStartRange("body_forces")
719709
if (bodyForces) call s_apply_bodyforces(q_cons_ts(2)%vf, q_prim_vf, rhs_vf, dt/4d0)
720-
call nvtxEndRange
721710

722711
if (grid_geometry == 3) call s_apply_fourier_filter(q_cons_ts(2)%vf)
723712

@@ -789,9 +778,7 @@ contains
789778
end do
790779
end if
791780

792-
call nvtxStartRange("body_forces")
793781
if (bodyForces) call s_apply_bodyforces(q_cons_ts(1)%vf, q_prim_vf, rhs_vf, 2d0*dt/3d0)
794-
call nvtxEndRange
795782

796783
if (grid_geometry == 3) call s_apply_fourier_filter(q_cons_ts(1)%vf)
797784

@@ -832,7 +819,7 @@ contains
832819

833820
call cpu_time(start)
834821

835-
call nvtxStartRange("Time_Step")
822+
call nvtxStartRange("TIMESTEP")
836823

837824
! Stage 1 of 3 =====================================================
838825
call s_adaptive_dt_bubble(t_step)
@@ -935,6 +922,7 @@ contains
935922

936923
integer :: i, j, k, l
937924

925+
call nvtxStartRange("RHS-BODYFORCES")
938926
call s_compute_body_forces_rhs(q_prim_vf, q_cons_vf, rhs_vf)
939927

940928
!$acc parallel loop collapse(4) gang vector default(present)
@@ -949,6 +937,8 @@ contains
949937
end do
950938
end do
951939

940+
call nvtxEndRange
941+
952942
end subroutine s_apply_bodyforces
953943

954944
!> This subroutine saves the temporary q_prim_vf vector

0 commit comments

Comments
 (0)