Skip to content

Commit ce5c8af

Browse files
max-Hawkinsmax-Hawkins
committed
Improve NVTX range tracking
1 parent e00bc64 commit ce5c8af

File tree

3 files changed

+54
-40
lines changed

3 files changed

+54
-40
lines changed

‎src/simulation/m_mpi_proxy.fpp‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ module m_mpi_proxy
2727
2828
use m_mpi_common
2929
30+
use m_nvtx
31+
3032
use ieee_arithmetic
3133
! ==========================================================================
3234
@@ -865,6 +867,7 @@ contains
865867
866868
#ifdef MFC_MPI
867869
870+
call nvtxStartRange("RHS-COMM-PACKBUF")
868871
!$acc update device(v_size)
869872
870873
if (qbmm .and. .not. polytropic) then
@@ -1057,6 +1060,7 @@ contains
10571060
#:endif
10581061
end if
10591062
#:endfor
1063+
call nvtxEndRange ! Packbuf
10601064
10611065
! Send/Recv
10621066
#:for rdma_mpi in [False, True]
@@ -1066,26 +1070,34 @@ contains
10661070
#:if rdma_mpi
10671071
!$acc data attach(p_send, p_recv)
10681072
!$acc host_data use_device(p_send, p_recv)
1073+
call nvtxStartRange("RHS-COMM-SENDRECV-RDMA")
10691074
#:else
1075+
call nvtxStartRange("RHS-COMM-DEV2HOST")
10701076
!$acc update host(q_cons_buff_send, ib_buff_send)
1077+
call nvtxEndRange
1078+
call nvtxStartRange("RHS-COMM-SENDRECV-NO-RMDA")
10711079
#:endif
10721080
10731081
call MPI_SENDRECV( &
10741082
p_send, buffer_count, MPI_DOUBLE_PRECISION, dst_proc, send_tag, &
10751083
p_recv, buffer_count, MPI_DOUBLE_PRECISION, src_proc, recv_tag, &
10761084
MPI_COMM_WORLD, MPI_STATUS_IGNORE, ierr)
1085+
call nvtxEndRange ! RHS-MPI-SENDRECV-(NO)-RDMA
10771086
10781087
#:if rdma_mpi
10791088
!$acc end host_data
10801089
!$acc end data
10811090
!$acc wait
10821091
#:else
1092+
call nvtxStartRange("RHS-COMM-HOST2DEV")
10831093
!$acc update device(q_cons_buff_recv)
1094+
call nvtxEndRange
10841095
#:endif
10851096
end if
10861097
#:endfor
10871098
10881099
! Unpack Received Buffer
1100+
call nvtxStartRange("RHS-COMM-UNPACKBUF")
10891101
#:for mpi_dir in [1, 2, 3]
10901102
if (mpi_dir == ${mpi_dir}$) then
10911103
#:if mpi_dir == 1
@@ -1254,6 +1266,7 @@ contains
12541266
#:endif
12551267
end if
12561268
#:endfor
1269+
call nvtxEndRange
12571270
12581271
#endif
12591272

‎src/simulation/m_rhs.fpp‎

Lines changed: 34 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -622,7 +622,7 @@ contains
622622
real(kind(0d0)) :: t_start, t_finish
623623
integer :: i, j, k, l, id !< Generic loop iterators
624624

625-
call nvtxStartRange("Compute_RHS")
625+
call nvtxStartRange("COMPUTE-RHS")
626626

627627
call cpu_time(t_start)
628628
! Association/Population of Working Variables ======================
@@ -683,8 +683,9 @@ contains
683683

684684
if (qbmm) call s_mom_inv(q_cons_qp%vf, q_prim_qp%vf, mom_sp, mom_3d, pb, rhs_pb, mv, rhs_mv, idwbuff(1), idwbuff(2), idwbuff(3), nbub)
685685

686-
call nvtxStartRange("Viscous")
687-
if (viscous) call s_get_viscous(qL_rsx_vf, qL_rsy_vf, qL_rsz_vf, &
686+
if (viscous) then
687+
call nvtxStartRange("RHS-VISCOUS")
688+
call s_get_viscous(qL_rsx_vf, qL_rsy_vf, qL_rsz_vf, &
688689
dqL_prim_dx_n, dqL_prim_dy_n, dqL_prim_dz_n, &
689690
qL_prim, &
690691
qR_rsx_vf, qR_rsy_vf, qR_rsz_vf, &
@@ -693,11 +694,15 @@ contains
693694
q_prim_qp, &
694695
dq_prim_dx_qp, dq_prim_dy_qp, dq_prim_dz_qp, &
695696
idwbuff(1), idwbuff(2), idwbuff(3))
696-
call nvtxEndRange
697+
call nvtxEndRange
698+
end if
697699

698-
call nvtxStartRange("Surface_Tension")
699-
if (surface_tension) call s_get_capilary(q_prim_qp%vf)
700-
call nvtxEndRange
700+
701+
if (surface_tension) then
702+
call nvtxStartRange("RHS-SURFACE-TENSION")
703+
call s_get_capilary(q_prim_qp%vf)
704+
call nvtxEndRange
705+
end if
701706
! Dimensional Splitting Loop =======================================
702707

703708
do id = 1, num_dims
@@ -777,7 +782,7 @@ contains
777782
irx%end = m; iry%end = n; irz%end = p
778783
! ===============================================================
779784

780-
call nvtxStartRange("RHS_riemann_solver")
785+
call nvtxStartRange("RHS-RIEMANN-SOLVER")
781786

782787
! Computing Riemann Solver Flux and Source Flux =================
783788

@@ -801,7 +806,7 @@ contains
801806
! Additional physics and source terms ==============================
802807

803808
! RHS addition for advection source
804-
call nvtxStartRange("RHS_advection_source")
809+
call nvtxStartRange("RHS-ADVECTION-SRC")
805810
call s_compute_advection_source_term(id, &
806811
rhs_vf, &
807812
q_cons_qp, &
@@ -810,34 +815,37 @@ contains
810815
call nvtxEndRange
811816

812817
! RHS additions for hypoelasticity
813-
call nvtxStartRange("RHS_Hypoelasticity")
818+
call nvtxStartRange("RHS-HYPOELASTICITY")
814819
if (hypoelasticity) call s_compute_hypoelastic_rhs(id, &
815820
q_prim_qp%vf, &
816821
rhs_vf)
817822
call nvtxEndRange
818823

819824
! RHS additions for viscosity
820-
call nvtxStartRange("RHS_add_phys")
821825
if (viscous .or. surface_tension) then
826+
call nvtxStartRange("RHS-ADD-PHYSICS")
822827
call s_compute_additional_physics_rhs(id, &
823828
q_prim_qp%vf, &
824829
rhs_vf, &
825830
flux_src_n(id)%vf, &
826831
dq_prim_dx_qp(1)%vf, &
827832
dq_prim_dy_qp(1)%vf, &
828833
dq_prim_dz_qp(1)%vf)
834+
call nvtxEndRange
829835
end if
830-
call nvtxEndRange
831836

832837
! RHS additions for sub-grid bubbles
833-
call nvtxStartRange("RHS_bubbles")
834-
if (bubbles) call s_compute_bubbles_rhs(id, &
835-
q_prim_qp%vf)
836-
call nvtxEndRange
838+
if (bubbles) then
839+
call nvtxStartRange("RHS-BUBBLES-COMPUTE")
840+
call s_compute_bubbles_rhs(id, q_prim_qp%vf)
841+
call nvtxEndRange
842+
end if
837843

838844
! RHS additions for qbmm bubbles
839-
call nvtxStartRange("RHS_qbmm")
840-
if (qbmm) call s_compute_qbmm_rhs(id, &
845+
846+
if (qbmm) then
847+
call nvtxStartRange("RHS-QBMM")
848+
call s_compute_qbmm_rhs(id, &
841849
q_cons_qp%vf, &
842850
q_prim_qp%vf, &
843851
rhs_vf, &
@@ -846,7 +854,8 @@ contains
846854
rhs_pb, &
847855
mv, &
848856
rhs_mv)
849-
call nvtxEndRange
857+
call nvtxEndRange
858+
end if
850859
! END: Additional physics and source terms =========================
851860

852861
#:if chemistry
@@ -873,7 +882,7 @@ contains
873882
! END: Dimensional Splitting Loop =================================
874883

875884
if (chemistry) then
876-
call nvtxStartRange("RHS_Chem_Advection")
885+
call nvtxStartRange("RHS-CHEM-ADVECTION")
877886
call s_compute_chemistry_advection_flux(flux_n, rhs_vf)
878887
call nvtxEndRange
879888
end if
@@ -895,12 +904,14 @@ contains
895904

896905
! Additional Physics and Source Temrs ==================================
897906
! Additions for acoustic_source
898-
call nvtxStartRange("RHS_acoustic_src")
899-
if (acoustic_source) call s_acoustic_src_calculations(q_cons_qp%vf(1:sys_size), &
907+
if (acoustic_source) then
908+
call nvtxStartRange("RHS-ACOUSTIC-SRC")
909+
call s_acoustic_src_calculations(q_cons_qp%vf(1:sys_size), &
900910
q_prim_qp%vf(1:sys_size), &
901911
t_step, &
902912
rhs_vf)
903-
call nvtxEndRange
913+
call nvtxEndRange
914+
end if
904915

905916
! Add bubles source term
906917
if (bubbles .and. (.not. adap_dt) .and. (.not. qbmm)) then

‎src/simulation/m_time_steppers.fpp‎

Lines changed: 7 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -279,7 +279,7 @@ contains
279279
integer :: i, j, k, l, q !< Generic loop iterator
280280

281281
! Stage 1 of 1 =====================================================
282-
call nvtxStartRange("Time_Step")
282+
call nvtxStartRange("TIMESTEP")
283283

284284
call s_compute_rhs(q_cons_ts(1)%vf, q_prim_vf, rhs_vf, pb_ts(1)%sf, rhs_pb, mv_ts(1)%sf, rhs_mv, t_step, time_avg)
285285

@@ -353,7 +353,6 @@ contains
353353
end do
354354
end if
355355

356-
call nvtxStartRange("body_forces")
357356
if (bodyForces) call s_apply_bodyforces(q_cons_ts(1)%vf, q_prim_vf, rhs_vf, dt)
358357
call nvtxEndRange
359358

@@ -371,8 +370,6 @@ contains
371370
end if
372371
end if
373372

374-
call nvtxEndRange
375-
376373
! ==================================================================
377374

378375
end subroutine s_1st_order_tvd_rk
@@ -391,7 +388,7 @@ contains
391388

392389
call cpu_time(start)
393390

394-
call nvtxStartRange("Time_Step")
391+
call nvtxStartRange("TIMESTEP")
395392

396393
call s_compute_rhs(q_cons_ts(1)%vf, q_prim_vf, rhs_vf, pb_ts(1)%sf, rhs_pb, mv_ts(1)%sf, rhs_mv, t_step, time_avg)
397394

@@ -457,9 +454,7 @@ contains
457454
end do
458455
end if
459456

460-
call nvtxStartRange("body_forces")
461457
if (bodyForces) call s_apply_bodyforces(q_cons_ts(2)%vf, q_prim_vf, rhs_vf, dt)
462-
call nvtxEndRange
463458

464459
if (grid_geometry == 3) call s_apply_fourier_filter(q_cons_ts(2)%vf)
465460

@@ -532,9 +527,7 @@ contains
532527
end do
533528
end if
534529

535-
call nvtxStartRange("body_forces")
536530
if (bodyForces) call s_apply_bodyforces(q_cons_ts(1)%vf, q_prim_vf, rhs_vf, 2d0*dt/3d0)
537-
call nvtxEndRange
538531

539532
if (grid_geometry == 3) call s_apply_fourier_filter(q_cons_ts(1)%vf)
540533

@@ -573,7 +566,7 @@ contains
573566

574567
if (.not. adap_dt) then
575568
call cpu_time(start)
576-
call nvtxStartRange("Time_Step")
569+
call nvtxStartRange("TIMESTEP")
577570
end if
578571

579572
call s_compute_rhs(q_cons_ts(1)%vf, q_prim_vf, rhs_vf, pb_ts(1)%sf, rhs_pb, mv_ts(1)%sf, rhs_mv, t_step, time_avg)
@@ -640,9 +633,7 @@ contains
640633
end do
641634
end if
642635

643-
call nvtxStartRange("body_forces")
644636
if (bodyForces) call s_apply_bodyforces(q_cons_ts(2)%vf, q_prim_vf, rhs_vf, dt)
645-
call nvtxEndRange
646637

647638
if (grid_geometry == 3) call s_apply_fourier_filter(q_cons_ts(2)%vf)
648639

@@ -715,9 +706,7 @@ contains
715706
end do
716707
end if
717708

718-
call nvtxStartRange("body_forces")
719709
if (bodyForces) call s_apply_bodyforces(q_cons_ts(2)%vf, q_prim_vf, rhs_vf, dt/4d0)
720-
call nvtxEndRange
721710

722711
if (grid_geometry == 3) call s_apply_fourier_filter(q_cons_ts(2)%vf)
723712

@@ -789,9 +778,7 @@ contains
789778
end do
790779
end if
791780

792-
call nvtxStartRange("body_forces")
793781
if (bodyForces) call s_apply_bodyforces(q_cons_ts(1)%vf, q_prim_vf, rhs_vf, 2d0*dt/3d0)
794-
call nvtxEndRange
795782

796783
if (grid_geometry == 3) call s_apply_fourier_filter(q_cons_ts(1)%vf)
797784

@@ -832,7 +819,7 @@ contains
832819

833820
call cpu_time(start)
834821

835-
call nvtxStartRange("Time_Step")
822+
call nvtxStartRange("TIMESTEP")
836823

837824
! Stage 1 of 3 =====================================================
838825
call s_adaptive_dt_bubble(t_step)
@@ -935,6 +922,7 @@ contains
935922

936923
integer :: i, j, k, l
937924

925+
call nvtxStartRange("RHS-BODYFORCES")
938926
call s_compute_body_forces_rhs(q_prim_vf, q_cons_vf, rhs_vf)
939927

940928
!$acc parallel loop collapse(4) gang vector default(present)
@@ -949,6 +937,8 @@ contains
949937
end do
950938
end do
951939

940+
call nvtxEndRange
941+
952942
end subroutine s_apply_bodyforces
953943

954944
!> This subroutine saves the temporary q_prim_vf vector

0 commit comments

Comments
 (0)