From 2f3be63bf1d1ed106836374aea1ae54cdeab0b5e Mon Sep 17 00:00:00 2001 From: Max Hawkins Date: Tue, 1 Oct 2024 09:25:00 -0400 Subject: [PATCH 1/9] Increase granularity of halo-exchange timing info --- src/simulation/m_mpi_proxy.fpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/simulation/m_mpi_proxy.fpp b/src/simulation/m_mpi_proxy.fpp index d2f86e4d97..b8592e1bcf 100644 --- a/src/simulation/m_mpi_proxy.fpp +++ b/src/simulation/m_mpi_proxy.fpp @@ -1070,6 +1070,7 @@ contains !$acc update host(q_cons_buff_send, ib_buff_send) #:endif + call nvtxStartRange("RHS-MPI-SENDRECV") call MPI_SENDRECV( & p_send, buffer_count, MPI_DOUBLE_PRECISION, dst_proc, send_tag, & p_recv, buffer_count, MPI_DOUBLE_PRECISION, src_proc, recv_tag, & From 2b7d551de64107c18e88680fa5ce5fc77bba0178 Mon Sep 17 00:00:00 2001 From: Max Hawkins Date: Thu, 17 Oct 2024 11:07:29 -0400 Subject: [PATCH 2/9] Standardize timing ranges --- src/common/m_mpi_common.fpp | 4 ++++ src/simulation/m_mpi_proxy.fpp | 1 - src/simulation/m_rhs.fpp | 24 ++++++++++++++++++++++-- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/common/m_mpi_common.fpp b/src/common/m_mpi_common.fpp index 42748e57b8..8cdf8bd1a2 100644 --- a/src/common/m_mpi_common.fpp +++ b/src/common/m_mpi_common.fpp @@ -16,6 +16,8 @@ module m_mpi_common use m_derived_types !< Definitions of the derived types use m_global_parameters !< Definitions of the global parameters + + use m_nvtx ! ========================================================================== implicit none @@ -239,6 +241,7 @@ contains #ifdef MFC_SIMULATION #ifdef MFC_MPI + call nvtxStartRange("TSTEP-RUNTIME-INFO-REDUCE-CRIT") ! Reducing local extrema of ICFL, VCFL, CCFL and Rc numbers to their ! global extrema and bookkeeping the results on the rank 0 processor call MPI_REDUCE(icfl_max_loc, icfl_max_glb, 1, & @@ -253,6 +256,7 @@ contains MPI_DOUBLE_PRECISION, MPI_MIN, 0, & MPI_COMM_WORLD, ierr) end if + call nvtxEndRange #else diff --git a/src/simulation/m_mpi_proxy.fpp b/src/simulation/m_mpi_proxy.fpp index b8592e1bcf..d2f86e4d97 100644 --- a/src/simulation/m_mpi_proxy.fpp +++ b/src/simulation/m_mpi_proxy.fpp @@ -1070,7 +1070,6 @@ contains !$acc update host(q_cons_buff_send, ib_buff_send) #:endif - call nvtxStartRange("RHS-MPI-SENDRECV") call MPI_SENDRECV( & p_send, buffer_count, MPI_DOUBLE_PRECISION, dst_proc, send_tag, & p_recv, buffer_count, MPI_DOUBLE_PRECISION, src_proc, recv_tag, & diff --git a/src/simulation/m_rhs.fpp b/src/simulation/m_rhs.fpp index 2628b024b2..8a1ec11145 100644 --- a/src/simulation/m_rhs.fpp +++ b/src/simulation/m_rhs.fpp @@ -669,7 +669,7 @@ contains gm_alpha_qp%vf) call nvtxEndRange - call nvtxStartRange("RHS-MPI") + call nvtxStartRange("RHS-COMMUNICATION") call s_populate_variables_buffers(q_prim_qp%vf, pb, mv) call nvtxEndRange @@ -849,6 +849,26 @@ contains call nvtxEndRange ! END: Additional physics and source terms ========================= + #:if chemistry + if (chem_params%advection) then + call nvtxStartRange("RHS-CHEM-ADVECTION") + + #:for NORM_DIR, XYZ in [(1, 'x'), (2, 'y'), (3, 'z')] + + if (id == ${NORM_DIR}$) then + call s_compute_chemistry_rhs_${XYZ}$ ( & + flux_n, & + rhs_vf, & + flux_src_n(${NORM_DIR}$)%vf, & + q_prim_vf) + end if + + #:endfor + + call nvtxEndRange + end if + #:endif + end do ! END: Dimensional Splitting Loop ================================= @@ -883,7 +903,7 @@ contains call nvtxEndRange ! Add bubles source term - call nvtxStartRange("RHS_bubbles") + call nvtxStartRange("RHS-BUBBLES-SRC") if (bubbles .and. (.not. adap_dt) .and. (.not. qbmm)) call s_compute_bubble_source( & q_cons_qp%vf(1:sys_size), & q_prim_qp%vf(1:sys_size), & From 8cfeee9b21520f27d4f053001778a1f1f1502378 Mon Sep 17 00:00:00 2001 From: Max Hawkins Date: Mon, 11 Nov 2024 17:08:52 -0500 Subject: [PATCH 3/9] Include NVTX range in conditionals --- src/simulation/m_rhs.fpp | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/simulation/m_rhs.fpp b/src/simulation/m_rhs.fpp index 8a1ec11145..26d95352d1 100644 --- a/src/simulation/m_rhs.fpp +++ b/src/simulation/m_rhs.fpp @@ -903,17 +903,19 @@ contains call nvtxEndRange ! Add bubles source term - call nvtxStartRange("RHS-BUBBLES-SRC") - if (bubbles .and. (.not. adap_dt) .and. (.not. qbmm)) call s_compute_bubble_source( & - q_cons_qp%vf(1:sys_size), & - q_prim_qp%vf(1:sys_size), & - t_step, & - rhs_vf) - call nvtxEndRange + if (bubbles .and. (.not. adap_dt) .and. (.not. qbmm)) then + call nvtxStartRange("RHS-BUBBLES-SRC") + call s_compute_bubble_source( & + q_cons_qp%vf(1:sys_size), & + q_prim_qp%vf(1:sys_size), & + t_step, & + rhs_vf) + call nvtxEndRange + end if if (chemistry) then if (chem_params%reactions) then - call nvtxStartRange("RHS_Chem_Reactions") + call nvtxStartRange("RHS-CHEM-REACTIONS") call s_compute_chemistry_reaction_flux(rhs_vf, q_cons_qp%vf, q_prim_qp%vf) call nvtxEndRange end if From 26af09c3815317419fde75735ff97058f817e867 Mon Sep 17 00:00:00 2001 From: Max Hawkins Date: Tue, 5 Nov 2024 10:34:06 -0500 Subject: [PATCH 4/9] Add global simulation nvtx range --- src/simulation/p_main.fpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/simulation/p_main.fpp b/src/simulation/p_main.fpp index 091538073e..f25414c23a 100644 --- a/src/simulation/p_main.fpp +++ b/src/simulation/p_main.fpp @@ -21,6 +21,8 @@ program p_main use m_start_up use m_time_steppers + + use m_nvtx ! ========================================================================== implicit none @@ -61,6 +63,7 @@ program p_main finaltime = t_step_stop*dt end if + call nvtxStartRange("Simulation") ! Time-stepping Loop ======================================================= do @@ -95,6 +98,8 @@ program p_main end do ! ========================================================================== + call nvtxEndRange ! Simulation + deallocate (proc_time, io_proc_time) call s_finalize_modules() From e00bc649f6f213870bc34be9617aaff42bdce0a3 Mon Sep 17 00:00:00 2001 From: Max Hawkins Date: Mon, 11 Nov 2024 15:39:39 -0500 Subject: [PATCH 5/9] Change NVTX names and add subranges --- src/common/m_mpi_common.fpp | 4 ---- src/simulation/m_start_up.fpp | 4 ++-- src/simulation/p_main.fpp | 2 +- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/common/m_mpi_common.fpp b/src/common/m_mpi_common.fpp index 8cdf8bd1a2..42748e57b8 100644 --- a/src/common/m_mpi_common.fpp +++ b/src/common/m_mpi_common.fpp @@ -16,8 +16,6 @@ module m_mpi_common use m_derived_types !< Definitions of the derived types use m_global_parameters !< Definitions of the global parameters - - use m_nvtx ! ========================================================================== implicit none @@ -241,7 +239,6 @@ contains #ifdef MFC_SIMULATION #ifdef MFC_MPI - call nvtxStartRange("TSTEP-RUNTIME-INFO-REDUCE-CRIT") ! Reducing local extrema of ICFL, VCFL, CCFL and Rc numbers to their ! global extrema and bookkeeping the results on the rank 0 processor call MPI_REDUCE(icfl_max_loc, icfl_max_glb, 1, & @@ -256,7 +253,6 @@ contains MPI_DOUBLE_PRECISION, MPI_MIN, 0, & MPI_COMM_WORLD, ierr) end if - call nvtxEndRange #else diff --git a/src/simulation/m_start_up.fpp b/src/simulation/m_start_up.fpp index 5be39477b6..ec8c59de4d 100644 --- a/src/simulation/m_start_up.fpp +++ b/src/simulation/m_start_up.fpp @@ -1239,7 +1239,7 @@ contains integer :: save_count call cpu_time(start) - ! call nvtxStartRange("I/O") + call nvtxStartRange("SAVE-DATA") do i = 1, sys_size !$acc update host(q_cons_ts(1)%vf(i)%sf) do l = 0, p @@ -1267,7 +1267,7 @@ contains call s_write_data_files(q_cons_ts(1)%vf, q_prim_vf, save_count) - ! call nvtxEndRange + call nvtxEndRange call cpu_time(finish) if (cfl_dt) then nt = mytime/t_save diff --git a/src/simulation/p_main.fpp b/src/simulation/p_main.fpp index f25414c23a..a83c92c581 100644 --- a/src/simulation/p_main.fpp +++ b/src/simulation/p_main.fpp @@ -63,7 +63,7 @@ program p_main finaltime = t_step_stop*dt end if - call nvtxStartRange("Simulation") + call nvtxStartRange("SIMULATION-TIME-MARCH") ! Time-stepping Loop ======================================================= do From ce5c8af1e8149c6856f13d63742ab8abfb30a7b3 Mon Sep 17 00:00:00 2001 From: Max Hawkins Date: Mon, 11 Nov 2024 19:11:47 -0500 Subject: [PATCH 6/9] Improve NVTX range tracking --- src/simulation/m_mpi_proxy.fpp | 13 +++++++ src/simulation/m_rhs.fpp | 57 ++++++++++++++++++------------ src/simulation/m_time_steppers.fpp | 24 ++++--------- 3 files changed, 54 insertions(+), 40 deletions(-) diff --git a/src/simulation/m_mpi_proxy.fpp b/src/simulation/m_mpi_proxy.fpp index d2f86e4d97..2c1e50d231 100644 --- a/src/simulation/m_mpi_proxy.fpp +++ b/src/simulation/m_mpi_proxy.fpp @@ -27,6 +27,8 @@ module m_mpi_proxy use m_mpi_common + use m_nvtx + use ieee_arithmetic ! ========================================================================== @@ -865,6 +867,7 @@ contains #ifdef MFC_MPI + call nvtxStartRange("RHS-COMM-PACKBUF") !$acc update device(v_size) if (qbmm .and. .not. polytropic) then @@ -1057,6 +1060,7 @@ contains #:endif end if #:endfor + call nvtxEndRange ! Packbuf ! Send/Recv #:for rdma_mpi in [False, True] @@ -1066,26 +1070,34 @@ contains #:if rdma_mpi !$acc data attach(p_send, p_recv) !$acc host_data use_device(p_send, p_recv) + call nvtxStartRange("RHS-COMM-SENDRECV-RDMA") #:else + call nvtxStartRange("RHS-COMM-DEV2HOST") !$acc update host(q_cons_buff_send, ib_buff_send) + call nvtxEndRange + call nvtxStartRange("RHS-COMM-SENDRECV-NO-RMDA") #:endif call MPI_SENDRECV( & p_send, buffer_count, MPI_DOUBLE_PRECISION, dst_proc, send_tag, & p_recv, buffer_count, MPI_DOUBLE_PRECISION, src_proc, recv_tag, & MPI_COMM_WORLD, MPI_STATUS_IGNORE, ierr) + call nvtxEndRange ! RHS-MPI-SENDRECV-(NO)-RDMA #:if rdma_mpi !$acc end host_data !$acc end data !$acc wait #:else + call nvtxStartRange("RHS-COMM-HOST2DEV") !$acc update device(q_cons_buff_recv) + call nvtxEndRange #:endif end if #:endfor ! Unpack Received Buffer + call nvtxStartRange("RHS-COMM-UNPACKBUF") #:for mpi_dir in [1, 2, 3] if (mpi_dir == ${mpi_dir}$) then #:if mpi_dir == 1 @@ -1254,6 +1266,7 @@ contains #:endif end if #:endfor + call nvtxEndRange #endif diff --git a/src/simulation/m_rhs.fpp b/src/simulation/m_rhs.fpp index 26d95352d1..59d3d2da4b 100644 --- a/src/simulation/m_rhs.fpp +++ b/src/simulation/m_rhs.fpp @@ -622,7 +622,7 @@ contains real(kind(0d0)) :: t_start, t_finish integer :: i, j, k, l, id !< Generic loop iterators - call nvtxStartRange("Compute_RHS") + call nvtxStartRange("COMPUTE-RHS") call cpu_time(t_start) ! Association/Population of Working Variables ====================== @@ -683,8 +683,9 @@ contains if (qbmm) call s_mom_inv(q_cons_qp%vf, q_prim_qp%vf, mom_sp, mom_3d, pb, rhs_pb, mv, rhs_mv, idwbuff(1), idwbuff(2), idwbuff(3), nbub) - call nvtxStartRange("Viscous") - if (viscous) call s_get_viscous(qL_rsx_vf, qL_rsy_vf, qL_rsz_vf, & + if (viscous) then + call nvtxStartRange("RHS-VISCOUS") + call s_get_viscous(qL_rsx_vf, qL_rsy_vf, qL_rsz_vf, & dqL_prim_dx_n, dqL_prim_dy_n, dqL_prim_dz_n, & qL_prim, & qR_rsx_vf, qR_rsy_vf, qR_rsz_vf, & @@ -693,11 +694,15 @@ contains q_prim_qp, & dq_prim_dx_qp, dq_prim_dy_qp, dq_prim_dz_qp, & idwbuff(1), idwbuff(2), idwbuff(3)) - call nvtxEndRange + call nvtxEndRange + end if - call nvtxStartRange("Surface_Tension") - if (surface_tension) call s_get_capilary(q_prim_qp%vf) - call nvtxEndRange + + if (surface_tension) then + call nvtxStartRange("RHS-SURFACE-TENSION") + call s_get_capilary(q_prim_qp%vf) + call nvtxEndRange + end if ! Dimensional Splitting Loop ======================================= do id = 1, num_dims @@ -777,7 +782,7 @@ contains irx%end = m; iry%end = n; irz%end = p ! =============================================================== - call nvtxStartRange("RHS_riemann_solver") + call nvtxStartRange("RHS-RIEMANN-SOLVER") ! Computing Riemann Solver Flux and Source Flux ================= @@ -801,7 +806,7 @@ contains ! Additional physics and source terms ============================== ! RHS addition for advection source - call nvtxStartRange("RHS_advection_source") + call nvtxStartRange("RHS-ADVECTION-SRC") call s_compute_advection_source_term(id, & rhs_vf, & q_cons_qp, & @@ -810,15 +815,15 @@ contains call nvtxEndRange ! RHS additions for hypoelasticity - call nvtxStartRange("RHS_Hypoelasticity") + call nvtxStartRange("RHS-HYPOELASTICITY") if (hypoelasticity) call s_compute_hypoelastic_rhs(id, & q_prim_qp%vf, & rhs_vf) call nvtxEndRange ! RHS additions for viscosity - call nvtxStartRange("RHS_add_phys") if (viscous .or. surface_tension) then + call nvtxStartRange("RHS-ADD-PHYSICS") call s_compute_additional_physics_rhs(id, & q_prim_qp%vf, & rhs_vf, & @@ -826,18 +831,21 @@ contains dq_prim_dx_qp(1)%vf, & dq_prim_dy_qp(1)%vf, & dq_prim_dz_qp(1)%vf) + call nvtxEndRange end if - call nvtxEndRange ! RHS additions for sub-grid bubbles - call nvtxStartRange("RHS_bubbles") - if (bubbles) call s_compute_bubbles_rhs(id, & - q_prim_qp%vf) - call nvtxEndRange + if (bubbles) then + call nvtxStartRange("RHS-BUBBLES-COMPUTE") + call s_compute_bubbles_rhs(id, q_prim_qp%vf) + call nvtxEndRange + end if ! RHS additions for qbmm bubbles - call nvtxStartRange("RHS_qbmm") - if (qbmm) call s_compute_qbmm_rhs(id, & + + if (qbmm) then + call nvtxStartRange("RHS-QBMM") + call s_compute_qbmm_rhs(id, & q_cons_qp%vf, & q_prim_qp%vf, & rhs_vf, & @@ -846,7 +854,8 @@ contains rhs_pb, & mv, & rhs_mv) - call nvtxEndRange + call nvtxEndRange + end if ! END: Additional physics and source terms ========================= #:if chemistry @@ -873,7 +882,7 @@ contains ! END: Dimensional Splitting Loop ================================= if (chemistry) then - call nvtxStartRange("RHS_Chem_Advection") + call nvtxStartRange("RHS-CHEM-ADVECTION") call s_compute_chemistry_advection_flux(flux_n, rhs_vf) call nvtxEndRange end if @@ -895,12 +904,14 @@ contains ! Additional Physics and Source Temrs ================================== ! Additions for acoustic_source - call nvtxStartRange("RHS_acoustic_src") - if (acoustic_source) call s_acoustic_src_calculations(q_cons_qp%vf(1:sys_size), & + if (acoustic_source) then + call nvtxStartRange("RHS-ACOUSTIC-SRC") + call s_acoustic_src_calculations(q_cons_qp%vf(1:sys_size), & q_prim_qp%vf(1:sys_size), & t_step, & rhs_vf) - call nvtxEndRange + call nvtxEndRange + end if ! Add bubles source term if (bubbles .and. (.not. adap_dt) .and. (.not. qbmm)) then diff --git a/src/simulation/m_time_steppers.fpp b/src/simulation/m_time_steppers.fpp index 288fed1056..5559ef2746 100644 --- a/src/simulation/m_time_steppers.fpp +++ b/src/simulation/m_time_steppers.fpp @@ -279,7 +279,7 @@ contains integer :: i, j, k, l, q !< Generic loop iterator ! Stage 1 of 1 ===================================================== - call nvtxStartRange("Time_Step") + call nvtxStartRange("TIMESTEP") call s_compute_rhs(q_cons_ts(1)%vf, q_prim_vf, rhs_vf, pb_ts(1)%sf, rhs_pb, mv_ts(1)%sf, rhs_mv, t_step, time_avg) @@ -353,7 +353,6 @@ contains end do end if - call nvtxStartRange("body_forces") if (bodyForces) call s_apply_bodyforces(q_cons_ts(1)%vf, q_prim_vf, rhs_vf, dt) call nvtxEndRange @@ -371,8 +370,6 @@ contains end if end if - call nvtxEndRange - ! ================================================================== end subroutine s_1st_order_tvd_rk @@ -391,7 +388,7 @@ contains call cpu_time(start) - call nvtxStartRange("Time_Step") + call nvtxStartRange("TIMESTEP") call s_compute_rhs(q_cons_ts(1)%vf, q_prim_vf, rhs_vf, pb_ts(1)%sf, rhs_pb, mv_ts(1)%sf, rhs_mv, t_step, time_avg) @@ -457,9 +454,7 @@ contains end do end if - call nvtxStartRange("body_forces") if (bodyForces) call s_apply_bodyforces(q_cons_ts(2)%vf, q_prim_vf, rhs_vf, dt) - call nvtxEndRange if (grid_geometry == 3) call s_apply_fourier_filter(q_cons_ts(2)%vf) @@ -532,9 +527,7 @@ contains end do end if - call nvtxStartRange("body_forces") if (bodyForces) call s_apply_bodyforces(q_cons_ts(1)%vf, q_prim_vf, rhs_vf, 2d0*dt/3d0) - call nvtxEndRange if (grid_geometry == 3) call s_apply_fourier_filter(q_cons_ts(1)%vf) @@ -573,7 +566,7 @@ contains if (.not. adap_dt) then call cpu_time(start) - call nvtxStartRange("Time_Step") + call nvtxStartRange("TIMESTEP") end if call s_compute_rhs(q_cons_ts(1)%vf, q_prim_vf, rhs_vf, pb_ts(1)%sf, rhs_pb, mv_ts(1)%sf, rhs_mv, t_step, time_avg) @@ -640,9 +633,7 @@ contains end do end if - call nvtxStartRange("body_forces") if (bodyForces) call s_apply_bodyforces(q_cons_ts(2)%vf, q_prim_vf, rhs_vf, dt) - call nvtxEndRange if (grid_geometry == 3) call s_apply_fourier_filter(q_cons_ts(2)%vf) @@ -715,9 +706,7 @@ contains end do end if - call nvtxStartRange("body_forces") if (bodyForces) call s_apply_bodyforces(q_cons_ts(2)%vf, q_prim_vf, rhs_vf, dt/4d0) - call nvtxEndRange if (grid_geometry == 3) call s_apply_fourier_filter(q_cons_ts(2)%vf) @@ -789,9 +778,7 @@ contains end do end if - call nvtxStartRange("body_forces") if (bodyForces) call s_apply_bodyforces(q_cons_ts(1)%vf, q_prim_vf, rhs_vf, 2d0*dt/3d0) - call nvtxEndRange if (grid_geometry == 3) call s_apply_fourier_filter(q_cons_ts(1)%vf) @@ -832,7 +819,7 @@ contains call cpu_time(start) - call nvtxStartRange("Time_Step") + call nvtxStartRange("TIMESTEP") ! Stage 1 of 3 ===================================================== call s_adaptive_dt_bubble(t_step) @@ -935,6 +922,7 @@ contains integer :: i, j, k, l + call nvtxStartRange("RHS-BODYFORCES") call s_compute_body_forces_rhs(q_prim_vf, q_cons_vf, rhs_vf) !$acc parallel loop collapse(4) gang vector default(present) @@ -949,6 +937,8 @@ contains end do end do + call nvtxEndRange + end subroutine s_apply_bodyforces !> This subroutine saves the temporary q_prim_vf vector From 095d89075dde34162030e115dda382d5b5350604 Mon Sep 17 00:00:00 2001 From: Max Hawkins Date: Mon, 11 Nov 2024 19:35:12 -0500 Subject: [PATCH 7/9] Add NVTX ranges for initialization and finalization --- src/simulation/p_main.fpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/simulation/p_main.fpp b/src/simulation/p_main.fpp index a83c92c581..18bab5a306 100644 --- a/src/simulation/p_main.fpp +++ b/src/simulation/p_main.fpp @@ -38,16 +38,24 @@ program p_main call system_clock(COUNT=cpu_start, COUNT_RATE=cpu_rate) + call nvtxStartRange("INIT") + !Initialize MPI + call nvtxStartRange("INIT-MPI") call s_initialize_mpi_domain() + call nvtxEndRange !Initialize Modules + call nvtxStartRange("INIT-MODULES") call s_initialize_modules() + call nvtxEndRange allocate (proc_time(0:num_procs - 1)) allocate (io_proc_time(0:num_procs - 1)) + call nvtxStartRange("INIT-GPU-VARS") call s_initialize_gpu_vars() + call nvtxEndRange ! Setting the time-step iterator to the first time-step if (cfl_dt) then @@ -63,6 +71,8 @@ program p_main finaltime = t_step_stop*dt end if + call nvtxEndRange ! INIT + call nvtxStartRange("SIMULATION-TIME-MARCH") ! Time-stepping Loop ======================================================= do @@ -102,6 +112,8 @@ program p_main deallocate (proc_time, io_proc_time) + call nvtxStartRange("FINALIZE-MODULES") call s_finalize_modules() + call nvtxEndRange end program p_main From 29f0b7b2a4bc1a63b9d2f43c4409c4c72f692c7b Mon Sep 17 00:00:00 2001 From: Max Hawkins Date: Mon, 11 Nov 2024 19:41:08 -0500 Subject: [PATCH 8/9] Fix formatting --- src/simulation/m_rhs.fpp | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/src/simulation/m_rhs.fpp b/src/simulation/m_rhs.fpp index 59d3d2da4b..0a863754cc 100644 --- a/src/simulation/m_rhs.fpp +++ b/src/simulation/m_rhs.fpp @@ -686,18 +686,17 @@ contains if (viscous) then call nvtxStartRange("RHS-VISCOUS") call s_get_viscous(qL_rsx_vf, qL_rsy_vf, qL_rsz_vf, & - dqL_prim_dx_n, dqL_prim_dy_n, dqL_prim_dz_n, & - qL_prim, & - qR_rsx_vf, qR_rsy_vf, qR_rsz_vf, & - dqR_prim_dx_n, dqR_prim_dy_n, dqR_prim_dz_n, & - qR_prim, & - q_prim_qp, & - dq_prim_dx_qp, dq_prim_dy_qp, dq_prim_dz_qp, & - idwbuff(1), idwbuff(2), idwbuff(3)) + dqL_prim_dx_n, dqL_prim_dy_n, dqL_prim_dz_n, & + qL_prim, & + qR_rsx_vf, qR_rsy_vf, qR_rsz_vf, & + dqR_prim_dx_n, dqR_prim_dy_n, dqR_prim_dz_n, & + qR_prim, & + q_prim_qp, & + dq_prim_dx_qp, dq_prim_dy_qp, dq_prim_dz_qp, & + idwbuff(1), idwbuff(2), idwbuff(3)) call nvtxEndRange end if - if (surface_tension) then call nvtxStartRange("RHS-SURFACE-TENSION") call s_get_capilary(q_prim_qp%vf) @@ -846,14 +845,14 @@ contains if (qbmm) then call nvtxStartRange("RHS-QBMM") call s_compute_qbmm_rhs(id, & - q_cons_qp%vf, & - q_prim_qp%vf, & - rhs_vf, & - flux_n(id)%vf, & - pb, & - rhs_pb, & - mv, & - rhs_mv) + q_cons_qp%vf, & + q_prim_qp%vf, & + rhs_vf, & + flux_n(id)%vf, & + pb, & + rhs_pb, & + mv, & + rhs_mv) call nvtxEndRange end if ! END: Additional physics and source terms ========================= @@ -907,9 +906,9 @@ contains if (acoustic_source) then call nvtxStartRange("RHS-ACOUSTIC-SRC") call s_acoustic_src_calculations(q_cons_qp%vf(1:sys_size), & - q_prim_qp%vf(1:sys_size), & - t_step, & - rhs_vf) + q_prim_qp%vf(1:sys_size), & + t_step, & + rhs_vf) call nvtxEndRange end if From 022f593dedb587fe21ef776cd32a0c5e4615f7fa Mon Sep 17 00:00:00 2001 From: Max Hawkins <37495064+max-Hawkins@users.noreply.github.com> Date: Mon, 11 Nov 2024 19:55:52 -0500 Subject: [PATCH 9/9] Remove chemistry straggler --- src/simulation/m_rhs.fpp | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/src/simulation/m_rhs.fpp b/src/simulation/m_rhs.fpp index 0a863754cc..95b8557d71 100644 --- a/src/simulation/m_rhs.fpp +++ b/src/simulation/m_rhs.fpp @@ -857,26 +857,6 @@ contains end if ! END: Additional physics and source terms ========================= - #:if chemistry - if (chem_params%advection) then - call nvtxStartRange("RHS-CHEM-ADVECTION") - - #:for NORM_DIR, XYZ in [(1, 'x'), (2, 'y'), (3, 'z')] - - if (id == ${NORM_DIR}$) then - call s_compute_chemistry_rhs_${XYZ}$ ( & - flux_n, & - rhs_vf, & - flux_src_n(${NORM_DIR}$)%vf, & - q_prim_vf) - end if - - #:endfor - - call nvtxEndRange - end if - #:endif - end do ! END: Dimensional Splitting Loop =================================