Skip to content

Commit 56c9ada

Browse files
wilfonbawilfonba
committed
Merge remote-tracking branch 'upstream/master' into MovingELBubblesSandia
2 parents fff74d8 + 336ec52 commit 56c9ada

File tree

8 files changed

+104
-70
lines changed

8 files changed

+104
-70
lines changed

‎.github/workflows/test.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ jobs:
5353
run: |
5454
brew update
5555
brew upgrade
56-
brew install coreutils python cmake fftw hdf5 gcc@15 boost open-mpi lapack
56+
brew install coreutils python fftw hdf5 gcc@15 boost open-mpi lapack
5757
echo "FC=gfortran-15" >> $GITHUB_ENV
5858
echo "BOOST_INCLUDE=/opt/homebrew/include/" >> $GITHUB_ENV
5959

‎.pr_agent.toml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,12 @@
44
pr_commands = ["/describe", "/review", "/improve"]
55

66
[pr_reviewer] # (all fields optional)
7-
num_max_findings = 5 # how many items to surface
8-
require_tests_review = true
7+
num_max_findings = 10 # how many items to surface
8+
require_tests_review = true
99
extra_instructions = """
1010
Focus on duplicate code, the possibility of bugs, and if the PR added appropriate tests if it added a simulation feature.
1111
"""
1212

1313
[pr_code_suggestions]
14-
commitable_code_suggestions = false # purely advisory, no write ops
15-
apply_suggestions_checkbox = false # hides the “Apply/Chat” boxes
14+
commitable_code_suggestions = true
15+
apply_suggestions_checkbox = true

‎src/common/m_helper_basic.fpp‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,14 +113,15 @@ contains
113113
pure subroutine s_configure_coordinate_bounds(recon_type, weno_polyn, muscl_polyn, &
114114
igr_order, buff_size, idwint, idwbuff, &
115115
viscous, bubbles_lagrange, m, n, p, num_dims, &
116-
igr, fd_number)
116+
igr, ib, fd_number)
117117

118118
integer, intent(in) :: recon_type, weno_polyn, muscl_polyn
119119
integer, intent(in) :: m, n, p, num_dims, igr_order, fd_number
120120
integer, intent(inout) :: buff_size
121121
type(int_bounds_info), dimension(3), intent(inout) :: idwint, idwbuff
122122
logical, intent(in) :: viscous, bubbles_lagrange
123123
logical, intent(in) :: igr
124+
logical, intent(in) :: ib
124125

125126
! Determining the number of cells that are needed in order to store
126127
! sufficient boundary conditions data as to iterate the solution in
@@ -143,6 +144,10 @@ contains
143144
buff_size = max(buff_size + fd_number, 6 + fd_number)
144145
end if
145146

147+
if (ib) then
148+
buff_size = max(buff_size, 10)
149+
end if
150+
146151
! Configuring Coordinate Direction Indexes
147152
idwint(1)%beg = 0; idwint(2)%beg = 0; idwint(3)%beg = 0
148153
idwint(1)%end = m; idwint(2)%end = n; idwint(3)%end = p

‎src/pre_process/m_global_parameters.fpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -938,7 +938,7 @@ contains
938938
igr_order, buff_size, &
939939
idwint, idwbuff, viscous, &
940940
bubbles_lagrange, m, n, p, &
941-
num_dims, igr, fd_number)
941+
num_dims, igr, ib, fd_number)
942942
943943
#ifdef MFC_MPI
944944

‎src/simulation/m_global_parameters.fpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1266,7 +1266,7 @@ contains
12661266
igr_order, buff_size, &
12671267
idwint, idwbuff, viscous, &
12681268
bubbles_lagrange, m, n, p, &
1269-
num_dims, igr, fd_number)
1269+
num_dims, igr, ib, fd_number)
12701270
$:GPU_UPDATE(device='[idwint, idwbuff]')
12711271
12721272
! Configuring Coordinate Direction Indexes

‎src/simulation/m_ibm.fpp‎

Lines changed: 20 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -53,19 +53,19 @@ contains
5353
impure subroutine s_initialize_ibm_module()
5454

5555
if (p > 0) then
56-
@:ALLOCATE(ib_markers%sf(-gp_layers:m+gp_layers, &
57-
-gp_layers:n+gp_layers, -gp_layers:p+gp_layers))
58-
@:ALLOCATE(levelset%sf(-gp_layers:m+gp_layers, &
59-
-gp_layers:n+gp_layers, -gp_layers:p+gp_layers, 1:num_ibs))
60-
@:ALLOCATE(levelset_norm%sf(-gp_layers:m+gp_layers, &
61-
-gp_layers:n+gp_layers, -gp_layers:p+gp_layers, 1:num_ibs, 1:3))
56+
@:ALLOCATE(ib_markers%sf(-buff_size:m+buff_size, &
57+
-buff_size:n+buff_size, -buff_size:p+buff_size))
58+
@:ALLOCATE(levelset%sf(-buff_size:m+buff_size, &
59+
-buff_size:n+buff_size, -buff_size:p+buff_size, 1:num_ibs))
60+
@:ALLOCATE(levelset_norm%sf(-buff_size:m+buff_size, &
61+
-buff_size:n+buff_size, -buff_size:p+buff_size, 1:num_ibs, 1:3))
6262
else
63-
@:ALLOCATE(ib_markers%sf(-gp_layers:m+gp_layers, &
64-
-gp_layers:n+gp_layers, 0:0))
65-
@:ALLOCATE(levelset%sf(-gp_layers:m+gp_layers, &
66-
-gp_layers:n+gp_layers, 0:0, 1:num_ibs))
67-
@:ALLOCATE(levelset_norm%sf(-gp_layers:m+gp_layers, &
68-
-gp_layers:n+gp_layers, 0:0, 1:num_ibs, 1:3))
63+
@:ALLOCATE(ib_markers%sf(-buff_size:m+buff_size, &
64+
-buff_size:n+buff_size, 0:0))
65+
@:ALLOCATE(levelset%sf(-buff_size:m+buff_size, &
66+
-buff_size:n+buff_size, 0:0, 1:num_ibs))
67+
@:ALLOCATE(levelset_norm%sf(-buff_size:m+buff_size, &
68+
-buff_size:n+buff_size, 0:0, 1:num_ibs, 1:3))
6969
end if
7070

7171
@:ACC_SETUP_SFs(ib_markers)
@@ -381,13 +381,13 @@ contains
381381
! s_cc points to the dim array we need
382382
if (dim == 1) then
383383
s_cc => x_cc
384-
bound = m
384+
bound = m + buff_size - 1
385385
elseif (dim == 2) then
386386
s_cc => y_cc
387-
bound = n
387+
bound = n + buff_size - 1
388388
else
389389
s_cc => z_cc
390-
bound = p
390+
bound = p + buff_size - 1
391391
end if
392392

393393
if (f_approx_equal(norm(dim), 0._wp)) then
@@ -402,9 +402,12 @@ contains
402402
index = ghost_points_in(q)%loc(dim)
403403
temp_loc = ghost_points_in(q)%ip_loc(dim)
404404
do while ((temp_loc < s_cc(index) &
405-
.or. temp_loc > s_cc(index + 1)) &
406-
.and. (index >= 0 .and. index <= bound))
405+
.or. temp_loc > s_cc(index + 1)))
407406
index = index + dir
407+
if (index < -buff_size .or. index > bound) then
408+
print *, "Increase buff_size further in m_helper_basic (currently set to a minimum of 10)"
409+
error stop "Increase buff_size"
410+
end if
408411
end do
409412
ghost_points_in(q)%ip_grid(dim) = index
410413
if (ghost_points_in(q)%DB(dim) == -1) then

‎src/simulation/m_mpi_proxy.fpp‎

Lines changed: 68 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -63,17 +63,17 @@ contains
6363
if (ib) then
6464
if (n > 0) then
6565
if (p > 0) then
66-
i_halo_size = -1 + gp_layers* &
67-
& (m + 2*gp_layers + 1)* &
68-
& (n + 2*gp_layers + 1)* &
69-
& (p + 2*gp_layers + 1)/ &
70-
& (cells_bounds%mnp_min + 2*gp_layers + 1)
66+
i_halo_size = -1 + buff_size* &
67+
& (m + 2*buff_size + 1)* &
68+
& (n + 2*buff_size + 1)* &
69+
& (p + 2*buff_size + 1)/ &
70+
& (cells_bounds%mnp_min + 2*buff_size + 1)
7171
else
72-
i_halo_size = -1 + gp_layers* &
73-
& (cells_bounds%mn_max + 2*gp_layers + 1)
72+
i_halo_size = -1 + buff_size* &
73+
& (cells_bounds%mn_max + 2*buff_size + 1)
7474
end if
7575
else
76-
i_halo_size = -1 + gp_layers
76+
i_halo_size = -1 + buff_size
7777
end if
7878
7979
$:GPU_UPDATE(device='[i_halo_size]')
@@ -313,9 +313,9 @@ contains
313313
call nvtxStartRange("IB-MARKER-COMM-PACKBUF")
314314
315315
buffer_counts = (/ &
316-
gp_layers*(n + 1)*(p + 1), &
317-
gp_layers*(m + 2*gp_layers + 1)*(p + 1), &
318-
gp_layers*(m + 2*gp_layers + 1)*(n + 2*gp_layers + 1) &
316+
buff_size*(n + 1)*(p + 1), &
317+
buff_size*(m + 2*buff_size + 1)*(p + 1), &
318+
buff_size*(m + 2*buff_size + 1)*(n + 2*buff_size + 1) &
319319
/)
320320
321321
buffer_count = buffer_counts(mpi_dir)
@@ -340,12 +340,12 @@ contains
340340
341341
pack_offset = 0
342342
if (f_xor(pbc_loc == 1, beg_end_geq_0)) then
343-
pack_offset = grid_dims(mpi_dir) - gp_layers + 1
343+
pack_offset = grid_dims(mpi_dir) - buff_size + 1
344344
end if
345345
346346
unpack_offset = 0
347347
if (pbc_loc == 1) then
348-
unpack_offset = grid_dims(mpi_dir) + gp_layers + 1
348+
unpack_offset = grid_dims(mpi_dir) + buff_size + 1
349349
end if
350350
351351
! Pack Buffer to Send
@@ -355,30 +355,30 @@ contains
355355
$:GPU_PARALLEL_LOOP(collapse=3,private='[r]')
356356
do l = 0, p
357357
do k = 0, n
358-
do j = 0, gp_layers - 1
359-
r = (j + gp_layers*(k + (n + 1)*l))
358+
do j = 0, buff_size - 1
359+
r = (j + buff_size*(k + (n + 1)*l))
360360
ib_buff_send(r) = ib_markers%sf(j + pack_offset, k, l)
361361
end do
362362
end do
363363
end do
364364
#:elif mpi_dir == 2
365365
$:GPU_PARALLEL_LOOP(collapse=3,private='[r]')
366366
do l = 0, p
367-
do k = 0, gp_layers - 1
368-
do j = -gp_layers, m + gp_layers
369-
r = ((j + gp_layers) + (m + 2*gp_layers + 1)* &
370-
(k + gp_layers*l))
367+
do k = 0, buff_size - 1
368+
do j = -buff_size, m + buff_size
369+
r = ((j + buff_size) + (m + 2*buff_size + 1)* &
370+
(k + buff_size*l))
371371
ib_buff_send(r) = ib_markers%sf(j, k + pack_offset, l)
372372
end do
373373
end do
374374
end do
375375
#:else
376376
$:GPU_PARALLEL_LOOP(collapse=3,private='[r]')
377-
do l = 0, gp_layers - 1
378-
do k = -gp_layers, n + gp_layers
379-
do j = -gp_layers, m + gp_layers
380-
r = ((j + gp_layers) + (m + 2*gp_layers + 1)* &
381-
((k + gp_layers) + (n + 2*gp_layers + 1)*l))
377+
do l = 0, buff_size - 1
378+
do k = -buff_size, n + buff_size
379+
do j = -buff_size, m + buff_size
380+
r = ((j + buff_size) + (m + 2*buff_size + 1)* &
381+
((k + buff_size) + (n + 2*buff_size + 1)*l))
382382
ib_buff_send(r) = ib_markers%sf(j, k, l + pack_offset)
383383
end do
384384
end do
@@ -388,12 +388,38 @@ contains
388388
#:endfor
389389
call nvtxEndRange ! Packbuf
390390
391-
call nvtxStartRange("IB-MARKER-SENDRECV")
392-
call MPI_SENDRECV( &
393-
ib_buff_send, buffer_count, MPI_INTEGER, dst_proc, send_tag, &
394-
ib_buff_recv, buffer_count, MPI_INTEGER, src_proc, recv_tag, &
395-
MPI_COMM_WORLD, MPI_STATUS_IGNORE, ierr)
396-
call nvtxEndRange ! RHS-MPI-SENDRECV-(NO)-RDMA
391+
#:for rdma_mpi in [False, True]
392+
if (rdma_mpi .eqv. ${'.true.' if rdma_mpi else '.false.'}$) then
393+
#:if rdma_mpi
394+
#:call GPU_HOST_DATA(use_device='[ib_buff_send, ib_buff_recv]')
395+
396+
call nvtxStartRange("IB-MARKER-SENDRECV-RDMA")
397+
call MPI_SENDRECV( &
398+
ib_buff_send, buffer_count, MPI_INTEGER, dst_proc, send_tag, &
399+
ib_buff_recv, buffer_count, MPI_INTEGER, src_proc, recv_tag, &
400+
MPI_COMM_WORLD, MPI_STATUS_IGNORE, ierr)
401+
call nvtxEndRange
402+
403+
#:endcall GPU_HOST_DATA
404+
$:GPU_WAIT()
405+
#:else
406+
call nvtxStartRange("IB-MARKER-DEV2HOST")
407+
$:GPU_UPDATE(host='[ib_buff_send]')
408+
call nvtxEndRange
409+
410+
call nvtxStartRange("IB-MARKER-SENDRECV-NO-RMDA")
411+
call MPI_SENDRECV( &
412+
ib_buff_send, buffer_count, MPI_INTEGER, dst_proc, send_tag, &
413+
ib_buff_recv, buffer_count, MPI_INTEGER, src_proc, recv_tag, &
414+
MPI_COMM_WORLD, MPI_STATUS_IGNORE, ierr)
415+
call nvtxEndRange
416+
417+
call nvtxStartRange("IB-MARKER-HOST2DEV")
418+
$:GPU_UPDATE(device='[ib_buff_recv]')
419+
call nvtxEndRange
420+
#:endif
421+
end if
422+
#:endfor
397423
398424
! Unpack Received Buffer
399425
call nvtxStartRange("IB-MARKER-COMM-UNPACKBUF")
@@ -403,32 +429,32 @@ contains
403429
$:GPU_PARALLEL_LOOP(collapse=3,private='[r]')
404430
do l = 0, p
405431
do k = 0, n
406-
do j = -gp_layers, -1
407-
r = (j + gp_layers*((k + 1) + (n + 1)*l))
432+
do j = -buff_size, -1
433+
r = (j + buff_size*((k + 1) + (n + 1)*l))
408434
ib_markers%sf(j + unpack_offset, k, l) = ib_buff_recv(r)
409435
end do
410436
end do
411437
end do
412438
#:elif mpi_dir == 2
413439
$:GPU_PARALLEL_LOOP(collapse=3,private='[r]')
414440
do l = 0, p
415-
do k = -gp_layers, -1
416-
do j = -gp_layers, m + gp_layers
417-
r = ((j + gp_layers) + (m + 2*gp_layers + 1)* &
418-
((k + gp_layers) + gp_layers*l))
441+
do k = -buff_size, -1
442+
do j = -buff_size, m + buff_size
443+
r = ((j + buff_size) + (m + 2*buff_size + 1)* &
444+
((k + buff_size) + buff_size*l))
419445
ib_markers%sf(j, k + unpack_offset, l) = ib_buff_recv(r)
420446
end do
421447
end do
422448
end do
423449
#:else
424450
! Unpacking buffer from bc_z%beg
425451
$:GPU_PARALLEL_LOOP(collapse=3,private='[r]')
426-
do l = -gp_layers, -1
427-
do k = -gp_layers, n + gp_layers
428-
do j = -gp_layers, m + gp_layers
429-
r = ((j + gp_layers) + (m + 2*gp_layers + 1)* &
430-
((k + gp_layers) + (n + 2*gp_layers + 1)* &
431-
(l + gp_layers)))
452+
do l = -buff_size, -1
453+
do k = -buff_size, n + buff_size
454+
do j = -buff_size, m + buff_size
455+
r = ((j + buff_size) + (m + 2*buff_size + 1)* &
456+
((k + buff_size) + (n + 2*buff_size + 1)* &
457+
(l + buff_size)))
432458
ib_markers%sf(j, k, l + unpack_offset) = ib_buff_recv(r)
433459
end do
434460
end do

‎src/simulation/m_start_up.fpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1368,14 +1368,14 @@ contains
13681368
call s_read_data_files(q_cons_ts(1)%vf)
13691369
end if
13701370

1371+
! Populating the buffers of the grid variables using the boundary conditions
1372+
call s_populate_grid_variables_buffers()
1373+
13711374
if (model_eqns == 3) call s_initialize_internal_energy_equations(q_cons_ts(1)%vf)
13721375
if (ib) call s_ibm_setup()
13731376
if (bodyForces) call s_initialize_body_forces_module()
13741377
if (acoustic_source) call s_precalculate_acoustic_spatial_sources()
13751378

1376-
! Populating the buffers of the grid variables using the boundary conditions
1377-
call s_populate_grid_variables_buffers()
1378-
13791379
! Initialize the Temperature cache.
13801380
if (chemistry) call s_compute_q_T_sf(q_T_sf, q_cons_ts(1)%vf, idwint)
13811381

0 commit comments

Comments
 (0)