Skip to content

Commit 363d584

Browse files
sbryngelsonsbryngelson
authored
Merge pull request #161 from anandrdbz/patch-1
2 parents c0e510e + be8dc4a commit 363d584

File tree

179 files changed

+683
-254
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

179 files changed

+683
-254
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 27 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ on:
1515
pull_request:
1616

1717
workflow_dispatch:
18-
18+
1919
jobs:
2020
github:
2121
name: Github
@@ -24,24 +24,25 @@ jobs:
2424
os: ['ubuntu', 'macos']
2525
mpi: ['mpi', 'no-mpi']
2626
debug: ['debug', 'no-debug']
27-
intel: [false]
27+
intel: [true, false]
2828
fail-fast: false
2929
continue-on-error: true
3030
runs-on: ${{ matrix.os }}-latest
3131
steps:
3232
- name: Clone
33+
if: matrix.os == 'ubuntu' || (matrix.os == 'macos' && matrix.intel == false)
3334
uses: actions/checkout@v3
3435

3536
- name: Setup MacOS
36-
if: matrix.os == 'macos'
37+
if: matrix.os == 'macos' && matrix.intel == false
3738
run: |
3839
echo "CC=gcc-13" >> $GITHUB_ENV
3940
echo "CXX=g++-13" >> $GITHUB_ENV
4041
echo "FC=gfortran-13" >> $GITHUB_ENV
4142
brew install wget make python make cmake coreutils gcc@13
4243
4344
- name: (MacOS) Build OpenMPI
44-
if: matrix.os == 'macos' && matrix.mpi == 'mpi'
45+
if: matrix.os == 'macos' && matrix.mpi == 'mpi' && matrix.intel == false
4546
run: |
4647
echo "OMPI_FC=gfortran-13" >> $GITHUB_ENV
4748
echo "OMPI_CXX=g++-13" >> $GITHUB_ENV
@@ -51,9 +52,9 @@ jobs:
5152
- name: Setup Ubuntu
5253
if: matrix.os == 'ubuntu' && matrix.intel == false
5354
run: |
54-
sudo apt update -y
55-
sudo apt install -y tar wget make cmake gcc g++ python3 python3-dev "openmpi-*" libopenmpi-dev
56-
55+
sudo apt update -y
56+
sudo apt install -y tar wget make cmake gcc g++ python3 python3-dev "openmpi-*" libopenmpi-dev
57+
5758
- name: Setup Ubuntu (Intel)
5859
if: matrix.os == 'ubuntu' && matrix.intel == true
5960
run: |
@@ -72,12 +73,29 @@ jobs:
7273
echo "OMPI_CXX=$(which icpc)" >> $GITHUB_ENV
7374
echo "OMPI_MPICC=$(which icc)" >> $GITHUB_ENV
7475
echo "MPI_HOME=/opt/intel/oneapi/mpi/2021.7.1/" >> $GITHUB_ENV
76+
echo "I_MPI_ROOT=/opt/intel/oneapi/mpi/2021.7.1/" >> $GITHUB_ENV
77+
78+
- name: Build-intel
79+
if: matrix.intel == true && matrix.os == 'ubuntu'
80+
run: |
81+
source /opt/intel/oneapi/setvars.sh
82+
/bin/bash mfc.sh build -j $(nproc) --${{ matrix.debug }} --${{ matrix.mpi }}
7583
7684
- name: Build
77-
run: /bin/bash mfc.sh build -j $(nproc) --${{ matrix.debug }} --${{ matrix.mpi }}
85+
if: matrix.intel == false
86+
run: |
87+
/bin/bash mfc.sh build -j $(nproc) --${{ matrix.debug }} --${{ matrix.mpi }}
88+
89+
- name: Test-intel
90+
if: matrix.intel == true && matrix.os == 'ubuntu'
91+
run: |
92+
source /opt/intel/oneapi/setvars.sh
93+
/bin/bash mfc.sh test -j $(nproc) $(if [ "${{ matrix.mpi }}" == "mpi" ]; then echo "--test-all"; fi)
7894
7995
- name: Test
80-
run: /bin/bash mfc.sh test -j $(nproc) $(if [ "${{ matrix.mpi }}" == "mpi" ]; then echo "--test-all"; fi)
96+
if: matrix.intel == false
97+
run: |
98+
/bin/bash mfc.sh test -j $(nproc) $(if [ "${{ matrix.mpi }}" == "mpi" ]; then echo "--test-all"; fi)
8199
82100
docker:
83101
name: Github | Docker

‎src/common/m_variables_conversion.fpp‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -690,7 +690,6 @@ contains
690690
s_convert_to_mixture_variables => &
691691
s_convert_species_to_mixture_variables
692692
end if
693-
694693
end subroutine s_initialize_variables_conversion_module ! --------------
695694

696695
!Initialize mv at the quadrature nodes based on the initialized moments and sigma

‎src/post_process/m_data_input.f90‎

Lines changed: 42 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -405,15 +405,15 @@ subroutine s_populate_grid_variables_buffer_regions() ! ----------------
405405

406406
call s_mpi_sendrecv_grid_vars_buffer_regions('beg', 'x')
407407

408-
do i = 1, offset_x%beg
409-
x_cb(-1 - i) = x_cb(-i) - dx(-i)
410-
end do
408+
end if
411409

412-
do i = 1, buff_size
413-
x_cc(-i) = x_cc(1 - i) - (dx(1 - i) + dx(-i))/2d0
414-
end do
410+
do i = 1, offset_x%beg
411+
x_cb(-1 - i) = x_cb(-i) - dx(-i)
412+
end do
415413

416-
end if
414+
do i = 1, buff_size
415+
x_cc(-i) = x_cc(1 - i) - (dx(1 - i) + dx(-i))/2d0
416+
end do
417417

418418
! Ghost-cell extrapolation BC at the end
419419
if (bc_x%end <= -3) then
@@ -441,15 +441,15 @@ subroutine s_populate_grid_variables_buffer_regions() ! ----------------
441441

442442
call s_mpi_sendrecv_grid_vars_buffer_regions('end', 'x')
443443

444-
do i = 1, offset_x%end
445-
x_cb(m + i) = x_cb(m + (i - 1)) + dx(m + i)
446-
end do
444+
end if
447445

448-
do i = 1, buff_size
449-
x_cc(m + i) = x_cc(m + (i - 1)) + (dx(m + (i - 1)) + dx(m + i))/2d0
450-
end do
446+
do i = 1, offset_x%end
447+
x_cb(m + i) = x_cb(m + (i - 1)) + dx(m + i)
448+
end do
451449

452-
end if
450+
do i = 1, buff_size
451+
x_cc(m + i) = x_cc(m + (i - 1)) + (dx(m + (i - 1)) + dx(m + i))/2d0
452+
end do
453453

454454
! END: Populating Buffer Regions in the x-direction ================
455455

@@ -483,15 +483,15 @@ subroutine s_populate_grid_variables_buffer_regions() ! ----------------
483483

484484
call s_mpi_sendrecv_grid_vars_buffer_regions('beg', 'y')
485485

486-
do i = 1, offset_y%beg
487-
y_cb(-1 - i) = y_cb(-i) - dy(-i)
488-
end do
486+
end if
489487

490-
do i = 1, buff_size
491-
y_cc(-i) = y_cc(1 - i) - (dy(1 - i) + dy(-i))/2d0
492-
end do
488+
do i = 1, offset_y%beg
489+
y_cb(-1 - i) = y_cb(-i) - dy(-i)
490+
end do
493491

494-
end if
492+
do i = 1, buff_size
493+
y_cc(-i) = y_cc(1 - i) - (dy(1 - i) + dy(-i))/2d0
494+
end do
495495

496496
! Ghost-cell extrapolation BC at the end
497497
if (bc_y%end <= -3) then
@@ -519,15 +519,15 @@ subroutine s_populate_grid_variables_buffer_regions() ! ----------------
519519

520520
call s_mpi_sendrecv_grid_vars_buffer_regions('end', 'y')
521521

522-
do i = 1, offset_y%end
523-
y_cb(n + i) = y_cb(n + (i - 1)) + dy(n + i)
524-
end do
522+
end if
525523

526-
do i = 1, buff_size
527-
y_cc(n + i) = y_cc(n + (i - 1)) + (dy(n + (i - 1)) + dy(n + i))/2d0
528-
end do
524+
do i = 1, offset_y%end
525+
y_cb(n + i) = y_cb(n + (i - 1)) + dy(n + i)
526+
end do
529527

530-
end if
528+
do i = 1, buff_size
529+
y_cc(n + i) = y_cc(n + (i - 1)) + (dy(n + (i - 1)) + dy(n + i))/2d0
530+
end do
531531

532532
! END: Populating Buffer Regions in the y-direction ================
533533

@@ -561,15 +561,15 @@ subroutine s_populate_grid_variables_buffer_regions() ! ----------------
561561

562562
call s_mpi_sendrecv_grid_vars_buffer_regions('beg', 'z')
563563

564-
do i = 1, offset_z%beg
565-
z_cb(-1 - i) = z_cb(-i) - dz(-i)
566-
end do
564+
end if
567565

568-
do i = 1, buff_size
569-
z_cc(-i) = z_cc(1 - i) - (dz(1 - i) + dz(-i))/2d0
570-
end do
566+
do i = 1, offset_z%beg
567+
z_cb(-1 - i) = z_cb(-i) - dz(-i)
568+
end do
571569

572-
end if
570+
do i = 1, buff_size
571+
z_cc(-i) = z_cc(1 - i) - (dz(1 - i) + dz(-i))/2d0
572+
end do
573573

574574
! Ghost-cell extrapolation BC at the end
575575
if (bc_z%end <= -3) then
@@ -597,15 +597,15 @@ subroutine s_populate_grid_variables_buffer_regions() ! ----------------
597597

598598
call s_mpi_sendrecv_grid_vars_buffer_regions('end', 'z')
599599

600-
do i = 1, offset_z%end
601-
z_cb(p + i) = z_cb(p + (i - 1)) + dz(p + i)
602-
end do
600+
end if
603601

604-
do i = 1, buff_size
605-
z_cc(p + i) = z_cc(p + (i - 1)) + (dz(p + (i - 1)) + dz(p + i))/2d0
606-
end do
602+
do i = 1, offset_z%end
603+
z_cb(p + i) = z_cb(p + (i - 1)) + dz(p + i)
604+
end do
607605

608-
end if
606+
do i = 1, buff_size
607+
z_cc(p + i) = z_cc(p + (i - 1)) + (dz(p + (i - 1)) + dz(p + i))/2d0
608+
end do
609609

610610
end if
611611

‎src/post_process/m_mpi_proxy.fpp‎

Lines changed: 45 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ module m_mpi_proxy
1919
use m_global_parameters !< Global parameters for the code
2020
2121
use m_mpi_common
22+
23+
use ieee_arithmetic
2224
! ==========================================================================
2325
2426
implicit none
@@ -516,8 +518,8 @@ contains
516518
! Boundary condition at the beginning
517519
if (proc_coords(2) > 0 .or. bc_y%beg == -1) then
518520
proc_coords(2) = proc_coords(2) - 1
519-
call MPI_CART_RANK(MPI_COMM_CART, proc_coords, bc_y%beg, &
520-
ierr)
521+
call MPI_CART_RANK(MPI_COMM_CART, proc_coords, &
522+
bc_y%beg, ierr)
521523
proc_coords(2) = proc_coords(2) + 1
522524
end if
523525
@@ -531,8 +533,8 @@ contains
531533
! Boundary condition at the end
532534
if (proc_coords(2) < num_procs_y - 1 .or. bc_y%end == -1) then
533535
proc_coords(2) = proc_coords(2) + 1
534-
call MPI_CART_RANK(MPI_COMM_CART, proc_coords, bc_y%end, &
535-
ierr)
536+
call MPI_CART_RANK(MPI_COMM_CART, proc_coords, &
537+
bc_y%end, ierr)
536538
proc_coords(2) = proc_coords(2) - 1
537539
end if
538540
@@ -607,7 +609,7 @@ contains
607609
else
608610
offset_x%beg = 0
609611
end if
610-
612+
611613
! Boundary condition at the end
612614
if (proc_coords(1) < num_procs_x - 1 .or. bc_x%end == -1) then
613615
proc_coords(1) = proc_coords(1) + 1
@@ -928,7 +930,13 @@ contains
928930
r = sys_size*(j + buff_size) &
929931
+ sys_size*buff_size*k + (i - 1) &
930932
+ sys_size*buff_size*(n + 1)*l
931-
q_cons_vf(i)%sf(j, k, l) = q_cons_buffer_in(r)
933+
q_cons_vf(i)%sf(j, k, l) = q_cons_buffer_in(r)
934+
#if defined(__INTEL_COMPILER)
935+
if(ieee_is_nan(q_cons_vf(i)%sf(j, k, l))) then
936+
print *, "Error", j, k, l, i
937+
error stop "NaN(s) in recv"
938+
end if
939+
#endif
932940
end do
933941
end do
934942
end do
@@ -1002,7 +1010,13 @@ contains
10021010
r = (i - 1) + sys_size*(j - m - 1) &
10031011
+ sys_size*buff_size*k &
10041012
+ sys_size*buff_size*(n + 1)*l
1005-
q_cons_vf(i)%sf(j, k, l) = q_cons_buffer_in(r)
1013+
q_cons_vf(i)%sf(j, k, l) = q_cons_buffer_in(r)
1014+
#if defined(__INTEL_COMPILER)
1015+
if(ieee_is_nan(q_cons_vf(i)%sf(j, k, l))) then
1016+
print *, "Error", j, k, l, i
1017+
error stop "NaN(s) in recv"
1018+
end if
1019+
#endif
10061020
end do
10071021
end do
10081022
end do
@@ -1091,6 +1105,12 @@ contains
10911105
(k + buff_size) + sys_size* &
10921106
(m + 2*buff_size + 1)*buff_size*l
10931107
q_cons_vf(i)%sf(j, k, l) = q_cons_buffer_in(r)
1108+
#if defined(__INTEL_COMPILER)
1109+
if(ieee_is_nan(q_cons_vf(i)%sf(j, k, l))) then
1110+
print *, "Error", j, k, l, i
1111+
error stop "NaN(s) in recv"
1112+
end if
1113+
#endif
10941114
end do
10951115
end do
10961116
end do
@@ -1171,6 +1191,12 @@ contains
11711191
(k - n - 1) + sys_size* &
11721192
(m + 2*buff_size + 1)*buff_size*l
11731193
q_cons_vf(i)%sf(j, k, l) = q_cons_buffer_in(r)
1194+
#if defined(__INTEL_COMPILER)
1195+
if(ieee_is_nan(q_cons_vf(i)%sf(j, k, l))) then
1196+
print *, "Error", j, k, l, i
1197+
error stop "NaN(s) in recv"
1198+
end if
1199+
#endif
11741200
end do
11751201
end do
11761202
end do
@@ -1264,6 +1290,12 @@ contains
12641290
+ sys_size*(m + 2*buff_size + 1)* &
12651291
(n + 2*buff_size + 1)*(l + buff_size)
12661292
q_cons_vf(i)%sf(j, k, l) = q_cons_buffer_in(r)
1293+
#if defined(__INTEL_COMPILER)
1294+
if(ieee_is_nan(q_cons_vf(i)%sf(j, k, l))) then
1295+
print *, "Error", j, k, l, i
1296+
error stop "NaN(s) in recv"
1297+
end if
1298+
#endif
12671299
end do
12681300
end do
12691301
end do
@@ -1349,6 +1381,12 @@ contains
13491381
+ sys_size*(m + 2*buff_size + 1)* &
13501382
(n + 2*buff_size + 1)*(l - p - 1)
13511383
q_cons_vf(i)%sf(j, k, l) = q_cons_buffer_in(r)
1384+
#if defined(__INTEL_COMPILER)
1385+
if(ieee_is_nan(q_cons_vf(i)%sf(j, k, l))) then
1386+
print *, "Error", j, k, l, i
1387+
error stop "NaN(s) in recv"
1388+
end if
1389+
#endif
13521390
end do
13531391
end do
13541392
end do

0 commit comments

Comments
 (0)