Skip to content

Commit 98eece0

Browse files
sbryngelsonsbryngelson
authored
Merge pull request #198 from henryleberre/master
2 parents 90e1968 + f8bf7d6 commit 98eece0

File tree

23 files changed

+330
-158
lines changed

23 files changed

+330
-158
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 11 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -25,24 +25,26 @@ jobs:
2525
mpi: ['mpi', 'no-mpi']
2626
debug: ['debug', 'no-debug']
2727
intel: [true, false]
28+
exclude:
29+
- intel: true
30+
os: macos
2831
fail-fast: false
2932
continue-on-error: true
3033
runs-on: ${{ matrix.os }}-latest
3134
steps:
3235
- name: Clone
33-
if: matrix.os == 'ubuntu' || (matrix.os == 'macos' && matrix.intel == false)
3436
uses: actions/checkout@v3
3537

3638
- name: Setup MacOS
37-
if: matrix.os == 'macos' && matrix.intel == false
39+
if: matrix.os == 'macos'
3840
run: |
3941
echo "CC=gcc-13" >> $GITHUB_ENV
4042
echo "CXX=g++-13" >> $GITHUB_ENV
4143
echo "FC=gfortran-13" >> $GITHUB_ENV
4244
brew install wget make python make cmake coreutils gcc@13
4345
4446
- name: (MacOS) Build OpenMPI
45-
if: matrix.os == 'macos' && matrix.mpi == 'mpi' && matrix.intel == false
47+
if: matrix.os == 'macos' && matrix.mpi == 'mpi'
4648
run: |
4749
echo "OMPI_FC=gfortran-13" >> $GITHUB_ENV
4850
echo "OMPI_CXX=g++-13" >> $GITHUB_ENV
@@ -75,27 +77,18 @@ jobs:
7577
echo "MPI_HOME=/opt/intel/oneapi/mpi/2021.7.1/" >> $GITHUB_ENV
7678
echo "I_MPI_ROOT=/opt/intel/oneapi/mpi/2021.7.1/" >> $GITHUB_ENV
7779
78-
- name: Build-intel
79-
if: matrix.intel == true && matrix.os == 'ubuntu'
80-
run: |
81-
source /opt/intel/oneapi/setvars.sh
82-
/bin/bash mfc.sh build -j $(nproc) --${{ matrix.debug }} --${{ matrix.mpi }}
83-
8480
- name: Build
85-
if: matrix.intel == false
8681
run: |
82+
if [ '${{ matrix.intel }}' == 'true' ]; then source /opt/intel/oneapi/setvars.sh; fi
8783
/bin/bash mfc.sh build -j $(nproc) --${{ matrix.debug }} --${{ matrix.mpi }}
8884
89-
- name: Test-intel
90-
if: matrix.intel == true && matrix.os == 'ubuntu'
91-
run: |
92-
source /opt/intel/oneapi/setvars.sh
93-
/bin/bash mfc.sh test -j $(nproc) $(if [ "${{ matrix.mpi }}" == "mpi" ]; then echo "--test-all"; fi)
94-
9585
- name: Test
96-
if: matrix.intel == false
9786
run: |
98-
/bin/bash mfc.sh test -j $(nproc) $(if [ "${{ matrix.mpi }}" == "mpi" ]; then echo "--test-all"; fi)
87+
if [ '${{ matrix.intel }}' == 'true' ]; then source /opt/intel/oneapi/setvars.sh; fi
88+
/bin/bash mfc.sh test -j $(nproc) $OPT1 $OPT2
89+
env:
90+
OPT1: ${{ matrix.mpi == 'mpi' && '--test-all' || '' }}
91+
OPT2: ${{ matrix.debug == 'debug' && '-% 75' || '' }}
9992

10093
docker:
10194
name: Github | Docker

‎CMakeLists.txt‎

Lines changed: 38 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ option(MFC_OpenACC "Build with OpenACC" OFF)
2222
option(MFC_PRE_PROCESS "Build pre_process" OFF)
2323
option(MFC_SIMULATION "Build simulation" OFF)
2424
option(MFC_POST_PROCESS "Build post_process" OFF)
25+
option(MFC_SYSCHECK "Build syscheck" OFF)
2526
option(MFC_DOCUMENTATION "Build documentation" OFF)
2627
option(MFC_ALL "Build everything" OFF)
2728

@@ -94,7 +95,8 @@ if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
9495

9596
if (CMAKE_BUILD_TYPE STREQUAL "Debug")
9697
add_compile_options(
97-
-fcheck=all -fbacktrace
98+
-fcheck=all,no-array-temps
99+
-fbacktrace
98100
-fimplicit-none
99101
)
100102
endif()
@@ -220,25 +222,36 @@ endif()
220222
# and generate documentation. Instead, we can simply include the list of .f90
221223
# files that will eventually be used to compile <target>.
222224

223-
macro(HANDLE_SOURCES target)
225+
macro(HANDLE_SOURCES target useCommon)
224226
set(${target}_DIR "${CMAKE_SOURCE_DIR}/src/${target}")
225227
set(common_DIR "${CMAKE_SOURCE_DIR}/src/common")
226228

227229
# Gather src/[<target>,common]/*.f90
228230
file(GLOB ${target}_F90s CONFIGURE_DEPENDS "${${target}_DIR}/*.f90")
229-
file(GLOB common_F90s CONFIGURE_DEPENDS "${common_DIR}/*.f90")
230-
set(${target}_SRCs ${${target}_F90s} ${common_F90s})
231+
set(${target}_SRCs ${${target}_F90s})
232+
if (${useCommon})
233+
file(GLOB common_F90s CONFIGURE_DEPENDS "${common_DIR}/*.f90")
234+
list(APPEND ${target}_SRCs ${common_F90s})
235+
endif()
231236

232237
# src/[<target>,common]/*.fpp -> src/<target>/autogen/*.f90
233238
file(GLOB ${target}_FPPs CONFIGURE_DEPENDS "${${target}_DIR}/*.fpp")
234-
file(GLOB common_FPPs CONFIGURE_DEPENDS "${common_DIR}/*.fpp")
239+
if (${useCommon})
240+
file(GLOB common_FPPs CONFIGURE_DEPENDS "${common_DIR}/*.fpp")
241+
list(APPEND ${target}_FPPs ${common_FPPs})
242+
endif()
235243

236244
# Locate src/[<target>,common]/include/*.fpp
237-
file(GLOB ${target}_incs CONFIGURE_DEPENDS "${${target}_DIR}/include/*.fpp")
238-
file(GLOB common_incs CONFIGURE_DEPENDS "${common_DIR}/include/*.fpp")
245+
if (EXISTS "${${target}_DIR}/include")
246+
file(GLOB ${target}_incs CONFIGURE_DEPENDS "${${target}_DIR}/include/*.fpp")
247+
endif()
248+
if (${useCommon})
249+
file(GLOB common_incs CONFIGURE_DEPENDS "${common_DIR}/include/*.fpp")
250+
list(APPEND ${target}_incs ${common_incs})
251+
endif()
239252

240253
file(MAKE_DIRECTORY "${${target}_DIR}/autogen")
241-
foreach(fpp ${${target}_FPPs} ${common_FPPs})
254+
foreach(fpp ${${target}_FPPs})
242255
cmake_path(GET fpp FILENAME fpp_filename)
243256
set(f90 "${${target}_DIR}/autogen/${fpp_filename}.f90")
244257

@@ -253,7 +266,7 @@ macro(HANDLE_SOURCES target)
253266
--line-numbering
254267
--no-folding
255268
"${fpp}" "${f90}"
256-
DEPENDS "${fpp};${${target}_incs};${common_incs}"
269+
DEPENDS "${fpp};${${target}_incs}"
257270
COMMENT "Preprocessing (Fypp) ${fpp_filename}"
258271
VERBATIM
259272
)
@@ -263,9 +276,10 @@ macro(HANDLE_SOURCES target)
263276
endmacro()
264277

265278

266-
HANDLE_SOURCES(pre_process)
267-
HANDLE_SOURCES(simulation)
268-
HANDLE_SOURCES(post_process)
279+
HANDLE_SOURCES(pre_process ON)
280+
HANDLE_SOURCES(simulation ON)
281+
HANDLE_SOURCES(post_process ON)
282+
HANDLE_SOURCES(syscheck OFF)
269283

270284

271285
# MFC_SETUP_TARGET: Given a target (herein <target>), this macro creates a new
@@ -293,8 +307,12 @@ function(MFC_SETUP_TARGET)
293307
target_include_directories(${ARGS_TARGET} PRIVATE
294308
"${CMAKE_SOURCE_DIR}/src/common"
295309
"${CMAKE_SOURCE_DIR}/src/common/include"
296-
"${CMAKE_SOURCE_DIR}/src/${ARGS_TARGET}"
297-
"${CMAKE_SOURCE_DIR}/src/${ARGS_TARGET}/include")
310+
"${CMAKE_SOURCE_DIR}/src/${ARGS_TARGET}")
311+
312+
if (EXISTS "${CMAKE_SOURCE_DIR}/src/${ARGS_TARGET}/include")
313+
target_include_directories(${ARGS_TARGET} PRIVATE
314+
"${CMAKE_SOURCE_DIR}/src/${ARGS_TARGET}/include")
315+
endif()
298316

299317
string(TOUPPER "${ARGS_TARGET}" ARGS_TARGET_UPPER)
300318
target_compile_definitions(
@@ -338,6 +356,7 @@ function(MFC_SETUP_TARGET)
338356
endif()
339357

340358
target_link_libraries(${ARGS_TARGET} PRIVATE OpenACC::OpenACC_Fortran)
359+
target_compile_definitions(${ARGS_TARGET} PRIVATE MFC_OpenACC)
341360

342361
if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
343362
# FIXME: This should work with other cards than gfx90a ones.
@@ -401,6 +420,11 @@ if (MFC_POST_PROCESS)
401420
target_compile_options(post_process PRIVATE -O0)
402421
endif()
403422

423+
if (MFC_SYSCHECK)
424+
MFC_SETUP_TARGET(TARGET syscheck
425+
SOURCES "${syscheck_SRCs}"
426+
MPI OpenACC)
427+
endif()
404428

405429
if (MFC_DOCUMENTATION)
406430
# GEN_DOCS: Given a target name (herein <target>), this macro sets up a

‎misc/run-phoenix-release-cpu.sh‎

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,16 @@
11
#!/bin/bash
2-
#SBATCH -Jshb-test-jobs # Job name
3-
#SBATCH --account=gts-sbryngelson3 # charge account
4-
#SBATCH -N1 --ntasks-per-node=12 # Number of nodes and cores per node required
5-
#SBATCH --mem-per-cpu=2G # Memory per core
6-
#SBATCH -t 04:00:00 # Duration of the job (Ex: 15 mins)
7-
#SBATCH -q embers # QOS Name
8-
#SBATCH -otest.out # Combined output and error messages file
9-
#SBATCH -W # Do not exit until the submitted job terminates.
2+
#SBATCH -Jshb-test-jobs # Job name
3+
#SBATCH --account=gts-sbryngelson3 # charge account
4+
#SBATCH -N1 --ntasks-per-node=12 # Number of nodes and cores per node required
5+
#SBATCH --mem-per-cpu=2G # Memory per core
6+
#SBATCH -t 04:00:00 # Duration of the job (Ex: 15 mins)
7+
#SBATCH -q embers # QOS Name
8+
#SBATCH -otest.out # Combined output and error messages file
9+
#SBATCH -W # Do not exit until the submitted job terminates.
10+
11+
cd "$SLURM_SUBMIT_DIR"
12+
echo "Running in $(pwd):"
1013

11-
cd $SLURM_SUBMIT_DIR # Change to working directory
12-
echo $(pwd)
1314
. ./mfc.sh load -c p -m gpu
14-
./mfc.sh test -j 12 -b mpirun -a
15+
./mfc.sh test -j $(nproc) -b mpirun -a
16+

‎misc/run-phoenix-release-gpu.sh‎

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,24 @@
11
#!/bin/bash
2-
#SBATCH -Jshb-test-jobs # Job name
3-
#SBATCH --account=gts-sbryngelson3 # charge account
4-
#SBATCH -N1 # Number of nodes and cores per node required
2+
#SBATCH -Jshb-test-jobs # Job name
3+
#SBATCH --account=gts-sbryngelson3 # charge account
4+
#SBATCH -N1 # Number of nodes and cores per node required
55
#SBATCH -CV100-16GB
66
#SBATCH -G2
7-
#SBATCH -t 02:00:00 # Duration of the job (Ex: 15 mins)
8-
#SBATCH -q embers # QOS Name
9-
#SBATCH -otest.out # Combined output and error messages file
10-
#SBATCH -W # Do not exit until the submitted job terminates.
7+
#SBATCH -t 02:00:00 # Duration of the job (Ex: 15 mins)
8+
#SBATCH -q embers # QOS Name
9+
#SBATCH -otest.out # Combined output and error messages file
10+
#SBATCH -W # Do not exit until the submitted job terminates.
1111

12-
cd $SLURM_SUBMIT_DIR # Change to working directory
13-
echo $(pwd)
12+
cd "$SLURM_SUBMIT_DIR"
13+
echo "Running in $(pwd):"
14+
15+
set -x
1416

1517
. ./mfc.sh load -c p -m GPU
1618

17-
nvidia-smi
18-
echo $(nproc)
19+
gpu_count=$(nvidia-smi -L | wc -l) # number of GPUs on node
20+
gpu_ids=$(seq -s ',' 0 $(($gpu_count-1))) # 0,1,2,...,gpu_count-1
1921

22+
./mfc.sh test -a -b mpirun -j $(nproc) \
23+
--gpu -g $gpu_ids
2024

21-
./mfc.sh test -b mpirun -a --gpu

‎src/simulation/m_checker.fpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ contains
2929
bub_fac = 0
3030
if (bubbles .and. (num_fluids == 1)) bub_fac = 1
3131

32-
#if !(defined(_OPENACC) && defined(__PGI))
32+
#if !(defined(MFC_OpenACC) && defined(__PGI))
3333
if (cu_mpi) then
3434
call s_mpi_abort('Unsupported value of cu_mpi. Exiting ...')
3535
end if

‎src/simulation/m_fftw.fpp‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ module m_fftw
1616

1717
use m_mpi_proxy !< Message passing interface (MPI) module proxy
1818

19-
#if defined(_OPENACC) && defined(__PGI)
19+
#if defined(MFC_OpenACC) && defined(__PGI)
2020
use cufft
2121
#endif
2222
! ==========================================================================
@@ -27,7 +27,7 @@ module m_fftw
2727
s_apply_fourier_filter, &
2828
s_finalize_fftw_module
2929

30-
#if !(defined(_OPENACC) && defined(__PGI))
30+
#if !(defined(MFC_OpenACC) && defined(__PGI))
3131
include 'fftw3.f03'
3232
#endif
3333

@@ -43,7 +43,7 @@ module m_fftw
4343
complex(c_double_complex), pointer :: data_fltr_cmplx(:) !<
4444
!! Filtered complex data in Fourier space
4545

46-
#if defined(_OPENACC) && defined(__PGI)
46+
#if defined(MFC_OpenACC) && defined(__PGI)
4747
!$acc declare create(real_size, cmplx_size, x_size, batch_size)
4848

4949
real(kind(0d0)), allocatable :: data_real_gpu(:)
@@ -65,7 +65,7 @@ contains
6565
!! applying the Fourier filter in the azimuthal direction.
6666
subroutine s_initialize_fftw_module() ! ----------------------------------
6767

68-
#if defined(_OPENACC) && !defined(__PGI)
68+
#if defined(MFC_OpenACC) && !defined(__PGI)
6969

7070
print *, "The FFTW module is not supported when using OpenACC with a compiler other than NVHPC/PGI."
7171
stop 1
@@ -81,7 +81,7 @@ contains
8181

8282
batch_size = x_size*sys_size
8383

84-
#if defined(_OPENACC) && defined(__PGI)
84+
#if defined(MFC_OpenACC) && defined(__PGI)
8585
rank = 1; istride = 1; ostride = 1
8686

8787
allocate(cufft_size(1:rank), iembed(1:rank), oembed(1:rank))
@@ -106,7 +106,7 @@ contains
106106
bwd_plan = fftw_plan_dft_c2r_1d(real_size, data_fltr_cmplx, data_real, FFTW_ESTIMATE)
107107
#endif
108108

109-
#if defined(_OPENACC) && defined(__PGI)
109+
#if defined(MFC_OpenACC) && defined(__PGI)
110110
@:ALLOCATE(data_real_gpu(1:real_size*x_size*sys_size))
111111
@:ALLOCATE(data_cmplx_gpu(1:cmplx_size*x_size*sys_size))
112112
@:ALLOCATE(data_fltr_cmplx_gpu(1:cmplx_size*x_size*sys_size))
@@ -132,7 +132,7 @@ contains
132132
! Restrict filter to processors that have cells adjacent to axis
133133
if (bc_y%beg >= 0) return
134134

135-
#if defined(_OPENACC) && defined(__PGI)
135+
#if defined(MFC_OpenACC) && defined(__PGI)
136136

137137
!$acc parallel loop collapse(3) gang vector default(present)
138138
do k = 1, sys_size
@@ -270,7 +270,7 @@ contains
270270
!! applying the Fourier filter in the azimuthal direction.
271271
subroutine s_finalize_fftw_module() ! ------------------------------------
272272

273-
#if defined(_OPENACC) && defined(__PGI)
273+
#if defined(MFC_OpenACC) && defined(__PGI)
274274
@:DEALLOCATE(data_real_gpu, data_fltr_cmplx_gpu, data_cmplx_gpu)
275275
ierr = cufftDestroy(fwd_plan_gpu)
276276
ierr = cufftDestroy(bwd_plan_gpu)

‎src/simulation/m_global_parameters.fpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ module m_global_parameters
2020

2121
use m_derived_types !< Definitions of the derived types
2222

23-
#ifdef _OPENACC
23+
#ifdef MFC_OpenACC
2424
use openacc
2525
#endif
2626

0 commit comments

Comments
 (0)