Skip to content

Commit 5896fd7

Browse files
hyeoksu-leehyeoksu-lee
committed
merge from MFCmerge from MFCmerge from MFCmerge from MFCmerge from MFCmerge from MFCmerge from MFCmerge from MFCmerge from MFC
2 parents 7ed6b2d + 1edcb49 commit 5896fd7

File tree

9 files changed

+196
-101
lines changed

9 files changed

+196
-101
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 116 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -98,86 +98,140 @@ jobs:
9898
${{ matrix.intel-command }}
9999
/bin/bash mfc.sh test -j $(nproc) -a
100100
101-
self-cpu:
102-
name: (Self) Test Suite - CPU
103-
strategy:
104-
matrix:
105-
include:
106-
- os: [self-hosted, wingtip]
107-
runs-on: ${{ matrix.os }}
101+
102+
self-cpu-release:
103+
name: Test CPU release - nvhpc
104+
runs-on:
105+
group: phoenix
106+
labels: self-hosted
108107
if: github.repository == 'MFlowCode/MFC'
109108
steps:
110109
- name: Clone
111110
uses: actions/checkout@v3
112-
113-
- name: Setup
114-
run: echo "test"
115-
111+
116112
- name: Build
117113
run: |
118-
module use /opt/nvidia/hpc_sdk/modulefiles/
119-
module load nvhpc
120-
/bin/bash mfc.sh build -j 8
114+
. ./mfc.sh load -c p -m g
115+
./mfc.sh build -j 2
121116
122-
- name: Test Suite
123-
run: |
124-
module use /opt/nvidia/hpc_sdk/modulefiles/
125-
module load nvhpc
126-
/bin/bash mfc.sh test -j 8 -a
127-
128-
self-cpu-debug:
129-
name: (Self) Test Suite - CPU - debug
130-
strategy:
131-
matrix:
132-
include:
133-
- os: [self-hosted, wingtip]
134-
runs-on: ${{ matrix.os }}
117+
- name: Test
118+
run: |
119+
. ./mfc.sh load -c p -m g
120+
mv misc/run-phoenix-release-cpu.sh ./
121+
sbatch run-phoenix-release-cpu.sh
122+
123+
- name: Print
124+
if: always()
125+
run: |
126+
cat test.out
127+
128+
self-gpu-release:
129+
name: Test GPU release - nvhpc
130+
runs-on:
131+
group: phoenix
132+
labels: self-hosted
135133
if: github.repository == 'MFlowCode/MFC'
136134
steps:
137135
- name: Clone
138136
uses: actions/checkout@v3
139-
140-
- name: Setup
141-
run: echo "test"
142-
137+
143138
- name: Build
144139
run: |
145-
module use /opt/nvidia/hpc_sdk/modulefiles/
146-
module load nvhpc
147-
/bin/bash mfc.sh build -j 8 --debug
140+
. ./mfc.sh load -c p -m g
141+
./mfc.sh build -j 2 --gpu
148142
149-
- name: Test Suite
150-
run: |
151-
module use /opt/nvidia/hpc_sdk/modulefiles/
152-
module load nvhpc
153-
/bin/bash mfc.sh test -j 8 --debug
143+
- name: Test
144+
run: |
145+
. ./mfc.sh load -c p -m g
146+
mv misc/run-phoenix-release-gpu.sh ./
147+
sbatch run-phoenix-release-gpu.sh
154148
155-
self-gpu:
156-
name: (Self) Test Suite - GPU
157-
strategy:
158-
matrix:
159-
include:
160-
- os: [self-hosted, wingtip]
161-
runs-on: ${{ matrix.os }}
162-
if: github.repository == 'MFlowCode/MFC'
163-
steps:
164-
- name: Clone
165-
uses: actions/checkout@v3
149+
- name: Print
150+
if: always()
151+
run: |
152+
cat test.out
153+
154+
155+
# self-cpu:
156+
# name: (Self) Test Suite - CPU
157+
# strategy:
158+
# matrix:
159+
# include:
160+
# - os: [self-hosted, wingtip]
161+
# runs-on: ${{ matrix.os }}
162+
# if: github.repository == 'MFlowCode/MFC'
163+
# steps:
164+
# - name: Clone
165+
# uses: actions/checkout@v3
166166

167-
- name: Setup
168-
run: echo "test"
167+
# - name: Setup
168+
# run: echo "test"
169169

170-
- name: Build
171-
run: |
172-
module use /opt/nvidia/hpc_sdk/modulefiles/
173-
module load nvhpc
174-
/bin/bash mfc.sh build -j 8 --gpu
170+
# - name: Build
171+
# run: |
172+
# module use /opt/nvidia/hpc_sdk/modulefiles/
173+
# module load nvhpc
174+
# /bin/bash mfc.sh build -j 8
175175

176-
- name: Test Suite
177-
run: |
178-
module use /opt/nvidia/hpc_sdk/modulefiles/
179-
module load nvhpc
180-
/bin/bash mfc.sh test -j 8 -b mpirun --gpu -a
176+
# - name: Test Suite
177+
# run: |
178+
# module use /opt/nvidia/hpc_sdk/modulefiles/
179+
# module load nvhpc
180+
# /bin/bash mfc.sh test -j 8 -a
181+
182+
# self-cpu-debug:
183+
# name: (Self) Test Suite - CPU - debug
184+
# strategy:
185+
# matrix:
186+
# include:
187+
# - os: [self-hosted, wingtip]
188+
# runs-on: ${{ matrix.os }}
189+
# if: github.repository == 'MFlowCode/MFC'
190+
# steps:
191+
# - name: Clone
192+
# uses: actions/checkout@v3
193+
194+
# - name: Setup
195+
# run: echo "test"
196+
197+
# - name: Build
198+
# run: |
199+
# module use /opt/nvidia/hpc_sdk/modulefiles/
200+
# module load nvhpc
201+
# /bin/bash mfc.sh build -j 8 --debug
202+
203+
# - name: Test Suite
204+
# run: |
205+
# module use /opt/nvidia/hpc_sdk/modulefiles/
206+
# module load nvhpc
207+
# /bin/bash mfc.sh test -j 8 --debug
208+
209+
# self-gpu:
210+
# name: (Self) Test Suite - GPU
211+
# strategy:
212+
# matrix:
213+
# include:
214+
# - os: [self-hosted, wingtip]
215+
# runs-on: ${{ matrix.os }}
216+
# if: github.repository == 'MFlowCode/MFC'
217+
# steps:
218+
# - name: Clone
219+
# uses: actions/checkout@v3
220+
221+
# - name: Setup
222+
# run: echo "test"
223+
224+
# - name: Build
225+
# run: |
226+
# module use /opt/nvidia/hpc_sdk/modulefiles/
227+
# module load nvhpc
228+
# /bin/bash mfc.sh build -j 8 --gpu
229+
230+
# - name: Test Suite
231+
# run: |
232+
# module use /opt/nvidia/hpc_sdk/modulefiles/
233+
# module load nvhpc
234+
# /bin/bash mfc.sh test -j 8 -b mpirun --gpu -a
181235

182236
# self-gpu-debug:
183237
# name: (Self) Test Suite - GPU - debug

‎examples/3D_sphbubcollapse/case.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@
108108
'patch_icpp(1)%vel(3)' : 0.E+00,
109109
'patch_icpp(1)%pres' : 1.E+05,
110110
'patch_icpp(1)%alpha_rho(1)' : 1000.E+00,
111-
'patch_icpp(1)%alpha_rho(2)' : 0.E+00,
111+
'patch_icpp(1)%alpha_rho(2)' : 0.1E+00,
112112
'patch_icpp(1)%alpha(1)' : 0.9E+00,
113113
'patch_icpp(1)%alpha(2)' : 0.1E+00,
114114
# ==========================================================================
@@ -129,10 +129,10 @@
129129
'patch_icpp(2)%vel(2)' : 0.E+00,
130130
'patch_icpp(2)%vel(3)' : 0.E+00,
131131
'patch_icpp(2)%pres' : 1.E+03,
132-
'patch_icpp(2)%alpha_rho(1)' : 0.E+00,
133-
'patch_icpp(2)%alpha_rho(2)' : 0.19E+00,
134-
'patch_icpp(2)%alpha(1)' : 0.9E+00,
135-
'patch_icpp(2)%alpha(2)' : 0.1E+00,
132+
'patch_icpp(2)%alpha_rho(1)' : 100.E+00,
133+
'patch_icpp(2)%alpha_rho(2)' : 0.9E+00,
134+
'patch_icpp(2)%alpha(1)' : 0.1E+00,
135+
'patch_icpp(2)%alpha(2)' : 0.9E+00,
136136
# ==========================================================================
137137

138138
# Fluids Physical Parameters ===============================================

‎mfc.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ if [ "$1" == "load" ]; then
132132
MODULES=("${MODULES[@]}" "python/3.8.5")
133133
elif [ "$u_computer" == "p" ]; then # Phoenix
134134
if [ "$u_cg" == "c" ]; then
135-
MODULES=("gcc/10.3.0-o57x6h" "mvapich2/2.3.6-ouywal")
135+
MODULES=("gcc/10.3.0-o57x6h" "openmpi/4.1.4")
136136
elif [ "$u_cg" == "g" ]; then
137137
MODULES=("cuda/11.7.0-7sdye3" "nvhpc/22.11")
138138
fi

‎misc/run-phoenix-release-cpu.sh‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/bin/bash
2+
#SBATCH -Jshb-test-jobs # Job name
3+
#SBATCH --account=gts-sbryngelson3 # charge account
4+
#SBATCH -N1 --ntasks-per-node=12 # Number of nodes and cores per node required
5+
#SBATCH --mem-per-cpu=2G # Memory per core
6+
#SBATCH -t 04:00:00 # Duration of the job (Ex: 15 mins)
7+
#SBATCH -q inferno # QOS Name
8+
#SBATCH -otest.out # Combined output and error messages file
9+
#SBATCH -W # Do not exit until the submitted job terminates.
10+
11+
cd $SLURM_SUBMIT_DIR # Change to working directory
12+
echo $(pwd)
13+
. ./mfc.sh load -c p -m g
14+
./mfc.sh test -j 12 -b mpirun -a

‎misc/run-phoenix-release-gpu.sh‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/bin/bash
2+
#SBATCH -Jshb-test-jobs # Job name
3+
#SBATCH --account=gts-sbryngelson3 # charge account
4+
#SBATCH -N1 # Number of nodes and cores per node required
5+
#SBATCH --gres=gpu:V100:2
6+
#SBATCH -t 02:00:00 # Duration of the job (Ex: 15 mins)
7+
#SBATCH -q inferno # QOS Name
8+
#SBATCH -otest.out # Combined output and error messages file
9+
#SBATCH -W # Do not exit until the submitted job terminates.
10+
11+
cd $SLURM_SUBMIT_DIR # Change to working directory
12+
echo $(pwd)
13+
. ./mfc.sh load -c p -m g
14+
./mfc.sh test -j 1 -b mpirun -a --gpu

‎toolchain/mfc/args.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@ def add_common_arguments(p, mask = None):
9595
run.add_argument( "--dry-run", action="store_true", default=False, help="(Batch) Run without submitting batch file.")
9696
run.add_argument("--case-optimization", action="store_true", default=False, help="(GPU Optimization) Compile MFC targets with some case parameters hard-coded.")
9797
run.add_argument( "--no-build", action="store_true", default=False, help="(Testing) Do not rebuild MFC.")
98+
run.add_argument("--wait", action="store_true", default=False, help="(Batch) Wait for the job to finish.")
9899

99100
# === BENCH ===
100101
add_common_arguments(bench, "t")

‎toolchain/mfc/run/queues.py‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import os, typing, dataclasses
22

33
from .. import common
4-
4+
from ..state import ARG
55

66
@dataclasses.dataclass
77
class QueueSystem:
@@ -27,6 +27,9 @@ def is_active(self) -> bool:
2727
return common.does_command_exist("qsub")
2828

2929
def gen_submit_cmd(self, filename: str) -> typing.List[str]:
30+
if ARG("wait"):
31+
raise common.MFCException("PBS Queue: Sorry, --wait is unimplemented.")
32+
3033
return ["qsub", filename]
3134

3235

@@ -38,7 +41,12 @@ def is_active(self) -> bool:
3841
return common.does_command_exist("bsub") and common.does_command_exist("bqueues")
3942

4043
def gen_submit_cmd(self, filename: str) -> None:
41-
return ["bsub", filename]
44+
cmd = ["bsub"]
45+
46+
if ARG("wait"):
47+
cmd += ["--wait"]
48+
49+
return cmd + [filename]
4250

4351

4452
class SLURMSystem(QueueSystem):
@@ -49,7 +57,12 @@ def is_active(self) -> bool:
4957
return common.does_command_exist("sbatch")
5058

5159
def gen_submit_cmd(self, filename: str) -> None:
52-
return ["sbatch", filename]
60+
cmd = ["sbatch"]
61+
62+
if ARG("wait"):
63+
cmd += ["--wait"]
64+
65+
return cmd + [filename]
5366

5467

5568
QUEUE_SYSTEMS = [ LSFSystem(), SLURMSystem(), PBSSystem() ]

‎toolchain/templates/pbs.sh‎

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -65,19 +65,21 @@ for binpath in {MFC::BINARIES}; do
6565

6666
echo -e ":) Running $binpath:"
6767

68-
srun \
69-
--nodes={nodes} \
70-
--ntasks-per-node {tasks_per_node} \
71-
{MFC::PROFILER} "$binpath"
72-
73-
#>
74-
#> srun --mpi=pmix \
75-
#> {MFC::PROFILER} "$binpath"
76-
#>
77-
#> mpirun \
78-
#> -np {tasks_per_node*nodes} \
79-
#> {MFC::PROFILER} "$binpath"
80-
#>
68+
if command -v srun > /dev/null 2>&1; then
69+
srun \
70+
--nodes {nodes} \
71+
--ntasks-per-node {tasks_per_node} \
72+
{MFC::PROFILER} "$binpath"
73+
74+
#>
75+
#> srun --mpi=pmix \
76+
#> {MFC::PROFILER} "$binpath"
77+
else
78+
mpirun \
79+
-np {tasks_per_node*nodes} \
80+
{MFC::PROFILER} "$binpath"
81+
82+
fi
8183

8284
done
8385

‎toolchain/templates/slurm.sh‎

Lines changed: 14 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -74,24 +74,21 @@ for binpath in {MFC::BINARIES}; do
7474

7575
echo -e ":) Running $binpath:"
7676

77-
#>
78-
#> Note: This MPI executable might not be well supported
79-
#> on your system - if at all. {MFC::BIN} refers to
80-
#> the path the MFC executable.
81-
#>
82-
#>srun \
83-
#> --nodes={nodes} \
84-
#> --ntasks-per-node {cpus_per_node} \
85-
#> --mpi=pmi2 \
86-
#> {MFC::PROFILER} "{MFC::BIN}"
87-
#>
88-
#> srun --mpi=pmix \
89-
#> {MFC::PROFILER} "{MFC::BIN}"
90-
#>
77+
if command -v srun > /dev/null 2>&1; then
78+
srun \
79+
--nodes {nodes} \
80+
--ntasks-per-node {tasks_per_node} \
81+
{MFC::PROFILER} "$binpath"
9182

92-
mpirun \
93-
-np {nodes*tasks_per_node} \
94-
{MFC::PROFILER} "$binpath"
83+
#>
84+
#> srun --mpi=pmix \
85+
#> {MFC::PROFILER} "$binpath"
86+
#>
87+
else
88+
mpirun \
89+
-np {nodes*tasks_per_node} \
90+
{MFC::PROFILER} "$binpath"
91+
fi
9592

9693
done
9794

0 commit comments

Comments
 (0)