Skip to content

Commit 5531fa2

Browse files
sbryngelsonsbryngelson
authored
Merge branch 'master' into CodeTidying
2 parents 446d38f + 6f58eec commit 5531fa2

File tree

8 files changed

+187
-141
lines changed

8 files changed

+187
-141
lines changed

‎.github/workflows/bench.yml‎

Lines changed: 39 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -23,19 +23,40 @@ jobs:
2323
filters: ".github/file-filter.yml"
2424

2525
self:
26-
name: Georgia Tech | Phoenix (NVHPC)
26+
name: "${{ matrix.name }} (${{ matrix.device }})"
2727
if: ${{ github.repository == 'MFlowCode/MFC' && needs.file-changes.outputs.checkall == 'true' && (
2828
(github.event_name == 'pull_request_review' && github.event.review.state == 'approved') ||
2929
(github.event_name == 'pull_request' && github.event.pull_request.user.login == 'sbryngelson')
30-
) }}
30+
) }}
3131
needs: file-changes
3232
strategy:
33-
matrix:
34-
device: ['cpu', 'gpu']
3533
fail-fast: false
34+
matrix:
35+
include:
36+
- cluster: phoenix
37+
name: Georgia Tech | Phoenix (NVHPC)
38+
group: phoenix
39+
labels: gt
40+
flag: p
41+
device: cpu
42+
build_script: ""
43+
- cluster: phoenix
44+
name: Georgia Tech | Phoenix (NVHPC)
45+
group: phoenix
46+
labels: gt
47+
flag: p
48+
device: gpu
49+
build_script: ""
50+
- cluster: frontier
51+
name: Oak Ridge | Frontier (CCE)
52+
group: phoenix
53+
labels: frontier
54+
flag: f
55+
device: gpu
56+
build_script: "bash .github/workflows/frontier/build.sh gpu bench"
3657
runs-on:
37-
group: phoenix
38-
labels: gt
58+
group: ${{ matrix.group }}
59+
labels: ${{ matrix.labels }}
3960
timeout-minutes: 1400
4061
env:
4162
ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16
@@ -53,15 +74,22 @@ jobs:
5374
ref: master
5475
path: master
5576

77+
- name: Setup & Build
78+
if: matrix.build_script != ''
79+
run: |
80+
(cd pr && ${{ matrix.build_script }}) &
81+
(cd master && ${{ matrix.build_script }}) &
82+
wait %1 && wait %2
83+
5684
- name: Bench (Master v. PR)
5785
run: |
58-
(cd pr && bash .github/workflows/phoenix/submit-bench.sh .github/workflows/phoenix/bench.sh ${{ matrix.device }}) &
59-
(cd master && bash .github/workflows/phoenix/submit-bench.sh .github/workflows/phoenix/bench.sh ${{ matrix.device }}) &
86+
(cd pr && bash .github/workflows/${{ matrix.cluster }}/submit-bench.sh .github/workflows/${{ matrix.cluster }}/bench.sh ${{ matrix.device }}) &
87+
(cd master && bash .github/workflows/${{ matrix.cluster }}/submit-bench.sh .github/workflows/${{ matrix.cluster }}/bench.sh ${{ matrix.device }}) &
6088
wait %1 && wait %2
6189
6290
- name: Generate & Post Comment
6391
run: |
64-
(cd pr && . ./mfc.sh load -c p -m g)
92+
(cd pr && . ./mfc.sh load -c ${{ matrix.flag }} -m g)
6593
(cd pr && ./mfc.sh bench_diff ../master/bench-${{ matrix.device }}.yaml ../pr/bench-${{ matrix.device }}.yaml)
6694
6795
- name: Print Logs
@@ -72,9 +100,9 @@ jobs:
72100
73101
- name: Archive Logs
74102
uses: actions/upload-artifact@v4
75-
if: always()
103+
if: always()
76104
with:
77-
name: logs-${{ matrix.device }}
105+
name: ${{ matrix.cluster }}-${{ matrix.device }}
78106
path: |
79107
pr/bench-${{ matrix.device }}.*
80108
pr/build/benchmarks/*

‎.github/workflows/frontier/bench.sh‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/bin/bash
2+
3+
n_ranks=12
4+
5+
if [ "$job_device" = "gpu" ]; then
6+
gpus=$(rocm-smi --showid | awk '{print $1}' | grep -Eo '[0-9]+' | uniq | tr '\n' ' ')
7+
n_ranks=$(echo "$gpus" | wc -w) # number of GPUs on node
8+
gpu_ids=$(echo "$gpus" | tr ' ' '\n' | tr '\n' ' ' | sed 's/ $//') # GPU IDs from rocm-smi
9+
device_opts="--gpu -g $gpu_ids"
10+
fi
11+
12+
if [ "$job_device" = "gpu" ]; then
13+
./mfc.sh bench --mem 12 -j $n_ranks -o "$job_slug.yaml" -- -c frontier $device_opts -n $n_ranks
14+
else
15+
./mfc.sh bench --mem 1 -j $(nproc) -o "$job_slug.yaml" -- -c frontier $device_opts -n $n_ranks
16+
fi

‎.github/workflows/frontier/build.sh‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,4 +6,12 @@ if [ "$1" = "gpu" ]; then
66
fi
77

88
. ./mfc.sh load -c f -m g
9-
./mfc.sh test --dry-run -j 8 $build_opts
9+
10+
if [ "$2" == "bench" ]; then
11+
for dir in benchmarks/*/; do
12+
dirname=$(basename "$dir")
13+
./mfc.sh run "$dir/case.py" --case-optimization -j 8 --dry-run $build_opts
14+
done
15+
else
16+
./mfc.sh test --dry-run -j 8 $build_opts
17+
fi

‎.github/workflows/frontier/submit-bench.sh‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
usage() {
6+
echo "Usage: $0 [script.sh] [cpu|gpu]"
7+
}
8+
9+
if [ ! -z "$1" ]; then
10+
sbatch_script_contents=`cat $1`
11+
else
12+
usage
13+
exit 1
14+
fi
15+
16+
if [ "$2" = "cpu" ]; then
17+
sbatch_device_opts="\
18+
#SBATCH -n 32 # Number of cores required"
19+
elif [ "$2" = "gpu" ]; then
20+
sbatch_device_opts="\
21+
#SBATCH -n 8 # Number of cores required"
22+
else
23+
usage; exit 1
24+
fi
25+
26+
27+
job_slug="`basename "$1" | sed 's/\.sh$//' | sed 's/[^a-zA-Z0-9]/-/g'`-$2"
28+
29+
sbatch <<EOT
30+
#!/bin/bash
31+
#SBATCH -JMFC-$job_slug # Job name
32+
#SBATCH -A CFD154 # charge account
33+
#SBATCH -N 1 # Number of nodes required
34+
$sbatch_device_opts
35+
#SBATCH -t 03:59:00 # Duration of the job (Ex: 15 mins)
36+
#SBATCH -o$job_slug.out # Combined output and error messages file
37+
#SBATCH -p extended # Extended partition for shorter queues
38+
#SBATCH -W # Do not exit until the submitted job terminates.
39+
40+
set -e
41+
set -x
42+
43+
cd "\$SLURM_SUBMIT_DIR"
44+
echo "Running in $(pwd):"
45+
46+
job_slug="$job_slug"
47+
job_device="$2"
48+
49+
. ./mfc.sh load -c f -m g
50+
51+
$sbatch_script_contents
52+
53+
EOT
54+

‎.github/workflows/test.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -128,4 +128,4 @@ jobs:
128128
if: always()
129129
with:
130130
name: logs-${{ strategy.job-index }}-${{ matrix.device }}
131-
path: test-${{ matrix.device }}.out
131+
path: test-${{ matrix.device }}.out

‎benchmarks/ibm/case.py‎

Lines changed: 32 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323

2424
dx = 1.0 / (1.0 * (Nx + 1))
2525

26-
Tend = 64e-06
26+
Tend = 1e-4
2727
Nt = 200
2828
mydt = Tend / (1.0 * Nt)
2929

@@ -48,21 +48,21 @@
4848
"t_step_stop": int(20 * (95 * size + 5)),
4949
"t_step_save": int(20 * (95 * size + 5)),
5050
# Simulation Algorithm Parameters
51-
"num_patches": 2,
51+
"num_patches": 1,
5252
"model_eqns": 2,
5353
"alt_soundspeed": "F",
54-
"num_fluids": 1,
54+
"num_fluids": 2,
5555
"mpp_lim": "F",
5656
"mixture_err": "F",
5757
"time_stepper": 3,
5858
"weno_order": 5,
5959
"weno_eps": 1.0e-16,
60-
"weno_Re_flux": "F",
61-
"weno_avg": "F",
62-
"mapped_weno": "F",
60+
"weno_Re_flux": "T",
61+
"weno_avg": "T",
62+
"mapped_weno": "T",
6363
"null_weights": "F",
6464
"mp_weno": "F",
65-
"riemann_solver": 1,
65+
"riemann_solver": 2,
6666
"wave_speeds": 1,
6767
"avg_state": 2,
6868
"bc_x%beg": -3,
@@ -71,48 +71,45 @@
7171
"bc_y%end": -3,
7272
"bc_z%beg": -3,
7373
"bc_z%end": -3,
74-
# Turning on Hypoelasticity
75-
"hypoelasticity": "T",
76-
"fd_order": 4,
74+
# Turn on IBM
75+
"ib": "T",
76+
"num_ibs": 1,
77+
"viscous": "T",
7778
# Formatted Database Files Structure Parameters
7879
"format": 1,
7980
"precision": 2,
8081
"prim_vars_wrt": "T",
8182
"parallel_io": "T",
8283
# Patch 1 L
8384
"patch_icpp(1)%geometry": 9,
84-
"patch_icpp(1)%x_centroid": 0.25,
85+
"patch_icpp(1)%x_centroid": 0.5,
8586
"patch_icpp(1)%y_centroid": 0.25,
8687
"patch_icpp(1)%z_centroid": 0.25,
87-
"patch_icpp(1)%length_x": 0.5,
88+
"patch_icpp(1)%length_x": 1.0,
8889
"patch_icpp(1)%length_y": 0.5,
8990
"patch_icpp(1)%length_z": 0.5,
90-
"patch_icpp(1)%vel(1)": 0.0,
91+
"patch_icpp(1)%vel(1)": 0.1,
9192
"patch_icpp(1)%vel(2)": 0,
9293
"patch_icpp(1)%vel(3)": 0,
93-
"patch_icpp(1)%pres": 1.0e8,
94-
"patch_icpp(1)%alpha_rho(1)": 1000,
95-
"patch_icpp(1)%alpha(1)": 1.0,
96-
"patch_icpp(1)%tau_e(1)": 0.0,
97-
# Patch 2 R
98-
"patch_icpp(2)%geometry": 9,
99-
"patch_icpp(2)%x_centroid": 0.75,
100-
"patch_icpp(2)%y_centroid": 0.25,
101-
"patch_icpp(2)%z_centroid": 0.25,
102-
"patch_icpp(2)%length_x": 0.5,
103-
"patch_icpp(2)%length_y": 0.5,
104-
"patch_icpp(2)%length_z": 0.5,
105-
"patch_icpp(2)%vel(1)": 0,
106-
"patch_icpp(2)%vel(2)": 0,
107-
"patch_icpp(2)%vel(3)": 0,
108-
"patch_icpp(2)%pres": 1.0e05,
109-
"patch_icpp(2)%alpha_rho(1)": 1000,
110-
"patch_icpp(2)%alpha(1)": 1.0,
111-
"patch_icpp(2)%tau_e(1)": 0.0,
94+
"patch_icpp(1)%pres": 1.0,
95+
"patch_icpp(1)%alpha_rho(1)": 0.8e00,
96+
"patch_icpp(1)%alpha(1)": 0.8e00,
97+
"patch_icpp(1)%alpha_rho(2)": 0.2e00,
98+
"patch_icpp(1)%alpha(2)": 0.2e00,
99+
# Patch: Sphere Immersed Boundary
100+
"patch_ib(1)%geometry": 8,
101+
"patch_ib(1)%x_centroid": 0.25,
102+
"patch_ib(1)%y_centroid": 0.25,
103+
"patch_ib(1)%z_centroid": 0.25,
104+
"patch_ib(1)%radius": 0.1,
112105
# Fluids Physical Parameters
113-
"fluid_pp(1)%gamma": 1.0e00 / (4.4e00 - 1.0e00),
114-
"fluid_pp(1)%pi_inf": 4.4e00 * 6.0e08 / (4.4e00 - 1.0e00),
115-
"fluid_pp(1)%G": 10e09,
106+
# Specify 2 fluids
107+
"fluid_pp(1)%gamma": 1.0e00 / (1.4 - 1.0e00),
108+
"fluid_pp(1)%pi_inf": 0,
109+
"fluid_pp(1)%Re(1)": 54000,
110+
"fluid_pp(2)%gamma": 1.0e00 / (1.4 - 1.0e00),
111+
"fluid_pp(2)%pi_inf": 0,
112+
"fluid_pp(2)%Re(1)": 54000,
116113
}
117114
)
118115
)

‎src/post_process/m_mpi_proxy.fpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ contains
107107
& 'file_per_process', 'relax', 'cf_wrt', &
108108
& 'adv_n', 'ib', 'cfl_adap_dt', 'cfl_const_dt', 'cfl_dt', &
109109
& 'surface_tension', 'hyperelasticity', 'bubbles_lagrange', &
110-
& 'output_partial_domain', 'relativity', 'cont_damage' ]
110+
& 'output_partial_domain', 'relativity', 'cont_damage', 'bc_io']
111111
call MPI_BCAST(${VAR}$, 1, MPI_LOGICAL, 0, MPI_COMM_WORLD, ierr)
112112
#:endfor
113113

0 commit comments

Comments
 (0)