Skip to content

Commit 7d71dfa

Browse files
henryleberresbryngelson
henryleberre
and
sbryngelson
authored
Handshake: Benchmarking (#316)
Co-authored-by: Spencer Bryngelson <[email protected]>
1 parent 1d27af6 commit 7d71dfa

File tree

15 files changed

+321
-120
lines changed

15 files changed

+321
-120
lines changed

‎.github/workflows/bench.yml‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
name: 'Benchmark'
2+
3+
on:
4+
pull_request:
5+
6+
jobs:
7+
self:
8+
name: Georgia Tech | Phoenix (NVHPC)
9+
if: github.repository == 'MFlowCode/MFC'
10+
strategy:
11+
matrix:
12+
device: ['cpu', 'gpu']
13+
runs-on:
14+
group: phoenix
15+
labels: gt
16+
steps:
17+
- name: Clone - PR
18+
uses: actions/checkout@v3
19+
with:
20+
path: pr
21+
22+
- name: Clone - Master
23+
uses: actions/checkout@v3
24+
with:
25+
repository: MFlowCode/MFC
26+
ref: master
27+
path: master
28+
29+
- name: Bench (Master v. PR)
30+
run: |
31+
(cd pr && bash .github/workflows/phoenix/submit.sh .github/workflows/phoenix/bench.sh ${{ matrix.device }}) &
32+
(cd master && bash .github/workflows/phoenix/submit.sh .github/workflows/phoenix/bench.sh ${{ matrix.device }}) &
33+
wait %1 && wait %2
34+
35+
- name: Generate Comment
36+
run: |
37+
COMMENT_MSG=`./mfc.sh bench_diff master/bench-${{ matrix.device }}.yaml pr/bench-${{ matrix.device }}.yaml`
38+
echo "COMMENT_MSG=\"$COMMENT_MSG\"" >> $GITHUB_ENV
39+
40+
- name: Post Comment
41+
run: echo "$COMMENT_MSG"
42+
43+
- name: Archive Logs
44+
uses: actions/upload-artifact@v3
45+
if: always()
46+
with:
47+
name: logs-${{ matrix.device }}
48+
path: |
49+
pr/bench-${{ matrix.device }}.*
50+
pr/build/benchmarks/*
51+
master/bench-${{ matrix.device }}.*
52+
master/build/benchmarks/*

‎.github/workflows/phoenix/bench.sh‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/bin/bash
2+
3+
n_ranks=4
4+
5+
if [ "$job_device" == "gpu" ]; then
6+
n_ranks=$(nvidia-smi -L | wc -l) # number of GPUs on node
7+
gpu_ids=$(seq -s ' ' 0 $(($n_ranks-1))) # 0,1,2,...,gpu_count-1
8+
device_opts="--gpu -g $gpu_ids"
9+
fi
10+
11+
./mfc.sh bench -j $(nproc) -o "$job_slug.yaml" -- -c phoenix $device_opts -n $n_ranks

‎.github/workflows/test.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,10 @@ jobs:
117117
- name: Build & Test
118118
run: bash .github/workflows/phoenix/submit.sh .github/workflows/phoenix/test.sh ${{ matrix.device }}
119119

120+
- name: Print Logs
121+
if: always()
122+
run: cat test-${{ matrix.device }}.out
123+
120124
- name: Archive Logs
121125
uses: actions/upload-artifact@v3
122126
if: always()

‎toolchain/bench.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
- slug: 3D_shockdroplet
2+
path: examples/3D_shockdroplet/case.py
3+
args: []
4+
- slug: 3D_turb_mixing
5+
path: examples/3D_turb_mixing/case.py
6+
args: []

‎toolchain/main.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,9 +47,9 @@ def __checks():
4747

4848

4949
def __run():
50-
{"test": test.test, "run": run.run, "build": build.build,
51-
"clean": build.clean, "bench": bench.bench, "count": count.count,
52-
"packer": packer.packer, "count_diff": count.count_diff
50+
{"test": test.test, "run": run.run, "build": build.build,
51+
"clean": build.clean, "bench": bench.bench, "count": count.count,
52+
"packer": packer.packer, "count_diff": count.count_diff, "bench_diff": bench.diff
5353
}[ARG("command")]()
5454

5555

‎toolchain/mfc/args.py‎

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,15 @@ def parse(config):
1818
)
1919

2020
parsers = parser.add_subparsers(dest="command")
21-
run = parsers.add_parser(name="run", help="Run a case with MFC.", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
22-
test = parsers.add_parser(name="test", help="Run MFC's test suite.", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
23-
build = parsers.add_parser(name="build", help="Build MFC and its dependencies.", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
24-
clean = parsers.add_parser(name="clean", help="Clean build artifacts.", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
25-
bench = parsers.add_parser(name="bench", help="Benchmark MFC (for CI).", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
26-
count = parsers.add_parser(name="count", help="Count LOC in MFC.", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
27-
count_diff = parsers.add_parser(name="count_diff", help="Count LOC in MFC.", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
28-
packer = parsers.add_parser(name="packer", help="Packer utility (pack/unpack/compare)", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
21+
run = parsers.add_parser(name="run", help="Run a case with MFC.", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
22+
test = parsers.add_parser(name="test", help="Run MFC's test suite.", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
23+
build = parsers.add_parser(name="build", help="Build MFC and its dependencies.", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
24+
clean = parsers.add_parser(name="clean", help="Clean build artifacts.", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
25+
bench = parsers.add_parser(name="bench", help="Benchmark MFC (for CI).", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
26+
bench_diff = parsers.add_parser(name="bench_diff", help="Compare MFC Benchmarks (for CI).", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
27+
count = parsers.add_parser(name="count", help="Count LOC in MFC.", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
28+
count_diff = parsers.add_parser(name="count_diff", help="Count LOC in MFC.", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
29+
packer = parsers.add_parser(name="packer", help="Packer utility (pack/unpack/compare)", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
2930

3031
packers = packer.add_subparsers(dest="packer")
3132
pack = packers.add_parser(name="pack", help="Pack a case into a single file.", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
@@ -120,9 +121,17 @@ def add_common_arguments(p, mask = None):
120121
run.add_argument("--wait", action="store_true", default=False, help="(Batch) Wait for the job to finish.")
121122
run.add_argument("-f", "--flags", metavar="FLAGS", dest="--", nargs=argparse.REMAINDER, type=str, default=[], help="Arguments to forward to the MPI invocation.")
122123
run.add_argument("-c", "--computer", metavar="COMPUTER", type=str, default="default", help=f"(Batch) Path to a custom submission file template or one of {format_list_to_string(list(get_baked_templates().keys()))}.")
124+
run.add_argument("-o", "--output-summary", metavar="OUTPUT", type=str, default=None, help="Output file (YAML) for summary.")
123125

124126
# === BENCH ===
125-
add_common_arguments(bench, "t")
127+
add_common_arguments(bench)
128+
bench.add_argument("-o", "--output", metavar="OUTPUT", default=None, type=str, required="True", help="Path to the YAML output file to write the results to.")
129+
bench.add_argument(metavar="FORWARDED", default=[], dest='--', nargs="*", help="Arguments to forward to the ./mfc.sh run invocations.")
130+
131+
# === BENCH_DIFF ===
132+
add_common_arguments(bench_diff, "t")
133+
bench_diff.add_argument("lhs", metavar="LHS", type=str, help="Path to a benchmark result YAML file.")
134+
bench_diff.add_argument("rhs", metavar="RHS", type=str, help="Path to a benchmark result YAML file.")
126135

127136
# === COUNT ===
128137
add_common_arguments(count, "g")
@@ -135,8 +144,7 @@ def add_common_arguments(p, mask = None):
135144

136145
# Add default arguments of other subparsers
137146
for name, parser in [("run", run), ("test", test), ("build", build),
138-
("clean", clean), ("bench", bench), ("count", count),
139-
("count_diff", count_diff)]:
147+
("clean", clean), ("count", count), ("count_diff", count_diff)]:
140148
if args["command"] == name:
141149
continue
142150

‎toolchain/mfc/bench.py‎

Lines changed: 113 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,114 @@
1-
from .common import MFCException
1+
import os, sys, uuid, subprocess, dataclasses
22

3-
def bench():
4-
raise MFCException("Benchmarks are currently disabled.")
3+
import rich.table
4+
5+
from .printer import cons
6+
from .state import ARG, CFG
7+
from .build import get_targets, DEFAULT_TARGETS
8+
from .common import system, MFC_BENCH_FILEPATH, MFC_SUBDIR, format_list_to_string
9+
from .common import file_load_yaml, file_dump_yaml, create_directory
10+
11+
12+
@dataclasses.dataclass
13+
class BenchCase:
14+
slug: str
15+
path: str
16+
args: list[str]
17+
18+
19+
def bench(targets = None):
20+
if targets is None:
21+
targets = ARG("targets")
22+
23+
targets = get_targets(targets)
24+
25+
bench_dirpath = os.path.join(MFC_SUBDIR, "benchmarks", str(uuid.uuid4())[:4])
26+
create_directory(bench_dirpath)
27+
28+
cons.print()
29+
cons.print(f"[bold]Benchmarking {format_list_to_string(ARG('targets'), 'magenta')} ([magenta]{os.path.relpath(bench_dirpath)}[/magenta]):[/bold]")
30+
cons.indent()
31+
cons.print()
32+
33+
CASES = [ BenchCase(**case) for case in file_load_yaml(MFC_BENCH_FILEPATH) ]
34+
35+
for case in CASES:
36+
case.args = case.args + ARG("--")
37+
case.path = os.path.abspath(case.path)
38+
39+
results = {
40+
"metadata": {
41+
"invocation": sys.argv[1:],
42+
"lock": dataclasses.asdict(CFG())
43+
},
44+
"cases": {},
45+
}
46+
47+
for i, case in enumerate(CASES):
48+
summary_filepath = os.path.join(bench_dirpath, f"{case.slug}.yaml")
49+
log_filepath = os.path.join(bench_dirpath, f"{case.slug}.out")
50+
51+
cons.print(f"{str(i+1).zfill(len(CASES) // 10 + 1)}/{len(CASES)}: {case.slug} @ [bold]{os.path.relpath(case.path)}[/bold]")
52+
cons.indent()
53+
cons.print()
54+
cons.print(f"> Log: [bold]{os.path.relpath(log_filepath)}[/bold]")
55+
cons.print(f"> Summary: [bold]{os.path.relpath(summary_filepath)}[/bold]")
56+
57+
with open(log_filepath, "w") as log_file:
58+
system(
59+
["./mfc.sh", "run", case.path, "--case-optimization"] +
60+
["--targets"] + [t.name for t in targets] +
61+
["--output-summary", summary_filepath] +
62+
case.args,
63+
stdout=log_file,
64+
stderr=subprocess.STDOUT)
65+
66+
results["cases"][case.slug] = {
67+
"description": dataclasses.asdict(case),
68+
"output_summary": file_load_yaml(summary_filepath),
69+
}
70+
71+
file_dump_yaml(ARG("output"), results)
72+
73+
cons.print(f"Wrote results to [bold magenta]{os.path.relpath(ARG('output'))}[/bold magenta].")
74+
75+
cons.unindent()
76+
77+
78+
def diff():
79+
lhs, rhs = file_load_yaml(ARG("lhs")), file_load_yaml(ARG("rhs"))
80+
81+
cons.print(f"[bold]Comparing Bencharks: [magenta]{os.path.relpath(ARG('lhs'))}[/magenta] is x times slower than [magenta]{os.path.relpath(ARG('rhs'))}[/magenta].[/bold]")
82+
83+
if lhs["metadata"] != rhs["metadata"]:
84+
cons.print(f"[bold yellow]Warning[/bold yellow]: Metadata of lhs and rhs are not equal.")
85+
quit(1)
86+
87+
slugs = set(lhs["cases"].keys()) & set(rhs["cases"].keys())
88+
if len(slugs) not in [len(lhs["cases"]), len(rhs["cases"])]:
89+
cons.print(f"[bold yellow]Warning[/bold yellow]: Cases of lhs and rhs are not equal.[/bold yellow]")
90+
cons.print(f"[bold yellow]lhs: {set(lhs['cases'].keys()) - slugs}[/bold yellow]")
91+
cons.print(f"[bold yellow]rhs: {set(rhs['cases'].keys()) - slugs}[/bold yellow]")
92+
cons.print(f"[bold yellow]Using intersection: {slugs}[/bold yellow]")
93+
94+
table = rich.table.Table(show_header=True, box=rich.table.box.SIMPLE)
95+
table.add_column("[bold]Case[/bold]", justify="left")
96+
table.add_column("[bold]Pre Process[/bold]", justify="right")
97+
table.add_column("[bold]Simulation[/bold]", justify="right")
98+
table.add_column("[bold]Post Process[/bold]", justify="right")
99+
100+
for slug in slugs:
101+
lhs_summary = lhs["cases"][slug]["output_summary"]
102+
rhs_summary = rhs["cases"][slug]["output_summary"]
103+
104+
speedups = ['N/A', 'N/A', 'N/A']
105+
106+
for i, target in enumerate(sorted(DEFAULT_TARGETS, key=lambda t: t.runOrder)):
107+
if target.name not in lhs_summary or target.name not in rhs_summary:
108+
continue
109+
110+
speedups[i] = f"{lhs_summary[target.name] / rhs_summary[target.name]:.2f}x"
111+
112+
table.add_row(f"[magenta]{slug}[/magenta]", *speedups)
113+
114+
cons.raw.print(table)

‎toolchain/mfc/run/case_dicts.py‎

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
from .. import common
21
from ..state import ARG
32

43

@@ -150,16 +149,11 @@
150149

151150

152151
def get_input_dict_keys(target_name: str) -> list:
153-
result = None
154-
if target_name == "pre_process":
155-
result = PRE_PROCESS.copy()
156-
if target_name == "simulation":
157-
result = SIMULATION.copy()
158-
if target_name == "post_process":
159-
result = POST_PROCESS.copy()
160-
161-
if result is None:
162-
raise common.MFCException(f"[INPUT DICTS] Target {target_name} doesn't have an input dict.")
152+
result = {
153+
"pre_process" : PRE_PROCESS,
154+
"simulation" : SIMULATION,
155+
"post_process" : POST_PROCESS
156+
}.get(target_name, {}).copy()
163157

164158
if not ARG("case_optimization") or target_name != "simulation":
165159
return result

‎toolchain/mfc/run/run.py‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,16 @@
1-
import re, os, typing
1+
import re, os, sys, typing, dataclasses
22

33
from glob import glob
44

55
from mako.lookup import TemplateLookup
66
from mako.template import Template
77

8-
from ..build import get_targets, build
8+
from ..build import get_targets, build, REQUIRED_TARGETS
99
from ..printer import cons
10-
from ..state import ARG, ARGS
10+
from ..state import ARG, ARGS, CFG
1111
from ..common import MFCException, isspace, file_read, does_command_exist
1212
from ..common import MFC_TEMPLATEDIR, file_write, system, MFC_ROOTDIR
13-
from ..common import format_list_to_string
13+
from ..common import format_list_to_string, file_dump_yaml
1414

1515
from . import queues, input
1616

@@ -84,12 +84,11 @@ def __generate_job_script(targets):
8484
env['CUDA_VISIBLE_DEVICES'] = ','.join([str(_) for _ in ARG('gpus')])
8585

8686
content = __get_template().render(
87-
**ARGS(),
87+
**{**ARGS(), 'targets': targets},
8888
ARG=ARG,
8989
env=env,
90-
rootdir=MFC_ROOTDIR,
90+
MFC_ROOTDIR=MFC_ROOTDIR,
9191
qsystem=queues.get_system(),
92-
binpaths=[target.get_install_binpath() for target in targets],
9392
profiler=__profiler_prepend(),
9493
)
9594

@@ -119,7 +118,7 @@ def __execute_job_script(qsystem: queues.QueueSystem):
119118

120119

121120
def run(targets = None):
122-
targets = get_targets(targets or ARG("targets"))
121+
targets = get_targets(list(REQUIRED_TARGETS) + (targets or ARG("targets")))
123122

124123
build(targets)
125124

@@ -134,4 +133,9 @@ def run(targets = None):
134133
__generate_input_files(targets)
135134

136135
if not ARG("dry_run"):
136+
if ARG("output_summary") is not None:
137+
file_dump_yaml(ARG("output_summary"), {
138+
"invocation": sys.argv[1:],
139+
"lock": dataclasses.asdict(CFG())
140+
})
137141
__execute_job_script(qsystem)

0 commit comments

Comments
 (0)