|
1 | | -import os, sys, uuid, subprocess, dataclasses, typing |
| 1 | +import os, sys, uuid, subprocess, dataclasses, typing, math |
| 2 | + |
| 3 | +import rich.table |
| 4 | + |
2 | 5 | from .printer import cons |
3 | | -from .state import ARG, CFG |
4 | | -from .build import get_targets |
5 | | -from .common import system, MFC_BENCH_FILEPATH, MFC_BUILD_DIR, format_list_to_string |
6 | | -from .common import file_load_yaml, file_dump_yaml, create_directory |
7 | | -from .common import MFCException |
| 6 | +from .state import ARG, CFG |
| 7 | +from .build import get_targets, DEFAULT_TARGETS, SIMULATION |
| 8 | +from .common import system, MFC_BENCH_FILEPATH, MFC_BUILD_DIR, format_list_to_string |
| 9 | +from .common import file_load_yaml, file_dump_yaml, create_directory |
| 10 | +from .common import MFCException |
| 11 | + |
8 | 12 |
|
9 | 13 | @dataclasses.dataclass |
10 | 14 | class BenchCase: |
11 | 15 | slug: str |
12 | 16 | path: str |
13 | 17 | args: typing.List[str] |
14 | 18 |
|
15 | | -def bench(targets=None): |
| 19 | + |
| 20 | +def bench(targets = None): |
16 | 21 | if targets is None: |
17 | 22 | targets = ARG("targets") |
18 | 23 |
|
19 | | - precision = "single" if ARG("single") else "double" |
20 | | - |
21 | | - additional_args = ARG("--") |
22 | | - |
23 | 24 | targets = get_targets(targets) |
| 25 | + |
24 | 26 | bench_dirpath = os.path.join(MFC_BUILD_DIR, "benchmarks", str(uuid.uuid4())[:4]) |
25 | 27 | create_directory(bench_dirpath) |
26 | 28 |
|
27 | | - cons.print(f"[bold]Benchmarking {format_list_to_string(ARG('targets'), 'magenta')} in '{precision}' precision " |
28 | | - f"([magenta]{os.path.relpath(bench_dirpath)}[/magenta]):[/bold]") |
| 29 | + cons.print() |
| 30 | + cons.print(f"[bold]Benchmarking {format_list_to_string(ARG('targets'), 'magenta')} ([magenta]{os.path.relpath(bench_dirpath)}[/magenta]):[/bold]") |
29 | 31 | cons.indent() |
30 | 32 | cons.print() |
31 | 33 |
|
32 | | - CASES = [BenchCase(**case) for case in file_load_yaml(MFC_BENCH_FILEPATH)] |
| 34 | + CASES = [ BenchCase(**case) for case in file_load_yaml(MFC_BENCH_FILEPATH) ] |
33 | 35 |
|
34 | 36 | for case in CASES: |
35 | | - case.args = case.args + additional_args |
36 | | - if precision == "single": |
37 | | - case.args.append("--single") |
| 37 | + case.args = case.args + ARG("--") |
38 | 38 | case.path = os.path.abspath(case.path) |
39 | 39 |
|
40 | 40 | results = { |
41 | 41 | "metadata": { |
42 | 42 | "invocation": sys.argv[1:], |
43 | | - "lock": dataclasses.asdict(CFG()), |
44 | | - "precision": precision |
| 43 | + "lock": dataclasses.asdict(CFG()) |
45 | 44 | }, |
46 | 45 | "cases": {}, |
47 | 46 | } |
48 | 47 |
|
49 | 48 | for i, case in enumerate(CASES): |
50 | | - summary_filepath = os.path.join(bench_dirpath, f"{case.slug}-{precision}.yaml") |
51 | | - log_filepath = os.path.join(bench_dirpath, f"{case.slug}-{precision}.out") |
| 49 | + summary_filepath = os.path.join(bench_dirpath, f"{case.slug}.yaml") |
| 50 | + log_filepath = os.path.join(bench_dirpath, f"{case.slug}.out") |
52 | 51 |
|
53 | | - cons.print(f"{str(i + 1).zfill(len(str(len(CASES))))}/{len(CASES)}: {case.slug} @ [bold]{os.path.relpath(case.path)}[/bold]") |
| 52 | + cons.print(f"{str(i+1).zfill(len(CASES) // 10 + 1)}/{len(CASES)}: {case.slug} @ [bold]{os.path.relpath(case.path)}[/bold]") |
54 | 53 | cons.indent() |
| 54 | + cons.print() |
55 | 55 | cons.print(f"> Log: [bold]{os.path.relpath(log_filepath)}[/bold]") |
56 | 56 | cons.print(f"> Summary: [bold]{os.path.relpath(summary_filepath)}[/bold]") |
57 | 57 |
|
58 | 58 | with open(log_filepath, "w") as log_file: |
59 | | - command = [ |
60 | | - "./mfc.sh", "run", case.path, "--case-optimization", |
61 | | - "--targets" |
62 | | - ] + [t.name for t in targets] + [ |
63 | | - "--output-summary", summary_filepath |
64 | | - ] + case.args |
65 | | - |
66 | | - cons.print(f"Case precision: {precision}") |
67 | | - cons.print(f"Case args: {case.args}") |
68 | | - cons.print(f"Running command: {' '.join(command)}") |
69 | | - |
70 | 59 | system( |
71 | | - command, |
| 60 | + ["./mfc.sh", "run", case.path, "--case-optimization"] + |
| 61 | + ["--targets"] + [t.name for t in targets] + |
| 62 | + ["--output-summary", summary_filepath] + |
| 63 | + case.args + |
| 64 | + ["--", "--gbpp", ARG('mem')], |
72 | 65 | stdout=log_file, |
73 | | - stderr=subprocess.STDOUT |
74 | | - ) |
| 66 | + stderr=subprocess.STDOUT) |
75 | 67 |
|
76 | 68 | results["cases"][case.slug] = { |
77 | | - "description": dataclasses.asdict(case), |
| 69 | + "description": dataclasses.asdict(case), |
78 | 70 | "output_summary": file_load_yaml(summary_filepath), |
79 | 71 | } |
80 | | - cons.unindent() |
81 | 72 |
|
82 | 73 | file_dump_yaml(ARG("output"), results) |
| 74 | + |
83 | 75 | cons.print(f"Wrote results to [bold magenta]{os.path.relpath(ARG('output'))}[/bold magenta].") |
| 76 | + |
84 | 77 | cons.unindent() |
85 | 78 |
|
| 79 | + |
| 80 | +# TODO: This function is too long and not nicely written at all. Someone should |
| 81 | +# refactor it... |
| 82 | +# pylint: disable=too-many-branches |
86 | 83 | def diff(): |
87 | | - """ |
88 | | - Compares the results between two benchmark YAML files (lhs vs rhs). |
89 | | - Checks both PR vs master and PR single vs PR double precision. |
90 | | - """ |
91 | 84 | lhs, rhs = file_load_yaml(ARG("lhs")), file_load_yaml(ARG("rhs")) |
92 | 85 |
|
93 | | - lhs_precision = lhs["metadata"].get("precision", "double") |
94 | | - rhs_precision = rhs["metadata"].get("precision", "double") |
95 | | - |
96 | | - is_pr_single_vs_double = lhs_precision == "double" and rhs_precision == "single" |
97 | | - |
98 | | - cons.print(f"[bold]Comparing Benchmarks: Speedups from [magenta]{os.path.relpath(ARG('lhs'))}[/magenta] to " |
99 | | - f"[magenta]{os.path.relpath(ARG('rhs'))}[/magenta][/bold]") |
100 | | - |
101 | | - if lhs["metadata"] != rhs["metadata"] and not is_pr_single_vs_double: |
| 86 | + cons.print(f"[bold]Comparing Benchmarks: Speedups from [magenta]{os.path.relpath(ARG('lhs'))}[/magenta] to [magenta]{os.path.relpath(ARG('rhs'))}[/magenta] are displayed below. Thus, numbers > 1 represent increases in performance.[/bold]") |
| 87 | + if lhs["metadata"] != rhs["metadata"]: |
102 | 88 | def _lock_to_str(lock): |
103 | 89 | return ' '.join([f"{k}={v}" for k, v in lock.items()]) |
104 | 90 |
|
105 | 91 | cons.print(f"""\ |
106 | 92 | [bold yellow]Warning[/bold yellow]: Metadata in lhs and rhs are not equal. |
| 93 | + This could mean that the benchmarks are not comparable (e.g. one was run on CPUs and the other on GPUs). |
107 | 94 | lhs: |
108 | 95 | * Invocation: [magenta]{' '.join(lhs['metadata']['invocation'])}[/magenta] |
109 | 96 | * Modes: {_lock_to_str(lhs['metadata']['lock'])} |
110 | | - * Precision: {lhs_precision} |
111 | 97 | rhs: |
112 | 98 | * Invocation: {' '.join(rhs['metadata']['invocation'])} |
113 | 99 | * Modes: [magenta]{_lock_to_str(rhs['metadata']['lock'])}[/magenta] |
114 | | - * Precision: {rhs_precision} |
115 | 100 | """) |
116 | 101 |
|
117 | 102 | slugs = set(lhs["cases"].keys()) & set(rhs["cases"].keys()) |
118 | 103 | if len(slugs) not in [len(lhs["cases"]), len(rhs["cases"])]: |
119 | 104 | cons.print(f"""\ |
120 | 105 | [bold yellow]Warning[/bold yellow]: Cases in lhs and rhs are not equal. |
121 | | - Using intersection: {slugs} with {len(slugs)} elements.""") |
| 106 | + * rhs cases: {', '.join(set(rhs['cases'].keys()) - slugs)}. |
| 107 | + * lhs cases: {', '.join(set(lhs['cases'].keys()) - slugs)}. |
| 108 | + Using intersection: {slugs} with {len(slugs)} elements. |
| 109 | + """) |
122 | 110 |
|
123 | 111 | table = rich.table.Table(show_header=True, box=rich.table.box.SIMPLE) |
124 | | - table.add_column("[bold]Case[/bold]", justify="left") |
125 | | - table.add_column("[bold]Speedup (Exec)[/bold]", justify="right") |
126 | | - table.add_column("[bold]Speedup (Grind)[/bold]", justify="right") |
| 112 | + table.add_column("[bold]Case[/bold]", justify="left") |
| 113 | + table.add_column("[bold]Pre Process[/bold]", justify="right") |
| 114 | + table.add_column("[bold]Simulation[/bold]", justify="right") |
| 115 | + table.add_column("[bold]Post Process[/bold]", justify="right") |
127 | 116 |
|
128 | 117 | err = 0 |
129 | 118 |
|
130 | 119 | for slug in slugs: |
131 | 120 | lhs_summary = lhs["cases"][slug]["output_summary"] |
132 | 121 | rhs_summary = rhs["cases"][slug]["output_summary"] |
133 | 122 |
|
134 | | - try: |
135 | | - exec_speedup = lhs_summary["exec"] / rhs_summary["exec"] |
136 | | - grind_speedup = lhs_summary["grind"] / rhs_summary["grind"] |
| 123 | + speedups = ['N/A', 'N/A', 'N/A'] |
| 124 | + |
| 125 | + for i, target in enumerate(sorted(DEFAULT_TARGETS, key=lambda t: t.runOrder)): |
| 126 | + if (target.name not in lhs_summary) or (target.name not in rhs_summary): |
137 | 127 |
|
138 | | - if is_pr_single_vs_double and exec_speedup < SINGLE_PRECISION_SPEEDUP_THRESHOLD: |
139 | | - cons.print(f"[bold red]Error[/bold red]: Case {slug} failed speedup requirement: " |
140 | | - f"Exec speedup {exec_speedup:.2f} < {SINGLE_PRECISION_SPEEDUP_THRESHOLD}.") |
141 | 128 | err = 1 |
142 | 129 |
|
143 | | - table.add_row(slug, f"{exec_speedup:.2f}x", f"{grind_speedup:.2f}x") |
| 130 | + if target.name not in lhs_summary: |
| 131 | + cons.print(f"{target.name} not present in lhs_summary - Case: {slug}") |
144 | 132 |
|
145 | | - except KeyError as e: |
146 | | - table.add_row(slug, "Error", "Error") |
147 | | - cons.print(f"[bold yellow]Warning[/bold yellow]: Missing key {e} for case {slug}.") |
148 | | - except ZeroDivisionError: |
149 | | - table.add_row(slug, "Inf", "Inf") |
150 | | - cons.print(f"[bold yellow]Warning[/bold yellow]: Zero execution time in case {slug}.") |
| 133 | + if target.name not in rhs_summary: |
| 134 | + cons.print(f"{target.name} not present in rhs_summary - Case: {slug}") |
151 | 135 |
|
152 | | - cons.raw.print(table) |
| 136 | + continue |
| 137 | + |
| 138 | + if not math.isfinite(lhs_summary[target.name]["exec"]) or not math.isfinite(rhs_summary[target.name]["exec"]): |
| 139 | + err = 1 |
| 140 | + cons.print(f"lhs_summary or rhs_summary reports non-real exec time for {target.name} - Case: {slug}") |
| 141 | + |
| 142 | + exec_time_speedup = "N/A" |
| 143 | + try: |
| 144 | + exec_time_speedup = f'{lhs_summary[target.name]["exec"] / rhs_summary[target.name]["exec"]:.2f}' |
| 145 | + except Exception as _: |
| 146 | + err = 1 |
| 147 | + cons.print(f"lhs_summary or rhs_summary reports non-real exec time for {target.name} - Case: {slug}") |
| 148 | + |
| 149 | + speedups[i] = f"Exec: {exec_time_speedup}" |
| 150 | + |
| 151 | + if target == SIMULATION: |
| 152 | + grind_time_speedup = "N/A" |
| 153 | + if not math.isfinite(lhs_summary[target.name]["grind"]) or not math.isfinite(rhs_summary[target.name]["grind"]): |
| 154 | + err = 1 |
| 155 | + cons.print(f"lhs_summary or rhs_summary reports non-real grind time for {target.name} - Case: {slug}") |
| 156 | + |
| 157 | + try: |
| 158 | + grind_time_speedup = f'{lhs_summary[target.name]["grind"] / rhs_summary[target.name]["grind"]:.2f}' |
| 159 | + except Exception as _: |
| 160 | + err = 1 |
| 161 | + cons.print(f"lhs_summary or rhs_summary reports non-real grind time for {target.name} - Case: {slug}") |
153 | 162 |
|
154 | | - if err: |
155 | | - raise MFCException("Benchmarking failed: Some cases did not meet the performance requirements.") |
| 163 | + speedups[i] += f" & Grind: {grind_time_speedup}" |
| 164 | + |
| 165 | + table.add_row(f"[magenta]{slug}[/magenta]", *speedups) |
| 166 | + |
| 167 | + cons.raw.print(table) |
156 | 168 |
|
| 169 | + if err != 0: |
| 170 | + raise MFCException("Benchmarking failed") |
0 commit comments