Skip to content

Commit 100bc95

Browse files
mateuszpnmateuszpn
authored
[UR][Benchmarks] Support for Unitrace for all benchmarks (#19320)
Run main.py script with `--unitrace` to get unitrace logs for every benchmark. Use `--unitrace inclusive` to get benchmark results and unitrace logs. --------- Signed-off-by: Mateusz P. Nowak <[email protected]>
1 parent 8fe0c4d commit 100bc95

File tree

16 files changed

+355
-68
lines changed

16 files changed

+355
-68
lines changed

‎devops/scripts/benchmarks/benches/base.py

Lines changed: 42 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
# See LICENSE.TXT
44
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
55

6-
from dataclasses import dataclass
76
import os
87
import shutil
98
import subprocess
@@ -12,6 +11,8 @@
1211
from options import options
1312
from utils.utils import download, run
1413
from abc import ABC, abstractmethod
14+
from utils.unitrace import get_unitrace
15+
from utils.logger import log
1516

1617
benchmark_tags = [
1718
BenchmarkTag("SYCL", "Benchmark uses SYCL runtime"),
@@ -61,6 +62,12 @@ def enabled(self) -> bool:
6162
By default, it returns True, but can be overridden to disable a benchmark."""
6263
return True
6364

65+
def traceable(self) -> bool:
66+
"""Returns whether this benchmark should be traced by Unitrace.
67+
By default, it returns True, but can be overridden to disable tracing for a benchmark.
68+
"""
69+
return True
70+
6471
@abstractmethod
6572
def setup(self):
6673
pass
@@ -70,11 +77,12 @@ def teardown(self):
7077
pass
7178

7279
@abstractmethod
73-
def run(self, env_vars) -> list[Result]:
80+
def run(self, env_vars, run_unitrace: bool = False) -> list[Result]:
7481
"""Execute the benchmark with the given environment variables.
7582
7683
Args:
7784
env_vars: Environment variables to use when running the benchmark.
85+
run_unitrace: Whether to run benchmark under Unitrace.
7886
7987
Returns:
8088
A list of Result objects with the benchmark results.
@@ -97,7 +105,14 @@ def get_adapter_full_path():
97105
), f"could not find adapter file {adapter_path} (and in similar lib paths)"
98106

99107
def run_bench(
100-
self, command, env_vars, ld_library=[], add_sycl=True, use_stdout=True
108+
self,
109+
command,
110+
env_vars,
111+
ld_library=[],
112+
add_sycl=True,
113+
use_stdout=True,
114+
run_unitrace=False,
115+
extra_unitrace_opt=None,
101116
):
102117
env_vars = env_vars.copy()
103118
if options.ur is not None:
@@ -110,13 +125,30 @@ def run_bench(
110125
ld_libraries = options.extra_ld_libraries.copy()
111126
ld_libraries.extend(ld_library)
112127

113-
result = run(
114-
command=command,
115-
env_vars=env_vars,
116-
add_sycl=add_sycl,
117-
cwd=options.benchmark_cwd,
118-
ld_library=ld_libraries,
119-
)
128+
if self.traceable() and run_unitrace:
129+
if extra_unitrace_opt is None:
130+
extra_unitrace_opt = []
131+
unitrace_output, command = get_unitrace().setup(
132+
self.name(), command, extra_unitrace_opt
133+
)
134+
log.debug(f"Unitrace output: {unitrace_output}")
135+
log.debug(f"Unitrace command: {' '.join(command)}")
136+
137+
try:
138+
result = run(
139+
command=command,
140+
env_vars=env_vars,
141+
add_sycl=add_sycl,
142+
cwd=options.benchmark_cwd,
143+
ld_library=ld_libraries,
144+
)
145+
except subprocess.CalledProcessError:
146+
if run_unitrace:
147+
get_unitrace().cleanup(options.benchmark_cwd, unitrace_output)
148+
raise
149+
150+
if self.traceable() and run_unitrace:
151+
get_unitrace().handle_output(unitrace_output)
120152

121153
if use_stdout:
122154
return result.stdout.decode()

‎devops/scripts/benchmarks/benches/benchdnn.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ def setup(self):
132132
if not self.bench_bin.exists():
133133
raise FileNotFoundError(f"Benchmark binary not found: {self.bench_bin}")
134134

135-
def run(self, env_vars):
135+
def run(self, env_vars, run_unitrace: bool = False) -> list[Result]:
136136
command = [
137137
str(self.bench_bin),
138138
*self.bench_args.split(),
@@ -151,6 +151,8 @@ def run(self, env_vars):
151151
add_sycl=True,
152152
ld_library=ld_library,
153153
use_stdout=True,
154+
run_unitrace=run_unitrace,
155+
extra_unitrace_opt=["--chrome-dnn-logging"],
154156
)
155157
result_value = self._extract_time(output)
156158

‎devops/scripts/benchmarks/benches/benchdnn_list.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@
6262
"graph",
6363
"sdpa-plain-f16",
6464
"--reset --dt=f16 --case=complex_fusion/mha/sdpa-plain-implicit-causal-mask-fp32-bs1.json",
65+
False, # Do not run SYCL graph for this benchmark
6566
],
6667
[
6768
"graph",

‎devops/scripts/benchmarks/benches/compute.py

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -317,6 +317,10 @@ def enabled(self) -> bool:
317317
# Check if the specific runtime is enabled (or no specific runtime required)
318318
return self.runtime is None or self.runtime in self.enabled_runtimes()
319319

320+
def name(self):
321+
"""Returns the name of the benchmark, can be overridden."""
322+
return self.bench_name
323+
320324
def bin_args(self) -> list[str]:
321325
return []
322326

@@ -334,7 +338,7 @@ def explicit_group(self):
334338
def description(self) -> str:
335339
return ""
336340

337-
def run(self, env_vars) -> list[Result]:
341+
def run(self, env_vars, run_unitrace: bool = False) -> list[Result]:
338342
command = [
339343
f"{self.benchmark_bin}",
340344
f"--test={self.test}",
@@ -345,7 +349,11 @@ def run(self, env_vars) -> list[Result]:
345349
command += self.bin_args()
346350
env_vars.update(self.extra_env_vars())
347351

348-
result = self.run_bench(command, env_vars)
352+
result = self.run_bench(
353+
command,
354+
env_vars,
355+
run_unitrace=run_unitrace,
356+
)
349357
parsed_results = self.parse_output(result)
350358
ret = []
351359
for label, median, stddev, unit in parsed_results:

‎devops/scripts/benchmarks/benches/gromacs.py

Lines changed: 25 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
from utils.result import Result
1414
from utils.oneapi import get_oneapi
1515
from utils.logger import log
16+
from utils.unitrace import get_unitrace
1617

1718

1819
class GromacsBench(Suite):
@@ -64,24 +65,29 @@ def setup(self) -> None:
6465

6566
self.oneapi = get_oneapi()
6667

68+
cmd_list = [
69+
"cmake",
70+
f"-S {self.gromacs_src}",
71+
f"-B {self.gromacs_build_path}",
72+
"-DCMAKE_BUILD_TYPE=Release",
73+
"-DCMAKE_CXX_COMPILER=clang++",
74+
"-DCMAKE_C_COMPILER=clang",
75+
"-DGMX_GPU=SYCL",
76+
"-DGMX_SYCL_ENABLE_GRAPHS=ON",
77+
"-DGMX_SYCL_ENABLE_EXPERIMENTAL_SUBMIT_API=ON",
78+
"-DGMX_FFT_LIBRARY=MKL",
79+
"-DGMX_GPU_FFT_LIBRARY=MKL",
80+
f"-DMKLROOT={self.oneapi.mkl_dir()}",
81+
"-DGMX_GPU_NB_CLUSTER_SIZE=8",
82+
"-DGMX_GPU_NB_NUM_CLUSTER_PER_CELL_X=1",
83+
"-DGMX_OPENMP=OFF",
84+
]
85+
86+
if options.unitrace:
87+
cmd_list.append("-DGMX_USE_ITT=ON")
88+
6789
run(
68-
[
69-
"cmake",
70-
f"-S {str(self.directory)}/gromacs-repo",
71-
f"-B {self.gromacs_build_path}",
72-
f"-DCMAKE_BUILD_TYPE=Release",
73-
f"-DCMAKE_CXX_COMPILER=clang++",
74-
f"-DCMAKE_C_COMPILER=clang",
75-
f"-DGMX_GPU=SYCL",
76-
f"-DGMX_SYCL_ENABLE_GRAPHS=ON",
77-
f"-DGMX_SYCL_ENABLE_EXPERIMENTAL_SUBMIT_API=ON",
78-
f"-DGMX_FFT_LIBRARY=MKL",
79-
f"-DGMX_GPU_FFT_LIBRARY=MKL",
80-
f"-DMKLROOT={self.oneapi.mkl_dir()}",
81-
f"-DGMX_GPU_NB_CLUSTER_SIZE=8",
82-
f"-DGMX_GPU_NB_NUM_CLUSTER_PER_CELL_X=1",
83-
f"-DGMX_OPENMP=OFF",
84-
],
90+
cmd_list,
8591
add_sycl=True,
8692
)
8793
run(
@@ -163,7 +169,7 @@ def setup(self):
163169
ld_library=self.suite.oneapi.ld_libraries(),
164170
)
165171

166-
def run(self, env_vars):
172+
def run(self, env_vars, run_unitrace: bool = False) -> list[Result]:
167173
model_dir = self.grappa_dir / self.model
168174

169175
env_vars.update({"SYCL_CACHE_PERSISTENT": "1"})
@@ -202,6 +208,7 @@ def run(self, env_vars):
202208
add_sycl=True,
203209
use_stdout=False,
204210
ld_library=self.suite.oneapi.ld_libraries(),
211+
run_unitrace=run_unitrace,
205212
)
206213

207214
if not self._validate_correctness(options.benchmark_cwd + "/md.log"):

‎devops/scripts/benchmarks/benches/llamacpp.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ def get_tags(self):
115115
def lower_is_better(self):
116116
return False
117117

118-
def run(self, env_vars) -> list[Result]:
118+
def run(self, env_vars, run_unitrace: bool = False) -> list[Result]:
119119
command = [
120120
f"{self.benchmark_bin}",
121121
"--output",
@@ -141,7 +141,10 @@ def run(self, env_vars) -> list[Result]:
141141
]
142142

143143
result = self.run_bench(
144-
command, env_vars, ld_library=self.bench.oneapi.ld_libraries()
144+
command,
145+
env_vars,
146+
ld_library=self.bench.oneapi.ld_libraries(),
147+
run_unitrace=run_unitrace,
145148
)
146149
parsed = self.parse_output(result)
147150
results = []

‎devops/scripts/benchmarks/benches/syclbench.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,7 @@ def setup(self):
137137
self.directory, "sycl-bench-build", self.bench_name
138138
)
139139

140-
def run(self, env_vars) -> list[Result]:
140+
def run(self, env_vars, run_unitrace: bool = False) -> list[Result]:
141141
self.outputfile = os.path.join(self.bench.directory, self.test + ".csv")
142142

143143
command = [
@@ -151,7 +151,11 @@ def run(self, env_vars) -> list[Result]:
151151
env_vars.update(self.extra_env_vars())
152152

153153
# no output to stdout, all in outputfile
154-
self.run_bench(command, env_vars)
154+
self.run_bench(
155+
command,
156+
env_vars,
157+
run_unitrace=run_unitrace,
158+
)
155159

156160
with open(self.outputfile, "r") as f:
157161
reader = csv.reader(f)

‎devops/scripts/benchmarks/benches/test.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ def notes(self) -> str:
8888
def unstable(self) -> str:
8989
return self.unstable_text
9090

91-
def run(self, env_vars) -> list[Result]:
91+
def run(self, env_vars, run_unitrace: bool = False) -> list[Result]:
9292
random_value = self.value + random.uniform(-1 * (self.diff), self.diff)
9393
return [
9494
Result(

‎devops/scripts/benchmarks/benches/umf.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,7 @@ def get_names_of_benchmarks_to_be_run(self, command, env_vars):
137137

138138
return all_names
139139

140-
def run(self, env_vars) -> list[Result]:
140+
def run(self, env_vars, run_unitrace: bool = False) -> list[Result]:
141141
command = [f"{self.benchmark_bin}"]
142142

143143
all_names = self.get_names_of_benchmarks_to_be_run(command, env_vars)
@@ -151,7 +151,11 @@ def run(self, env_vars) -> list[Result]:
151151
specific_benchmark = command + ["--benchmark_filter=^" + name + "$"]
152152

153153
result = self.run_bench(
154-
specific_benchmark, env_vars, add_sycl=False, ld_library=[self.umf_lib]
154+
specific_benchmark,
155+
env_vars,
156+
add_sycl=False,
157+
ld_library=[self.umf_lib],
158+
run_unitrace=run_unitrace,
155159
)
156160

157161
parsed = self.parse_output(result)

‎devops/scripts/benchmarks/benches/velocity.py

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -130,15 +130,20 @@ def description(self) -> str:
130130
def get_tags(self):
131131
return ["SYCL", "application"]
132132

133-
def run(self, env_vars) -> list[Result]:
133+
def run(self, env_vars, run_unitrace: bool = False) -> list[Result]:
134134
env_vars.update(self.extra_env_vars())
135135

136136
command = [
137137
f"{self.benchmark_bin}",
138138
]
139139
command += self.bin_args()
140140

141-
result = self.run_bench(command, env_vars, ld_library=self.ld_libraries())
141+
result = self.run_bench(
142+
command,
143+
env_vars,
144+
ld_library=self.ld_libraries(),
145+
run_unitrace=run_unitrace,
146+
)
142147

143148
return [
144149
Result(
@@ -282,7 +287,7 @@ class QuickSilver(VelocityBase):
282287
def __init__(self, vb: VelocityBench):
283288
super().__init__("QuickSilver", "qs", vb, "MMS/CTT")
284289

285-
def run(self, env_vars) -> list[Result]:
290+
def run(self, env_vars, run_unitrace: bool = False) -> list[Result]:
286291
# TODO: fix the crash in QuickSilver when UR_L0_USE_IMMEDIATE_COMMANDLISTS=0
287292
if (
288293
"UR_L0_USE_IMMEDIATE_COMMANDLISTS" in env_vars

0 commit comments

Comments
 (0)