[UR][Benchmarks] Add flamegraphs to benchmark results

devops/scripts/benchmarks/benches/base.py

@@ -7,12 +7,22 @@
 import shutil
 import subprocess
 from pathlib import Path
+from enum import Enum
 from utils.result import BenchmarkMetadata, BenchmarkTag, Result
 from options import options
 from utils.utils import download, run
 from abc import ABC, abstractmethod
 from utils.unitrace import get_unitrace
 from utils.logger import log
+from utils.flamegraph import get_flamegraph
+
+
+class TracingType(Enum):
+    """Enumeration of available tracing types."""
+
+    UNITRACE = "unitrace"
+    FLAMEGRAPH = "flamegraph"
+
 
 benchmark_tags = [
     BenchmarkTag("SYCL", "Benchmark uses SYCL runtime"),
@@ -62,9 +72,10 @@ def enabled(self) -> bool:
         By default, it returns True, but can be overridden to disable a benchmark."""
         return True
 
-    def traceable(self) -> bool:
-        """Returns whether this benchmark should be traced by Unitrace.
-        By default, it returns True, but can be overridden to disable tracing for a benchmark.
+    def traceable(self, tracing_type: TracingType) -> bool:
+        """Returns whether this benchmark should be traced by the specified tracing method.
+        By default, it returns True for all tracing types, but can be overridden
+        to disable specific tracing methods for a benchmark.
         """
         return True
 
@@ -77,12 +88,15 @@ def teardown(self):
         pass
 
     @abstractmethod
-    def run(self, env_vars, run_unitrace: bool = False) -> list[Result]:
+    def run(
+        self, env_vars, run_unitrace: bool = False, run_flamegraph: bool = False
+    ) -> list[Result]:
         """Execute the benchmark with the given environment variables.
 
         Args:
             env_vars: Environment variables to use when running the benchmark.
             run_unitrace: Whether to run benchmark under Unitrace.
+            run_flamegraph: Whether to run benchmark under FlameGraph.
 
         Returns:
             A list of Result objects with the benchmark results.
@@ -113,6 +127,8 @@ def run_bench(
         use_stdout=True,
         run_unitrace=False,
         extra_unitrace_opt=None,
+        run_flamegraph=False,
+        extra_perf_opt=None,  # VERIFY
     ):
         env_vars = env_vars.copy()
         if options.ur is not None:
@@ -125,7 +141,7 @@ def run_bench(
         ld_libraries = options.extra_ld_libraries.copy()
         ld_libraries.extend(ld_library)
 
-        if self.traceable() and run_unitrace:
+        if self.traceable(TracingType.UNITRACE) and run_unitrace:
             if extra_unitrace_opt is None:
                 extra_unitrace_opt = []
             unitrace_output, command = get_unitrace().setup(
@@ -147,9 +163,41 @@ def run_bench(
                 get_unitrace().cleanup(options.benchmark_cwd, unitrace_output)
             raise
 
-        if self.traceable() and run_unitrace:
+        if self.traceable(TracingType.UNITRACE) and run_unitrace:
             get_unitrace().handle_output(unitrace_output)
 
+        # flamegraph run
+
+        ld_libraries = options.extra_ld_libraries.copy()
+        ld_libraries.extend(ld_library)
+
+        perf_data_file = None
+        if self.traceable(TracingType.FLAMEGRAPH) and run_flamegraph:
+            if extra_perf_opt is None:
+                extra_perf_opt = []
+            perf_data_file, command = get_flamegraph().setup(
+                self.name(), command, extra_perf_opt
+            )
+            log.debug(f"FlameGraph perf data: {perf_data_file}")
+            log.debug(f"FlameGraph command: {' '.join(command)}")
+
+        try:
+            result = run(
+                command=command,
+                env_vars=env_vars,
+                add_sycl=add_sycl,
+                cwd=options.benchmark_cwd,
+                ld_library=ld_libraries,
+            )
+        except subprocess.CalledProcessError:
+            if run_flamegraph and perf_data_file:
+                get_flamegraph().cleanup(options.benchmark_cwd, perf_data_file)
+            raise
+
+        if self.traceable(TracingType.FLAMEGRAPH) and run_flamegraph and perf_data_file:
+            svg_file = get_flamegraph().handle_output(self.name(), perf_data_file)
+            log.info(f"FlameGraph generated: {svg_file}")
+
         if use_stdout:
             return result.stdout.decode()
         else:

devops/scripts/benchmarks/benches/benchdnn.py

@@ -132,7 +132,9 @@ def setup(self):
         if not self.bench_bin.exists():
             raise FileNotFoundError(f"Benchmark binary not found: {self.bench_bin}")
 
-    def run(self, env_vars, run_unitrace: bool = False) -> list[Result]:
+    def run(
+        self, env_vars, run_flamegraph: bool = False, run_unitrace: bool = False
+    ) -> list[Result]:
         command = [
             str(self.bench_bin),
             *self.bench_args.split(),
@@ -153,6 +155,7 @@ def run(self, env_vars, run_unitrace: bool = False) -> list[Result]:
             use_stdout=True,
             run_unitrace=run_unitrace,
             extra_unitrace_opt=["--chrome-dnn-logging"],
+            run_flamegraph=run_flamegraph,
         )
         result_value = self._extract_time(output)
 

devops/scripts/benchmarks/benches/benchdnn_list.py

@@ -14,29 +14,29 @@
 
 # the final choice of benchmarks to run, used in CI and other environments
 benches_final_set = [
-    [
-        "sum",
-        "f16-1",
-        "--sdt=f16:f16:f16 --stag=abx:abx:abx --scales=1.25:3:0.5 16x2x6x4x3",
-        False,  # Do not run graph for this benchmark
-    ],
-    [
-        "sum",
-        "f16-2",
-        "--reset --ddt=f16 \
-            --sdt=f16:f16:f16:f16:f16:f16:f16:f16:f16:f16 \
-            --stag=abx:aBx16b:ABx16a16b:ABcd16b16a:BAcd16a16b:BAcd16b16a:aBCd16b16c:aBCd16c16b:aCBd16b16c:aCBd16c16b \
-            --dtag=abx,aBx16b,ABx16a16b,ABcd16b16a,BAcd16a16b,BAcd16b16a,aBCd16b16c,aBCd16c16b,aCBd16b16c,aCBd16c16b \
-            --scales=1.25:3:0.5:2:0.5:2:0.5:2:0.5:2 \
-            16x32x48x5",
-        False,  # Do not run graph for this benchmark
-    ],
-    [
-        "sum",
-        "f32-1",
-        "--sdt=bf16:bf16:bf16 --stag=abx:abx:abx --scales=0.5:2:0.5    16x2x6x4x3",
-        False,  # Do not run graph for this benchmark
-    ],
+    # [
+    #     "sum",
+    #     "f16-1",
+    #     "--sdt=f16:f16:f16 --stag=abx:abx:abx --scales=1.25:3:0.5 16x2x6x4x3",
+    #     False,  # Do not run graph for this benchmark
+    # ],
+    # [
+    #     "sum",
+    #     "f16-2",
+    #     "--reset --ddt=f16 \
+    #         --sdt=f16:f16:f16:f16:f16:f16:f16:f16:f16:f16 \
+    #         --stag=abx:aBx16b:ABx16a16b:ABcd16b16a:BAcd16a16b:BAcd16b16a:aBCd16b16c:aBCd16c16b:aCBd16b16c:aCBd16c16b \
+    #         --dtag=abx,aBx16b,ABx16a16b,ABcd16b16a,BAcd16a16b,BAcd16b16a,aBCd16b16c,aBCd16c16b,aCBd16b16c,aCBd16c16b \
+    #         --scales=1.25:3:0.5:2:0.5:2:0.5:2:0.5:2 \
+    #         16x32x48x5",
+    #     False,  # Do not run graph for this benchmark
+    # ],
+    # [
+    #     "sum",
+    #     "f32-1",
+    #     "--sdt=bf16:bf16:bf16 --stag=abx:abx:abx --scales=0.5:2:0.5    16x2x6x4x3",
+    #     False,  # Do not run graph for this benchmark
+    # ],
     [
         "sum",
         "f32-2",

devops/scripts/benchmarks/benches/compute.py

@@ -339,7 +339,9 @@ def explicit_group(self):
     def description(self) -> str:
         return ""
 
-    def run(self, env_vars, run_unitrace: bool = False) -> list[Result]:
+    def run(
+        self, env_vars, run_unitrace: bool = False, run_flamegraph: bool = False
+    ) -> list[Result]:
         command = [
             f"{self.benchmark_bin}",
             f"--test={self.test}",
@@ -351,9 +353,7 @@ def run(self, env_vars, run_unitrace: bool = False) -> list[Result]:
         env_vars.update(self.extra_env_vars())
 
         result = self.run_bench(
-            command,
-            env_vars,
-            run_unitrace=run_unitrace,
+            command, env_vars, run_unitrace=run_unitrace, run_flamegraph=run_flamegraph
         )
         parsed_results = self.parse_output(result)
         ret = []

devops/scripts/benchmarks/benches/gromacs.py

@@ -169,7 +169,9 @@ def setup(self):
             ld_library=self.suite.oneapi.ld_libraries(),
         )
 
-    def run(self, env_vars, run_unitrace: bool = False) -> list[Result]:
+    def run(
+        self, env_vars, run_flamegraph: bool = False, run_unitrace: bool = False
+    ) -> list[Result]:
         model_dir = self.grappa_dir / self.model
 
         env_vars.update({"SYCL_CACHE_PERSISTENT": "1"})
@@ -209,6 +211,7 @@ def run(self, env_vars, run_unitrace: bool = False) -> list[Result]:
             use_stdout=False,
             ld_library=self.suite.oneapi.ld_libraries(),
             run_unitrace=run_unitrace,
+            run_flamegraph=run_flamegraph,
         )
 
         if not self._validate_correctness(options.benchmark_cwd + "/md.log"):

devops/scripts/benchmarks/benches/llamacpp.py

@@ -115,7 +115,9 @@ def get_tags(self):
     def lower_is_better(self):
         return False
 
-    def run(self, env_vars, run_unitrace: bool = False) -> list[Result]:
+    def run(
+        self, env_vars, run_unitrace: bool = False, run_flamegraph: bool = False
+    ) -> list[Result]:
         command = [
             f"{self.benchmark_bin}",
             "--output",
@@ -145,6 +147,7 @@ def run(self, env_vars, run_unitrace: bool = False) -> list[Result]:
             env_vars,
             ld_library=self.bench.oneapi.ld_libraries(),
             run_unitrace=run_unitrace,
+            run_flamegraph=run_flamegraph,
         )
         parsed = self.parse_output(result)
         results = []

devops/scripts/benchmarks/benches/syclbench.py

@@ -137,7 +137,9 @@ def setup(self):
             self.directory, "sycl-bench-build", self.bench_name
         )
 
-    def run(self, env_vars, run_unitrace: bool = False) -> list[Result]:
+    def run(
+        self, env_vars, run_unitrace: bool = False, run_flamegraph: bool = False
+    ) -> list[Result]:
         self.outputfile = os.path.join(self.bench.directory, self.test + ".csv")
 
         command = [
@@ -152,9 +154,7 @@ def run(self, env_vars, run_unitrace: bool = False) -> list[Result]:
 
         # no output to stdout, all in outputfile
         self.run_bench(
-            command,
-            env_vars,
-            run_unitrace=run_unitrace,
+            command, env_vars, run_unitrace=run_unitrace, run_flamegraph=run_flamegraph
         )
 
         with open(self.outputfile, "r") as f:

devops/scripts/benchmarks/benches/test.py

@@ -88,7 +88,9 @@ def notes(self) -> str:
     def unstable(self) -> str:
         return self.unstable_text
 
-    def run(self, env_vars, run_unitrace: bool = False) -> list[Result]:
+    def run(
+        self, env_vars, run_unitrace: bool = False, run_flamegraph: bool = False
+    ) -> list[Result]:
         random_value = self.value + random.uniform(-1 * (self.diff), self.diff)
         return [
             Result(

devops/scripts/benchmarks/benches/umf.py

@@ -137,7 +137,9 @@ def get_names_of_benchmarks_to_be_run(self, command, env_vars):
 
         return all_names
 
-    def run(self, env_vars, run_unitrace: bool = False) -> list[Result]:
+    def run(
+        self, env_vars, run_unitrace: bool = False, run_flamegraph: bool = False
+    ) -> list[Result]:
         command = [f"{self.benchmark_bin}"]
 
         all_names = self.get_names_of_benchmarks_to_be_run(command, env_vars)
@@ -156,6 +158,7 @@ def run(self, env_vars, run_unitrace: bool = False) -> list[Result]:
                 add_sycl=False,
                 ld_library=[self.umf_lib],
                 run_unitrace=run_unitrace,
+                run_flamegraph=run_flamegraph,
             )
 
             parsed = self.parse_output(result)

devops/scripts/benchmarks/benches/velocity.py

@@ -130,7 +130,9 @@ def description(self) -> str:
     def get_tags(self):
         return ["SYCL", "application"]
 
-    def run(self, env_vars, run_unitrace: bool = False) -> list[Result]:
+    def run(
+        self, env_vars, run_unitrace: bool = False, run_flamegraph: bool = False
+    ) -> list[Result]:
         env_vars.update(self.extra_env_vars())
 
         command = [
@@ -143,6 +145,7 @@ def run(self, env_vars, run_unitrace: bool = False) -> list[Result]:
             env_vars,
             ld_library=self.ld_libraries(),
             run_unitrace=run_unitrace,
+            run_flamegraph=run_flamegraph,
         )
 
         return [
@@ -287,7 +290,9 @@ class QuickSilver(VelocityBase):
     def __init__(self, vb: VelocityBench):
         super().__init__("QuickSilver", "qs", vb, "MMS/CTT")
 
-    def run(self, env_vars, run_unitrace: bool = False) -> list[Result]:
+    def run(
+        self, env_vars, run_unitrace: bool = False, run_flamegraph: bool = False
+    ) -> list[Result]:
         # TODO: fix the crash in QuickSilver when UR_L0_USE_IMMEDIATE_COMMANDLISTS=0
         if (
             "UR_L0_USE_IMMEDIATE_COMMANDLISTS" in env_vars

devops/scripts/benchmarks/html/data.js

@@ -8,3 +8,8 @@
 benchmarkRuns = [];
 
 defaultCompareNames = [];
+
+benchmarkMetadata = {};
+
+benchmarkTags = {};
+

devops/scripts/benchmarks/html/index.html

@@ -57,6 +57,10 @@ <h3>Display Options</h3>
                             <input type="checkbox" id="show-archived-data">
                             Include archived runs
                         </label>
+                        <label title="Show flamegraph SVG files instead of benchmark charts.">
+                            <input type="checkbox" id="show-flamegraph">
+                            Show flamegraph
+                        </label>
                     </div>
                 </div>