significantly rebuild

Signed-off-by: Mateusz P. Nowak <[email protected]>

Author: mateuszpn

devops/scripts/benchmarks/benches/base.py

@@ -13,13 +13,14 @@
 from utils.utils import download, run
 from abc import ABC, abstractmethod
 from utils.unitrace import get_unitrace
-from utils.logger import log
 from utils.flamegraph import get_flamegraph
+from utils.logger import log
 
 
 class TracingType(Enum):
     """Enumeration of available tracing types."""
 
+    NONE = ""
     UNITRACE = "unitrace"
     FLAMEGRAPH = "flamegraph"
 
@@ -88,15 +89,12 @@ def teardown(self):
         pass
 
     @abstractmethod
-    def run(
-        self, env_vars, run_unitrace: bool = False, run_flamegraph: bool = False
-    ) -> list[Result]:
+    def run(self, env_vars, run_trace: TracingType = TracingType.NONE) -> list[Result]:
         """Execute the benchmark with the given environment variables.
 
         Args:
             env_vars: Environment variables to use when running the benchmark.
-            run_unitrace: Whether to run benchmark under Unitrace.
-            run_flamegraph: Whether to run benchmark under FlameGraph.
+            run_trace: The type of tracing to run (NONE, UNITRACE, or FLAMEGRAPH).
 
         Returns:
             A list of Result objects with the benchmark results.
@@ -125,10 +123,8 @@ def run_bench(
         ld_library=[],
         add_sycl=True,
         use_stdout=True,
-        run_unitrace=False,
-        extra_unitrace_opt=None,
-        run_flamegraph=False,
-        extra_perf_opt=None,  # VERIFY
+        run_trace: TracingType = TracingType.NONE,
+        extra_trace_opt=None,
     ):
         env_vars = env_vars.copy()
         if options.ur is not None:
@@ -141,11 +137,11 @@ def run_bench(
         ld_libraries = options.extra_ld_libraries.copy()
         ld_libraries.extend(ld_library)
 
-        if self.traceable(TracingType.UNITRACE) and run_unitrace:
-            if extra_unitrace_opt is None:
-                extra_unitrace_opt = []
+        if self.traceable(TracingType.UNITRACE) and run_trace == TracingType.UNITRACE:
+            if extra_trace_opt is None:
+                extra_trace_opt = []
             unitrace_output, command = get_unitrace().setup(
-                self.name(), command, extra_unitrace_opt
+                self.name(), command, extra_trace_opt
             )
             log.debug(f"Unitrace output: {unitrace_output}")
             log.debug(f"Unitrace command: {' '.join(command)}")
@@ -159,24 +155,22 @@ def run_bench(
                 ld_library=ld_libraries,
             )
         except subprocess.CalledProcessError:
-            if run_unitrace:
+            if run_trace == TracingType.UNITRACE:
                 get_unitrace().cleanup(options.benchmark_cwd, unitrace_output)
             raise
 
-        if self.traceable(TracingType.UNITRACE) and run_unitrace:
+        if self.traceable(TracingType.UNITRACE) and run_trace == TracingType.UNITRACE:
             get_unitrace().handle_output(unitrace_output)
 
         # flamegraph run
 
-        ld_libraries = options.extra_ld_libraries.copy()
-        ld_libraries.extend(ld_library)
-
         perf_data_file = None
-        if self.traceable(TracingType.FLAMEGRAPH) and run_flamegraph:
-            if extra_perf_opt is None:
-                extra_perf_opt = []
+        if (
+            self.traceable(TracingType.FLAMEGRAPH)
+            and run_trace == TracingType.FLAMEGRAPH
+        ):
             perf_data_file, command = get_flamegraph().setup(
-                self.name(), command, extra_perf_opt
+                self.name(), self.get_suite_name(), command
             )
             log.debug(f"FlameGraph perf data: {perf_data_file}")
             log.debug(f"FlameGraph command: {' '.join(command)}")
@@ -190,11 +184,15 @@ def run_bench(
                 ld_library=ld_libraries,
             )
         except subprocess.CalledProcessError:
-            if run_flamegraph and perf_data_file:
-                get_flamegraph().cleanup(options.benchmark_cwd, perf_data_file)
+            if run_trace == TracingType.FLAMEGRAPH and perf_data_file:
+                get_flamegraph().cleanup(perf_data_file)
             raise
 
-        if self.traceable(TracingType.FLAMEGRAPH) and run_flamegraph and perf_data_file:
+        if (
+            self.traceable(TracingType.FLAMEGRAPH)
+            and run_trace == TracingType.FLAMEGRAPH
+            and perf_data_file
+        ):
             svg_file = get_flamegraph().handle_output(
                 self.name(), perf_data_file, self.get_suite_name()
             )

devops/scripts/benchmarks/benches/benchdnn.py

@@ -6,7 +6,7 @@
 
 from pathlib import Path
 
-from .base import Suite, Benchmark
+from .base import Suite, Benchmark, TracingType
 from options import options
 from utils.utils import git_clone, run, create_build_path
 from utils.result import Result
@@ -132,9 +132,13 @@ def setup(self):
         if not self.bench_bin.exists():
             raise FileNotFoundError(f"Benchmark binary not found: {self.bench_bin}")
 
-    def run(
-        self, env_vars, run_flamegraph: bool = False, run_unitrace: bool = False
-    ) -> list[Result]:
+    def run(self, env_vars, run_trace: TracingType = TracingType.NONE) -> list[Result]:
+        # Determine extra trace options based on tracing type
+        if run_trace == TracingType.UNITRACE:
+            extra_trace_opt = ["--chrome-dnn-logging"]
+        else:
+            extra_trace_opt = None
+
         command = [
             str(self.bench_bin),
             *self.bench_args.split(),
@@ -153,9 +157,8 @@ def run(
             add_sycl=True,
             ld_library=ld_library,
             use_stdout=True,
-            run_unitrace=run_unitrace,
-            extra_unitrace_opt=["--chrome-dnn-logging"],
-            run_flamegraph=run_flamegraph,
+            run_trace=run_trace,
+            extra_trace_opt=extra_trace_opt,
         )
         result_value = self._extract_time(output)
 

devops/scripts/benchmarks/benches/benchdnn_list.py

@@ -48,16 +48,16 @@
             16x32x48x5",
         False,  # Do not run graph for this benchmark
     ],
-    [
-        "sum",
-        "padding-1",
-        "--ddt=f32 --sdt=f32:f32 --stag=aBcd16b --dtag=aBcd16b 1x8x64x64 1x8x640x1024 1x24x640x1024",
-    ],
-    [
-        "sum",
-        "padding-2",
-        "--sdt=bf16:bf16 --ddt=bf16 --stag=AB48a16b:AB48a16b --dtag=AB48a16b 512x1024",
-    ],
+    # [
+    #     "sum",
+    #     "padding-1",
+    #     "--ddt=f32 --sdt=f32:f32 --stag=aBcd16b --dtag=aBcd16b 1x8x64x64 1x8x640x1024 1x24x640x1024",
+    # ],
+    # [
+    #     "sum",
+    #     "padding-2",
+    #     "--sdt=bf16:bf16 --ddt=bf16 --stag=AB48a16b:AB48a16b --dtag=AB48a16b 512x1024",
+    # ],
     [
         "graph",
         "sdpa-plain-f16",

devops/scripts/benchmarks/benches/compute.py

@@ -8,7 +8,7 @@
 import io
 import copy
 from utils.utils import run, git_clone, create_build_path
-from .base import Benchmark, Suite
+from .base import Benchmark, Suite, TracingType
 from utils.result import BenchmarkMetadata, Result
 from options import options
 from enum import Enum
@@ -339,9 +339,7 @@ def explicit_group(self):
     def description(self) -> str:
         return ""
 
-    def run(
-        self, env_vars, run_unitrace: bool = False, run_flamegraph: bool = False
-    ) -> list[Result]:
+    def run(self, env_vars, run_trace: TracingType = TracingType.NONE) -> list[Result]:
         command = [
             f"{self.benchmark_bin}",
             f"--test={self.test}",
@@ -352,9 +350,7 @@ def run(
         command += self.bin_args()
         env_vars.update(self.extra_env_vars())
 
-        result = self.run_bench(
-            command, env_vars, run_unitrace=run_unitrace, run_flamegraph=run_flamegraph
-        )
+        result = self.run_bench(command, env_vars, run_trace=run_trace)
         parsed_results = self.parse_output(result)
         ret = []
         for label, median, stddev, unit in parsed_results:

devops/scripts/benchmarks/benches/gromacs.py

@@ -7,7 +7,7 @@
 from pathlib import Path
 import re
 
-from .base import Suite, Benchmark
+from .base import Suite, Benchmark, TracingType
 from options import options
 from utils.utils import git_clone, download, run, create_build_path
 from utils.result import Result
@@ -169,9 +169,7 @@ def setup(self):
             ld_library=self.suite.oneapi.ld_libraries(),
         )
 
-    def run(
-        self, env_vars, run_flamegraph: bool = False, run_unitrace: bool = False
-    ) -> list[Result]:
+    def run(self, env_vars, run_trace: TracingType = TracingType.NONE) -> list[Result]:
         model_dir = self.grappa_dir / self.model
 
         env_vars.update({"SYCL_CACHE_PERSISTENT": "1"})
@@ -210,8 +208,7 @@ def run(
             add_sycl=True,
             use_stdout=False,
             ld_library=self.suite.oneapi.ld_libraries(),
-            run_unitrace=run_unitrace,
-            run_flamegraph=run_flamegraph,
+            run_trace=run_trace,
         )
 
         if not self._validate_correctness(options.benchmark_cwd + "/md.log"):

devops/scripts/benchmarks/benches/llamacpp.py

@@ -7,7 +7,7 @@
 import io
 from pathlib import Path
 from utils.utils import download, git_clone
-from .base import Benchmark, Suite
+from .base import Benchmark, Suite, TracingType
 from utils.result import Result
 from utils.utils import run, create_build_path
 from options import options
@@ -115,9 +115,7 @@ def get_tags(self):
     def lower_is_better(self):
         return False
 
-    def run(
-        self, env_vars, run_unitrace: bool = False, run_flamegraph: bool = False
-    ) -> list[Result]:
+    def run(self, env_vars, run_trace: TracingType = TracingType.NONE) -> list[Result]:
         command = [
             f"{self.benchmark_bin}",
             "--output",
@@ -146,8 +144,7 @@ def run(
             command,
             env_vars,
             ld_library=self.bench.oneapi.ld_libraries(),
-            run_unitrace=run_unitrace,
-            run_flamegraph=run_flamegraph,
+            run_trace=run_trace,
         )
         parsed = self.parse_output(result)
         results = []

devops/scripts/benchmarks/benches/syclbench.py

@@ -7,7 +7,7 @@
 import csv
 import io
 from utils.utils import run, git_clone, create_build_path
-from .base import Benchmark, Suite
+from .base import Benchmark, Suite, TracingType
 from utils.result import Result
 from options import options
 
@@ -137,9 +137,7 @@ def setup(self):
             self.directory, "sycl-bench-build", self.bench_name
         )
 
-    def run(
-        self, env_vars, run_unitrace: bool = False, run_flamegraph: bool = False
-    ) -> list[Result]:
+    def run(self, env_vars, run_trace: TracingType = TracingType.NONE) -> list[Result]:
         self.outputfile = os.path.join(self.bench.directory, self.test + ".csv")
 
         command = [
@@ -153,9 +151,7 @@ def run(
         env_vars.update(self.extra_env_vars())
 
         # no output to stdout, all in outputfile
-        self.run_bench(
-            command, env_vars, run_unitrace=run_unitrace, run_flamegraph=run_flamegraph
-        )
+        self.run_bench(command, env_vars, run_trace=run_trace)
 
         with open(self.outputfile, "r") as f:
             reader = csv.reader(f)

devops/scripts/benchmarks/benches/test.py

@@ -5,7 +5,7 @@
 
 import random
 from utils.utils import git_clone
-from .base import Benchmark, Suite
+from .base import Benchmark, Suite, TracingType
 from utils.result import BenchmarkMetadata, Result
 from utils.utils import run, create_build_path
 from options import options
@@ -88,9 +88,7 @@ def notes(self) -> str:
     def unstable(self) -> str:
         return self.unstable_text
 
-    def run(
-        self, env_vars, run_unitrace: bool = False, run_flamegraph: bool = False
-    ) -> list[Result]:
+    def run(self, env_vars, run_trace: TracingType = TracingType.NONE) -> list[Result]:
         random_value = self.value + random.uniform(-1 * (self.diff), self.diff)
         return [
             Result(

devops/scripts/benchmarks/benches/umf.py

@@ -8,7 +8,7 @@
 import io
 import re
 
-from .base import Benchmark, Suite
+from .base import Benchmark, Suite, TracingType
 from utils.result import Result
 from options import options
 from utils.oneapi import get_oneapi
@@ -137,9 +137,7 @@ def get_names_of_benchmarks_to_be_run(self, command, env_vars):
 
         return all_names
 
-    def run(
-        self, env_vars, run_unitrace: bool = False, run_flamegraph: bool = False
-    ) -> list[Result]:
+    def run(self, env_vars, run_trace: TracingType = TracingType.NONE) -> list[Result]:
         command = [f"{self.benchmark_bin}"]
 
         all_names = self.get_names_of_benchmarks_to_be_run(command, env_vars)
@@ -157,8 +155,7 @@ def run(
                 env_vars,
                 add_sycl=False,
                 ld_library=[self.umf_lib],
-                run_unitrace=run_unitrace,
-                run_flamegraph=run_flamegraph,
+                run_trace=run_trace,
             )
 
             parsed = self.parse_output(result)