add force option, omitting traceable()

Signed-off-by: Mateusz P. Nowak <[email protected]>

Author: mateuszpn

devops/scripts/benchmarks/benches/base.py

@@ -89,12 +89,18 @@ def teardown(self):
         pass
 
     @abstractmethod
-    def run(self, env_vars, run_trace: TracingType = TracingType.NONE) -> list[Result]:
+    def run(
+        self,
+        env_vars,
+        run_trace: TracingType = TracingType.NONE,
+        force_trace: bool = False,
+    ) -> list[Result]:
         """Execute the benchmark with the given environment variables.
 
         Args:
             env_vars: Environment variables to use when running the benchmark.
             run_trace: The type of tracing to run (NONE, UNITRACE, or FLAMEGRAPH).
+            force_trace: If True, ignore the traceable() method and force tracing.
 
         Returns:
             A list of Result objects with the benchmark results.
@@ -125,6 +131,7 @@ def run_bench(
         use_stdout=True,
         run_trace: TracingType = TracingType.NONE,
         extra_trace_opt=None,
+        force_trace: bool = False,
     ):
         env_vars = env_vars.copy()
         if options.ur is not None:
@@ -137,7 +144,10 @@ def run_bench(
         ld_libraries = options.extra_ld_libraries.copy()
         ld_libraries.extend(ld_library)
 
-        if self.traceable(TracingType.UNITRACE) and run_trace == TracingType.UNITRACE:
+        unitrace_output = None
+        if (
+            self.traceable(TracingType.UNITRACE) or force_trace
+        ) and run_trace == TracingType.UNITRACE:
             if extra_trace_opt is None:
                 extra_trace_opt = []
             unitrace_output, command = get_unitrace().setup(
@@ -146,29 +156,12 @@ def run_bench(
             log.debug(f"Unitrace output: {unitrace_output}")
             log.debug(f"Unitrace command: {' '.join(command)}")
 
-        try:
-            result = run(
-                command=command,
-                env_vars=env_vars,
-                add_sycl=add_sycl,
-                cwd=options.benchmark_cwd,
-                ld_library=ld_libraries,
-            )
-        except subprocess.CalledProcessError:
-            if run_trace == TracingType.UNITRACE:
-                get_unitrace().cleanup(options.benchmark_cwd, unitrace_output)
-            raise
-
-        if self.traceable(TracingType.UNITRACE) and run_trace == TracingType.UNITRACE:
-            get_unitrace().handle_output(unitrace_output)
-
         # flamegraph run
 
         perf_data_file = None
         if (
-            self.traceable(TracingType.FLAMEGRAPH)
-            and run_trace == TracingType.FLAMEGRAPH
-        ):
+            self.traceable(TracingType.FLAMEGRAPH) or force_trace
+        ) and run_trace == TracingType.FLAMEGRAPH:
             perf_data_file, command = get_flamegraph().setup(
                 self.name(), self.get_suite_name(), command
             )
@@ -184,12 +177,21 @@ def run_bench(
                 ld_library=ld_libraries,
             )
         except subprocess.CalledProcessError:
+            if run_trace == TracingType.UNITRACE and unitrace_output:
+                get_unitrace().cleanup(options.benchmark_cwd, unitrace_output)
             if run_trace == TracingType.FLAMEGRAPH and perf_data_file:
                 get_flamegraph().cleanup(perf_data_file)
             raise
 
         if (
-            self.traceable(TracingType.FLAMEGRAPH)
+            (self.traceable(TracingType.UNITRACE) or force_trace)
+            and run_trace == TracingType.UNITRACE
+            and unitrace_output
+        ):
+            get_unitrace().handle_output(unitrace_output)
+
+        if (
+            (self.traceable(TracingType.FLAMEGRAPH) or force_trace)
             and run_trace == TracingType.FLAMEGRAPH
             and perf_data_file
         ):

devops/scripts/benchmarks/benches/benchdnn.py

@@ -132,7 +132,12 @@ def setup(self):
         if not self.bench_bin.exists():
             raise FileNotFoundError(f"Benchmark binary not found: {self.bench_bin}")
 
-    def run(self, env_vars, run_trace: TracingType = TracingType.NONE) -> list[Result]:
+    def run(
+        self,
+        env_vars,
+        run_trace: TracingType = TracingType.NONE,
+        force_trace: bool = False,
+    ) -> list[Result]:
         # Determine extra trace options based on tracing type
         if run_trace == TracingType.UNITRACE:
             extra_trace_opt = ["--chrome-dnn-logging"]
@@ -159,6 +164,7 @@ def run(self, env_vars, run_trace: TracingType = TracingType.NONE) -> list[Resul
             use_stdout=True,
             run_trace=run_trace,
             extra_trace_opt=extra_trace_opt,
+            force_trace=force_trace,
         )
         result_value = self._extract_time(output)
 

devops/scripts/benchmarks/benches/compute.py

@@ -339,7 +339,12 @@ def explicit_group(self):
     def description(self) -> str:
         return ""
 
-    def run(self, env_vars, run_trace: TracingType = TracingType.NONE) -> list[Result]:
+    def run(
+        self,
+        env_vars,
+        run_trace: TracingType = TracingType.NONE,
+        force_trace: bool = False,
+    ) -> list[Result]:
         command = [
             f"{self.benchmark_bin}",
             f"--test={self.test}",
@@ -350,7 +355,9 @@ def run(self, env_vars, run_trace: TracingType = TracingType.NONE) -> list[Resul
         command += self.bin_args()
         env_vars.update(self.extra_env_vars())
 
-        result = self.run_bench(command, env_vars, run_trace=run_trace)
+        result = self.run_bench(
+            command, env_vars, run_trace=run_trace, force_trace=force_trace
+        )
         parsed_results = self.parse_output(result)
         ret = []
         for label, median, stddev, unit in parsed_results:

devops/scripts/benchmarks/benches/gromacs.py

@@ -169,7 +169,12 @@ def setup(self):
             ld_library=self.suite.oneapi.ld_libraries(),
         )
 
-    def run(self, env_vars, run_trace: TracingType = TracingType.NONE) -> list[Result]:
+    def run(
+        self,
+        env_vars,
+        run_trace: TracingType = TracingType.NONE,
+        force_trace: bool = False,
+    ) -> list[Result]:
         model_dir = self.grappa_dir / self.model
 
         env_vars.update({"SYCL_CACHE_PERSISTENT": "1"})
@@ -209,6 +214,7 @@ def run(self, env_vars, run_trace: TracingType = TracingType.NONE) -> list[Resul
             use_stdout=False,
             ld_library=self.suite.oneapi.ld_libraries(),
             run_trace=run_trace,
+            force_trace=force_trace,
         )
 
         if not self._validate_correctness(options.benchmark_cwd + "/md.log"):

devops/scripts/benchmarks/benches/llamacpp.py

@@ -115,7 +115,12 @@ def get_tags(self):
     def lower_is_better(self):
         return False
 
-    def run(self, env_vars, run_trace: TracingType = TracingType.NONE) -> list[Result]:
+    def run(
+        self,
+        env_vars,
+        run_trace: TracingType = TracingType.NONE,
+        force_trace: bool = False,
+    ) -> list[Result]:
         command = [
             f"{self.benchmark_bin}",
             "--output",
@@ -145,6 +150,7 @@ def run(self, env_vars, run_trace: TracingType = TracingType.NONE) -> list[Resul
             env_vars,
             ld_library=self.bench.oneapi.ld_libraries(),
             run_trace=run_trace,
+            force_trace=force_trace,
         )
         parsed = self.parse_output(result)
         results = []

devops/scripts/benchmarks/benches/syclbench.py

@@ -137,7 +137,12 @@ def setup(self):
             self.directory, "sycl-bench-build", self.bench_name
         )
 
-    def run(self, env_vars, run_trace: TracingType = TracingType.NONE) -> list[Result]:
+    def run(
+        self,
+        env_vars,
+        run_trace: TracingType = TracingType.NONE,
+        force_trace: bool = False,
+    ) -> list[Result]:
         self.outputfile = os.path.join(self.bench.directory, self.test + ".csv")
 
         command = [
@@ -151,7 +156,7 @@ def run(self, env_vars, run_trace: TracingType = TracingType.NONE) -> list[Resul
         env_vars.update(self.extra_env_vars())
 
         # no output to stdout, all in outputfile
-        self.run_bench(command, env_vars, run_trace=run_trace)
+        self.run_bench(command, env_vars, run_trace=run_trace, force_trace=force_trace)
 
         with open(self.outputfile, "r") as f:
             reader = csv.reader(f)

devops/scripts/benchmarks/benches/test.py

@@ -88,7 +88,12 @@ def notes(self) -> str:
     def unstable(self) -> str:
         return self.unstable_text
 
-    def run(self, env_vars, run_trace: TracingType = TracingType.NONE) -> list[Result]:
+    def run(
+        self,
+        env_vars,
+        run_trace: TracingType = TracingType.NONE,
+        force_trace: bool = False,
+    ) -> list[Result]:
         random_value = self.value + random.uniform(-1 * (self.diff), self.diff)
         return [
             Result(

devops/scripts/benchmarks/benches/umf.py

@@ -137,7 +137,12 @@ def get_names_of_benchmarks_to_be_run(self, command, env_vars):
 
         return all_names
 
-    def run(self, env_vars, run_trace: TracingType = TracingType.NONE) -> list[Result]:
+    def run(
+        self,
+        env_vars,
+        run_trace: TracingType = TracingType.NONE,
+        force_trace: bool = False,
+    ) -> list[Result]:
         command = [f"{self.benchmark_bin}"]
 
         all_names = self.get_names_of_benchmarks_to_be_run(command, env_vars)
@@ -156,6 +161,7 @@ def run(self, env_vars, run_trace: TracingType = TracingType.NONE) -> list[Resul
                 add_sycl=False,
                 ld_library=[self.umf_lib],
                 run_trace=run_trace,
+                force_trace=force_trace,
             )
 
             parsed = self.parse_output(result)

devops/scripts/benchmarks/benches/velocity.py

@@ -130,7 +130,12 @@ def description(self) -> str:
     def get_tags(self):
         return ["SYCL", "application"]
 
-    def run(self, env_vars, run_trace: TracingType = TracingType.NONE) -> list[Result]:
+    def run(
+        self,
+        env_vars,
+        run_trace: TracingType = TracingType.NONE,
+        force_trace: bool = False,
+    ) -> list[Result]:
         env_vars.update(self.extra_env_vars())
 
         command = [
@@ -143,6 +148,7 @@ def run(self, env_vars, run_trace: TracingType = TracingType.NONE) -> list[Resul
             env_vars,
             ld_library=self.ld_libraries(),
             run_trace=run_trace,
+            force_trace=force_trace,
         )
 
         return [
@@ -287,15 +293,20 @@ class QuickSilver(VelocityBase):
     def __init__(self, vb: VelocityBench):
         super().__init__("QuickSilver", "qs", vb, "MMS/CTT")
 
-    def run(self, env_vars, run_trace: TracingType = TracingType.NONE) -> list[Result]:
+    def run(
+        self,
+        env_vars,
+        run_trace: TracingType = TracingType.NONE,
+        force_trace: bool = False,
+    ) -> list[Result]:
         # TODO: fix the crash in QuickSilver when UR_L0_USE_IMMEDIATE_COMMANDLISTS=0
         if (
             "UR_L0_USE_IMMEDIATE_COMMANDLISTS" in env_vars
             and env_vars["UR_L0_USE_IMMEDIATE_COMMANDLISTS"] == "0"
         ):
             return None
 
-        return super().run(env_vars, run_trace)
+        return super().run(env_vars, run_trace, force_trace)
 
     def name(self):
         return "Velocity-Bench QuickSilver"