Benchmark updates for faster run and more reliable results (#2164)

Author: mateuszpn

scripts/benchmarks/benches/SobelFilter.py

@@ -35,5 +35,5 @@ def parse_output(self, stdout: str) -> float:
         if match:
             return round(float(match.group(1)) * 1000, 3)
         else:
-            raise ValueError("Failed to parse benchmark output.")
+            raise ValueError("{self.__class__.__name__}: Failed to parse benchmark output.")
 

scripts/benchmarks/benches/bitcracker.py

@@ -31,4 +31,4 @@ def parse_output(self, stdout: str) -> float:
         if match:
             return float(match.group(1))
         else:
-            raise ValueError("Failed to parse benchmark output.")
+            raise ValueError("{self.__class__.__name__}: Failed to parse benchmark output.")

scripts/benchmarks/benches/compute.py

@@ -20,7 +20,7 @@ def setup(self):
         if self.built:
             return
 
-        repo_path = git_clone(self.directory, "compute-benchmarks-repo", "https://github.com/intel/compute-benchmarks.git", "f6882552215736f90295244046fcb6e17fe53e83")
+        repo_path = git_clone(self.directory, "compute-benchmarks-repo", "https://github.com/intel/compute-benchmarks.git", "08c41bb8bc1762ad53c6194df6d36bfcceff4aa2")
         build_path = create_build_path(self.directory, 'compute-benchmarks-build')
 
         configure_command = [
@@ -34,12 +34,13 @@ def setup(self):
             f"-DBUILD_UR=ON",
             f"-Dunified-runtime_DIR={options.ur_dir}/lib/cmake/unified-runtime",
         ]
-        run(configure_command, add_sycl=True)
 
+        print(f"{self.__class__.__name__}: Run {configure_command}")
+        run(configure_command, add_sycl=True)
+        print(f"{self.__class__.__name__}: Run cmake --build {build_path} -j")
         run(f"cmake --build {build_path} -j", add_sycl=True)
 
         self.built = True
-        self.bins = os.path.join(build_path, 'bin')
 
 class ComputeBenchmark(Benchmark):
     def __init__(self, bench, name, test):
@@ -58,8 +59,8 @@ def unit(self):
         return "μs"
 
     def setup(self):
+        self.benchmark_bin = os.path.join(self.bench.directory, 'compute-benchmarks-build', 'bin', self.bench_name)
         self.bench.setup()
-        self.benchmark_bin = os.path.join(self.bench.bins, self.bench_name)
 
     def run(self, env_vars) -> list[Result]:
         command = [

scripts/benchmarks/benches/hashtable.py

@@ -31,4 +31,4 @@ def parse_output(self, stdout: str) -> float:
         if match:
             return float(match.group(1))
         else:
-            raise ValueError("Failed to parse keys per second from benchmark output.")
+            raise ValueError("{self.__class__.__name__}: Failed to parse keys per second from benchmark output.")

scripts/benchmarks/benches/quicksilver.py

@@ -42,4 +42,4 @@ def parse_output(self, stdout: str) -> float:
         if match:
             return float(match.group(1))
         else:
-            raise ValueError("Failed to parse benchmark output.")
+            raise ValueError("{self.__class__.__name__}: Failed to parse benchmark output.")

scripts/benchmarks/benches/result.py

@@ -14,6 +14,7 @@ class Result:
     command: str
     env: str
     stdout: str
+    passed: bool = True
     unit: str = ""
     name: str = ""
     lower_is_better: bool = True