better and more tags

Author: pbalcer

devops/scripts/benchmarks/benches/base.py

@@ -11,18 +11,25 @@
 from options import options
 from utils.utils import download, run
 
-benchmark_tags = [BenchmarkTag('sycl', 'Benchmark uses SYCL RT'),
-                  BenchmarkTag('ur', 'Benchmark uses Unified Runtime'),
-                  BenchmarkTag('L0', 'Benchmark uses L0 directly'),
-                  BenchmarkTag('umf', 'Benchmark uses UMF directly'),
-                  BenchmarkTag('micro', 'Microbenchmark focusing on a specific niche'),
-                  BenchmarkTag('application', 'Real application-based performance test'),
-                  BenchmarkTag('proxy', 'Benchmark that tries to implement a real application use-case'),
-                  BenchmarkTag('submit', 'Benchmark tests the kernel submit path'),
-                  BenchmarkTag('math', 'Benchmark tests math compute performance'),
-                  BenchmarkTag('memory', 'Benchmark tests memory transfer performance'),
-                  BenchmarkTag('allocation', 'Benchmark tests memory allocation performance'),
-                  BenchmarkTag('graph', 'Benchmark tests graph performance'),]
+benchmark_tags = [
+    BenchmarkTag('SYCL', 'Benchmark uses SYCL runtime'),
+    BenchmarkTag('UR', 'Benchmark uses Unified Runtime API'),
+    BenchmarkTag('L0', 'Benchmark uses Level Zero API directly'),
+    BenchmarkTag('UMF', 'Benchmark uses Unified Memory Framework directly'),
+    BenchmarkTag('micro', 'Microbenchmark focusing on a specific functionality'),
+    BenchmarkTag('application', 'Real application-based performance test'),
+    BenchmarkTag('proxy', 'Benchmark that simulates real application use-cases'),
+    BenchmarkTag('submit', 'Tests kernel submission performance'),
+    BenchmarkTag('math', 'Tests math computation performance'),
+    BenchmarkTag('memory', 'Tests memory transfer or bandwidth performance'),
+    BenchmarkTag('allocation', 'Tests memory allocation performance'),
+    BenchmarkTag('graph', 'Tests graph-based execution performance'),
+    BenchmarkTag('latency', 'Measures operation latency'),
+    BenchmarkTag('throughput', 'Measures operation throughput'),
+    BenchmarkTag('inference', 'Tests ML/AI inference performance'),
+    BenchmarkTag('image', 'Image processing benchmark'),
+    BenchmarkTag('simulation', 'Physics or scientific simulation benchmark'),
+]
 
 benchmark_tags_dict = {tag.name: tag for tag in benchmark_tags}
 

devops/scripts/benchmarks/benches/compute.py

@@ -28,9 +28,9 @@ def runtime_to_name(runtime: RUNTIMES) -> str:
 
 def runtime_to_tag_name(runtime: RUNTIMES) -> str:
     return {
-        RUNTIMES.SYCL: "sycl",
+        RUNTIMES.SYCL: "SYCL",
         RUNTIMES.LEVEL_ZERO: "L0",
-        RUNTIMES.UR: "ur",
+        RUNTIMES.UR: "UR",
     }[runtime]
 
 
@@ -84,16 +84,16 @@ def additionalMetadata(self) -> dict[str, BenchmarkMetadata]:
                 "The first layer is the Level Zero API, the second is the Unified Runtime API, and the third is the SYCL API.\n"
                 "The UR v2 adapter noticeably reduces UR layer overhead, also improving SYCL performance.\n"
                 "Work is ongoing to reduce the overhead of the SYCL API\n",
-                tags=['submit', 'micro', 'sycl', 'ur', 'L0']
+                tags=['submit', 'micro', 'SYCL', 'UR', 'L0']
             ),
             "SinKernelGraph": BenchmarkMetadata(
                 type="group",
                 unstable="This benchmark combines both eager and graph execution, and may not be representative of real use cases.",
-                tags=['submit', 'memory', 'proxy', 'sycl', 'ur', 'L0', 'graph']
+                    tags=['submit', 'memory', 'proxy', 'SYCL', 'UR', 'L0', 'graph']
             ),
             "SubmitGraph": BenchmarkMetadata(
                 type="group",
-                tags=['submit', 'micro', 'sycl', 'ur', 'L0', 'graph']
+                tags=['submit', 'micro', 'SYCL', 'UR', 'L0', 'graph']
             ),
         }
 
@@ -279,7 +279,7 @@ def __init__(self, bench, runtime: RUNTIMES, ioq, measure_completion=0):
         )
 
     def get_tags(self):
-        return ['submit', runtime_to_tag_name(self.runtime), 'micro']
+        return ['submit', 'latency', runtime_to_tag_name(self.runtime), 'micro']
 
     def name(self):
         order = "in order" if self.ioq else "out of order"
@@ -344,7 +344,7 @@ def description(self) -> str:
         )
 
     def get_tags(self):
-        return ['memory', 'sycl', 'micro']
+        return ['memory', 'submit', 'latency', 'SYCL', 'micro']
 
     def bin_args(self) -> list[str]:
         return [
@@ -377,7 +377,7 @@ def description(self) -> str:
         )
 
     def get_tags(self):
-        return ['memory', 'sycl', 'micro']
+        return ['memory', 'latency', 'SYCL', 'micro']
 
     def bin_args(self) -> list[str]:
         return [
@@ -407,7 +407,7 @@ def description(self) -> str:
         )
 
     def get_tags(self):
-        return ['memory', 'sycl', 'micro']
+        return ['memory', 'latency', 'SYCL', 'micro']
 
     def bin_args(self) -> list[str]:
         return [
@@ -439,7 +439,7 @@ def lower_is_better(self):
         return False
 
     def get_tags(self):
-        return ['memory', 'sycl', 'micro']
+        return ['memory', 'throughput', 'SYCL', 'micro']
 
     def bin_args(self) -> list[str]:
         return [
@@ -468,7 +468,7 @@ def description(self) -> str:
         )
 
     def get_tags(self):
-        return ['math', 'sycl', 'micro']
+        return ['math', 'throughput', 'SYCL', 'micro']
 
     def bin_args(self) -> list[str]:
         return [
@@ -517,7 +517,7 @@ def description(self) -> str:
         )
 
     def get_tags(self):
-        return ['memory', 'ur', 'micro']
+        return ['memory', 'latency', 'UR', 'micro']
 
     def bin_args(self) -> list[str]:
         return [
@@ -560,7 +560,7 @@ def unstable(self) -> str:
         return "This benchmark combines both eager and graph execution, and may not be representative of real use cases."
 
     def get_tags(self):
-        return ['graph', runtime_to_tag_name(self.runtime), 'proxy', 'submit', 'memory']
+        return ['graph', runtime_to_tag_name(self.runtime), 'proxy', 'submit', 'memory', 'latency']
 
     def bin_args(self) -> list[str]:
         return [
@@ -595,7 +595,7 @@ def name(self):
         return f"graph_api_benchmark_{self.runtime.value} SubmitGraph numKernels:{self.numKernels} ioq {self.inOrderQueue} measureCompletion {self.measureCompletionTime}"
 
     def get_tags(self):
-        return ['graph', runtime_to_tag_name(self.runtime), 'micro', 'submit']
+        return ['graph', runtime_to_tag_name(self.runtime), 'micro', 'submit', 'latency']
 
     def bin_args(self) -> list[str]:
         return [
@@ -625,7 +625,7 @@ def name(self):
         return f"ulls_benchmark_{self.runtime.value} EmptyKernel wgc:{self.wgc}, wgs:{self.wgs}"
 
     def get_tags(self):
-        return [runtime_to_tag_name(self.runtime), 'micro']
+        return [runtime_to_tag_name(self.runtime), 'micro', 'latency', 'submit']
 
     def bin_args(self) -> list[str]:
         return [
@@ -666,7 +666,7 @@ def name(self):
         return f"ulls_benchmark_{self.runtime.value} KernelSwitch count {self.count} kernelTime {self.kernelTime}"
 
     def get_tags(self):
-        return [runtime_to_tag_name(self.runtime), 'micro']
+        return [runtime_to_tag_name(self.runtime), 'micro', 'latency', 'submit']
 
     def bin_args(self) -> list[str]:
         return [

devops/scripts/benchmarks/benches/llamacpp.py

@@ -102,7 +102,7 @@ def description(self) -> str:
         )
 
     def get_tags(self):
-        return ['sycl', 'application']
+        return ['SYCL', 'application', 'inference', 'throughput']
 
     def lower_is_better(self):
         return False

devops/scripts/benchmarks/benches/syclbench.py

@@ -113,7 +113,16 @@ def extra_env_vars(self) -> dict:
         return {}
 
     def get_tags(self):
-        return ['sycl', 'micro']
+        base_tags = ['SYCL', 'micro']
+        if "Memory" in self.bench_name or "mem" in self.bench_name.lower():
+            base_tags.append('memory')
+        if "Reduction" in self.bench_name:
+            base_tags.append('math')
+        if "Bandwidth" in self.bench_name:
+            base_tags.append('throughput')
+        if "Latency" in self.bench_name:
+            base_tags.append('latency')
+        return base_tags
 
     def setup(self):
         self.benchmark_bin = os.path.join(

devops/scripts/benchmarks/benches/umf.py

@@ -75,7 +75,7 @@ def setup(self):
         self.benchmark_bin = os.path.join(options.umf, "benchmark", self.bench_name)
 
     def get_tags(self):
-        return ['umf', 'allocation']
+        return ['UMF', 'allocation', 'latency', 'micro']
 
     def run(self, env_vars) -> list[Result]:
         command = [

devops/scripts/benchmarks/benches/velocity.py

@@ -119,7 +119,7 @@ def description(self) -> str:
         return ""
 
     def get_tags(self):
-        return ['sycl', 'application']
+        return ['SYCL', 'application']
 
     def run(self, env_vars) -> list[Result]:
         env_vars.update(self.extra_env_vars())
@@ -175,6 +175,9 @@ def parse_output(self, stdout: str) -> float:
                 "{self.__class__.__name__}: Failed to parse keys per second from benchmark output."
             )
 
+    def get_tags(self):
+        return ['SYCL', 'application', 'throughput']
+
 
 class Bitcracker(VelocityBase):
     def __init__(self, vb: VelocityBench):
@@ -213,6 +216,9 @@ def parse_output(self, stdout: str) -> float:
                 "{self.__class__.__name__}: Failed to parse benchmark output."
             )
 
+    def get_tags(self):
+        return ['SYCL', 'application', 'throughput']
+
 
 class SobelFilter(VelocityBase):
     def __init__(self, vb: VelocityBench):
@@ -259,6 +265,9 @@ def parse_output(self, stdout: str) -> float:
                 "{self.__class__.__name__}: Failed to parse benchmark output."
             )
 
+    def get_tags(self):
+        return ['SYCL', 'application', 'image', 'throughput']
+
 
 class QuickSilver(VelocityBase):
     def __init__(self, vb: VelocityBench):
@@ -306,6 +315,9 @@ def parse_output(self, stdout: str) -> float:
                 "{self.__class__.__name__}: Failed to parse benchmark output."
             )
 
+    def get_tags(self):
+        return ['SYCL', 'application', 'simulation', 'throughput']
+
 
 class Easywave(VelocityBase):
     def __init__(self, vb: VelocityBench):
@@ -370,6 +382,9 @@ def parse_output(self, stdout: str) -> float:
             os.path.join(options.benchmark_cwd, "easywave.log")
         )
 
+    def get_tags(self):
+        return ['SYCL', 'application', 'simulation']
+
 
 class CudaSift(VelocityBase):
     def __init__(self, vb: VelocityBench):
@@ -398,6 +413,9 @@ def parse_output(self, stdout: str) -> float:
         else:
             raise ValueError("Failed to parse benchmark output.")
 
+    def get_tags(self):
+        return ['SYCL', 'application', 'image']
+
 
 class DLCifar(VelocityBase):
     def __init__(self, vb: VelocityBench):
@@ -449,6 +467,9 @@ def parse_output(self, stdout: str) -> float:
         else:
             raise ValueError("Failed to parse benchmark output.")
 
+    def get_tags(self):
+        return ['SYCL', 'application', 'inference', 'image']
+
 
 class DLMnist(VelocityBase):
     def __init__(self, vb: VelocityBench):
@@ -534,6 +555,9 @@ def parse_output(self, stdout: str) -> float:
         else:
             raise ValueError("Failed to parse benchmark output.")
 
+    def get_tags(self):
+        return ['SYCL', 'application', 'inference', 'image']
+
 
 class SVM(VelocityBase):
     def __init__(self, vb: VelocityBench):
@@ -576,3 +600,6 @@ def parse_output(self, stdout: str) -> float:
             return float(match.group(1))
         else:
             raise ValueError("Failed to parse benchmark output.")
+
+    def get_tags(self):
+        return ['SYCL', 'application', 'inference']