add support for benchmark tags

Author: pbalcer

devops/scripts/benchmarks/benches/base.py

@@ -3,15 +3,36 @@
 # See LICENSE.TXT
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+from dataclasses import dataclass
 import os
 import shutil
 from pathlib import Path
-from utils.result import BenchmarkMetadata, Result
+from utils.result import BenchmarkMetadata, BenchmarkTag, Result
 from options import options
 from utils.utils import download, run
-import urllib.request
-import tarfile
 
+benchmark_tags = [BenchmarkTag('sycl', 'Benchmark uses SYCL RT'),
+                  BenchmarkTag('ur', 'Benchmark uses Unified Runtime'),
+                  BenchmarkTag('L0', 'Benchmark uses L0 directly'),
+                  BenchmarkTag('umf', 'Benchmark uses UMF directly'),
+                  BenchmarkTag('micro', 'Microbenchmark focusing on a specific niche'),
+                  BenchmarkTag('application', 'Real application-based performance test'),
+                  BenchmarkTag('proxy', 'Benchmark that tries to implement a real application use-case'),
+                  BenchmarkTag('submit', 'Benchmark tests the kernel submit path'),
+                  BenchmarkTag('math', 'Benchmark tests math compute performance'),
+                  BenchmarkTag('memory', 'Benchmark tests memory transfer performance'),
+                  BenchmarkTag('allocation', 'Benchmark tests memory allocation performance'),
+                  BenchmarkTag('graph', 'Benchmark tests graph performance'),]
+
+def translate_tags(tag_names: list[str]) -> list[BenchmarkTag]:
+    tags = []
+    for tag_name in tag_names:
+        for tag in benchmark_tags:
+            if tag.name == tag_name:
+                tags.append(tag)
+                break
+
+    return tags
 
 class Benchmark:
     def __init__(self, directory, suite):
@@ -105,15 +126,18 @@ def notes(self) -> str:
     def unstable(self) -> str:
         return None
 
+    def get_tags(self) -> list[str]:
+        return []
+
     def get_metadata(self) -> BenchmarkMetadata:
         return BenchmarkMetadata(
             type="benchmark",
             description=self.description(),
             notes=self.notes(),
             unstable=self.unstable(),
+            tags=translate_tags(self.get_tags())
         )
 
-
 class Suite:
     def benchmarks(self) -> list[Benchmark]:
         raise NotImplementedError()

devops/scripts/benchmarks/benches/compute.py

@@ -7,7 +7,7 @@
 import csv
 import io
 from utils.utils import run, git_clone, create_build_path
-from .base import Benchmark, Suite
+from .base import Benchmark, Suite, translate_tags
 from utils.result import BenchmarkMetadata, Result
 from options import options
 from enum import Enum
@@ -26,6 +26,13 @@ def runtime_to_name(runtime: RUNTIMES) -> str:
         RUNTIMES.UR: "Unified Runtime",
     }[runtime]
 
+def runtime_to_tag_name(runtime: RUNTIMES) -> str:
+    return {
+        RUNTIMES.SYCL: "sycl",
+        RUNTIMES.LEVEL_ZERO: "L0",
+        RUNTIMES.UR: "ur",
+    }[runtime]
+
 
 class ComputeBench(Suite):
     def __init__(self, directory):
@@ -77,10 +84,12 @@ def additionalMetadata(self) -> dict[str, BenchmarkMetadata]:
                 "The first layer is the Level Zero API, the second is the Unified Runtime API, and the third is the SYCL API.\n"
                 "The UR v2 adapter noticeably reduces UR layer overhead, also improving SYCL performance.\n"
                 "Work is ongoing to reduce the overhead of the SYCL API\n",
+                tags=translate_tags(['submit', 'micro'])
             ),
             "SinKernelGraph": BenchmarkMetadata(
                 type="group",
                 unstable="This benchmark combines both eager and graph execution, and may not be representative of real use cases.",
+                tags=translate_tags(['submit', 'micro'])
             ),
         }
 
@@ -265,6 +274,9 @@ def __init__(self, bench, runtime: RUNTIMES, ioq, measure_completion=0):
             bench, f"api_overhead_benchmark_{runtime.value}", "SubmitKernel"
         )
 
+    def get_tags(self):
+        return ['submit', runtime_to_tag_name(self.runtime), 'micro']
+
     def name(self):
         order = "in order" if self.ioq else "out of order"
         completion_str = " with measure completion" if self.measure_completion else ""
@@ -327,6 +339,9 @@ def description(self) -> str:
             f"{self.destination} memory with {self.size} bytes. Tests immediate execution overheads."
         )
 
+    def get_tags(self):
+        return ['memory', 'sycl', 'micro']
+
     def bin_args(self) -> list[str]:
         return [
             "--iterations=100000",
@@ -357,6 +372,9 @@ def description(self) -> str:
             f"{self.source} to {self.destination} with {self.size} bytes, executed 100 times per iteration."
         )
 
+    def get_tags(self):
+        return ['memory', 'sycl', 'micro']
+
     def bin_args(self) -> list[str]:
         return [
             "--iterations=10000",
@@ -384,6 +402,9 @@ def description(self) -> str:
             f"{self.destination} with {self.size} bytes per operation."
         )
 
+    def get_tags(self):
+        return ['memory', 'sycl', 'micro']
+
     def bin_args(self) -> list[str]:
         return [
             "--iterations=10000",
@@ -413,6 +434,9 @@ def description(self) -> str:
     def lower_is_better(self):
         return False
 
+    def get_tags(self):
+        return ['memory', 'sycl', 'micro']
+
     def bin_args(self) -> list[str]:
         return [
             "--iterations=10000",
@@ -439,6 +463,9 @@ def description(self) -> str:
             "using SYCL."
         )
 
+    def get_tags(self):
+        return ['math', 'sycl', 'micro']
+
     def bin_args(self) -> list[str]:
         return [
             "--iterations=1000",
@@ -485,6 +512,9 @@ def description(self) -> str:
             f"from {src_type} to {dst_type} memory {events} events."
         )
 
+    def get_tags(self):
+        return ['memory', 'ur', 'micro']
+
     def bin_args(self) -> list[str]:
         return [
             "--Ioq=1",
@@ -525,6 +555,9 @@ def name(self):
     def unstable(self) -> str:
         return "This benchmark combines both eager and graph execution, and may not be representative of real use cases."
 
+    def get_tags(self):
+        return ['graph', runtime_to_tag_name(self.runtime), 'proxy', 'submit', 'memory']
+
     def bin_args(self) -> list[str]:
         return [
             "--iterations=10000",
@@ -557,6 +590,9 @@ def description(self) -> str:
     def name(self):
         return f"graph_api_benchmark_{self.runtime.value} SubmitGraph numKernels:{self.numKernels} ioq {self.inOrderQueue} measureCompletion {self.measureCompletionTime}"
 
+    def get_tags(self):
+        return ['graph', runtime_to_tag_name(self.runtime), 'micro', 'submit']
+
     def bin_args(self) -> list[str]:
         return [
             "--iterations=10000",
@@ -584,6 +620,9 @@ def description(self) -> str:
     def name(self):
         return f"ulls_benchmark_{self.runtime.value} EmptyKernel wgc:{self.wgc}, wgs:{self.wgs}"
 
+    def get_tags(self):
+        return [runtime_to_tag_name(self.runtime), 'micro']
+
     def bin_args(self) -> list[str]:
         return [
             "--iterations=10000",
@@ -622,6 +661,9 @@ def description(self) -> str:
     def name(self):
         return f"ulls_benchmark_{self.runtime.value} KernelSwitch count {self.count} kernelTime {self.kernelTime}"
 
+    def get_tags(self):
+        return [runtime_to_tag_name(self.runtime), 'micro']
+
     def bin_args(self) -> list[str]:
         return [
             "--iterations=1000",

devops/scripts/benchmarks/benches/llamacpp.py

@@ -101,6 +101,9 @@ def description(self) -> str:
             "quantized model and leverages SYCL with oneDNN for acceleration."
         )
 
+    def get_tags(self):
+        return ['sycl', 'application']
+
     def lower_is_better(self):
         return False
 

devops/scripts/benchmarks/benches/syclbench.py

@@ -112,6 +112,9 @@ def bin_args(self) -> list[str]:
     def extra_env_vars(self) -> dict:
         return {}
 
+    def get_tags(self):
+        return ['sycl', 'micro']
+
     def setup(self):
         self.benchmark_bin = os.path.join(
             self.directory, "sycl-bench-build", self.bench_name

devops/scripts/benchmarks/benches/umf.py

@@ -74,6 +74,9 @@ def setup(self):
 
         self.benchmark_bin = os.path.join(options.umf, "benchmark", self.bench_name)
 
+    def get_tags(self):
+        return ['umf', 'allocation']
+
     def run(self, env_vars) -> list[Result]:
         command = [
             f"{self.benchmark_bin}",

devops/scripts/benchmarks/benches/velocity.py

@@ -118,6 +118,9 @@ def parse_output(self, stdout: str) -> float:
     def description(self) -> str:
         return ""
 
+    def get_tags(self):
+        return ['sycl', 'application']
+
     def run(self, env_vars) -> list[Result]:
         env_vars.update(self.extra_env_vars())
 

devops/scripts/benchmarks/utils/result.py

@@ -27,7 +27,6 @@ class Result:
     name: str = ""
     lower_is_better: bool = True
     suite: str = "Unknown"
-    description: str = "No description provided."
 
 
 @dataclass_json
@@ -44,10 +43,18 @@ class BenchmarkRun:
     )
 
 
+@dataclass_json
+@dataclass
+class BenchmarkTag:
+    name: str
+    description: str = ""
+
+
 @dataclass_json
 @dataclass
 class BenchmarkMetadata:
     type: str = "benchmark"  # or 'group'
     description: Optional[str] = None
     notes: Optional[str] = None
     unstable: Optional[str] = None
+    tags: list[BenchmarkTag] = field(default_factory=list)