add benchmark metadata

Author: pbalcer

devops/scripts/benchmarks/benches/base.py

@@ -6,7 +6,7 @@
 import os
 import shutil
 from pathlib import Path
-from utils.result import Result
+from utils.result import BenchmarkMetadata, Result
 from options import options
 from utils.utils import download, run
 import urllib.request
@@ -78,6 +78,9 @@ def download(
     def name(self):
         raise NotImplementedError()
 
+    def description(self):
+        return "No description provided."
+
     def lower_is_better(self):
         return True
 
@@ -96,6 +99,23 @@ def stddev_threshold(self):
     def get_suite_name(self) -> str:
         return self.suite.name()
 
+    def result_names(self) -> list[str]:
+        return [self.name()]
+
+    def notes(self) -> str:
+        return None
+
+    def unstable(self) -> str:
+        return None
+
+    def get_metadata(self) -> BenchmarkMetadata:
+        return BenchmarkMetadata(
+            type='benchmark',
+            description=self.description(),
+            notes=self.notes(),
+            unstable=self.unstable(),
+        )
+
 
 class Suite:
     def benchmarks(self) -> list[Benchmark]:
@@ -106,3 +126,6 @@ def name(self) -> str:
 
     def setup(self):
         return
+
+    def additionalMetadata(self) -> dict[str, BenchmarkMetadata]:
+        return {}

devops/scripts/benchmarks/benches/compute.py

@@ -8,7 +8,7 @@
 import io
 from utils.utils import run, git_clone, create_build_path
 from .base import Benchmark, Suite
-from utils.result import Result
+from utils.result import BenchmarkMetadata, Result
 from options import options
 from enum import Enum
 
@@ -54,6 +54,23 @@ def setup(self):
 
         self.built = True
 
+    def additionalMetadata(self) -> dict[str, BenchmarkMetadata]:
+        return {
+            "SubmitKernel" : BenchmarkMetadata(
+                type="group",
+                description="Measures CPU time overhead of submitting kernels through different APIs.",
+                notes="Each layer builds on top of the previous layer, adding functionality and overhead. "
+                      "The first layer is the Level Zero API, the second is the Unified Runtime API, and the third is the SYCL API. "
+                      "The UR v2 adapter noticeably reduces UR layer overhead, also improving SYCL performance."
+                      "Work is ongoing to reduce the overhead of the SYCL API",
+            ),
+            "SinKernelGraph" : BenchmarkMetadata(
+                type="group",
+                unstable="This benchmark combines both eager and graph execution, and may not be representative of real use cases.",
+            ),
+        }
+
+
     def benchmarks(self) -> list[Benchmark]:
         if options.sycl is None:
             return []
@@ -106,14 +123,7 @@ def benchmarks(self) -> list[Benchmark]:
                 SubmitKernelUR(self, 1, 0),
                 SubmitKernelUR(self, 1, 1),
                 MemcpyExecute(self, 400, 1, 102400, 10, 1, 1, 1),
-                MemcpyExecute(self, 100, 8, 102400, 10, 1, 1, 1),
-                MemcpyExecute(self, 400, 8, 1024, 1000, 1, 1, 1),
-                MemcpyExecute(self, 10, 16, 1024, 10000, 1, 1, 1),
                 MemcpyExecute(self, 400, 1, 102400, 10, 0, 1, 1),
-                MemcpyExecute(self, 100, 8, 102400, 10, 0, 1, 1),
-                MemcpyExecute(self, 400, 8, 1024, 1000, 0, 1, 1),
-                MemcpyExecute(self, 10, 16, 1024, 10000, 0, 1, 1),
-                MemcpyExecute(self, 4096, 1, 1024, 10, 0, 1, 0),
                 MemcpyExecute(self, 4096, 4, 1024, 10, 0, 1, 0),
                 GraphApiSinKernelGraph(self, RUNTIMES.UR, 0, 5),
                 GraphApiSinKernelGraph(self, RUNTIMES.UR, 1, 5),
@@ -540,6 +550,9 @@ def description(self) -> str:
     def name(self):
         return f"graph_api_benchmark_{self.runtime.value} SinKernelGraph graphs:{self.withGraphs}, numKernels:{self.numKernels}"
 
+    def unstable(self) -> str:
+        return "This benchmark combines both eager and graph execution, and may not be representative of real use cases."
+
     def bin_args(self) -> list[str]:
         return [
             "--iterations=10000",

devops/scripts/benchmarks/benches/test.py

@@ -6,7 +6,7 @@
 import random
 from utils.utils import git_clone
 from .base import Benchmark, Suite
-from utils.result import Result
+from utils.result import BenchmarkMetadata, Result
 from utils.utils import run, create_build_path
 from options import options
 import os
@@ -24,33 +24,49 @@ def name(self) -> str:
 
     def benchmarks(self) -> list[Benchmark]:
         bench_configs = [
-            ("Memory Bandwidth", 2000, 200, "Foo Group"),
-            ("Latency", 100, 20, "Bar Group"),
-            ("Throughput", 1500, 150, "Foo Group"),
-            ("FLOPS", 3000, 300, "Foo Group"),
-            ("Cache Miss Rate", 250, 25, "Bar Group"),
+            ("Memory Bandwidth", 2000, 200, "Foo Group", None, None),
+            ("Latency", 100, 20, "Bar Group", "A Latency test note!", None),
+            ("Throughput", 1500, 150, "Foo Group", None, None),
+            ("FLOPS", 3000, 300, "Foo Group", None, "Unstable FLOPS test!"),
+            ("Cache Miss Rate", 250, 25, "Bar Group", "Test Note", "And another note!"),
         ]
 
         result = []
-        for base_name, base_value, base_diff, group in bench_configs:
+        for base_name, base_value, base_diff, group, notes, unstable in bench_configs:
             for variant in range(6):
                 value_multiplier = 1.0 + (variant * 0.2)
                 name = f"{base_name} {variant+1}"
                 value = base_value * value_multiplier
                 diff = base_diff * value_multiplier
 
-                result.append(TestBench(self, name, value, diff, group))
+                result.append(TestBench(self, name, value, diff, group, notes, unstable))
 
         return result
 
+    def additionalMetadata(self) -> dict[str, BenchmarkMetadata]:
+        return {
+            "Foo Group" : BenchmarkMetadata(
+                type="group",
+                description="This is a test benchmark for Foo Group.",
+                notes="This is a test note for Foo Group.",
+            ),
+            "Bar Group" : BenchmarkMetadata(
+                type="group",
+                description="This is a test benchmark for Bar Group.",
+                unstable="This is an unstable note for Bar Group.",
+            ),
+        }
+
 
 class TestBench(Benchmark):
-    def __init__(self, suite, name, value, diff, group=""):
+    def __init__(self, suite, name, value, diff, group="", notes=None, unstable=None):
         super().__init__("", suite)
         self.bname = name
         self.value = value
         self.diff = diff
         self.group = group
+        self.notes_text = notes
+        self.unstable_text = unstable
 
     def name(self):
         return self.bname
@@ -64,6 +80,12 @@ def setup(self):
     def description(self) -> str:
         return f"This is a test benchmark for {self.bname}."
 
+    def notes(self) -> str:
+        return self.notes_text
+
+    def unstable(self) -> str:
+        return self.unstable_text
+
     def run(self, env_vars) -> list[Result]:
         random_value = self.value + random.uniform(-1 * (self.diff), self.diff)
         return [

devops/scripts/benchmarks/html/index.html

@@ -171,7 +171,98 @@
         .extra-info-entry em {
             color: #555;
         }
-</style>
+        .display-options-container {
+            text-align: center;
+            margin-bottom: 24px;
+            padding: 16px;
+            background: #e9ecef;
+            border-radius: 8px;
+        }
+        .display-options-container label {
+            margin: 0 12px;
+            cursor: pointer;
+        }
+        .display-options-container input {
+            margin-right: 8px;
+        }
+        .benchmark-note {
+            background-color: #cfe2ff;
+            color: #084298;
+            padding: 10px;
+            margin-bottom: 10px;
+            border-radius: 5px;
+            border-left: 4px solid #084298;
+        }
+        .benchmark-unstable {
+            background-color: #f8d7da;
+            color: #842029;
+            padding: 10px;
+            margin-bottom: 10px;
+            border-radius: 5px;
+            border-left: 4px solid #842029;
+        }
+        .note-text {
+            color: #084298;
+        }
+        .unstable-warning {
+            color: #842029;
+            font-weight: bold;
+        }
+        .unstable-text {
+            color: #842029;
+        }
+        .options-container {
+            margin-bottom: 24px;
+            background: #e9ecef;
+            border-radius: 8px;
+            overflow: hidden;
+        }
+        .options-container summary {
+            padding: 12px 16px;
+            font-weight: 500;
+            cursor: pointer;
+            background: #dee2e6;
+            user-select: none;
+        }
+        .options-container summary:hover {
+            background: #ced4da;
+        }
+        .options-content {
+            padding: 16px;
+            display: flex;
+            flex-wrap: wrap;
+            gap: 24px;
+        }
+        .filter-section {
+            flex: 1;
+            min-width: 300px;
+        }
+        .filter-section h3 {
+            margin-top: 0;
+            margin-bottom: 12px;
+            font-size: 18px;
+            font-weight: 500;
+            text-align: left;
+        }
+        #suite-filters {
+            display: flex;
+            flex-wrap: wrap;
+            gap: 8px;
+        }
+        .display-options {
+            display: flex;
+            flex-direction: column;
+            gap: 8px;
+        }
+        .display-options label {
+            display: flex;
+            align-items: center;
+            cursor: pointer;
+        }
+        .display-options input {
+            margin-right: 8px;
+        }
+    </style>
 </head>
 <body>
     <div class="container">
@@ -182,16 +273,38 @@ <h1>Benchmark Results</h1>
         <div class="filter-container">
             <input type="text" id="bench-filter" placeholder="Regex...">
         </div>
-        <div class="suite-filter-container" id="suite-filters">
-            <!-- Suite checkboxes will be generated by JavaScript -->
-        </div>
         <div class="run-selector">
             <select id="run-select">
                 <option value="">Select a run to compare...</option>
             </select>
             <button onclick="addSelectedRun()">Add</button>
             <div id="selected-runs" class="selected-runs"></div>
         </div>
+        <details class="options-container">
+            <summary>Options</summary>
+            <div class="options-content">
+                <div class="filter-section">
+                    <h3>Suites</h3>
+                    <div id="suite-filters">
+                        <!-- Suite checkboxes will be generated by JavaScript -->
+                    </div>
+                </div>
+
+                <div class="filter-section">
+                    <h3>Display Options</h3>
+                    <div class="display-options">
+                        <label>
+                            <input type="checkbox" id="show-notes" checked>
+                            Director's commentary
+                        </label>
+                        <label>
+                            <input type="checkbox" id="show-unstable">
+                            Show 'it works on my machine' scenarios
+                        </label>
+                    </div>
+                </div>
+            </div>
+        </details>
         <details class="timeseries" open>
             <summary>Historical Results</summary>
             <div class="charts"></div>