[CI][Benchmarks] add an option to adjust y-axis per benchmark

pbalcer · pbalcer · commit 8c4b869ca6c9 · 2025-04-02T12:06:36.000Z
This is useful for more accurate comparisons between multiple labels.
diff --git a/devops/scripts/benchmarks/benches/base.py b/devops/scripts/benchmarks/benches/base.py
@@ -129,13 +129,20 @@ def unstable(self) -> str:
     def get_tags(self) -> list[str]:
         return []
 
+    def range(self) -> tuple[float, float]:
+        return None
+
     def get_metadata(self) -> BenchmarkMetadata:
+        range = self.range()
+
         return BenchmarkMetadata(
             type="benchmark",
             description=self.description(),
             notes=self.notes(),
             unstable=self.unstable(),
             tags=self.get_tags(),
+            range_min=range[0] if range else None,
+            range_max=range[1] if range else None,
         )
 
 
diff --git a/devops/scripts/benchmarks/benches/compute.py b/devops/scripts/benchmarks/benches/compute.py
@@ -92,6 +92,7 @@ def additionalMetadata(self) -> dict[str, BenchmarkMetadata]:
                 "The UR v2 adapter noticeably reduces UR layer overhead, also improving SYCL performance.\n"
                 "Work is ongoing to reduce the overhead of the SYCL API\n",
                 tags=["submit", "micro", "SYCL", "UR", "L0"],
+                range_min=0.0,
             ),
             "SinKernelGraph": BenchmarkMetadata(
                 type="group",
@@ -317,6 +318,9 @@ def description(self) -> str:
             f"Runs 10 simple kernels with minimal execution time to isolate API overhead from kernel execution time. {l0_specific}"
         )
 
+    def range(self) -> tuple[float, float]:
+        return (0.0, None)
+
     def bin_args(self) -> list[str]:
         return [
             f"--Ioq={self.ioq}",
diff --git a/devops/scripts/benchmarks/html/index.html b/devops/scripts/benchmarks/html/index.html
@@ -47,6 +47,10 @@ <h3>Display Options</h3>
                             <input type="checkbox" id="show-unstable">
                             Show unstable scenarios
                         </label>
+                        <label>
+                            <input type="checkbox" id="custom-range">
+                            Adjust Y-axis for comparisons
+                        </label>
                     </div>
                 </div>
 
diff --git a/devops/scripts/benchmarks/html/scripts.js b/devops/scripts/benchmarks/html/scripts.js
@@ -97,7 +97,7 @@ function createChart(data, containerId, type) {
                             ];
                         } else {
                             return [`${context.dataset.label}:`,
-                                `Value: ${context.parsed.y.toFixed(2)} ${data.unit}`,
+                            `Value: ${context.parsed.y.toFixed(2)} ${data.unit}`,
                             ];
                         }
                     }
@@ -111,6 +111,8 @@ function createChart(data, containerId, type) {
                     text: data.unit
                 },
                 grace: '20%',
+                min: isCustomRangesEnabled() ? data.range_min : null,
+                max: isCustomRangesEnabled() ? data.range_max : null
             }
         }
     };
@@ -182,7 +184,7 @@ function updateCharts() {
     const filterRunData = (chart) => ({
         ...chart,
         runs: Object.fromEntries(
-            Object.entries(chart.runs).filter(([_, data]) => 
+            Object.entries(chart.runs).filter(([_, data]) =>
                 activeRuns.has(data.runName)
             )
         )
@@ -276,25 +278,25 @@ function createChartContainer(data, canvasId, type) {
     // Add tags if present
     if (metadata && metadata.tags) {
         container.setAttribute('data-tags', metadata.tags.join(','));
-        
+
         // Add tags display
         const tagsContainer = document.createElement('div');
         tagsContainer.className = 'benchmark-tags';
-        
+
         metadata.tags.forEach(tag => {
             const tagElement = document.createElement('span');
             tagElement.className = 'tag';
             tagElement.textContent = tag;
             tagElement.setAttribute('data-tag', tag);
-            
+
             // Add tooltip with tag description
             if (benchmarkTags[tag]) {
                 tagElement.setAttribute('title', benchmarkTags[tag].description);
             }
-            
+
             tagsContainer.appendChild(tagElement);
         });
-        
+
         container.appendChild(tagsContainer);
     }
 
@@ -493,6 +495,12 @@ function updateURL() {
         url.searchParams.set('unstable', 'true');
     }
 
+    if (!isCustomRangesEnabled()) {
+        url.searchParams.delete('customRange');
+    } else {
+        url.searchParams.set('customRange', 'true');
+    }
+
     history.replaceState(null, '', url);
 }
 
@@ -505,12 +513,12 @@ function filterCharts() {
         const label = container.getAttribute('data-label');
         const suite = container.getAttribute('data-suite');
         const isUnstable = container.getAttribute('data-unstable') === 'true';
-        const tags = container.getAttribute('data-tags') ? 
-                    container.getAttribute('data-tags').split(',') : [];
+        const tags = container.getAttribute('data-tags') ?
+            container.getAttribute('data-tags').split(',') : [];
 
         // Check if benchmark has all active tags (if any are selected)
-        const hasAllActiveTags = activeTags.size === 0 || 
-                               Array.from(activeTags).every(tag => tags.includes(tag));
+        const hasAllActiveTags = activeTags.size === 0 ||
+            Array.from(activeTags).every(tag => tags.includes(tag));
 
         // Hide unstable benchmarks if showUnstable is false
         const shouldShow = regex.test(label) &&
@@ -535,16 +543,19 @@ function processTimeseriesData(benchmarkRuns) {
 
     benchmarkRuns.forEach(run => {
         run.results.forEach(result => {
+            const metadata = metadataForLabel(result.label, 'benchmark');
+
             if (!resultsByLabel[result.label]) {
                 resultsByLabel[result.label] = {
                     label: result.label,
                     suite: result.suite,
                     unit: result.unit,
                     lower_is_better: result.lower_is_better,
+                    range_min: metadata?.range_min ?? null, // can't use || because js treats 0 as null
+                    range_max: metadata?.range_max ?? null,
                     runs: {}
                 };
             }
-
             addRunDataPoint(resultsByLabel[result.label], run, result, run.name);
         });
     });
@@ -568,6 +579,8 @@ function processBarChartsData(benchmarkRuns) {
                     suite: result.suite,
                     unit: result.unit,
                     lower_is_better: result.lower_is_better,
+                    range_min: groupMetadata?.range_min ?? null, // can't use || because js treats 0 as null
+                    range_max: groupMetadata?.range_max ?? null,
                     labels: [],
                     datasets: [],
                     // Add metadata if available
@@ -654,6 +667,8 @@ function processLayerComparisonsData(benchmarkRuns) {
                     suite: result.suite,
                     unit: result.unit,
                     lower_is_better: result.lower_is_better,
+                    range_min: metadata?.range_min ?? null, // can't use || because js treats 0 as null
+                    range_max: metadata?.range_max ?? null,
                     runs: {},
                     benchmarkLabels: [],
                     description: metadata?.description || null,
@@ -760,26 +775,37 @@ function isUnstableEnabled() {
     return unstableToggle.checked;
 }
 
+function isCustomRangesEnabled() {
+    const rangesToggle = document.getElementById('custom-range');
+    return rangesToggle.checked;
+}
+
 function setupToggles() {
     const notesToggle = document.getElementById('show-notes');
     const unstableToggle = document.getElementById('show-unstable');
+    const customRangeToggle = document.getElementById('custom-range');
 
-    notesToggle.addEventListener('change', function() {
+    notesToggle.addEventListener('change', function () {
         // Update all note elements visibility
         document.querySelectorAll('.benchmark-note').forEach(note => {
             note.style.display = isNotesEnabled() ? 'block' : 'none';
         });
         updateURL();
     });
 
-    unstableToggle.addEventListener('change', function() {
+    unstableToggle.addEventListener('change', function () {
         // Update all unstable warning elements visibility
         document.querySelectorAll('.benchmark-unstable').forEach(warning => {
             warning.style.display = isUnstableEnabled() ? 'block' : 'none';
         });
         filterCharts();
     });
 
+    customRangeToggle.addEventListener('change', function () {
+        // redraw all charts
+        updateCharts();
+    });
+
     // Initialize from URL params if present
     const notesParam = getQueryParam('notes');
     const unstableParam = getQueryParam('unstable');
@@ -793,13 +819,18 @@ function setupToggles() {
         let showUnstable = unstableParam === 'true';
         unstableToggle.checked = showUnstable;
     }
+
+    const customRangesParam = getQueryParam('customRange');
+    if (customRangesParam !== null) {
+        customRangeToggle.checked = customRangesParam === 'true';
+    }
 }
 
 function setupTagFilters() {
     tagFiltersContainer = document.getElementById('tag-filters');
 
     const allTags = [];
-    
+
     if (benchmarkTags) {
         for (const tag in benchmarkTags) {
             if (!allTags.includes(tag)) {
@@ -812,17 +843,17 @@ function setupTagFilters() {
     allTags.forEach(tag => {
         const tagContainer = document.createElement('div');
         tagContainer.className = 'tag-filter';
-        
+
         const checkbox = document.createElement('input');
         checkbox.type = 'checkbox';
         checkbox.id = `tag-${tag}`;
         checkbox.className = 'tag-checkbox';
         checkbox.dataset.tag = tag;
-        
+
         const label = document.createElement('label');
         label.htmlFor = `tag-${tag}`;
         label.textContent = tag;
-        
+
         // Add info icon with tooltip if tag description exists
         if (benchmarkTags[tag]) {
             const infoIcon = document.createElement('span');
@@ -831,16 +862,16 @@ function setupTagFilters() {
             infoIcon.title = benchmarkTags[tag].description;
             label.appendChild(infoIcon);
         }
-        
-        checkbox.addEventListener('change', function() {
+
+        checkbox.addEventListener('change', function () {
             if (this.checked) {
                 activeTags.add(tag);
             } else {
                 activeTags.delete(tag);
             }
             filterCharts();
         });
-        
+
         tagContainer.appendChild(checkbox);
         tagContainer.appendChild(label);
         tagFiltersContainer.appendChild(tagContainer);
@@ -849,18 +880,18 @@ function setupTagFilters() {
 
 function toggleAllTags(select) {
     const checkboxes = document.querySelectorAll('.tag-checkbox');
-    
+
     checkboxes.forEach(checkbox => {
         checkbox.checked = select;
         const tag = checkbox.dataset.tag;
-        
+
         if (select) {
             activeTags.add(tag);
         } else {
             activeTags.delete(tag);
         }
     });
-    
+
     filterCharts();
 }
 
diff --git a/devops/scripts/benchmarks/utils/result.py b/devops/scripts/benchmarks/utils/result.py
@@ -59,6 +59,8 @@ class BenchmarkMetadata:
     notes: str = None
     unstable: str = None
     tags: list[str] = field(default_factory=list)
+    range_min: float = None
+    range_max: float = None
 
 
 @dataclass_json
