Merge pull request #2458 from pbalcer/filter-suites

[benchmarks] add ability to filter benchmarks by suite

Author: pbalcer

scripts/benchmarks/benches/base.py

@@ -13,8 +13,9 @@
 import tarfile
 
 class Benchmark:
-    def __init__(self, directory):
+    def __init__(self, directory, suite):
         self.directory = directory
+        self.suite = suite
 
     @staticmethod
     def get_adapter_full_path():
@@ -74,9 +75,15 @@ def teardown(self):
     def stddev_threshold(self):
         return None
 
+    def get_suite_name(self) -> str:
+        return self.suite.name()
+
 class Suite:
     def benchmarks(self) -> list[Benchmark]:
         raise NotImplementedError()
 
+    def name(self) -> str:
+        raise NotImplementedError()
+
     def setup(self):
         return

scripts/benchmarks/benches/compute.py

@@ -15,6 +15,9 @@ class ComputeBench(Suite):
     def __init__(self, directory):
         self.directory = directory
 
+    def name(self) -> str:
+        return "Compute Benchmarks"
+
     def setup(self):
         if options.sycl is None:
             return
@@ -90,10 +93,10 @@ def parse_unit_type(compute_unit):
 
 class ComputeBenchmark(Benchmark):
     def __init__(self, bench, name, test):
+        super().__init__(bench.directory, bench)
         self.bench = bench
         self.bench_name = name
         self.test = test
-        super().__init__(bench.directory)
 
     def bin_args(self) -> list[str]:
         return []

scripts/benchmarks/benches/llamacpp.py

@@ -21,6 +21,9 @@ def __init__(self, directory):
 
         self.directory = directory
 
+    def name(self) -> str:
+        return "llama.cpp bench"
+
     def setup(self):
         if options.sycl is None:
             return
@@ -64,8 +67,8 @@ def benchmarks(self) -> list[Benchmark]:
 
 class LlamaBench(Benchmark):
     def __init__(self, bench):
+        super().__init__(bench.directory, bench)
         self.bench = bench
-        super().__init__(bench.directory)
 
     def setup(self):
         self.benchmark_bin = os.path.join(self.bench.build_path, 'bin', 'llama-bench')

scripts/benchmarks/benches/result.py

@@ -25,8 +25,9 @@ class Result:
     # values below should not be set by the benchmark
     name: str = ""
     lower_is_better: bool = True
-    git_hash: str = ''
+    git_hash: str = ""
     date: Optional[datetime] = None
+    suite: str = ""
 
 @dataclass_json
 @dataclass

scripts/benchmarks/benches/syclbench.py

@@ -19,6 +19,9 @@ def __init__(self, directory):
         self.directory = directory
         return
 
+    def name(self) -> str:
+        return "SYCL-Bench"
+
     def setup(self):
         if options.sycl is None:
             return
@@ -87,11 +90,11 @@ def benchmarks(self) -> list[Benchmark]:
 
 class SyclBenchmark(Benchmark):
     def __init__(self, bench, name, test):
+        super().__init__(bench.directory, bench)
         self.bench = bench
         self.bench_name = name
         self.test = test
         self.done = False
-        super().__init__(bench.directory)
 
     def bin_args(self) -> list[str]:
         return []

scripts/benchmarks/benches/umf.py

@@ -22,7 +22,10 @@ def __init__(self, directory):
         self.directory = directory
         if not isUMFAvailable():
             print("UMF not provided. Related benchmarks will not run")
-    
+
+    def name(self) -> str:
+        return "UMF"
+
     def setup(self):
         if not isUMFAvailable():
             return []

scripts/benchmarks/benches/velocity.py

@@ -22,6 +22,9 @@ def __init__(self, directory):
 
         self.directory = directory
 
+    def name(self) -> str:
+        return "Velocity Bench"
+
     def setup(self):
         if options.sycl is None:
             return
@@ -46,7 +49,7 @@ def benchmarks(self) -> list[Benchmark]:
 
 class VelocityBase(Benchmark):
     def __init__(self, name: str, bin_name: str, vb: VelocityBench, unit: str):
-        super().__init__(vb.directory)
+        super().__init__(vb.directory, vb)
         self.vb = vb
         self.bench_name = name
         self.bin_name = bin_name

scripts/benchmarks/main.py

@@ -42,6 +42,7 @@ def run_iterations(benchmark: Benchmark, env_vars, iters: int, results: dict[str
 
             bench_result.name = bench_result.label
             bench_result.lower_is_better = benchmark.lower_is_better()
+            bench_result.suite = benchmark.get_suite_name()
 
             if bench_result.label not in results:
                 results[bench_result.label] = []

scripts/benchmarks/output_html.py

@@ -15,6 +15,7 @@
 @dataclass
 class BenchmarkMetadata:
     unit: str
+    suite: str
     lower_is_better: bool
 
 @dataclass
@@ -26,6 +27,7 @@ class BenchmarkSeries:
 @dataclass
 class BenchmarkChart:
     label: str
+    suite: str
     html: str
 
 def tooltip_css() -> str:
@@ -74,13 +76,6 @@ def create_time_series_chart(benchmarks: list[BenchmarkSeries], github_repo: str
                 targets=targets)
             mpld3.plugins.connect(fig, tooltip)
 
-        # This is so that the stddev doesn't fill the entire y axis on the chart
-        if all_values and all_stddevs:
-            max_value = max(all_values)
-            min_value = min(all_values)
-            max_stddev = max(all_stddevs)
-            ax.set_ylim(min_value - 3 * max_stddev, max_value + 3 * max_stddev)
-
         ax.set_title(benchmark.label, pad=20)
         performance_indicator = "lower is better" if benchmark.metadata.lower_is_better else "higher is better"
         ax.text(0.5, 1.05, f"({performance_indicator})",
@@ -98,7 +93,7 @@ def create_time_series_chart(benchmarks: list[BenchmarkSeries], github_repo: str
         ax.xaxis.set_major_formatter(mdates.ConciseDateFormatter('%Y-%m-%d %H:%M:%S'))
 
         plt.tight_layout()
-        html_charts.append(BenchmarkChart(html=mpld3.fig_to_html(fig), label=benchmark.label))
+        html_charts.append(BenchmarkChart(html=mpld3.fig_to_html(fig), label=benchmark.label, suite=benchmark.metadata.suite))
         plt.close(fig)
 
     return html_charts
@@ -119,7 +114,7 @@ def create_explicit_groups(benchmark_runs: list[BenchmarkRun], compare_names: li
                 if res.explicit_group != '':
                     if res.explicit_group not in groups:
                         groups[res.explicit_group] = ExplicitGroup(name=res.explicit_group, nnames=len(compare_names),
-                                metadata=BenchmarkMetadata(unit=res.unit, lower_is_better=res.lower_is_better),
+                                metadata=BenchmarkMetadata(unit=res.unit, lower_is_better=res.lower_is_better, suite=res.suite),
                                 runs={})
 
                     group = groups[res.explicit_group]
@@ -207,7 +202,7 @@ def create_grouped_bar_charts(groups: list[ExplicitGroup]) -> list[BenchmarkChar
                 color='#666666')
 
         plt.tight_layout()
-        html_charts.append(BenchmarkChart(label=group.name, html=mpld3.fig_to_html(fig)))
+        html_charts.append(BenchmarkChart(label=group.name, html=mpld3.fig_to_html(fig), suite=group.metadata.suite))
         plt.close(fig)
 
     return html_charts
@@ -224,7 +219,8 @@ def process_benchmark_data(benchmark_runs: list[BenchmarkRun], compare_names: li
             if result.label not in benchmark_metadata:
                 benchmark_metadata[result.label] = BenchmarkMetadata(
                     unit=result.unit,
-                    lower_is_better=result.lower_is_better
+                    lower_is_better=result.lower_is_better,
+                    suite=result.suite
                 )
 
             result.date = run.date
@@ -249,12 +245,15 @@ def generate_html(benchmark_runs: list[BenchmarkRun], github_repo: str, compare_
     benchmarks = process_benchmark_data(benchmark_runs, compare_names)
 
     timeseries = create_time_series_chart(benchmarks, github_repo)
-    timeseries_charts_html = '\n'.join(f'<div class="chart" data-label="{ts.label}"><div>{ts.html}</div></div>' for ts in timeseries)
+    timeseries_charts_html = '\n'.join(f'<div class="chart" data-label="{ts.label}" data-suite="{ts.suite}"><div>{ts.html}</div></div>' for ts in timeseries)
 
     explicit_groups = create_explicit_groups(benchmark_runs, compare_names)
 
     bar_charts = create_grouped_bar_charts(explicit_groups)
-    bar_charts_html = '\n'.join(f'<div class="chart" data-label="{bc.label}"><div>{bc.html}</div></div>' for bc in bar_charts)
+    bar_charts_html = '\n'.join(f'<div class="chart" data-label="{bc.label}" data-suite="{bc.suite}"><div>{bc.html}</div></div>' for bc in bar_charts)
+
+    suite_names = {t.suite for t in timeseries}
+    suite_checkboxes_html = ' '.join(f'<label><input type="checkbox" class="suite-checkbox" data-suite="{suite}" checked> {suite}</label>' for suite in suite_names)
 
     html_template = f"""
     <!DOCTYPE html>
@@ -317,6 +316,16 @@ def generate_html(benchmark_runs: list[BenchmarkRun], github_repo: str, compare_
                 width: 400px;
                 max-width: 100%;
             }}
+            .suite-filter-container {{
+                text-align: center;
+                margin-bottom: 24px;
+                padding: 16px;
+                background: #e9ecef;
+                border-radius: 8px;
+            }}
+            .suite-checkbox {{
+                margin: 0 8px;
+            }}
             details {{
                 margin-bottom: 24px;
             }}
@@ -342,54 +351,87 @@ def generate_html(benchmark_runs: list[BenchmarkRun], github_repo: str, compare_
             function filterCharts() {{
                 const regexInput = document.getElementById('bench-filter').value;
                 const regex = new RegExp(regexInput, 'i');
+                const activeSuites = Array.from(document.querySelectorAll('.suite-checkbox:checked')).map(checkbox => checkbox.getAttribute('data-suite'));
                 const charts = document.querySelectorAll('.chart');
-                let timeseriesVisible = false;
-                let barChartsVisible = false;
 
                 charts.forEach(chart => {{
                     const label = chart.getAttribute('data-label');
-                    if (regex.test(label)) {{
+                    const suite = chart.getAttribute('data-suite');
+                    if (regex.test(label) && activeSuites.includes(suite)) {{
                         chart.style.display = '';
-                        if (chart.closest('.timeseries')) {{
-                            timeseriesVisible = true;
-                        }} else if (chart.closest('.bar-charts')) {{
-                            barChartsVisible = true;
-                        }}
                     }} else {{
                         chart.style.display = 'none';
                     }}
                 }});
 
-                updateURL(regexInput);
-
-                document.querySelector('.timeseries').open = timeseriesVisible;
-                document.querySelector('.bar-charts').open = barChartsVisible;
+                updateURL();
             }}
 
-            function updateURL(regex) {{
+            function updateURL() {{
                 const url = new URL(window.location);
+                const regex = document.getElementById('bench-filter').value;
+                const activeSuites = Array.from(document.querySelectorAll('.suite-checkbox:checked')).map(checkbox => checkbox.getAttribute('data-suite'));
+
                 if (regex) {{
                     url.searchParams.set('regex', regex);
                 }} else {{
                     url.searchParams.delete('regex');
                 }}
+
+                if (activeSuites.length > 0) {{
+                    url.searchParams.set('suites', activeSuites.join(','));
+                }} else {{
+                    url.searchParams.delete('suites');
+                }}
+
                 history.replaceState(null, '', url);
             }}
 
             document.addEventListener('DOMContentLoaded', (event) => {{
                 const regexParam = getQueryParam('regex');
+                const suitesParam = getQueryParam('suites');
+
                 if (regexParam) {{
                     document.getElementById('bench-filter').value = regexParam;
-                    filterCharts();
                 }}
+
+                const suiteCheckboxes = document.querySelectorAll('.suite-checkbox');
+                if (suitesParam) {{
+                    const suites = suitesParam.split(',');
+                    suiteCheckboxes.forEach(checkbox => {{
+                        if (suites.includes(checkbox.getAttribute('data-suite'))) {{
+                            checkbox.checked = true;
+                        }} else {{
+                            checkbox.checked = false;
+                        }}
+                    }});
+                }} else {{
+                    suiteCheckboxes.forEach(checkbox => {{
+                        checkbox.checked = true;
+                    }});
+                }}
+                filterCharts();
+
+                suiteCheckboxes.forEach(checkbox => {{
+                    checkbox.addEventListener('change', () => {{
+                        filterCharts();
+                    }});
+                }});
+
+                document.getElementById('bench-filter').addEventListener('input', () => {{
+                    filterCharts();
+                }});
             }});
         </script>
     </head>
     <body>
         <div class="container">
             <h1>Benchmark Results</h1>
             <div class="filter-container">
-                <input type="text" id="bench-filter" placeholder="Regex..." oninput="filterCharts()">
+                <input type="text" id="bench-filter" placeholder="Regex...">
+            </div>
+            <div class="suite-filter-container">
+                {suite_checkboxes_html}
             </div>
             <details class="timeseries">
                 <summary>Historical Results</summary>