simplify presets, remove suites if all set

Author: pbalcer

devops/scripts/benchmarks/html/scripts.js

@@ -6,6 +6,7 @@
 // Core state
 let activeRuns = new Set(defaultCompareNames);
 let chartInstances = new Map();
+let suiteNames = new Set();
 let timeseriesData, barChartsData, allRunNames;
 
 // DOM Elements
@@ -306,7 +307,7 @@ function updateURL() {
         url.searchParams.delete('regex');
     }
 
-    if (activeSuites.length > 0) {
+    if (activeSuites.length > 0 && activeSuites.length != suiteNames.size) {
         url.searchParams.set('suites', activeSuites.join(','));
     } else {
         url.searchParams.delete('suites');
@@ -444,7 +445,6 @@ function setupRunSelector() {
 function setupSuiteFilters() {
     suiteFiltersContainer = document.getElementById('suite-filters');
 
-    const suiteNames = new Set();
     benchmarkRuns.forEach(run => {
         run.results.forEach(result => {
             suiteNames.add(result.suite);

devops/scripts/benchmarks/main.py

@@ -17,7 +17,7 @@
 from history import BenchmarkHistory
 from utils.utils import prepare_workdir
 from utils.compute_runtime import *
-from presets import preset_get_by_name, presets
+from presets import enabled_suites, presets
 
 import argparse
 import re
@@ -164,7 +164,7 @@ def main(directory, additional_env_vars, save_name, compare_names, filter):
     failures = {}
 
     for s in suites:
-        if s.name() not in options.preset.suites():
+        if s.name() not in enabled_suites(options.preset):
             continue
 
         suite_benchmarks = s.benchmarks()
@@ -443,9 +443,9 @@ def validate_and_parse_env_args(env_args):
     parser.add_argument(
         "--preset",
         type=str,
-        choices=[p.name() for p in presets],
+        choices=[p for p in presets.keys()],
         help="Benchmark preset to run.",
-        default=options.preset.name(),
+        default=options.preset,
     )
     parser.add_argument(
         "--results-dir",
@@ -478,7 +478,7 @@ def validate_and_parse_env_args(env_args):
     options.current_run_name = args.relative_perf
     options.cudnn_directory = args.cudnn_directory
     options.cublas_directory = args.cublas_directory
-    options.preset = preset_get_by_name(args.preset)
+    options.preset = args.preset
     options.custom_results_dir = args.results_dir
 
     if args.build_igc and args.compute_runtime is None:

devops/scripts/benchmarks/options.py

@@ -1,7 +1,7 @@
 from dataclasses import dataclass, field
 from enum import Enum
-from presets import Preset, presets
 
+from presets import presets
 
 class Compare(Enum):
     LATEST = "latest"
@@ -40,7 +40,7 @@ class Options:
     compute_runtime_tag: str = "25.05.32567.18"
     build_igc: bool = False
     current_run_name: str = "This PR"
-    preset: Preset = presets[0]
+    preset: str = "Full"
     custom_results_dir = None
 
 

devops/scripts/benchmarks/presets.py

@@ -3,63 +3,36 @@
 # See LICENSE.TXT
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-from typing import List, Type
+presets: dict[str, list[str]] = {
+    "Full": [
+        "Compute Benchmarks",
+        "llama.cpp bench",
+        "SYCL-Bench",
+        "Velocity Bench",
+        "UMF",
+    ],
+    "SYCL": [
+        "Compute Benchmarks",
+        "llama.cpp bench",
+        "SYCL-Bench",
+        "Velocity Bench",
+    ],
+    "Minimal": [
+        "Compute Benchmarks",
+    ],
+    "Normal": [
+        "Compute Benchmarks",
+        "llama.cpp bench",
+        "Velocity Bench",
+    ],
+    "Test": [
+        "Test Suite",
+    ],
+}
+
+def enabled_suites(preset: str) -> list[str]:
+    try:
+        return presets[preset]
+    except KeyError:
+        raise ValueError(f"Preset '{preset}' not found.")
 
-class Preset:
-    def description(self) -> str:
-        raise NotImplementedError
-
-    def name(self) -> str:
-        return self.__class__.__name__
-
-    def suites(self) -> List[str]:
-        raise NotImplementedError
-
-class Full(Preset):
-    def description(self) -> str:
-        return "All available benchmarks."
-
-    def suites(self) -> List[str]:
-        return [
-            "Compute Benchmarks",
-            "llama.cpp bench",
-            "SYCL-Bench",
-            "Velocity Bench",
-            "UMF",
-        ]
-
-class SYCL(Preset):
-    def description(self) -> str:
-        return "All available benchmarks related to SYCL."
-
-    def suites(self) -> List[str]:
-        return ["Compute Benchmarks", "llama.cpp bench", "SYCL-Bench", "Velocity Bench"]
-
-class Minimal(Preset):
-    def description(self) -> str:
-        return "Short microbenchmarks."
-
-    def suites(self) -> List[str]:
-        return ["Compute Benchmarks"]
-
-class Normal(Preset):
-    def description(self) -> str:
-        return "Comprehensive mix of microbenchmarks and real applications."
-
-    def suites(self) -> List[str]:
-        return ["Compute Benchmarks", "llama.cpp bench", "Velocity Bench"]
-
-class Test(Preset):
-    def description(self) -> str:
-        return "Noop benchmarks for framework testing."
-
-    def suites(self) -> List[str]:
-        return ["Test Suite"]
-
-presets = [Full(), SYCL(), Minimal(), Normal(), Test()]
-
-def preset_get_by_name(name: str) -> Preset:
-    for p in presets:
-        if p.name() == name:
-            return p
-    raise ValueError(f"Preset '{name}' not found.")