simplify preset implementation and fix normal preset

Author: pbalcer

devops/scripts/benchmarks/main.py

@@ -17,7 +17,7 @@
 from history import BenchmarkHistory
 from utils.utils import prepare_workdir
 from utils.compute_runtime import *
-from presets import Presets
+from presets import preset_get_by_name, presets
 
 import argparse
 import re
@@ -440,9 +440,9 @@ def validate_and_parse_env_args(env_args):
     parser.add_argument(
         "--preset",
         type=str,
-        choices=[p.name for p in Presets],
+        choices=[p.name() for p in presets],
         help="Benchmark preset to run.",
-        default="FULL",
+        default=options.preset.name(),
     )
 
     args = parser.parse_args()
@@ -469,7 +469,7 @@ def validate_and_parse_env_args(env_args):
     options.current_run_name = args.relative_perf
     options.cudnn_directory = args.cudnn_directory
     options.cublas_directory = args.cublas_directory
-    options.preset = Presets[args.preset].value()
+    options.preset = preset_get_by_name(args.preset)
 
     if args.build_igc and args.compute_runtime is None:
         parser.error("--build-igc requires --compute-runtime to be set")

devops/scripts/benchmarks/options.py

@@ -1,6 +1,6 @@
 from dataclasses import dataclass, field
 from enum import Enum
-from presets import Preset
+from presets import Preset, presets
 
 
 class Compare(Enum):
@@ -40,7 +40,7 @@ class Options:
     compute_runtime_tag: str = "25.05.32567.18"
     build_igc: bool = False
     current_run_name: str = "This PR"
-    preset: Preset = None
+    preset: Preset = presets[0]
 
 
 options = Options()

devops/scripts/benchmarks/presets.py

@@ -3,22 +3,23 @@
 # See LICENSE.TXT
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-from enum import Enum
-
+from typing import List, Type
 
 class Preset:
-    def description(self):
-        pass
+    def description(self) -> str:
+        raise NotImplementedError
 
-    def suites(self) -> list[str]:
-        return []
+    def name(self) -> str:
+        return self.__class__.__name__
 
+    def suites(self) -> List[str]:
+        raise NotImplementedError
 
 class Full(Preset):
-    def description(self):
+    def description(self) -> str:
         return "All available benchmarks."
 
-    def suites(self) -> list[str]:
+    def suites(self) -> List[str]:
         return [
             "Compute Benchmarks",
             "llama.cpp bench",
@@ -27,42 +28,38 @@ def suites(self) -> list[str]:
             "UMF",
         ]
 
-
 class SYCL(Preset):
-    def description(self):
+    def description(self) -> str:
         return "All available benchmarks related to SYCL."
 
-    def suites(self) -> list[str]:
+    def suites(self) -> List[str]:
         return ["Compute Benchmarks", "llama.cpp bench", "SYCL-Bench", "Velocity Bench"]
 
-
 class Minimal(Preset):
-    def description(self):
+    def description(self) -> str:
         return "Short microbenchmarks."
 
-    def suites(self) -> list[str]:
+    def suites(self) -> List[str]:
         return ["Compute Benchmarks"]
 
-
 class Normal(Preset):
-    def description(self):
+    def description(self) -> str:
         return "Comprehensive mix of microbenchmarks and real applications."
 
-    def suites(self) -> list[str]:
-        return ["Compute Benchmarks"]
-
+    def suites(self) -> List[str]:
+        return ["Compute Benchmarks", "llama.cpp bench", "Velocity Bench"]
 
 class Test(Preset):
-    def description(self):
+    def description(self) -> str:
         return "Noop benchmarks for framework testing."
 
-    def suites(self) -> list[str]:
+    def suites(self) -> List[str]:
         return ["Test Suite"]
 
+presets = [Full(), SYCL(), Minimal(), Normal(), Test()]
 
-class Presets(Enum):
-    FULL = Full
-    SYCL = SYCL  # Nightly
-    NORMAL = Normal  # PR
-    MINIMAL = Minimal  # Quick smoke tests
-    TEST = Test
+def preset_get_by_name(name: str) -> Preset:
+    for p in presets:
+        if p.name().upper() == name.upper():
+            return p
+    raise ValueError(f"Preset '{name}' not found.")