[Benchmarks] Metadata generator

devops/scripts/benchmarks/benches/compute.py

@@ -16,6 +16,7 @@
 from utils.result import BenchmarkMetadata, Result
 
 from .base import Benchmark, Suite, TracingType
+from .compute_metadata import ComputeMetadataGenerator
 
 
 class RUNTIMES(Enum):
@@ -99,61 +100,20 @@ def setup(self) -> None:
         self.project.build(add_sycl=True)
 
     def additional_metadata(self) -> dict[str, BenchmarkMetadata]:
-        metadata = {
-            "SinKernelGraph": BenchmarkMetadata(
-                type="group",
-                unstable="This benchmark combines both eager and graph execution, and may not be representative of real use cases.",
-                tags=["submit", "memory", "proxy", "SYCL", "UR", "L0", "graph"],
-            ),
-            "FinalizeGraph": BenchmarkMetadata(
-                type="group", tags=["finalize", "micro", "SYCL", "graph"]
-            ),
-        }
-
-        # Add metadata for all SubmitKernel group variants
-        submit_kernel_metadata = BenchmarkMetadata(
-            type="group",
-            notes="Each layer builds on top of the previous layer, adding functionality and overhead.\n"
-            "The first layer is the Level Zero API, the second is the Unified Runtime API, and the third is the SYCL API.\n"
-            "The UR v2 adapter noticeably reduces UR layer overhead, also improving SYCL performance.\n"
-            "Work is ongoing to reduce the overhead of the SYCL API\n",
-            tags=["submit", "micro", "SYCL", "UR", "L0"],
-            range_min=0.0,
-        )
-        for order in ["in order", "out of order"]:
-            for completion in ["", " with completion"]:
-                for events in ["", " using events"]:
-                    group_name = f"SubmitKernel {order}{completion}{events} long kernel"
-                    metadata[group_name] = copy.deepcopy(submit_kernel_metadata)
-                    metadata[group_name].description = (
-                        f"Measures CPU time overhead of submitting {order} kernels with longer execution times through different APIs."
-                    )
-                    # CPU count variants
-                    cpu_count_group = f"{group_name}, CPU count"
-                    metadata[cpu_count_group] = copy.deepcopy(submit_kernel_metadata)
-                    metadata[cpu_count_group].description = (
-                        f"Measures CPU instruction count overhead of submitting {order} kernels with longer execution times through different APIs."
-                    )
-
-        # Add metadata for all SubmitGraph group variants
-        submit_graph_metadata = BenchmarkMetadata(
-            type="group", tags=["submit", "micro", "SYCL", "UR", "L0", "graph"]
-        )
-        for order in ["in order", "out of order"]:
-            for completion in ["", " with completion"]:
-                for events in ["", " using events"]:
-                    for num_kernels in self.submit_graph_num_kernels:
-                        for host_tasks in ["", " use host tasks"]:
-                            group_name = f"SubmitGraph {order}{completion}{events}{host_tasks}, {num_kernels} kernels"
-                            metadata[group_name] = copy.deepcopy(submit_graph_metadata)
-                            # CPU count variants
-                            cpu_count_group = f"{group_name}, CPU count"
-                            metadata[cpu_count_group] = copy.deepcopy(
-                                submit_graph_metadata
-                            )
-        return metadata
+        """
+        Returns:
+            Dictionary mapping group names to their metadata
+        """
+        # Generate metadata based on actual benchmark instances
+        generator = ComputeMetadataGenerator()
+        benchmarks = self.benchmarks()
+        return generator.generate_metadata_from_benchmarks(benchmarks)
 
     def benchmarks(self) -> list[Benchmark]:
+        """
+        Returns:
+            List of all possible benchmark instances
+        """
         benches = []
 
         # hand-picked value so that total execution time of the benchmark is

devops/scripts/benchmarks/benches/compute_metadata.py

@@ -0,0 +1,138 @@
+# Copyright (C) 2025 Intel Corporation
+# Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
+# See LICENSE.TXT
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+"""
+Metadata generator for Compute Benchmarks.
+
+This module provides centralized metadata generation for Compute Benchmark groups,
+ensuring consistency between benchmark group membership and group metadata definitions.
+"""
+
+from collections import namedtuple
+from typing import Dict, List
+
+from utils.result import BenchmarkMetadata
+
+from .base import Benchmark
+
+
+def string_consts(cls):
+    """Decorator to convert string-annotated class attributes to string constants."""
+    for key, value in cls.__annotations__.items():
+        if value is str:
+            setattr(cls, key, key)
+    return cls
+
+
+@string_consts
+class Tags:
+    """String constants for benchmark tags to prevent typos."""
+
+    submit: str
+    micro: str
+    SYCL: str
+    UR: str
+    L0: str
+    graph: str
+    memory: str
+    proxy: str
+    finalize: str
+
+
+BaseGroupMetadata = namedtuple(
+    "BaseGroupMetadata",
+    [
+        "description",
+        "notes",
+        "unstable",
+        "tags",
+        "range_min",
+        "range_max",
+    ],
+    defaults=(None, None, None, [], None, None),
+)
+
+class ComputeMetadataGenerator:
+    """
+    Generates metadata for Compute Benchmark groups.
+
+    This class keeps the logic for creating group metadata, ensuring that
+    all possible benchmark group configurations have corresponding metadata entries.
+    """
+
+    def __init__(self):
+        # Base metadata for core groups
+        self._base_group_metadata = {
+            "SubmitKernel": BaseGroupMetadata(
+                description="Measures CPU time overhead of submitting kernels through different APIs.",
+                notes=(
+                    "Each layer builds on top of the previous layer, adding functionality and overhead.\n"
+                    "The first layer is the Level Zero API, the second is the Unified Runtime API, and the third is the SYCL API.\n"
+                    "The UR v2 adapter noticeably reduces UR layer overhead, also improving SYCL performance.\n"
+                    "Work is ongoing to reduce the overhead of the SYCL API\n"
+                ),
+                tags=[Tags.submit, Tags.micro, Tags.SYCL, Tags.UR, Tags.L0],
+                range_min=0.0,
+            ),
+            "SinKernelGraph": BaseGroupMetadata(
+                unstable="This benchmark combines both eager and graph execution, and may not be representative of real use cases.",
+                tags=[
+                    Tags.submit,
+                    Tags.memory,
+                    Tags.proxy,
+                    Tags.SYCL,
+                    Tags.UR,
+                    Tags.L0,
+                    Tags.graph,
+                ],
+            ),
+            "SubmitGraph": BaseGroupMetadata(
+                tags=[Tags.submit, Tags.micro, Tags.SYCL, Tags.UR, Tags.L0, Tags.graph]
+            ),
+            "FinalizeGraph": BaseGroupMetadata(
+                tags=[Tags.finalize, Tags.micro, Tags.SYCL, Tags.graph]
+            ),
+        }
+
+    def generate_metadata_from_benchmarks(
+        self, benchmarks: List[Benchmark]
+    ) -> Dict[str, BenchmarkMetadata]:
+        """
+        Generate group metadata based on actual benchmark configurations.
+
+        Args:
+            benchmarks: List of benchmark instances to analyze
+
+        Returns:
+            Dictionary mapping group names to their metadata
+        """
+        metadata = {}
+        # Discover all group names from actual benchmarks
+        for benchmark in benchmarks:
+            group_name = benchmark.explicit_group()
+            if group_name and group_name not in metadata:
+                metadata[group_name] = self._generate_metadata(group_name)
+
+        return metadata
+
+    def _generate_metadata(self, group_name: str) -> BenchmarkMetadata:
+        """
+        Generate metadata for a specific benchmark group.
+        Args:
+            group_name: Name of the benchmark group
+        """
+        base_metadata = self._base_group_metadata.get(
+            group_name.split()[0], BaseGroupMetadata()
+        )
+        return BenchmarkMetadata(
+            type="group",
+            description=base_metadata.description,
+            notes=base_metadata.notes,
+            unstable=base_metadata.unstable,
+            tags=base_metadata.tags,
+            range_min=base_metadata.range_min,
+            range_max=base_metadata.range_max,
+            explicit_group=group_name,
+        )

devops/scripts/benchmarks/utils/result.py

@@ -75,8 +75,8 @@ class BenchmarkMetadata:
     notes: str = None
     unstable: str = None
     tags: list[str] = field(default_factory=list)
-    range_min: float = None
-    range_max: float = None
+    range_min: float | None = None
+    range_max: float | None = None
     display_name: str = None
     explicit_group: str = None