Skip to content

Commit 5c53e77

Browse files
PatKaminlslusarczykCopilot
authored
[Benchmarks] Metadata generator (#20294)
Generate Compute Benchmarks scenarios' metadata automatically based on `Benchmark.explicit_group`. --------- Co-authored-by: Łukasz Ślusarczyk <[email protected]> Co-authored-by: Copilot <[email protected]>
1 parent 0a02b82 commit 5c53e77

File tree

3 files changed

+167
-55
lines changed

3 files changed

+167
-55
lines changed

‎devops/scripts/benchmarks/benches/compute.py‎

Lines changed: 13 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
from utils.result import BenchmarkMetadata, Result
1717

1818
from .base import Benchmark, Suite, TracingType
19+
from .compute_metadata import ComputeMetadataGenerator
1920

2021

2122
class RUNTIMES(Enum):
@@ -99,61 +100,20 @@ def setup(self) -> None:
99100
self.project.build(add_sycl=True)
100101

101102
def additional_metadata(self) -> dict[str, BenchmarkMetadata]:
102-
metadata = {
103-
"SinKernelGraph": BenchmarkMetadata(
104-
type="group",
105-
unstable="This benchmark combines both eager and graph execution, and may not be representative of real use cases.",
106-
tags=["submit", "memory", "proxy", "SYCL", "UR", "L0", "graph"],
107-
),
108-
"FinalizeGraph": BenchmarkMetadata(
109-
type="group", tags=["finalize", "micro", "SYCL", "graph"]
110-
),
111-
}
112-
113-
# Add metadata for all SubmitKernel group variants
114-
submit_kernel_metadata = BenchmarkMetadata(
115-
type="group",
116-
notes="Each layer builds on top of the previous layer, adding functionality and overhead.\n"
117-
"The first layer is the Level Zero API, the second is the Unified Runtime API, and the third is the SYCL API.\n"
118-
"The UR v2 adapter noticeably reduces UR layer overhead, also improving SYCL performance.\n"
119-
"Work is ongoing to reduce the overhead of the SYCL API\n",
120-
tags=["submit", "micro", "SYCL", "UR", "L0"],
121-
range_min=0.0,
122-
)
123-
for order in ["in order", "out of order"]:
124-
for completion in ["", " with completion"]:
125-
for events in ["", " using events"]:
126-
group_name = f"SubmitKernel {order}{completion}{events} long kernel"
127-
metadata[group_name] = copy.deepcopy(submit_kernel_metadata)
128-
metadata[group_name].description = (
129-
f"Measures CPU time overhead of submitting {order} kernels with longer execution times through different APIs."
130-
)
131-
# CPU count variants
132-
cpu_count_group = f"{group_name}, CPU count"
133-
metadata[cpu_count_group] = copy.deepcopy(submit_kernel_metadata)
134-
metadata[cpu_count_group].description = (
135-
f"Measures CPU instruction count overhead of submitting {order} kernels with longer execution times through different APIs."
136-
)
137-
138-
# Add metadata for all SubmitGraph group variants
139-
submit_graph_metadata = BenchmarkMetadata(
140-
type="group", tags=["submit", "micro", "SYCL", "UR", "L0", "graph"]
141-
)
142-
for order in ["in order", "out of order"]:
143-
for completion in ["", " with completion"]:
144-
for events in ["", " using events"]:
145-
for num_kernels in self.submit_graph_num_kernels:
146-
for host_tasks in ["", " use host tasks"]:
147-
group_name = f"SubmitGraph {order}{completion}{events}{host_tasks}, {num_kernels} kernels"
148-
metadata[group_name] = copy.deepcopy(submit_graph_metadata)
149-
# CPU count variants
150-
cpu_count_group = f"{group_name}, CPU count"
151-
metadata[cpu_count_group] = copy.deepcopy(
152-
submit_graph_metadata
153-
)
154-
return metadata
103+
"""
104+
Returns:
105+
Dictionary mapping group names to their metadata
106+
"""
107+
# Generate metadata based on actual benchmark instances
108+
generator = ComputeMetadataGenerator()
109+
benchmarks = self.benchmarks()
110+
return generator.generate_metadata_from_benchmarks(benchmarks)
155111

156112
def benchmarks(self) -> list[Benchmark]:
113+
"""
114+
Returns:
115+
List of all possible benchmark instances
116+
"""
157117
benches = []
158118

159119
# hand-picked value so that total execution time of the benchmark is

‎devops/scripts/benchmarks/benches/compute_metadata.py‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
# Copyright (C) 2025 Intel Corporation
2+
# Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions.
3+
# See LICENSE.TXT
4+
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5+
6+
"""
7+
Metadata generator for Compute Benchmarks.
8+
9+
This module provides centralized metadata generation for Compute Benchmark groups,
10+
ensuring consistency between benchmark group membership and group metadata definitions.
11+
"""
12+
13+
from collections import namedtuple
14+
from typing import Dict, List
15+
16+
from utils.result import BenchmarkMetadata
17+
18+
from .base import Benchmark
19+
20+
21+
def string_consts(cls):
22+
"""Decorator to convert string-annotated class attributes to string constants."""
23+
for key, value in cls.__annotations__.items():
24+
if value is str:
25+
setattr(cls, key, key)
26+
return cls
27+
28+
29+
@string_consts
30+
class Tags:
31+
"""String constants for benchmark tags to prevent typos."""
32+
33+
submit: str
34+
micro: str
35+
SYCL: str
36+
UR: str
37+
L0: str
38+
graph: str
39+
memory: str
40+
proxy: str
41+
finalize: str
42+
43+
44+
BaseGroupMetadata = namedtuple(
45+
"BaseGroupMetadata",
46+
[
47+
"description",
48+
"notes",
49+
"unstable",
50+
"tags",
51+
"range_min",
52+
"range_max",
53+
],
54+
defaults=(None, None, None, [], None, None),
55+
)
56+
57+
58+
class ComputeMetadataGenerator:
59+
"""
60+
Generates metadata for Compute Benchmark groups.
61+
62+
This class keeps the logic for creating group metadata, ensuring that
63+
all possible benchmark group configurations have corresponding metadata entries.
64+
"""
65+
66+
def __init__(self):
67+
# Base metadata for core groups
68+
self._base_group_metadata = {
69+
"SubmitKernel": BaseGroupMetadata(
70+
description="Measures CPU time overhead of submitting kernels through different APIs.",
71+
notes=(
72+
"Each layer builds on top of the previous layer, adding functionality and overhead.\n"
73+
"The first layer is the Level Zero API, the second is the Unified Runtime API, and the third is the SYCL API.\n"
74+
"The UR v2 adapter noticeably reduces UR layer overhead, also improving SYCL performance.\n"
75+
"Work is ongoing to reduce the overhead of the SYCL API\n"
76+
),
77+
tags=[Tags.submit, Tags.micro, Tags.SYCL, Tags.UR, Tags.L0],
78+
range_min=0.0,
79+
),
80+
"SinKernelGraph": BaseGroupMetadata(
81+
unstable="This benchmark combines both eager and graph execution, and may not be representative of real use cases.",
82+
tags=[
83+
Tags.submit,
84+
Tags.memory,
85+
Tags.proxy,
86+
Tags.SYCL,
87+
Tags.UR,
88+
Tags.L0,
89+
Tags.graph,
90+
],
91+
),
92+
"SubmitGraph": BaseGroupMetadata(
93+
tags=[Tags.submit, Tags.micro, Tags.SYCL, Tags.UR, Tags.L0, Tags.graph]
94+
),
95+
"FinalizeGraph": BaseGroupMetadata(
96+
tags=[Tags.finalize, Tags.micro, Tags.SYCL, Tags.graph]
97+
),
98+
}
99+
100+
def generate_metadata_from_benchmarks(
101+
self, benchmarks: List[Benchmark]
102+
) -> Dict[str, BenchmarkMetadata]:
103+
"""
104+
Generate group metadata based on actual benchmark configurations.
105+
106+
Args:
107+
benchmarks: List of benchmark instances to analyze
108+
109+
Returns:
110+
Dictionary mapping group names to their metadata
111+
"""
112+
metadata = {}
113+
# Discover all group names from actual benchmarks
114+
for benchmark in benchmarks:
115+
group_name = benchmark.explicit_group()
116+
if group_name and group_name not in metadata:
117+
metadata[group_name] = self._generate_metadata(group_name)
118+
119+
return metadata
120+
121+
def _generate_metadata(self, group_name: str) -> BenchmarkMetadata:
122+
"""
123+
Generate metadata for a specific benchmark group.
124+
125+
Args:
126+
group_name: Name of the benchmark group
127+
128+
Returns:
129+
BenchmarkMetadata: Metadata object describing the specified benchmark group.
130+
"""
131+
base_group_name = self._extract_base_group_name(group_name)
132+
base_metadata = self._base_group_metadata.get(
133+
base_group_name, BaseGroupMetadata()
134+
)
135+
return BenchmarkMetadata(
136+
type="group",
137+
description=base_metadata.description,
138+
notes=base_metadata.notes,
139+
unstable=base_metadata.unstable,
140+
tags=base_metadata.tags,
141+
range_min=base_metadata.range_min,
142+
range_max=base_metadata.range_max,
143+
explicit_group=group_name,
144+
)
145+
146+
def _extract_base_group_name(self, group_name: str) -> str:
147+
"""
148+
Extracts the base group name from a group name string.
149+
Assumes group names are in the format 'BaseGroupName [Variant]'.
150+
If the format changes, this method should be updated accordingly.
151+
"""
152+
return group_name.split()[0]

‎devops/scripts/benchmarks/utils/result.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,8 +75,8 @@ class BenchmarkMetadata:
7575
notes: str = None
7676
unstable: str = None
7777
tags: list[str] = field(default_factory=list)
78-
range_min: float = None
79-
range_max: float = None
78+
range_min: float | None = None
79+
range_max: float | None = None
8080
display_name: str = None
8181
explicit_group: str = None
8282

0 commit comments

Comments
 (0)