diff --git a/devops/scripts/benchmarks/benches/base.py b/devops/scripts/benchmarks/benches/base.py index ffa177c736a04..5aa7bf754824a 100644 --- a/devops/scripts/benchmarks/benches/base.py +++ b/devops/scripts/benchmarks/benches/base.py @@ -45,6 +45,12 @@ def __init__(self, directory, suite): def name(self) -> str: pass + def display_name(self) -> str: + """Returns a user-friendly name for display in charts. + By default returns the same as name(), but can be overridden. + """ + return self.name() + @abstractmethod def setup(self): pass @@ -145,18 +151,21 @@ def get_tags(self) -> list[str]: def range(self) -> tuple[float, float]: return None - def get_metadata(self) -> BenchmarkMetadata: + def get_metadata(self) -> dict[str, BenchmarkMetadata]: range = self.range() - return BenchmarkMetadata( - type="benchmark", - description=self.description(), - notes=self.notes(), - unstable=self.unstable(), - tags=self.get_tags(), - range_min=range[0] if range else None, - range_max=range[1] if range else None, - ) + return { + self.name(): BenchmarkMetadata( + type="benchmark", + description=self.description(), + notes=self.notes(), + unstable=self.unstable(), + tags=self.get_tags(), + range_min=range[0] if range else None, + range_max=range[1] if range else None, + display_name=self.display_name(), + ) + } class Suite(ABC): diff --git a/devops/scripts/benchmarks/benches/compute.py b/devops/scripts/benchmarks/benches/compute.py index 3181c0e51eaf1..ad01acde61c95 100644 --- a/devops/scripts/benchmarks/benches/compute.py +++ b/devops/scripts/benchmarks/benches/compute.py @@ -6,6 +6,7 @@ import os import csv import io +import copy from utils.utils import run, git_clone, create_build_path from .base import Benchmark, Suite from utils.result import BenchmarkMetadata, Result @@ -317,6 +318,7 @@ def __init__(self, bench, runtime: RUNTIMES, ioq, MeasureCompletion=0, UseEvents self.runtime = runtime self.MeasureCompletion = MeasureCompletion self.UseEvents = UseEvents + self.NumKernels = 10 super().__init__( bench, f"api_overhead_benchmark_{runtime.value}", "SubmitKernel" ) @@ -334,6 +336,16 @@ def name(self): return f"api_overhead_benchmark_{self.runtime.value} SubmitKernel {order}{completion_str}{events_str}" + def display_name(self) -> str: + order = "in order" if self.ioq else "out of order" + info = [] + if self.MeasureCompletion: + info.append("with measure completion") + if self.UseEvents: + info.append("using events") + additional_info = f" {' '.join(info)}" if info else "" + return f"{self.runtime.value.upper()} SubmitKernel {order}{additional_info}, NumKernels {self.NumKernels}" + def explicit_group(self): order = "In Order" if self.ioq else "Out Of Order" completion_str = " With Completion" if self.MeasureCompletion else "" @@ -354,7 +366,7 @@ def description(self) -> str: return ( f"Measures CPU time overhead of submitting {order} kernels through {runtime_name} API{completion_desc}. " - f"Runs 10 simple kernels with minimal execution time to isolate API overhead from kernel execution time." + f"Runs {self.NumKernels} simple kernels with minimal execution time to isolate API overhead from kernel execution time." ) def range(self) -> tuple[float, float]: @@ -366,11 +378,23 @@ def bin_args(self) -> list[str]: f"--MeasureCompletion={self.MeasureCompletion}", "--iterations=100000", "--Profiling=0", - "--NumKernels=10", + f"--NumKernels={self.NumKernels}", "--KernelExecTime=1", f"--UseEvents={self.UseEvents}", ] + def get_metadata(self) -> dict[str, BenchmarkMetadata]: + metadata_dict = super().get_metadata() + + # Create CPU count variant with modified display name + cpu_count_name = self.name() + " CPU count" + cpu_count_metadata = copy.deepcopy(metadata_dict[self.name()]) + cpu_count_display_name = self.display_name() + ", CPU count" + cpu_count_metadata.display_name = cpu_count_display_name + metadata_dict[cpu_count_name] = cpu_count_metadata + + return metadata_dict + class ExecImmediateCopyQueue(ComputeBenchmark): def __init__(self, bench, ioq, isCopyOnly, source, destination, size): @@ -385,6 +409,10 @@ def name(self): order = "in order" if self.ioq else "out of order" return f"api_overhead_benchmark_sycl ExecImmediateCopyQueue {order} from {self.source} to {self.destination}, size {self.size}" + def display_name(self) -> str: + order = "in order" if self.ioq else "out of order" + return f"SYCL ExecImmediateCopyQueue {order} from {self.source} to {self.destination}, size {self.size}" + def description(self) -> str: order = "in-order" if self.ioq else "out-of-order" operation = "copy-only" if self.isCopyOnly else "copy and command submission" @@ -419,6 +447,9 @@ def __init__(self, bench, isCopyOnly, source, destination, size): def name(self): return f"memory_benchmark_sycl QueueInOrderMemcpy from {self.source} to {self.destination}, size {self.size}" + def display_name(self) -> str: + return f"SYCL QueueInOrderMemcpy from {self.source} to {self.destination}, size {self.size}" + def description(self) -> str: operation = "copy-only" if self.isCopyOnly else "copy and command submission" return ( @@ -450,6 +481,9 @@ def __init__(self, bench, source, destination, size): def name(self): return f"memory_benchmark_sycl QueueMemcpy from {self.source} to {self.destination}, size {self.size}" + def display_name(self) -> str: + return f"SYCL QueueMemcpy from {self.source} to {self.destination}, size {self.size}" + def description(self) -> str: return ( f"Measures general SYCL queue memory copy performance from {self.source} to " @@ -478,6 +512,9 @@ def __init__(self, bench, type, size, placement): def name(self): return f"memory_benchmark_sycl StreamMemory, placement {self.placement}, type {self.type}, size {self.size}" + def display_name(self) -> str: + return f"SYCL StreamMemory, placement {self.placement}, type {self.type}, size {self.size}" + def description(self) -> str: return ( f"Measures {self.placement} memory bandwidth using {self.type} pattern with " @@ -511,6 +548,9 @@ def __init__(self, bench): def name(self): return f"miscellaneous_benchmark_sycl VectorSum" + def display_name(self) -> str: + return f"SYCL VectorSum" + def description(self) -> str: return ( "Measures performance of vector addition across 3D grid (512x256x256 elements) " @@ -565,6 +605,19 @@ def name(self): + (" without copy offload" if not self.useCopyOffload else "") ) + def display_name(self) -> str: + info = [] + if not self.useEvents: + info.append("without events") + if not self.useCopyOffload: + info.append("without copy offload") + additional_info = f", {' '.join(info)}" if info else "" + return ( + f"UR MemcpyExecute, opsPerThread {self.numOpsPerThread}, " + f"numThreads {self.numThreads}, allocSize {self.allocSize}, srcUSM {self.srcUSM}, " + f"dstUSM {self.dstUSM}{additional_info}" + ) + def explicit_group(self): return ( "MemcpyExecute opsPerThread: " @@ -624,6 +677,9 @@ def description(self) -> str: def name(self): return f"graph_api_benchmark_{self.runtime.value} SinKernelGraph graphs:{self.withGraphs}, numKernels:{self.numKernels}" + def display_name(self) -> str: + return f"{self.runtime.value.upper()} SinKernelGraph, graphs {self.withGraphs}, numKernels {self.numKernels}" + def unstable(self) -> str: return "This benchmark combines both eager and graph execution, and may not be representative of real use cases." @@ -672,6 +728,9 @@ def description(self) -> str: def name(self): return f"graph_api_benchmark_{self.runtime.value} SubmitGraph numKernels:{self.numKernels} ioq {self.inOrderQueue} measureCompletion {self.measureCompletionTime}" + def display_name(self) -> str: + return f"{self.runtime.value.upper()} SubmitGraph, numKernels {self.numKernels}, ioq {self.inOrderQueue}, measureCompletion {self.measureCompletionTime}" + def get_tags(self): return [ "graph", @@ -710,6 +769,11 @@ def description(self) -> str: def name(self): return f"ulls_benchmark_{self.runtime.value} EmptyKernel wgc:{self.wgc}, wgs:{self.wgs}" + def display_name(self) -> str: + return ( + f"{self.runtime.value.upper()} EmptyKernel, wgc {self.wgc}, wgs {self.wgs}" + ) + def get_tags(self): return [runtime_to_tag_name(self.runtime), "micro", "latency", "submit"] @@ -751,6 +815,9 @@ def description(self) -> str: def name(self): return f"ulls_benchmark_{self.runtime.value} KernelSwitch count {self.count} kernelTime {self.kernelTime}" + def display_name(self) -> str: + return f"{self.runtime.value.upper()} KernelSwitch, count {self.count}, kernelTime {self.kernelTime}" + def get_tags(self): return [runtime_to_tag_name(self.runtime), "micro", "latency", "submit"] @@ -787,6 +854,12 @@ def name(self): f"usmMemoryPlacement:{self.usm_memory_placement} size:{self.size} measureMode:{self.measure_mode}" ) + def display_name(self) -> str: + return ( + f"{self.runtime.value.upper()} UsmMemoryAllocation, " + f"usmMemoryPlacement {self.usm_memory_placement}, size {self.size}, measureMode {self.measure_mode}" + ) + def explicit_group(self): return f"UsmMemoryAllocation" @@ -839,6 +912,12 @@ def name(self): f"usmMemoryPlacement:{self.usm_memory_placement} allocationCount:{self.allocation_count} size:{self.size} measureMode:{self.measure_mode}" ) + def display_name(self) -> str: + return ( + f"{self.runtime.value.upper()} UsmBatchMemoryAllocation, " + f"usmMemoryPlacement {self.usm_memory_placement}, allocationCount {self.allocation_count}, size {self.size}, measureMode {self.measure_mode}" + ) + def explicit_group(self): return f"UsmBatchMemoryAllocation" diff --git a/devops/scripts/benchmarks/html/scripts.js b/devops/scripts/benchmarks/html/scripts.js index 38c5eab6a12dd..18ade174bc4e5 100644 --- a/devops/scripts/benchmarks/html/scripts.js +++ b/devops/scripts/benchmarks/html/scripts.js @@ -78,7 +78,7 @@ function createChart(data, containerId, type) { plugins: { title: { display: true, - text: data.label + text: data.display_label || data.label }, subtitle: { display: true, @@ -148,7 +148,14 @@ function createChart(data, containerId, type) { const chartConfig = { type: type === 'time' ? 'line' : 'bar', data: type === 'time' ? { - datasets: Object.values(data.runs) + datasets: Object.values(data.runs).map(runData => ({ + ...runData, + // For timeseries (historical results charts) use runName, + // otherwise use displayLabel (for layer comparison charts) + label: containerId.startsWith('timeseries') ? + runData.runName : + (runData.displayLabel || runData.label) + })) } : { labels: data.labels, datasets: data.datasets @@ -163,9 +170,9 @@ function createChart(data, containerId, type) { function createTimeseriesDatasets(data) { return Object.entries(data.runs).map(([name, runData], index) => ({ - label: name, + label: runData.runName, // Use run name for legend data: runData.points.map(p => ({ - seriesName: name, + seriesName: runData.runName, // Use run name for tooltips x: p.date, y: p.value, gitHash: p.git_hash, @@ -334,12 +341,17 @@ function createChartContainer(data, canvasId, type) { } function metadataForLabel(label, type) { + // First try exact match + if (benchmarkMetadata[label]?.type === type) { + return benchmarkMetadata[label]; + } + + // Then fall back to prefix match for backward compatibility for (const [key, metadata] of Object.entries(benchmarkMetadata)) { if (metadata.type === type && label.startsWith(key)) { return metadata; } } - return null; } @@ -372,24 +384,40 @@ function extractLabels(data) { // For bar charts if (data.datasets) { - return data.datasets.map(dataset => dataset.label); + // Use the unique lookupLabel for filtering and lookup purposes + return data.datasets.map(dataset => dataset.lookupLabel || dataset.label); } // For time series charts return [data.label]; } -function generateExtraInfo(latestRunsLookup, data) { +function getDisplayLabel(label, data, metadata) { + if (data.datasets) { + // For bar charts, find the corresponding dataset and use its display label + const dataset = data.datasets.find(d => (d.lookupLabel || d.label) === label); + if (dataset) { + return dataset.label; + } + } else if (metadata && metadata.display_name) { + // For other chart types + return metadata.display_name; + } + return label; +} + +function generateExtraInfo(latestRunsLookup, data, type = 'benchmark') { const labels = extractLabels(data); return labels.map(label => { - const metadata = metadataForLabel(label, 'benchmark'); + const metadata = metadataForLabel(label, type); const latestRun = latestRunsLookup.get(label); + const displayLabel = getDisplayLabel(label, data, metadata); let html = '
'; if (metadata && latestRun) { - html += `${label}: ${formatCommand(latestRun.result)}
`; + html += `${displayLabel}: ${formatCommand(latestRun.result)}
`; if (metadata.description) { html += `Description: ${metadata.description}`; @@ -403,7 +431,7 @@ function generateExtraInfo(latestRunsLookup, data) { html += `
⚠️ Unstable: ${metadata.unstable}`; } } else { - html += `${label}: No data available`; + html += `${displayLabel}: No data available`; } html += '
'; @@ -549,6 +577,7 @@ function processTimeseriesData(benchmarkRuns) { if (!resultsByLabel[result.label]) { resultsByLabel[result.label] = { label: result.label, + display_label: metadata?.display_name || result.label, suite: result.suite, unit: result.unit, lower_is_better: result.lower_is_better, @@ -577,6 +606,7 @@ function processBarChartsData(benchmarkRuns) { groupedResults[result.explicit_group] = { label: result.explicit_group, + display_label: groupMetadata?.display_name || result.explicit_group, // Use display_name if available suite: result.suite, unit: result.unit, lower_is_better: result.lower_is_better, @@ -597,11 +627,18 @@ function processBarChartsData(benchmarkRuns) { group.labels.push(run.name); } - let dataset = group.datasets.find(d => d.label === result.label); + // Store the label we'll use for lookup and the display label separately + const lookupLabel = result.label; + // First try to get display name from metadata using the actual label + const metadata = benchmarkMetadata[result.label]; + const displayLabel = metadata?.display_name || result.label; + + let dataset = group.datasets.find(d => d.lookupLabel === lookupLabel); if (!dataset) { const datasetIndex = group.datasets.length; dataset = { - label: result.label, + lookupLabel: lookupLabel, // Store the original label for lookup + label: displayLabel, // Use display label for rendering data: new Array(group.labels.length).fill(null), backgroundColor: colorPalette[datasetIndex % colorPalette.length], borderColor: colorPalette[datasetIndex % colorPalette.length], @@ -698,8 +735,11 @@ function addRunDataPoint(group, run, result, name = null) { if (!group.runs[runKey]) { const datasetIndex = Object.keys(group.runs).length; + const metadata = benchmarkMetadata[result.name]; + const displayName = metadata?.display_name || result.label; group.runs[runKey] = { label: runKey, + displayLabel: displayName + ' (' + run.name + ')', // Format for layer comparison charts runName: run.name, data: [], borderColor: colorPalette[datasetIndex % colorPalette.length], @@ -712,7 +752,8 @@ function addRunDataPoint(group, run, result, name = null) { } group.runs[runKey].data.push({ - seriesName: runKey, + // For historical results use only run.name, for layer comparisons use displayLabel + seriesName: name === run.name ? run.name : group.runs[runKey].displayLabel, x: new Date(run.date), y: result.value, stddev: result.stddev, diff --git a/devops/scripts/benchmarks/main.py b/devops/scripts/benchmarks/main.py index d50894d5c6acf..826678ef34740 100755 --- a/devops/scripts/benchmarks/main.py +++ b/devops/scripts/benchmarks/main.py @@ -146,7 +146,8 @@ def collect_metadata(suites): metadata.update(s.additional_metadata()) suite_benchmarks = s.benchmarks() for benchmark in suite_benchmarks: - metadata[benchmark.name()] = benchmark.get_metadata() + results = benchmark.get_metadata() + metadata.update(results) return metadata @@ -309,6 +310,7 @@ def main(directory, additional_env_vars, save_name, compare_names, filter): html_path = options.output_directory if options.output_directory is None: html_path = os.path.join(os.path.dirname(__file__), "html") + generate_html(history.runs, compare_names, html_path, metadata) diff --git a/devops/scripts/benchmarks/utils/result.py b/devops/scripts/benchmarks/utils/result.py index 4d70095d1f267..3d994e3d0a811 100644 --- a/devops/scripts/benchmarks/utils/result.py +++ b/devops/scripts/benchmarks/utils/result.py @@ -62,6 +62,7 @@ class BenchmarkMetadata: tags: list[str] = field(default_factory=list) range_min: float = None range_max: float = None + display_name: str = None @dataclass_json