Merge pull request #2433 from pbalcer/bench-stddev

[benchmarks] add support for stddev

Author: pbalcer

scripts/benchmarks/benches/compute.py

@@ -118,9 +118,9 @@ def run(self, env_vars) -> list[Result]:
         result = self.run_bench(command, env_vars)
         parsed_results = self.parse_output(result)
         ret = []
-        for label, mean, unit in parsed_results:
+        for label, median, stddev, unit in parsed_results:
             extra_label = " CPU count" if parse_unit_type(unit) == "instr" else ""
-            ret.append(Result(label=self.name() + extra_label, value=mean, command=command, env=env_vars, stdout=result, unit=parse_unit_type(unit)))
+            ret.append(Result(label=self.name() + extra_label, value=median, stddev=stddev, command=command, env=env_vars, stdout=result, unit=parse_unit_type(unit)))
         return ret
 
     def parse_output(self, output):
@@ -135,8 +135,11 @@ def parse_output(self, output):
             try:
                 label = data_row[0]
                 mean = float(data_row[1])
+                median = float(data_row[2])
+                # compute benchmarks report stddev as %
+                stddev = mean * (float(data_row[3].strip('%')) / 100.0)
                 unit = data_row[7]
-                results.append((label, mean, unit))
+                results.append((label, median, stddev, unit))
             except (ValueError, IndexError) as e:
                 raise ValueError(f"Error parsing output: {e}")
         if len(results) == 0:

scripts/benchmarks/benches/result.py

@@ -18,12 +18,14 @@ class Result:
     stdout: str
     passed: bool = True
     unit: str = ""
-    # values should not be set by the benchmark
+    # stddev can be optionally set by the benchmark,
+    # if not set, it will be calculated automatically.
+    stddev: float = 0.0
+    # values below should not be set by the benchmark
     name: str = ""
     lower_is_better: bool = True
     git_hash: str = ''
     date: Optional[datetime] = None
-    stddev: float = 0.0
 
 @dataclass_json
 @dataclass

scripts/benchmarks/main.py

@@ -103,7 +103,10 @@ def process_results(results: dict[str, list[Result]]) -> tuple[bool, list[Result
         rlist.sort(key=lambda res: res.value)
         median_index = len(rlist) // 2
         median_result = rlist[median_index]
-        median_result.stddev = stddev
+
+        # only override the stddev if not already set
+        if median_result.stddev == 0.0:
+            median_result.stddev = stddev
 
         processed.append(median_result)
 
@@ -160,7 +163,6 @@ def main(directory, additional_env_vars, save_name, compare_names, filter):
                 if valid:
                     break
             results += processed
-
         except Exception as e:
             if options.exit_on_failure:
                 raise e

scripts/benchmarks/output_html.py

@@ -32,24 +32,32 @@ def create_time_series_chart(benchmarks: list[BenchmarkSeries], github_repo: str
 
     num_benchmarks = len(benchmarks)
     if num_benchmarks == 0:
-        return
+        return []
 
     html_charts = []
 
     for _, benchmark in enumerate(benchmarks):
         fig, ax = plt.subplots(figsize=(10, 4))
 
+        all_values = []
+        all_stddevs = []
+
         for run in benchmark.runs:
             sorted_points = sorted(run.results, key=lambda x: x.date)
             dates = [point.date for point in sorted_points]
             values = [point.value for point in sorted_points]
+            stddevs = [point.stddev for point in sorted_points]
+
+            all_values.extend(values)
+            all_stddevs.extend(stddevs)
 
-            ax.plot_date(dates, values, '-', label=run.name, alpha=0.5)
+            ax.errorbar(dates, values, yerr=stddevs, fmt='-', label=run.name, alpha=0.5)
             scatter = ax.scatter(dates, values, picker=True)
 
             tooltip_labels = [
                 f"Date: {point.date.strftime('%Y-%m-%d %H:%M:%S')}\n"
-                f"Value: {point.value:.2f}\n"
+                f"Value: {point.value:.2f} {benchmark.metadata.unit}\n"
+                f"Stddev: {point.stddev:.2f} {benchmark.metadata.unit}\n"
                 f"Git Hash: {point.git_hash}"
                 for point in sorted_points
             ]
@@ -62,6 +70,13 @@ def create_time_series_chart(benchmarks: list[BenchmarkSeries], github_repo: str
                 targets=targets)
             mpld3.plugins.connect(fig, tooltip)
 
+        # This is so that the stddev doesn't fill the entire y axis on the chart
+        if all_values and all_stddevs:
+            max_value = max(all_values)
+            min_value = min(all_values)
+            max_stddev = max(all_stddevs)
+            ax.set_ylim(min_value - 3 * max_stddev, max_value + 3 * max_stddev)
+
         ax.set_title(benchmark.label, pad=20)
         performance_indicator = "lower is better" if benchmark.metadata.lower_is_better else "higher is better"
         ax.text(0.5, 1.05, f"({performance_indicator})",
@@ -79,7 +94,7 @@ def create_time_series_chart(benchmarks: list[BenchmarkSeries], github_repo: str
         ax.xaxis.set_major_formatter(mdates.ConciseDateFormatter('%Y-%m-%d %H:%M:%S'))
 
         plt.tight_layout()
-        html_charts.append(BenchmarkTimeSeries(html= mpld3.fig_to_html(fig), label= benchmark.label))
+        html_charts.append(BenchmarkTimeSeries(html=mpld3.fig_to_html(fig), label=benchmark.label))
         plt.close(fig)
 
     return html_charts