First pass at Web UI

Author: mdboom

bench_runner/__main__.py

@@ -1,10 +1,3 @@
-import importlib
-import sys
-
-
-import rich
-
-
 COMMANDS = {
     "backfill": "Schedule benchmarking a number of commits",
     "compare": "Compare a matrix of specific results",
@@ -23,15 +16,20 @@
 }
 
 if __name__ == "__main__":
+    import importlib
+    import sys
+
+    from .util import rich_print
+
     command = len(sys.argv) >= 2 and sys.argv[1] or ""
 
     if command not in COMMANDS:
         command_length = max(len(k) for k in COMMANDS.keys())
-        rich.print(f"Unknown command '{command}'.", file=sys.stderr)
-        rich.print(file=sys.stderr)
-        rich.print("Valid commands are:", file=sys.stderr)
+        rich_print(f"Unknown command '{command}'.", file=sys.stderr)
+        rich_print(file=sys.stderr)
+        rich_print("Valid commands are:", file=sys.stderr)
         for k, v in COMMANDS.items():
-            rich.print(f"  [blue]{k:{command_length}}[/blue]: {v}")
+            rich_print(f"  [blue]{k:{command_length}}[/blue]: {v}")
         sys.exit(1)
 
     sys.argv = [sys.argv[0], *sys.argv[2:]]

bench_runner/config.py

@@ -15,5 +15,17 @@
 def get_bench_runner_config(
     filepath: Path | str = Path("bench_runner.toml"),
 ):
-    with Path(filepath).open("rb") as fd:
-        return tomllib.load(fd)
+    filepath = Path(filepath)
+    if filepath.is_file():
+        with filepath.open("rb") as fd:
+            return tomllib.load(fd)
+    else:
+        try:
+            from pyodide.http import pyfetch  # type: ignore[import]
+        except ImportError:
+            raise RuntimeError("bench_runner.toml not found")
+        toml_content = pyfetch(str(filepath)).string()
+        if toml_content:
+            return tomllib.loads(toml_content)
+        else:
+            raise RuntimeError("bench_runner.toml not found")

bench_runner/git.py

@@ -7,10 +7,8 @@
 import subprocess
 
 
-import rich
-
-
 from .util import PathLike
+from .util import rich_print
 
 
 def get_log(
@@ -98,7 +96,7 @@ def get_git_merge_base(dirname: PathLike) -> str | None:
             encoding="utf-8",
         ).strip()
     except subprocess.CalledProcessError:
-        rich.print("[red]Failed to get merge base[/red]")
+        rich_print("[red]Failed to get merge base[/red]")
         return None
 
     if merge_base == commit_hash:

bench_runner/plot.py

@@ -8,14 +8,17 @@
 from pathlib import Path
 import re
 import tempfile
-from typing import Callable, Iterable, Sequence
+from typing import Callable, Iterable, Sequence, TextIO
 
 
 from matplotlib import pyplot as plt
 import matplotlib
 import numpy as np
-import rich_argparse
-from scour import scour
+
+try:
+    from scour import scour
+except ImportError:
+    scour = None
 
 
 matplotlib.use("agg")
@@ -49,7 +52,7 @@
 }
 
 
-def savefig(output_filename: PathLike, **kwargs):
+def savefig(output: PathLike | TextIO, **kwargs):
     class Options:
         quiet = True
         remove_descriptive_elements = True
@@ -59,19 +62,19 @@ class Options:
         shorten_ids = True
         digits = 3
 
-    output_filename = Path(output_filename)
-
-    plt.savefig(output_filename, **kwargs)
+    plt.savefig(output, format="svg", **kwargs)
     plt.close("all")
 
-    if output_filename.suffix == ".svg":
-        with tempfile.NamedTemporaryFile(
-            dir=output_filename.parent, delete=False
-        ) as tmp:
-            with open(output_filename) as fd:
-                scour.start(Options(), fd, tmp)
-            output_filename.unlink()
-            Path(tmp.name).rename(output_filename)
+    if isinstance(output, (str, Path)):
+        output = Path(output)
+        if output.suffix == ".svg" and scour:
+            with tempfile.NamedTemporaryFile(
+                dir=output_filename.parent, delete=False
+            ) as tmp:
+                with open(output) as fd:
+                    scour.start(Options(), fd, tmp)
+                output.unlink()
+                Path(tmp.name).rename(output)
 
 
 @functools.cache
@@ -478,6 +481,8 @@ def get_comparison_value(ref, r):
 
 
 if __name__ == "__main__":
+    import rich_argparse
+
     parser = argparse.ArgumentParser(
         "Compare two benchmark .json files",
         formatter_class=rich_argparse.ArgumentDefaultsRichHelpFormatter,

bench_runner/result.py

@@ -17,9 +17,6 @@
 
 
 import numpy as np
-from packaging import version
-import pyperf
-import rich.progress
 
 
 from . import bases as mbases
@@ -30,7 +27,7 @@
 from . import plot
 from . import runners
 from . import util
-from .util import PathLike
+from .util import PathLike, rich_track
 
 
 CombinedData = list[tuple[str, np.ndarray | None, float]]
@@ -190,14 +187,18 @@ def write_table(self, filename: PathLike) -> str | None:
     def _get_combined_data(
         self, ref_data: dict[str, np.ndarray], head_data: dict[str, np.ndarray]
     ) -> CombinedData:
+        import pyperf
+
         def remove_outliers(values, m=2):
             return values[
                 abs(values - np.mean(values)) < np.multiply(m, np.std(values))
             ]
 
         def calculate_diffs(ref_values, head_values) -> tuple[np.ndarray | None, float]:
             if len(ref_values) > 3 and len(head_values) > 3:
-                sig, t_score = pyperf._utils.is_significant(ref_values, head_values)
+                sig, t_score = pyperf._utils.is_significant(  # pyright: ignore
+                    ref_values, head_values
+                )
                 if not sig:
                     return None, 0.0
                 else:
@@ -461,6 +462,7 @@ def __init__(
         self._commit_datetime = commit_datetime
         self._filename = None
         self.bases = {}
+        self._contents: None | dict[str, Any] = None
 
     @classmethod
     def from_filename(cls, filename: PathLike) -> "Result":
@@ -496,6 +498,22 @@ def from_filename(cls, filename: PathLike) -> "Result":
         obj._filename = filename
         return obj
 
+    @classmethod
+    def from_online_json(cls, url: str, json_content: str | dict) -> "Result":
+        from urllib.parse import urlparse
+
+        parsed_url = urlparse(url)
+        path = "/".join(parsed_url.path.split("/")[-2:])
+
+        result = cls.from_filename(Path(unquote(path)))
+        result._filename = None
+        if isinstance(json_content, str):
+            result._contents = json.loads(json_content)
+        else:
+            result._contents = json_content
+
+        return result
+
     @classmethod
     def from_arbitrary_filename(cls, filename: PathLike) -> "Result":
         filename = Path(filename)
@@ -600,10 +618,14 @@ def result_info(self) -> tuple[str | None, str | None, str | None]:
             f"Unknown result type (extra={self.extra} suffix={self.suffix})"
         )
 
-    @functools.cached_property
+    @property
     def contents(self) -> dict[str, Any]:
+        if self._contents is not None:
+            return self._contents
         with self.filename.open("rb") as fd:
-            return json.load(fd)
+            self._contents = json.load(fd)
+        assert self._contents is not None
+        return self._contents
 
     @property
     def metadata(self) -> dict[str, Any]:
@@ -705,6 +727,8 @@ def get_timing_data(self) -> dict[str, np.ndarray]:
         return data
 
     def get_memory_data(self) -> dict[str, np.ndarray]:
+        from packaging import version
+
         data = {}
         excluded = util.get_excluded_benchmarks()
 
@@ -800,7 +824,7 @@ def find_match(result, candidates, base, func):
         bases = []
 
     if progress:
-        track = rich.progress.track  # type: ignore
+        track = rich_track  # type: ignore
     else:
 
         def track(it, *_args, **_kwargs):

bench_runner/scripts/generate_results.py

@@ -5,6 +5,7 @@
 from collections import defaultdict
 import datetime
 import io
+import json
 from pathlib import Path
 import sys
 from typing import Iterable, TextIO, Sequence
@@ -17,6 +18,7 @@
 
 
 from bench_runner.bases import get_bases
+from bench_runner import config as mconfig
 from bench_runner import flags as mflags
 from bench_runner import plot
 from bench_runner.result import (
@@ -371,6 +373,51 @@ def filter_broken_memory_results(results):
     return [r for r in results if r.nickname != "darwin"]
 
 
+def generate_webui_index(bases: Iterable[str], results: Iterable[Result]) -> None:
+    """
+    Generate a machine readable index of all the available results.
+    """
+
+    def make_path(result):
+        return "/".join([result.filename.parent.name, result.filename.name])
+
+    row_data = []
+    index = {}
+    base_results = []
+    for i, result in enumerate(results):
+        row_data.append(
+            [
+                result.commit_date,
+                result.cpython_hash,
+                unquote(result.fork),
+                unquote(result.ref),
+                result.version,
+                result.runner,
+                ",".join(result.flags),
+            ]
+        )
+        index[make_path(result)] = i
+
+    for result in results:
+        if "base" in result.bases:
+            base = result.bases["base"].ref
+            base_results.append(index[make_path(base)])
+        else:
+            base_results.append(None)
+
+    content = {
+        "rowData": row_data,
+        "_index": list(index.keys()),
+        "_base_results": base_results,
+        "_bases": bases,
+    }
+
+    with Path("pages/index.json").open("w") as fd:
+        # By using indent=0, git can diff the file much better than if
+        # it were on a single line
+        json.dump(content, fd, indent=0)
+
+
 def _main(repo_dir: PathLike, force: bool = False, bases: Sequence[str] | None = None):
     repo_dir = Path(repo_dir)
     results_dir = repo_dir / "results"
@@ -385,6 +432,8 @@ def _main(repo_dir: PathLike, force: bool = False, bases: Sequence[str] | None =
     benchmarking_results = [r for r in results if r.result_info[0] == "raw results"]
     generate_indices(bases, results, benchmarking_results, repo_dir)
     generate_directory_indices(benchmarking_results)
+    if mconfig.get_bench_runner_config().get("webui", False):
+        generate_webui_index(bases, benchmarking_results)
 
     memory_benchmarking_results = filter_broken_memory_results(benchmarking_results)