Merge pull request #445 from mdboom/fix-bisect

Fix the bisect command to not cache results

Author: mdboom

bench_runner/result.py

@@ -40,6 +40,10 @@ def _load_contents(filename: Path) -> dict[str, Any]:
         return json.load(fd)
 
 
+def clear_contents_cache() -> None:
+    _load_contents.cache_clear()
+
+
 def _clean(string: str) -> str:
     """
     Clean an arbitrary string to be suitable for a filename.

bench_runner/scripts/bisect.py

@@ -45,6 +45,7 @@ def parse_result(benchmark_json: PathLike) -> float:
     # The name of the benchmark in the JSON file may be different from the one
     # used to select the benchmark. Therefore, just take the mean of all the
     # benchmarks in the JSON file.
+    result.clear_contents_cache()
     r = result.Result.from_arbitrary_filename(benchmark_json)
     timing_data = r.get_timing_data()
     return float(np.mean([x.mean() for x in timing_data.values()]))
@@ -123,6 +124,12 @@ def _main(
     bad_timing = get_result(benchmark, pgo, flags, cpython=cpython)
     log(f"KNOWN  BAD  ({bad[:7]}): {format_seconds(bad_timing)}")
 
+    if good_timing >= bad_timing:
+        show_log()
+        raise ValueError(
+            f"Good timing ({good_timing}) must be less than bad timing ({bad_timing})."
+        )
+
     try:
         with contextlib.chdir(cpython):
             subprocess.run(["git", "bisect", "start"])