No longer use ASV --strict. (SciTools#5496)

Author: trexfeathers

benchmarks/bm_runner.py

@@ -32,8 +32,7 @@
 
 # Common ASV arguments for all run_types except `custom`.
 ASV_HARNESS = (
-    "run {posargs} --attribute rounds=4 --interleave-rounds --strict "
-    "--show-stderr"
+    "run {posargs} --attribute rounds=4 --interleave-rounds --show-stderr"
 )
 
 
@@ -501,14 +500,19 @@ def csperf(
         asv_command = (
             ASV_HARNESS.format(posargs=commit_range) + f" --bench={run_type}"
         )
-        # C/SPerf benchmarks are much bigger than the CI ones:
-        # Don't fail the whole run if memory blows on 1 benchmark.
-        asv_command = asv_command.replace(" --strict", "")
+
         # Only do a single round.
         asv_command = shlex.split(
             re.sub(r"rounds=\d", "rounds=1", asv_command)
         )
-        _subprocess_runner([*asv_command, *args.asv_args], asv=True)
+        try:
+            _subprocess_runner([*asv_command, *args.asv_args], asv=True)
+        except subprocess.CalledProcessError as err:
+            # C/SPerf benchmarks are much bigger than the CI ones:
+            # Don't fail the whole run if memory blows on 1 benchmark.
+            # ASV produces return code of 2 if the run includes crashes.
+            if err.returncode != 2:
+                raise
 
         asv_command = shlex.split(
             f"publish {commit_range} --html-dir={publish_subdir}"

docs/src/whatsnew/latest.rst

@@ -79,6 +79,9 @@ This document explains the changes made to Iris for this release
    working properly. (Main pull request: :pull:`5437`, more detail:
    :pull:`5430`, :pull:`5431`, :pull:`5432`, :pull:`5434`, :pull:`5436`)
 
+#. `@trexfeathers`_ adapted benchmarking to work with ASV ``>=v0.6`` by no
+   longer using the ``--strict`` argument. (:pull:`5496`)
+
 
 .. comment
     Whatsnew author names (@github name) in alphabetical order. Note that,