Fix error_on_regression only using last check (#2390)

This commit ensures that error_on_regression checker checks every one of the checks specified in the config file. Prior to this commit it would only check the last one.

The commit also increases the threshold that benchcomp uses to report a regression. Benchmarks that take less than 10 seconds (previously 2 seconds) are now ignored, and a 50% slowdown (previously 10%) is considered a regression.

Author: karkhaz

tools/benchcomp/benchcomp/visualizers/__init__.py

@@ -49,8 +49,8 @@ def __call__(self, results):
             any_benchmark_regressed = viz_utils.AnyBenchmarkRegressedChecker(
                     self.variant_pairs, **check)
 
-        if any_benchmark_regressed(results):
-            viz_utils.EXIT_CODE = 1
+            if any_benchmark_regressed(results):
+                viz_utils.EXIT_CODE = 1
 
 
 

tools/benchcomp/configs/perf-regression.yaml

@@ -36,6 +36,6 @@ visualize:
         # benchmark has regressed if the lambda returns true.
         test: "lambda old, new: False if not old else not new"
       - metric: solver_runtime
-        test: "lambda old, new: False if new < 2 else new/old > 1.1"
+        test: "lambda old, new: False if new < 10 else new/old > 1.5"
       - metric: symex_runtime
-        test: "lambda old, new: False if new < 2 else new/old > 1.1"
+        test: "lambda old, new: False if new < 10 else new/old > 1.5"