Add --only and --exclude to benchcomp visualize (#2409)

These flags allow users to select which visualizations to run---a subset
of those in the configuration file. Typically this would be used when
the `error_on_regression` is used in CI, or any other visualization that
terminates benchcomp with a non-zero return code.

The idea is that users can first run all visualizations, except for ones
that cause benchcomp to terminate, using `--except error_on_regression`.
Users can then save any output files from visualizations, before running
`benchcomp visualize --only error_on_regression`.

Author: karkhaz

tools/benchcomp/benchcomp/cmd_args.py

@@ -174,6 +174,19 @@ def _get_args_dict():
                         "type": pathlib.Path,
                         "help":
                             "read result from F instead of %(default)s. "
+                    }, {
+                        "flags": ["--only"],
+                        "nargs": "+",
+                        "metavar": "V",
+                        "help":
+                            "Only run visualization V; ignore others in "
+                            "config file"
+                    }, {
+                        "flags": ["--except"],
+                        "dest": "except_for",
+                        "nargs": "+",
+                        "metavar": "V",
+                        "help": "Run all visualizations except V",
                     }],
                 },
             }

tools/benchcomp/benchcomp/entry/visualize.py

@@ -14,6 +14,7 @@ def main(args):
     with open(args.result_file, encoding="utf-8") as handle:
         results = yaml.safe_load(handle)
 
-    generate_visualizations = benchcomp.visualizers.utils.Generator(args.config)
+    generate_visualizations = benchcomp.visualizers.utils.Generator(
+        args.config, args.except_for, args.only)
     generate_visualizations(results)
     sys.exit(benchcomp.visualizers.utils.EXIT_CODE)

tools/benchcomp/benchcomp/visualizers/utils.py

@@ -96,10 +96,26 @@ class Generator:
     """Generate all visualizations in a config file given a dict of results"""
 
     config: benchcomp.ConfigFile
+    except_for: list
+    only: list
 
 
     def __call__(self, results):
-        for viz in self.config["visualize"]:
+        visualizations = self.config["visualize"]
+
+        if self.except_for:
+            for viz_name in self.except_for:
+                vizs = [v for v in visualizations if v["type"] == viz_name]
+                for viz in vizs:
+                    visualizations.remove(viz)
+        if self.only:
+            for viz_name in [v["type"] for v in visualizations]:
+                if viz_name not in self.only:
+                    vizs = [v for v in visualizations if v["type"] == viz_name]
+                    for viz in vizs:
+                        visualizations.remove(viz)
+
+        for viz in visualizations:
             viz_type = viz.pop("type")
             klass = getattr(benchcomp.visualizers, viz_type)
             visualize = klass(**viz)

tools/benchcomp/test/test_regression.py

@@ -30,12 +30,14 @@ def __init__(self, config):
         wd = tempfile.mkdtemp()
         self.working_directory = pathlib.Path(wd)
 
-    def __call__(self, subcommand=None, default_flags=None, *flags):
+    def __call__(self, subcommand=None, default_flags=None, flags=None):
         subcommand = subcommand or []
         default_flags = default_flags or [
             "--out-prefix", "/tmp/benchcomp/test"]
         config_flags = ["--config", str(self.config_file)]
 
+        flags = flags or []
+
         cmd = [self.bc, *config_flags, *subcommand, *default_flags, *flags]
         self.proc = subprocess.Popen(
             cmd, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
@@ -386,6 +388,99 @@ def test_error_on_regression_visualization_ratio_regressed(self):
                 run_bc.proc.returncode, 1, msg=run_bc.stderr)
 
 
+    def test_only_dump_yaml(self):
+        """Ensure that benchcomp terminates with return code 0 when `--only dump_yaml` is passed, even if the error_on_regression visualization would have resulted in a return code of 1"""
+
+        with tempfile.TemporaryDirectory() as tmp:
+            run_bc = Benchcomp({
+                "variants": {
+                    "passed": {
+                        "config": {
+                            "directory": str(tmp),
+                            "command_line":
+                                "mkdir bench_1 bench_2 && "
+                                "echo true > bench_1/success &&"
+                                "echo true > bench_2/success"
+                        },
+                    },
+                    "failed": {
+                        "config": {
+                            "directory": str(tmp),
+                            "command_line":
+                                "mkdir bench_1 bench_2 && "
+                                "echo true > bench_1/success &&"
+                                "echo false > bench_2/success"
+                        }
+                    }
+                },
+                "run": {
+                    "suites": {
+                        "suite_1": {
+                            "parser": { "module": "test_file_to_metric" },
+                            "variants": ["passed", "failed"]
+                        }
+                    }
+                },
+                "visualize": [{
+                    "type": "dump_yaml",
+                }, {
+                    "type": "error_on_regression",
+                    "variant_pairs": [["passed", "failed"]],
+                    "checks": [{
+                        "metric": "success",
+                        "test":
+                            "lambda old, new: True"
+                    }]
+                }]
+            })
+            run_bc(flags=["--only", "dump_yaml"])
+
+            self.assertEqual(
+                run_bc.proc.returncode, 0, msg=run_bc.stderr)
+
+            with open(run_bc.working_directory / "result.yaml") as handle:
+                result = yaml.safe_load(handle)
+
+
+    def test_ignore_dump_yaml(self):
+        """Ensure that benchcomp does not print any YAML output even with the dump_yaml visualization when the `--except dump_yaml` flag is passed"""
+
+        with tempfile.TemporaryDirectory() as tmp:
+            run_bc = Benchcomp({
+                "variants": {
+                    "variant_1": {
+                        "config": {
+                            "directory": tmp,
+                            "command_line": "true",
+                        }
+                    },
+                    "variant_2": {
+                        "config": {
+                            "directory": tmp,
+                            "command_line": "true",
+                        }
+                    }
+                },
+                "run": {
+                    "suites": {
+                        "suite_1": {
+                            "parser": {"module": "test"},
+                            "variants": ["variant_1", "variant_2"]
+                        }
+                    }
+                },
+                "visualize": [{
+                    "type": "dump_yaml",
+                }],
+            })
+            run_bc(flags=["--except", "dump_yaml"])
+
+            self.assertEqual(
+                run_bc.stdout, "", msg=run_bc.stdout)
+
+            with open(run_bc.working_directory / "result.yaml") as handle:
+                result = yaml.safe_load(handle)
+
 
     def test_return_0(self):
         """Ensure that benchcomp terminates with return code 0"""
@@ -423,6 +518,7 @@ def test_return_0(self):
             with open(run_bc.working_directory / "result.yaml") as handle:
                 result = yaml.safe_load(handle)
 
+
     def test_return_0_on_fail(self):
         """Ensure that benchcomp terminates with 0 even if a suite fails"""
 
@@ -459,6 +555,7 @@ def test_return_0_on_fail(self):
             with open(run_bc.working_directory / "result.yaml") as handle:
                 result = yaml.safe_load(handle)
 
+
     def test_env(self):
         """Ensure that benchcomp reads the 'env' key of variant config"""