[CI][Bench] Add a filter to comparison scripts (#18852)

The nightly SYCL benchmarking CI currently fails on any benchmark.
However, not all benchmarks are specific to SYCL. This PR adds a filter
and sets the nightly SYCL CI to only fail on compute benchmark tests
marked with `_sycl`.

**Note** to llvm-reviewers-benchmarking: Observe that changes to the
code here are disconnected from the main benchmarking scripts. This PR
should functionally have no effect on the main benchmarking scripts.

Author: ianayl

devops/actions/run-tests/benchmark/action.yml

@@ -161,7 +161,8 @@ runs:
       python3 ./devops/scripts/benchmarks/compare.py to_hist \
         --name "$SAVE_NAME" \
         --compare-file "./llvm-ci-perf-results/results/${SAVE_NAME}_${SAVE_TIMESTAMP}.json" \
-        --results-dir "./llvm-ci-perf-results/results/"
+        --results-dir "./llvm-ci-perf-results/results/" \
+        --regression-filter '^[a-z_]_sycl '
       echo "-----"
 
   - name: Cache changes to benchmark folder for archival purposes

devops/scripts/benchmarks/compare.py

@@ -4,6 +4,7 @@
 from options import options
 
 import os
+import re
 import sys
 import json
 import argparse
@@ -319,6 +320,12 @@ def to_hist(
         help="Timestamp (in YYYYMMDD_HHMMSS) of oldest result to include in historic average calculation",
         default="20000101_010101",
     )
+    parser_avg.add_argument(
+        "--regression-filter",
+        type=str,
+        help="If provided, only regressions matching provided regex will cause exit status 1.",
+        default=None,
+    )
 
     args = parser.parse_args()
 
@@ -333,24 +340,40 @@ def to_hist(
             "median", args.name, args.compare_file, args.results_dir, args.cutoff
         )
 
+        # Not all regressions are of concern: if a filter is provided, filter
+        # regressions using filter
+        regressions_ignored = []
+        regressions_of_concern = []
+        if args.regression_filter is not None:
+            filter_pattern = re.compile(args.regression_filter)
+            for test in regressions:
+                if filter_pattern.search(test["name"]):
+                    regressions_of_concern.append(test)
+                else:
+                    regressions_ignored.append(test)
+
         def print_regression(entry: dict):
             """Print an entry outputted from Compare.to_hist"""
             print(f"Test: {entry['name']}")
             print(f"-- Historic {entry['avg_type']}: {entry['hist_avg']}")
-            print(f"-- Run result: {test['value']}")
-            print(f"-- Delta: {test['delta']}")
+            print(f"-- Run result: {entry['value']}")
+            print(f"-- Delta: {entry['delta']}")
             print("")
 
         if improvements:
             print("#\n# Improvements:\n#\n")
             for test in improvements:
                 print_regression(test)
-        if regressions:
+        if regressions_ignored:
+            print("#\n# Regressions (filtered out by regression-filter):\n#\n")
+            for test in regressions_ignored:
+                print_regression(test)
+        if regressions_of_concern:
             print("#\n# Regressions:\n#\n")
-            for test in regressions:
+            for test in regressions_of_concern:
                 print_regression(test)
             exit(1)  # Exit 1 to trigger github test failure
-        print("\nNo regressions found!")
+        print("\nNo unexpected regressions found!")
     else:
         print("Unsupported operation: exiting.")
         exit(1)