[CI][Bench] Create summary reports for benchmarking CI run results

devops/actions/run-tests/benchmark/action.yml

@@ -169,15 +169,19 @@ runs:
         --compare-file "./llvm-ci-perf-results/results/${SAVE_NAME}_${SAVE_TIMESTAMP}.json" \
         --results-dir "./llvm-ci-perf-results/results/" \
         --regression-filter '^[a-z_]+_sycl ' \
+        --regression-filter-type 'SYCL' \
         --verbose \
+        --produce-github-summary "github_summary.md" \
         ${{ inputs.dry_run == 'true' && '--dry-run' || '' }} \
 
       echo "-----"
 
-  - name: Cache changes to benchmark folder for archival purposes
+  - name: Cache changes and upload github summary
     if: always()
     shell: bash
     run: | 
+      [ -f "github_summary.md" ] && cat github_summary.md >> $GITHUB_STEP_SUMMARY
+
       cd "./llvm-ci-perf-results"
       git add .
       for diff in $(git diff HEAD --name-only); do

devops/scripts/benchmarks/compare.py

@@ -340,14 +340,26 @@ def to_hist(
     parser_avg.add_argument(
         "--regression-filter",
         type=str,
-        help="If provided, only regressions matching provided regex will cause exit status 1.",
+        help="If provided, only regressions in tests matching provided regex will cause exit status 1.",
         default=None,
     )
+    parser_avg.add_argument(
+        "--regression-filter-type",
+        type=str,
+        help="Name to use in logging for tests that fall within the filter defined by --regression-filter; i.e. if --regression-filter filters for SYCL benchmarks, --regression-filter-type could be 'SYCL'.",
+        default="filtered",
+    )
     parser_avg.add_argument(
         "--dry-run",
         action="store_true",
         help="Do not return error upon regressions.",
     )
+    parser_avg.add_argument(
+        "--produce-github-summary",
+        type=str,
+        help="Create a github CI summary file using the provided filename",
+        default="",
+    )
 
     args = parser.parse_args()
 
@@ -365,6 +377,18 @@ def to_hist(
             args.avg_type, args.name, args.compare_file, args.results_dir, args.cutoff
         )
 
+        # Initialize github summary variables:
+        if args.produce_github_summary:
+            gh_summary = []
+
+            filter_type_capitalized = (
+                args.regression_filter_type[0].upper() + args.regression_filter_type[1:]
+            )
+
+            def write_summary_to_file(summary: list[str]):
+                with open(args.produce_github_summary, "w") as f:
+                    f.write("\n".join(summary))
+
         # Not all regressions are of concern: if a filter is provided, filter
         # regressions using filter
         regressions_ignored = []
@@ -378,7 +402,7 @@ def to_hist(
                     regressions_ignored.append(test)
 
         def print_regression(entry: dict, is_warning: bool = False):
-            """Print an entry outputted from Compare.to_hist
+            """Print an entry outputted from Compare.to_hist[github_summary.md")
 
             Args:
                 entry (dict): The entry to print
@@ -390,28 +414,72 @@ def print_regression(entry: dict, is_warning: bool = False):
             log_func(f"-- Run result: {entry['value']}")
             log_func(f"-- Delta: {entry['delta']}")
             log_func("")
+            if args.produce_github_summary:
+                gh_summary.append(f"#### {entry['name']}:")
+                gh_summary.append(
+                    f"- Historic {entry['avg_type']}: {entry['hist_avg']}"
+                )
+                gh_summary.append(f"- Run result: {entry['value']}")
+                gh_summary.append(
+                    f"- Delta: {round(entry['delta']*100, 2)}% ({entry['delta']})"
+                )
+                gh_summary.append("")
 
         if improvements:
             log.info("#")
             log.info("# Improvements:")
             log.info("#")
+            if args.produce_github_summary:
+                gh_summary.append(f"### Improvements")
+                gh_summary.append(
+                    f"<details><summary>{len(improvements)} improved tests:</summary>"
+                )
+                gh_summary.append("")
             for test in improvements:
                 print_regression(test)
+            if args.produce_github_summary:
+                gh_summary.append("</details>")
+                gh_summary.append("")
         if regressions_ignored:
             log.info("#")
-            log.info("# Regressions (filtered out by regression-filter):")
+            log.info("# Regressions (filtered out by --regression-filter):")
             log.info("#")
+            if args.produce_github_summary:
+                gh_summary.append(f"### Non-{filter_type_capitalized} Regressions")
+                gh_summary.append(
+                    f"<details><summary>{len(regressions_ignored)} non-{args.regression_filter_type} regressions:</summary>"
+                )
+                gh_summary.append("")
             for test in regressions_ignored:
                 print_regression(test)
+            if args.produce_github_summary:
+                gh_summary.append("</details>")
+                gh_summary.append("")
         if regressions_of_concern:
             log.warning("#")
             log.warning("# Regressions:")
             log.warning("#")
+            if args.produce_github_summary:
+                gh_summary.append(f"### {filter_type_capitalized} Regressions")
+                gh_summary.append(
+                    f"{len(regressions_of_concern)} {args.regression_filter_type} regressions. These regressions warrant a CI failure:"
+                )
+                gh_summary.append("")
             for test in regressions_of_concern:
                 print_regression(test, is_warning=True)
+            if args.produce_github_summary:
+                gh_summary.append("")
+
             if not args.dry_run:
+                if args.produce_github_summary:
+                    write_summary_to_file(gh_summary)
                 exit(1)  # Exit 1 to trigger github test failure
+
         log.info("No unexpected regressions found!")
+        if args.produce_github_summary:
+            gh_summary.append("No unexpected regressions found!")
+            write_summary_to_file(gh_summary)
+
     else:
         log.error("Unsupported operation: exiting.")
         exit(1)