Introduce presets

Author: ianayl

.github/workflows/sycl-linux-run-tests.yml

@@ -122,6 +122,10 @@ on:
         type: string
         default: ''
         required: False
+      benchmark_preset:
+        type: string
+        default: 'Minimal'
+        required: False
 
   workflow_dispatch:
     inputs:
@@ -356,6 +360,7 @@ jobs:
         target_devices: ${{ inputs.target_devices }}
         upload_results: ${{ inputs.benchmark_upload_results }}
         save_name: ${{ inputs.benchmark_save_name }}
+        preset: ${{ inputs.benchmark_preset }}
       env:
         RUNNER_TAG: ${{ inputs.runner }}
         GITHUB_TOKEN: ${{ secrets.LLVM_SYCL_BENCHMARK_TOKEN }}

.github/workflows/sycl-ur-perf-benchmarking.yml

@@ -5,6 +5,12 @@ on:
     - cron: '0 1 * * *'  # 2 hrs earlier than sycl-nightly.yml
   workflow_call:
     inputs:
+      preset:
+        type: string
+        description: |
+          Benchmark presets to run: See /devops/scripts/benchmarks/presets.py
+        required: false
+        default: 'Minimal'  # Only compute-benchmarks
       pr_no:
         type: string
         description: |
@@ -40,6 +46,17 @@ on:
 
   workflow_dispatch:
     inputs:
+      preset:
+        type: choice
+        description: |
+          Benchmark presets to run, See /devops/scripts/benchmarks/presets.py. Hint: Minimal is compute-benchmarks only.
+        options:
+          - Full
+          - SYCL
+          - Minimal
+          - Normal
+          - Test
+        default: 'Minimal'  # Only compute-benchmarks
       pr_no:
         type: string
         description: |
@@ -125,6 +142,7 @@ jobs:
       tests_selector: benchmark_v2
       benchmark_upload_results: ${{ inputs.upload_results }}
       benchmark_save_name: ${{ matrix.save_name }}
+      benchmark_preset: ${{ inputs.preset }}
       repo_ref: ${{ matrix.ref }}
       devops_ref: ${{ github.ref }}
       sycl_toolchain_artifact: sycl_linux_default
@@ -154,4 +172,5 @@ jobs:
       tests_selector: benchmark_v2
       benchmark_save_name: Baseline
       benchmark_upload_results: ${{ inputs.upload_results }}
+      benchmark_preset: ${{ inputs.preset }}
       repo_ref: ${{ github.ref }} # TODO figure out nightly commit hash

devops/actions/run-tests/benchmark_v2/action.yml

@@ -22,7 +22,9 @@ inputs:
   save_name:
     type: string
     required: True
-    default: ''
+  preset:
+    type: string
+    required: True
 
 runs:
   using: "composite"
@@ -32,6 +34,7 @@ runs:
     env:
       TARGET_DEVICE: ${{ inputs.target_devices }}
       RUNNER_NAME: ${{ runner.name }}
+      PRESET: ${{ inputs.preset }}
     run: |
       case "$RUNNER_TAG" in
         '["PVC_PERF"]' ) ;;
@@ -61,6 +64,10 @@ runs:
       esac
       echo "ONEAPI_DEVICE_SELECTOR=$TARGET_DEVICE" >> $GITHUB_ENV 
 
+      # Make sure specified preset is a known value and is not malicious
+      python3 ./devops/scripts/benchmarks/preset.py "$PRESET"
+      [ "$?" -ne 0 ] && exit 1  # Stop workflow if invalid preset
+      echo "PRESET=$PRESET" >> $GITHUB_ENV
   - name: Compute CPU core range to run benchmarks on
     shell: bash
     run: |
@@ -117,7 +124,7 @@ runs:
         --output-html remote \
         --results-dir "./llvm-ci-perf-results/$RUNNER_NAME" \
         --output-dir "./llvm-ci-perf-results/$RUNNER_NAME" \
-        --preset Minimal \
+        --preset "$PRESET" \
         --timestamp-override "$SAVE_TIMESTAMP"
       echo "-----"
       python3 ./devops/scripts/benchmarks/compare.py to_hist \

devops/scripts/benchmarks/presets.py

@@ -3,6 +3,8 @@
 # See LICENSE.TXT
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
+import argparse
+
 presets: dict[str, list[str]] = {
     "Full": [
         "Compute Benchmarks",
@@ -36,3 +38,38 @@ def enabled_suites(preset: str) -> list[str]:
         return presets[preset]
     except KeyError:
         raise ValueError(f"Preset '{preset}' not found.")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Benchmark Preset Utilities")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    query_parser = subparsers.add_parser(
+        "query",
+        help="Query benchmarks ran by a preset (as defined in presets.py)"
+    )
+    validate_parser.add_argument(
+        "preset_to_query",
+        type=str,
+        help="preset name to query"
+    )
+    validate_parser.add_argument(
+        "-q", "--quiet",
+        action="store_true",
+        help="Disable stdout messages: Useful if you want to check if a preset exists within a shell script."
+    )
+
+    args = parser.parse_args()
+    if args.command == 'query':
+        if args.preset_to_query in presets:
+            if not args.quiet:
+                print(f"Benchmark suites to be ran in {args.preset_to_query}:")
+                for suite in presets[args.preset_to_query]:
+                    print(suite)
+            exit(0)
+        else:
+            if not args.quiet: print(f"Error: No preset named '{args.preset_to_query}'.")
+            exit(1)
+
+if __name__ == "__main__":
+    main()