[CI][Bench] Add concurrency group #499
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # A combined workflow for all benchmarks-related jobs for SYCL and UR. | |
| # Supports both manual triggering (dispatch) and nightly runs. | |
| # It also tests changes to benchmark scripts/framework in PR, if modified. | |
| name: SYCL Run Benchmarks | |
| on: | |
| schedule: | |
| # 3 hours ahead of SYCL nightly | |
| - cron: '0 0 * * *' | |
| # Run on pull requests only when a benchmark-related files were changed. | |
| pull_request: | |
| # These paths are exactly the same as in sycl-linux/windows-precommit.yml (to ignore over there) | |
| paths: | |
| - 'devops/scripts/benchmarks/**' | |
| - 'devops/actions/run-tests/benchmark/**' | |
| - '.github/workflows/sycl-ur-perf-benchmarking.yml' | |
| workflow_dispatch: | |
| inputs: | |
| preset: | |
| type: choice | |
| description: | | |
| Benchmark presets to run, See /devops/scripts/benchmarks/presets.py. Hint: Minimal is compute-benchmarks only. | |
| options: | |
| - Full | |
| - SYCL | |
| - Minimal | |
| - Normal | |
| - Test | |
| - Gromacs | |
| - OneDNN | |
| default: 'Minimal' # Only compute-benchmarks | |
| pr_no: | |
| type: string | |
| description: | | |
| PR no. to build SYCL from - it will be built from HEAD of incoming branch. | |
| Leave both pr_no and commit_hash empty to use the latest commit from branch/tag this workflow started from. | |
| required: false | |
| default: '' | |
| commit_hash: | |
| type: string | |
| description: | | |
| Commit hash (within intel/llvm) to build SYCL from. | |
| Leave both pr_no and commit_hash empty to use the latest commit from branch/tag this workflow started from. | |
| required: false | |
| default: '' | |
| save_name: | |
| type: string | |
| description: | | |
| Name to use for the benchmark result | |
| required: false | |
| default: '' | |
| upload_results: | |
| description: Save and upload results (to https://intel.github.io/llvm/benchmarks) | |
| type: choice | |
| options: | |
| - false | |
| - true | |
| default: true | |
| exit_on_failure: | |
| description: Fail benchmark script on any error. Limit number of iterations to just test correctness. | |
| type: choice | |
| options: | |
| - false | |
| - true | |
| default: false | |
| runner: | |
| description: Self-hosted runner to use for the benchmarks | |
| type: choice | |
| options: | |
| - '["PVC_PERF"]' | |
| - '["BMG_PERF"]' | |
| backend: | |
| description: Backend to use | |
| type: choice | |
| options: | |
| - 'level_zero:gpu' | |
| - 'level_zero_v2:gpu' | |
| concurrency: | |
| # Cancel a currently running workflow for: | |
| # - the same PR | |
| # - manual dispatch if pr_no or commit_hash is repeated, and the runner & backend are the same | |
| # - scheduled runs if they overlap in time | |
| group: >- | |
| ${{ github.workflow }}-${{ github.event_name }}- | |
| ${{ github.event.pull_request.number || inputs.pr_no || inputs.commit_hash || github.ref }}- | |
| ${{ inputs.runner }}-${{ inputs.backend }} | |
| cancel-in-progress: true | |
| permissions: read-all | |
| jobs: | |
| # Manual trigger (dispatch) path: | |
| sanitize_inputs_dispatch: | |
| name: '[Dispatch] Sanitize inputs' | |
| if: github.event_name == 'workflow_dispatch' | |
| runs-on: ubuntu-latest | |
| env: | |
| COMMIT_HASH: ${{ inputs.commit_hash }} | |
| PR_NO: ${{ inputs.pr_no }} | |
| SAVE_NAME: ${{ inputs.save_name }} | |
| outputs: | |
| benchmark_save_name: ${{ steps.sanitize.outputs.benchmark_save_name }} | |
| build_ref: ${{ steps.sanitize.outputs.build_ref }} | |
| steps: | |
| - id: sanitize | |
| run: | | |
| # Validate user inputs: | |
| # usage: check_if_nonempty <var> <regex to check var against> <err message> | |
| check_nonempty() { | |
| [ -z "$1" ] && return | |
| if [ -z "$(echo "$1" | grep -P "$2")" ]; then | |
| echo "$3" | |
| exit 1 | |
| fi | |
| } | |
| check_nonempty "$COMMIT_HASH" '^[0-9a-f]{7,}$' "Bad commit hash (or hash short)." | |
| check_nonempty "$PR_NO" '^[0-9]+$' "Bad PR number." | |
| check_nonempty "$SAVE_NAME" '^[A-Za-z][A-Za-z0-9_-]+$' "Bad save name." | |
| BENCHMARK_SAVE_NAME="" | |
| BUILD_REF="${{ github.ref }}" | |
| if [ -n "$SAVE_NAME" ]; then | |
| BENCHMARK_SAVE_NAME="$(echo "$SAVE_NAME" | tr -cd 'A-Za-z0-9_-')" | |
| fi; | |
| if [ -n "$COMMIT_HASH" ]; then | |
| echo "Using commit hash $COMMIT_HASH for build..." | |
| BUILD_REF="$COMMIT_HASH" | |
| shortened_commit="$(echo "$COMMIT_HASH" | cut -c 1-7)" | |
| [ -z "$BENCHMARK_SAVE_NAME" ] && BENCHMARK_SAVE_NAME="Commit_${shortened_commit}" | |
| elif [ -n "$PR_NO" ]; then | |
| echo "Using PR no. $PR_NO for build..." | |
| BUILD_REF="refs/pull/$PR_NO/head" | |
| [ -z "$BENCHMARK_SAVE_NAME" ] && BENCHMARK_SAVE_NAME="PR_${PR_NO}" | |
| fi | |
| [ -z "$BENCHMARK_SAVE_NAME" ] && BENCHMARK_SAVE_NAME="Baseline" | |
| echo "benchmark_save_name=$BENCHMARK_SAVE_NAME" >> $GITHUB_OUTPUT | |
| echo "build_ref=$BUILD_REF" >> $GITHUB_OUTPUT | |
| echo "Final sanitized values:" | |
| cat $GITHUB_OUTPUT | |
| build_sycl_dispatch: | |
| name: '[Dispatch] Build SYCL' | |
| needs: [ sanitize_inputs_dispatch ] | |
| uses: ./.github/workflows/sycl-linux-build.yml | |
| with: | |
| build_ref: ${{ needs.sanitize_inputs_dispatch.outputs.build_ref }} | |
| build_cache_root: "/__w/" | |
| build_cache_suffix: "prod_noassert" | |
| build_configure_extra_args: "--no-assertions" | |
| build_image: "ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest" | |
| cc: clang | |
| cxx: clang++ | |
| changes: '[]' | |
| toolchain_artifact: sycl_linux_prod_noassert | |
| benchmark_dispatch: | |
| name: '[Dispatch] Benchmarks' | |
| needs: [ build_sycl_dispatch, sanitize_inputs_dispatch ] | |
| if: always() && !cancelled() && needs.build_sycl_dispatch.outputs.build_conclusion == 'success' | |
| strategy: | |
| matrix: | |
| include: | |
| # Set default values if not specified: | |
| - runner: ${{ inputs.runner || '["PVC_PERF"]' }} | |
| backend: ${{ inputs.backend || 'level_zero:gpu' }} | |
| uses: ./.github/workflows/sycl-linux-run-tests.yml | |
| secrets: inherit | |
| with: | |
| name: "Benchmarks (${{ matrix.runner }}, ${{ matrix.backend }}, preset: ${{ inputs.preset }})" | |
| runner: ${{ matrix.runner }} | |
| image: ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest | |
| image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN | |
| target_devices: ${{ matrix.backend }} | |
| tests_selector: benchmarks | |
| benchmark_upload_results: ${{ inputs.upload_results }} | |
| benchmark_save_name: ${{ needs.sanitize_inputs_dispatch.outputs.benchmark_save_name }} | |
| benchmark_preset: ${{ inputs.preset }} | |
| benchmark_exit_on_failure: ${{ inputs.exit_on_failure }} | |
| repo_ref: ${{ needs.sanitize_inputs_dispatch.outputs.build_ref }} | |
| toolchain_artifact: ${{ needs.build_sycl_dispatch.outputs.toolchain_artifact }} | |
| toolchain_artifact_filename: ${{ needs.build_sycl_dispatch.outputs.toolchain_artifact_filename }} | |
| toolchain_decompress_command: ${{ needs.build_sycl_dispatch.outputs.toolchain_decompress_command }} | |
| # END manual trigger (dispatch) path | |
| # Nightly benchmarking path: | |
| build_nightly: | |
| name: '[Nightly] Build SYCL' | |
| if: github.repository == 'intel/llvm' && github.event_name == 'schedule' | |
| uses: ./.github/workflows/sycl-linux-build.yml | |
| secrets: inherit | |
| with: | |
| build_cache_root: "/__w/" | |
| build_configure_extra_args: '--no-assertions' | |
| build_image: ghcr.io/intel/llvm/ubuntu2404_build:latest | |
| toolchain_artifact: sycl_linux_default | |
| toolchain_artifact_filename: sycl_linux.tar.gz | |
| benchmark_nightly: | |
| name: '[Nightly] Benchmarks' | |
| needs: [build_nightly] | |
| if: always() && !cancelled() && needs.build_nightly.outputs.build_conclusion == 'success' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| runner: ['["PVC_PERF"]', '["BMG_PERF"]'] | |
| backend: ['level_zero:gpu', 'level_zero_v2:gpu'] | |
| include: | |
| - ref: ${{ github.sha }} | |
| save_name: 'Baseline' | |
| preset: 'Full' | |
| uses: ./.github/workflows/sycl-linux-run-tests.yml | |
| secrets: inherit | |
| with: | |
| name: "Benchmarks (${{ matrix.runner }}, ${{ matrix.backend }}, preset: ${{ matrix.preset }})" | |
| runner: ${{ matrix.runner }} | |
| image: ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest | |
| image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN | |
| target_devices: ${{ matrix.backend }} | |
| tests_selector: benchmarks | |
| benchmark_upload_results: true | |
| benchmark_save_name: ${{ matrix.save_name }} | |
| benchmark_preset: ${{ matrix.preset }} | |
| repo_ref: ${{ matrix.ref }} | |
| toolchain_artifact: ${{ needs.build_nightly.outputs.toolchain_artifact }} | |
| toolchain_artifact_filename: ${{ needs.build_nightly.outputs.toolchain_artifact_filename }} | |
| toolchain_decompress_command: ${{ needs.build_nightly.outputs.toolchain_decompress_command }} | |
| # END nightly benchmarking path | |
| # Benchmark framework builds and runs on PRs path: | |
| build_pr: | |
| name: '[PR] Build SYCL' | |
| if: github.event_name == 'pull_request' | |
| uses: ./.github/workflows/sycl-linux-build.yml | |
| with: | |
| build_ref: ${{ github.sha }} | |
| build_cache_root: "/__w/" | |
| build_cache_suffix: "default" | |
| # Docker image has last nightly pre-installed and added to the PATH | |
| build_image: "ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest" | |
| cc: clang | |
| cxx: clang++ | |
| changes: '[]' | |
| toolchain_artifact: sycl_linux_default | |
| # TODO: When we have stable BMG runner(s), consider moving this job to that runner. | |
| test_benchmark_framework: | |
| name: '[PR] Benchmark suite testing' | |
| needs: [build_pr] | |
| if: always() && !cancelled() && needs.build_pr.outputs.build_conclusion == 'success' | |
| uses: ./.github/workflows/sycl-linux-run-tests.yml | |
| with: | |
| name: 'Framework test: PVC_PERF, L0, Minimal preset' | |
| runner: '["PVC_PERF"]' | |
| image: ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest | |
| image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN | |
| target_devices: 'level_zero:gpu' | |
| tests_selector: benchmarks | |
| benchmark_upload_results: false | |
| benchmark_preset: 'Minimal' | |
| benchmark_dry_run: true | |
| benchmark_exit_on_failure: true | |
| repo_ref: ${{ github.sha }} | |
| toolchain_artifact: ${{ needs.build_pr.outputs.toolchain_artifact }} | |
| toolchain_artifact_filename: ${{ needs.build_pr.outputs.toolchain_artifact_filename }} | |
| toolchain_decompress_command: ${{ needs.build_pr.outputs.toolchain_decompress_command }} | |
| # END benchmark framework builds and runs on PRs path |