Skip to content

Commit f90ca8e

Browse files
ianaylpbalcerlukaszstolarczuk
authored
[CI][Benchmark] Add comparison script to benchmark scripts + benchmark dashboard via sycl-docs.yml (#17522)
This PR: - Introduces new CI to replace existing nightly benchmarking workflow - Adds a preliminary implementation of a comparison script alongside the existing benchmarking scripts: This comparison script is used by the CI to decide whether or not a benchmark result is over a (predefined) tolerance above historical median of benchmark results This is not a permanent solution: I am looking into exponential moving averages in the future - Adds code in sycl-docs.yml for bringing up new benchmark results dashboard on https://intel.github.io --------- Co-authored-by: Piotr Balcer <[email protected]> Co-authored-by: Łukasz Stolarczuk <[email protected]>
1 parent 384025f commit f90ca8e

File tree

22 files changed

+1011
-1342
lines changed

22 files changed

+1011
-1342
lines changed

‎.github/workflows/sycl-docs.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,13 @@ jobs:
4949
mkdir clang
5050
mv $GITHUB_WORKSPACE/build/tools/sycl/doc/html/* .
5151
mv $GITHUB_WORKSPACE/build/tools/clang/docs/html/* clang/
52+
cp -r $GITHUB_WORKSPACE/repo/devops/scripts/benchmarks/html benchmarks
5253
touch .nojekyll
54+
# Update benchmarking dashboard configuration
55+
cat << EOF > benchmarks/config.js
56+
remoteDataUrl = 'https://raw.githubusercontent.com/intel/llvm-ci-perf-results/refs/heads/unify-ci/data.json';
57+
defaultCompareNames = ["Baseline_PVC_L0"];
58+
EOF
5359
# Upload the generated docs as an artifact and deploy to GitHub Pages.
5460
- name: Upload artifact
5561
uses: actions/upload-pages-artifact@v3

‎.github/workflows/sycl-linux-run-tests.yml‎

Lines changed: 34 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ on:
2525
required: False
2626
tests_selector:
2727
description: |
28-
Three possible options: "e2e", "cts", and "compute-benchmarks".
28+
Three possible options: "e2e", "cts", and "benchmarks".
2929
type: string
3030
default: "e2e"
3131

@@ -111,6 +111,33 @@ on:
111111
default: ''
112112
required: False
113113

114+
benchmark_upload_results:
115+
description: |
116+
Set to true to upload results to git repository storing benchmarking
117+
results.
118+
type: string
119+
default: 'false'
120+
required: False
121+
benchmark_save_name:
122+
description: |
123+
Save name to use for benchmark results: Save names are stored in
124+
metadata of result file, and are used to identify benchmark results in
125+
the same series (e.g. same configuration, same device, etc.).
126+
127+
Note: Currently, benchmark result filenames are in the format of
128+
<benchmark_save_name>_<Device>_<Backend>_YYYYMMDD_HHMMSS.json
129+
type: string
130+
default: ''
131+
required: False
132+
benchmark_preset:
133+
description: |
134+
Name of benchmark preset to run.
135+
136+
See /devops/scripts/benchmarks/presets.py for all presets available.
137+
type: string
138+
default: 'Minimal'
139+
required: False
140+
114141
workflow_dispatch:
115142
inputs:
116143
runner:
@@ -150,7 +177,7 @@ on:
150177
options:
151178
- e2e
152179
- cts
153-
- compute-benchmarks
180+
- benchmarks
154181

155182
env:
156183
description: |
@@ -303,11 +330,14 @@ jobs:
303330
target_devices: ${{ inputs.target_devices }}
304331
retention-days: ${{ inputs.retention-days }}
305332

306-
- name: Run compute-benchmarks on SYCL
307-
if: inputs.tests_selector == 'compute-benchmarks'
333+
- name: Run benchmarks
334+
if: inputs.tests_selector == 'benchmarks'
308335
uses: ./devops/actions/run-tests/benchmark
309336
with:
310337
target_devices: ${{ inputs.target_devices }}
338+
upload_results: ${{ inputs.benchmark_upload_results }}
339+
save_name: ${{ inputs.benchmark_save_name }}
340+
preset: ${{ inputs.benchmark_preset }}
311341
env:
312342
RUNNER_TAG: ${{ inputs.runner }}
313343
GITHUB_TOKEN: ${{ secrets.LLVM_SYCL_BENCHMARK_TOKEN }}

‎.github/workflows/sycl-nightly.yml‎

Lines changed: 14 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -277,35 +277,30 @@ jobs:
277277
sycl_toolchain_archive: ${{ needs.build-win.outputs.artifact_archive_name }}
278278
sycl_cts_artifact: sycl_cts_bin_win
279279

280-
aggregate_benchmark_results:
281-
if: github.repository == 'intel/llvm' && !cancelled()
282-
name: Aggregate benchmark results and produce historical averages
283-
uses: ./.github/workflows/sycl-benchmark-aggregate.yml
284-
secrets:
285-
LLVM_SYCL_BENCHMARK_TOKEN: ${{ secrets.LLVM_SYCL_BENCHMARK_TOKEN }}
286-
with:
287-
lookback_days: 100
288-
289280
run-sycl-benchmarks:
290-
needs: [ubuntu2204_build, aggregate_benchmark_results]
281+
needs: [ubuntu2204_build]
291282
if: ${{ always() && !cancelled() && needs.ubuntu2204_build.outputs.build_conclusion == 'success' }}
292283
strategy:
293-
fail-fast: false
294284
matrix:
295285
include:
296-
- name: Run compute-benchmarks on L0 PVC
286+
- ref: ${{ github.sha }}
287+
save_name: Baseline
297288
runner: '["PVC_PERF"]'
298-
image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
299-
target_devices: level_zero:gpu
289+
backend: 'level_zero:gpu'
290+
preset: Minimal
300291
uses: ./.github/workflows/sycl-linux-run-tests.yml
301292
secrets: inherit
302293
with:
303-
name: ${{ matrix.name }}
294+
name: Run compute-benchmarks (${{ matrix.runner }}, ${{ matrix.backend }})
304295
runner: ${{ matrix.runner }}
305-
image_options: ${{ matrix.image_options }}
306-
target_devices: ${{ matrix.target_devices }}
307-
tests_selector: compute-benchmarks
308-
repo_ref: ${{ github.sha }}
296+
image: ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest
297+
image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
298+
target_devices: ${{ matrix.backend }}
299+
tests_selector: benchmarks
300+
benchmark_upload_results: true
301+
benchmark_save_name: ${{ matrix.save_name }}
302+
benchmark_preset: ${{ matrix.preset }}
303+
repo_ref: ${{ matrix.ref }}
309304
sycl_toolchain_artifact: sycl_linux_default
310305
sycl_toolchain_archive: ${{ needs.ubuntu2204_build.outputs.artifact_archive_name }}
311306
sycl_toolchain_decompress_command: ${{ needs.ubuntu2204_build.outputs.artifact_decompress_command }}

‎.github/workflows/sycl-ur-perf-benchmarking.yml‎

Lines changed: 133 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,138 @@
1-
name: Benchmarks
1+
name: Run Benchmarks
22

3-
# This workflow is a WIP: this workflow file acts as a placeholder.
3+
on:
4+
workflow_call:
5+
inputs:
6+
preset:
7+
type: string
8+
description: |
9+
Benchmark presets to run: See /devops/scripts/benchmarks/presets.py
10+
required: false
11+
default: 'Minimal' # Only compute-benchmarks
12+
pr_no:
13+
type: string
14+
description: |
15+
PR no. to build SYCL from if specified: SYCL will be built from HEAD
16+
of incoming branch used by the specified PR no.
417
5-
on: [ workflow_dispatch ]
18+
If both pr_no and commit_hash are empty, the latest SYCL nightly build
19+
will be used.
20+
required: false
21+
default: ''
22+
commit_hash:
23+
type: string
24+
description: |
25+
Commit hash (within intel/llvm) to build SYCL from if specified.
26+
27+
If both pr_no and commit_hash are empty, the latest commit in
28+
deployment branch will be used.
29+
required: false
30+
default: ''
31+
upload_results:
32+
type: string # true/false: workflow_dispatch does not support booleans
33+
required: true
34+
runner:
35+
type: string
36+
required: true
37+
backend:
38+
type: string
39+
required: true
40+
41+
workflow_dispatch:
42+
inputs:
43+
preset:
44+
type: choice
45+
description: |
46+
Benchmark presets to run, See /devops/scripts/benchmarks/presets.py. Hint: Minimal is compute-benchmarks only.
47+
options:
48+
- Full
49+
- SYCL
50+
- Minimal
51+
- Normal
52+
- Test
53+
default: 'Minimal' # Only compute-benchmarks
54+
pr_no:
55+
type: string
56+
description: |
57+
PR no. to build SYCL from:
58+
59+
SYCL will be built from HEAD of incoming branch.
60+
required: false
61+
default: ''
62+
commit_hash:
63+
type: string
64+
description: |
65+
Commit hash (within intel/llvm) to build SYCL from:
66+
67+
Leave both pr_no and commit_hash empty to use latest commit.
68+
required: false
69+
default: ''
70+
upload_results:
71+
description: 'Save and upload results'
72+
type: choice
73+
options:
74+
- false
75+
- true
76+
default: true
77+
runner:
78+
type: choice
79+
options:
80+
- '["PVC_PERF"]'
81+
backend:
82+
description: Backend to use
83+
type: choice
84+
options:
85+
- 'level_zero:gpu'
86+
- 'level_zero_v2:gpu'
87+
# As of #17407, sycl-linux-build now builds v2 by default
88+
89+
permissions: read-all
690

791
jobs:
8-
do-nothing:
9-
runs-on: ubuntu-latest
10-
steps:
11-
- run: echo 'This workflow is a WIP.'
92+
build_sycl:
93+
name: Build SYCL
94+
uses: ./.github/workflows/sycl-linux-build.yml
95+
with:
96+
build_ref: |
97+
${{
98+
inputs.commit_hash != '' && inputs.commit_hash ||
99+
inputs.pr_no != '' && format('refs/pull/{0}/head', inputs.pr_no) ||
100+
github.ref
101+
}}
102+
build_cache_root: "/__w/"
103+
build_artifact_suffix: "prod_noassert"
104+
build_cache_suffix: "prod_noassert"
105+
build_configure_extra_args: "--no-assertions"
106+
build_image: "ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest"
107+
cc: clang
108+
cxx: clang++
109+
changes: '[]'
12110

111+
run_benchmarks_build:
112+
name: Run Benchmarks on Build
113+
needs: [ build_sycl ]
114+
strategy:
115+
matrix:
116+
include:
117+
- ref: ${{ inputs.commit_hash != '' && inputs.commit_hash || format('refs/pull/{0}/head', inputs.pr_no) }}
118+
save_name: ${{ inputs.commit_hash != '' && format('Commit{0}', inputs.commit_hash) || format('PR{0}', inputs.pr_no) }}
119+
# Set default values if not specified:
120+
runner: ${{ inputs.runner || '["PVC_PERF"]' }}
121+
backend: ${{ inputs.backend || 'level_zero:gpu' }}
122+
uses: ./.github/workflows/sycl-linux-run-tests.yml
123+
secrets: inherit
124+
with:
125+
name: Run compute-benchmarks (${{ matrix.save_name }}, ${{ matrix.runner }}, ${{ matrix.backend }})
126+
runner: ${{ matrix.runner }}
127+
image: ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest
128+
image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN
129+
target_devices: ${{ matrix.backend }}
130+
tests_selector: benchmarks
131+
benchmark_upload_results: ${{ inputs.upload_results }}
132+
benchmark_save_name: ${{ matrix.save_name }}
133+
benchmark_preset: ${{ inputs.preset }}
134+
repo_ref: ${{ matrix.ref }}
135+
devops_ref: ${{ github.ref }}
136+
sycl_toolchain_artifact: sycl_linux_prod_noassert
137+
sycl_toolchain_archive: ${{ needs.build_sycl.outputs.artifact_archive_name }}
138+
sycl_toolchain_decompress_command: ${{ needs.build_sycl.outputs.artifact_decompress_command }}

‎devops/actions/benchmarking/aggregate/action.yml‎

Lines changed: 0 additions & 95 deletions
This file was deleted.

0 commit comments

Comments
 (0)