diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 7eb3c7b06a..b18a41c4b3 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -7,18 +7,22 @@ on: description: PR number (if 0, it'll run on the main) type: number bench_script_params: + # If you want to save the results of the manual run in 'benchmark-results' branch, + # you have to pass '--save XXX', where XXX is the label of your results. description: Parameters passed to script executing benchmark type: string required: false default: '' - upload_report: - description: 'Upload HTML report' - type: boolean - required: false - default: false + runner: + description: Runner + type: choice + required: true + default: 'L0_PERF' + options: + - L0_PERF permissions: - contents: read + contents: write pull-requests: write jobs: @@ -28,4 +32,4 @@ jobs: with: pr_no: ${{ inputs.pr_no }} bench_script_params: ${{ inputs.bench_script_params }} - upload_report: ${{ inputs.upload_report }} + runner: ${{ inputs.runner }} diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 1317482fd4..b11d17fa46 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -248,9 +248,9 @@ jobs: call "C:\Program Files (x86)\Intel\oneAPI\setvars-vcvarsall.bat" ctest -C ${{matrix.build_type}} --output-on-failure --test-dir test - hwloc-fallback: # Scenarios where UMF_LINK_HWLOC_STATICALLY is set to OFF and hwloc is not installed in the system # The hwloc library is fetched implicitly + hwloc-fallback: name: "Fallback to static hwloc build" strategy: matrix: @@ -317,9 +317,8 @@ jobs: Benchmarks: uses: ./.github/workflows/reusable_benchmarks.yml permissions: - contents: read + contents: write pull-requests: write with: pr_no: '0' - bench_script_params: '--save baseline' - upload_report: true + bench_script_params: '--save Baseline_PVC' diff --git a/.github/workflows/reusable_benchmarks.yml b/.github/workflows/reusable_benchmarks.yml index 3953e98de5..26f9c348bc 100644 --- a/.github/workflows/reusable_benchmarks.yml +++ b/.github/workflows/reusable_benchmarks.yml @@ -1,5 +1,5 @@ # Executes benchmarks implemented in this repository using scripts -# for results visualization from intel/llvm (unified-runtime dir). +# for results visualization from intel/llvm. name: Benchmarks on: @@ -14,13 +14,13 @@ on: required: false type: string default: '' - upload_report: + runner: required: false - type: boolean - default: false + type: string + default: 'L0_PERF' permissions: - contents: read + contents: write pull-requests: write env: @@ -32,17 +32,9 @@ jobs: name: Benchmarks # run only on upstream; forks will not have the HW if: github.repository == 'oneapi-src/unified-memory-framework' - runs-on: L0_PERF + runs-on: ${{ inputs.runner }} steps: - # Workspace on self-hosted runners is not cleaned automatically. - # We have to delete the files created outside of using actions. - - name: Cleanup self-hosted workspace - if: always() - run: | - ls -la ./ - rm -rf ./* || true - - name: Add comment to PR uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 if: ${{ always() && inputs.pr_no != 0 }} @@ -97,23 +89,32 @@ jobs: - name: Build UMF run: cmake --build ${{env.BUILD_DIR}} -j $(nproc) - # Get scripts for benchmark data visualization. - # Use specific tag, as the scripts or files' location may change. - - name: Checkout SYCL + - name: Checkout UMF results branch + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: benchmark-results + path: results-repo + + # Get scripts for benchmark data visualization (from SYCL repo). + # Use specific ref, as the scripts or files' location may change. + - name: Checkout benchmark scripts uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: intel/llvm - # [BENCHMARK] fix default timeout parameter - # https://github.com/intel/llvm/pull/17412 - ref: 357e9e0b253b7eba105d044e38452b3c09169f8a - path: sycl-repo - fetch-depth: 1 + # Note: The same ref is used in docs build (for dashboard generation)! + # + # 20.03.2025 + # branch: unify-benchmark-ci + ref: cae7049c78c697b3ac94f931716d9efb53addcd8 + path: sc + sparse-checkout: | + devops/scripts/benchmarks - name: Install benchmarking scripts deps run: | python -m venv .venv source .venv/bin/activate - pip install -r ${{github.workspace}}/sycl-repo/unified-runtime/third_party/benchmark_requirements.txt + pip install -r ${{github.workspace}}/sc/devops/scripts/benchmarks/requirements.txt - name: Set core range and GPU mask run: | @@ -135,22 +136,21 @@ jobs: - name: Run UMF benchmarks id: benchmarks - working-directory: ${{env.BUILD_DIR}} run: > - source ${{github.workspace}}/.venv/bin/activate && - taskset -c ${{ env.CORES }} ${{ github.workspace }}/sycl-repo/unified-runtime/scripts/benchmarks/main.py + source .venv/bin/activate && + taskset -c ${{ env.CORES }} ./sc/devops/scripts/benchmarks/main.py ~/bench_workdir_umf --umf ${{env.BUILD_DIR}} - --compare baseline --timeout 3000 - ${{ inputs.upload_report && '--output-html' || '' }} - ${{ inputs.pr_no != 0 && '--output-markdown' || '' }} + --output-html remote + --results-dir ${{ github.workspace }}/results-repo + --output-markdown ${{ inputs.bench_script_params }} # In case it failed to add a comment, we can still print the results. - name: Print benchmark results - if: ${{ always() && inputs.pr_no != 0 }} - run: cat ${{env.BUILD_DIR}}/benchmark_results.md + if: ${{ always() }} + run: cat ${{ github.workspace }}/benchmark_results.md || true - name: Add comment to PR uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 @@ -160,7 +160,7 @@ jobs: let markdown = "" try { const fs = require('fs'); - markdown = fs.readFileSync('${{env.BUILD_DIR}}/benchmark_results.md', 'utf8'); + markdown = fs.readFileSync('${{ github.workspace }}/benchmark_results.md', 'utf8'); } catch(err) { } @@ -177,15 +177,42 @@ jobs: repo: context.repo.repo, body: body }) - - - name: Upload HTML report - if: ${{ always() && inputs.upload_report }} - uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 - with: - path: umf-repo/build/benchmark_results.html - key: benchmark-results-${{ github.run_id }} - - name: Get information about platform - if: ${{ always() }} - working-directory: ${{env.UMF_DIR}} - run: .github/scripts/get_system_info.sh + - name: Commit data.json and results directory + working-directory: results-repo + run: | + git config --global user.name "GitHub Actions Bot" + git config --global user.email "actions@github.com" + + for attempt in {1..5}; do + echo "Attempt #$attempt to push changes" + + rm -f data.json + cp ${{ github.workspace }}/sc/devops/scripts/benchmarks/html/data.json . + + git add data.json results/ + git commit -m "Add benchmark results and data.json" + + results_file=$(git diff HEAD~1 --name-only -- results/ | head -n 1) + + if git push origin benchmark-results; then + echo "Push succeeded" + break + fi + + echo "Push failed, retrying..." + + if [ -n "$results_file" ]; then + mv $results_file ${{ github.workspace }}/temp_$(basename $results_file) + + git reset --hard origin/benchmark-results + git pull origin benchmark-results + + new_file="results/$(basename "$results_file")" + mv ${{ github.workspace }}/temp_$(basename $results_file) $new_file + fi + + echo "Regenerating data.json" + (cd ${{ github.workspace }} && ${{ github.workspace }}/sc/devops/scripts/benchmarks/main.py ~/bench_workdir_umf --dry-run --results-dir ${{ github.workspace }}/results-repo --output-html remote) + + done diff --git a/.github/workflows/reusable_docs_build.yml b/.github/workflows/reusable_docs_build.yml index 9317478bb4..e12895aebc 100644 --- a/.github/workflows/reusable_docs_build.yml +++ b/.github/workflows/reusable_docs_build.yml @@ -45,19 +45,37 @@ jobs: -DUMF_DISABLE_HWLOC=ON cmake --build build --target docs - # If we upload HTML docs, we want to include benchmark results as well - - name: Download benchmark HTML before uploading docs + # + # Documentation is built. Now we want to add benchmark dashboard. + # We only do it if inputs.upload is set, as this job is also used for testing docs build. + # + - name: Checkout benchmark scripts if: ${{ inputs.upload == true }} - id: download-bench-html - uses: actions/cache/restore@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: - path: umf-repo/build/benchmark_results.html - key: benchmark-results- + repository: intel/llvm + # 20.03.2025 + # branch: unify-benchmark-ci + ref: cae7049c78c697b3ac94f931716d9efb53addcd8 + path: sc + sparse-checkout: | + devops/scripts/benchmarks - - name: Move benchmark HTML - if: ${{ inputs.upload == true && steps.download-bench-html.outputs.cache-hit != '' }} + - name: Move benchmark HTML files + if: ${{ inputs.upload == true }} + working-directory: ${{ github.workspace }}/build/docs_build/generated/html + run: | + mkdir performance + mv ${{ github.workspace }}/sc/devops/scripts/benchmarks/html/* performance/ + + - name: Replace config.js + if: ${{ inputs.upload == true }} + working-directory: ${{ github.workspace }}/build/docs_build/generated/html run: | - mv umf-repo/build/benchmark_results.html ${{github.workspace}}/build/docs_build/generated/html + cat << 'EOF' > ./performance/config.js + remoteDataUrl = 'https://raw.githubusercontent.com/oneapi-src/unified-memory-framework/refs/heads/benchmark-results/data.json'; + defaultCompareNames = ["Baseline_PVC"]; + EOF - name: Upload artifact if: ${{ inputs.upload == true }}