diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 0000000000..7eb3c7b06a --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,31 @@ +name: Compute Benchmarks + +on: + workflow_dispatch: + inputs: + pr_no: + description: PR number (if 0, it'll run on the main) + type: number + bench_script_params: + description: Parameters passed to script executing benchmark + type: string + required: false + default: '' + upload_report: + description: 'Upload HTML report' + type: boolean + required: false + default: false + +permissions: + contents: read + pull-requests: write + +jobs: + manual: + name: Compute Benchmarks + uses: ./.github/workflows/reusable_benchmarks.yml + with: + pr_no: ${{ inputs.pr_no }} + bench_script_params: ${{ inputs.bench_script_params }} + upload_report: ${{ inputs.upload_report }} diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 46543fac83..28149c3a10 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -194,3 +194,13 @@ jobs: # Beside the 2 LTS Ubuntu, we also test this on the latest Ubuntu - to be updated # every 6 months, so we verify the latest version of packages (compilers, etc.). os: "['ubuntu-22.04', 'ubuntu-24.04', 'ubuntu-24.10']" + + Benchmarks: + uses: ./.github/workflows/reusable_benchmarks.yml + permissions: + contents: read + pull-requests: write + with: + pr_no: '0' + bench_script_params: '' + upload_report: true diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml deleted file mode 100644 index 6057df5f0a..0000000000 --- a/.github/workflows/performance.yml +++ /dev/null @@ -1,115 +0,0 @@ -name: Performance - -on: - # Can be triggered via manual "dispatch" (from workflow view in GitHub Actions tab) - workflow_dispatch: - inputs: - pr_no: - description: PR number (if 0, it'll run on the main) - type: number - required: true - -permissions: - contents: read - pull-requests: write - -env: - BUILD_DIR : "${{github.workspace}}/build" - -jobs: - perf-l0: - name: Build UMF and run performance tests - runs-on: "L0_PERF" - - steps: - # Workspace on self-hosted runners is not cleaned automatically. - # We have to delete the files created outside of using actions. - - name: Cleanup self-hosted workspace - if: always() - run: | - ls -la ./ - rm -rf ./* || true - - - name: Add comment to PR - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 - if: ${{ always() && inputs.pr_no != 0 }} - with: - script: | - const pr_no = '${{ inputs.pr_no }}'; - const provider = 'LEVEL_ZERO'; - const url = '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}'; - const body = `Performance workflow for ${provider}_PROVIDER run:\n${url}`; - - github.rest.issues.createComment({ - issue_number: pr_no, - owner: context.repo.owner, - repo: context.repo.repo, - body: body - }) - - - name: Checkout UMF - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - - name: Get information about platform - run: .github/scripts/get_system_info.sh - - # We need to fetch special ref for proper PR's merge commit. Note, this ref may be absent if the PR is already merged. - - name: Fetch PR's merge commit - if: ${{ inputs.pr_no != 0 }} - working-directory: ${{github.workspace}} - env: - PR_NO: ${{ inputs.pr_no }} - run: | - git fetch -- https://github.com/${{github.repository}} +refs/pull/${PR_NO}/*:refs/remotes/origin/pr/${PR_NO}/* - git checkout origin/pr/${PR_NO}/merge - git rev-parse origin/pr/${PR_NO}/merge - - - name: Configure build - run: > - cmake - -B ${{env.BUILD_DIR}} - -DCMAKE_BUILD_TYPE=Release - -DUMF_BUILD_SHARED_LIBRARY=ON - -DUMF_BUILD_BENCHMARKS=ON - -DUMF_BUILD_BENCHMARKS_MT=ON - -DUMF_BUILD_TESTS=OFF - -DUMF_FORMAT_CODE_STYLE=OFF - -DUMF_DEVELOPER_MODE=OFF - -DUMF_BUILD_LEVEL_ZERO_PROVIDER=ON - -DUMF_BUILD_CUDA_PROVIDER=ON - -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON - -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON - - - name: Build - run: cmake --build ${{env.BUILD_DIR}} -j $(nproc) - - - name: Run benchmarks - working-directory: ${{env.BUILD_DIR}} - id: benchmarks - run: numactl -N 1 ctest -V --test-dir benchmark -C Release - - - name: Add comment to PR - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 - if: ${{ always() && inputs.pr_no != 0 }} - with: - script: | - let markdown = "" - try { - const fs = require('fs'); - markdown = fs.readFileSync('umf_perf_results.md', 'utf8'); - } catch(err) { - } - - const pr_no = '${{ inputs.pr_no }}'; - const provider = 'LEVEL_ZERO'; - const url = '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}'; - const test_status = '${{ steps.benchmarks.outcome }}'; - const job_status = '${{ job.status }}'; - const body = `Performance workflow for ${provider}_PROVIDER run:\n${url}\nJob status: ${job_status}. Test status: ${test_status}.\n ${markdown}`; - - github.rest.issues.createComment({ - issue_number: pr_no, - owner: context.repo.owner, - repo: context.repo.repo, - body: body - }) diff --git a/.github/workflows/pr_push.yml b/.github/workflows/pr_push.yml index 9623b69f1b..cfc4a04b97 100644 --- a/.github/workflows/pr_push.yml +++ b/.github/workflows/pr_push.yml @@ -54,9 +54,6 @@ jobs: uses: ./.github/workflows/reusable_qemu.yml with: short_run: true - Benchmarks: - needs: [Build] - uses: ./.github/workflows/reusable_benchmarks.yml ProxyLib: needs: [Build] uses: ./.github/workflows/reusable_proxy_lib.yml diff --git a/.github/workflows/reusable_benchmarks.yml b/.github/workflows/reusable_benchmarks.yml index ed6a482947..028b974ef9 100644 --- a/.github/workflows/reusable_benchmarks.yml +++ b/.github/workflows/reusable_benchmarks.yml @@ -1,80 +1,163 @@ # Executes benchmarks implemented in this repository +# using scripts for benchmark results visualisation, +# which are downloaded from Unified Runtime repository. name: Benchmarks -on: workflow_call +on: + workflow_call: + inputs: + pr_no: + # even though this is a number, this is a workaround for issues with + # reusable workflow calls that result in "Unexpected value '0'" error. + type: string + default: '0' + bench_script_params: + required: false + type: string + default: '' + upload_report: + required: false + type: boolean + default: false permissions: contents: read + pull-requests: write env: - BUILD_DIR : "${{github.workspace}}/build" - INSTL_DIR : "${{github.workspace}}/../install-dir" + UMF_DIR: "${{github.workspace}}/umf-repo" + BUILD_DIR : "${{github.workspace}}/umf-repo/build" jobs: benchmarks: name: Benchmarks - env: - VCPKG_PATH: "${{github.workspace}}/build/vcpkg/packages/hwloc_x64-windows;${{github.workspace}}/build/vcpkg/packages/tbb_x64-windows;${{github.workspace}}/build/vcpkg/packages/jemalloc_x64-windows" strategy: matrix: - os: ['ubuntu-latest', 'windows-latest'] - include: - # Windows doesn't recognize 'CMAKE_BUILD_TYPE', it uses '--config' param in build command to determine the build type - - os: ubuntu-latest - extra_build_option: '-DCMAKE_BUILD_TYPE=Release' + os: ['ubuntu-latest'] runs-on: ${{matrix.os}} steps: - - name: Checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - fetch-depth: 0 - - - name: Install apt packages - if: matrix.os == 'ubuntu-latest' - run: | - sudo apt-get update - sudo apt-get install -y cmake libhwloc-dev libnuma-dev libtbb-dev - - - name: Initialize vcpkg - if: matrix.os == 'windows-latest' - uses: lukka/run-vcpkg@5e0cab206a5ea620130caf672fce3e4a6b5666a1 # v11.5 - with: - vcpkgGitCommitId: 3dd44b931481d7a8e9ba412621fa810232b66289 - vcpkgDirectory: ${{env.BUILD_DIR}}/vcpkg - vcpkgJsonGlob: '**/vcpkg.json' - - - name: Install vcpkg packages - if: matrix.os == 'windows-latest' - run: vcpkg install - shell: pwsh # Specifies PowerShell as the shell for running the script. - - - name: Configure build - run: > - cmake - -B ${{env.BUILD_DIR}} - ${{matrix.extra_build_option}} - -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}" - -DCMAKE_PREFIX_PATH="${{env.VCPKG_PATH}}" - -DUMF_BUILD_SHARED_LIBRARY=ON - -DUMF_BUILD_BENCHMARKS=ON - -DUMF_BUILD_BENCHMARKS_MT=ON - -DUMF_BUILD_TESTS=OFF - -DUMF_FORMAT_CODE_STYLE=OFF - -DUMF_DEVELOPER_MODE=OFF - -DUMF_BUILD_LEVEL_ZERO_PROVIDER=ON - -DUMF_BUILD_CUDA_PROVIDER=ON - -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON - -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON - - - name: Build UMF on Linux - if: matrix.os == 'ubuntu-latest' - run: cmake --build ${{env.BUILD_DIR}} -j $(nproc) - - - name: Build UMF on Windows - if: matrix.os == 'windows-latest' - run: cmake --build ${{env.BUILD_DIR}} --config Release -j $Env:NUMBER_OF_PROCESSORS - - - name: Run benchmarks - working-directory: ${{env.BUILD_DIR}} - run: ctest -V --test-dir benchmark -C Release + # Workspace on self-hosted runners is not cleaned automatically. + # We have to delete the files created outside of using actions. + - name: Cleanup self-hosted workspace + if: always() + run: | + ls -la ./ + rm -rf ./* || true + + - name: Add comment to PR + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + if: ${{ always() && inputs.pr_no != 0 }} + with: + script: | + const pr_no = '${{ inputs.pr_no }}'; + const url = '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}'; + const params = '${{ inputs.bench_script_params }}'; + const body = `Compute Benchmarks run (with params: ${params}):\n${url}`; + + github.rest.issues.createComment({ + issue_number: pr_no, + owner: context.repo.owner, + repo: context.repo.repo, + body: body + }) + + - name: Checkout UMF + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: ${{env.UMF_DIR}} + fetch-depth: 0 + + # We need to fetch special ref for proper PR's merge commit. Note, this ref may be absent if the PR is already merged. + - name: Fetch PR's merge commit + if: ${{ inputs.pr_no != 0 }} + working-directory: ${{env.UMF_DIR}} + env: + PR_NO: ${{ inputs.pr_no }} + run: | + git fetch -- https://github.com/${{github.repository}} +refs/pull/${PR_NO}/*:refs/remotes/origin/pr/${PR_NO}/* + git checkout origin/pr/${PR_NO}/merge + git rev-parse origin/pr/${PR_NO}/merge + + - name: Install apt packages + run: | + sudo apt-get update + sudo apt-get install -y cmake libhwloc-dev libnuma-dev libtbb-dev + + - name: Configure build + run: > + cmake + -S ${{env.UMF_DIR}} + -B ${{env.BUILD_DIR}} + -DCMAKE_BUILD_TYPE=Release + -DUMF_BUILD_SHARED_LIBRARY=ON + -DUMF_BUILD_BENCHMARKS=ON + -DUMF_BUILD_BENCHMARKS_MT=ON + -DUMF_BUILD_TESTS=OFF + -DUMF_FORMAT_CODE_STYLE=OFF + -DUMF_DEVELOPER_MODE=OFF + -DUMF_BUILD_LEVEL_ZERO_PROVIDER=ON + -DUMF_BUILD_CUDA_PROVIDER=ON + -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON + -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON + -DUMF_BUILD_EXAMPLES=OFF + + - name: Build UMF + run: cmake --build ${{env.BUILD_DIR}} -j $(nproc) + + # We are going to clone Unified Runtime repository in order to run + # the most up-to-date UR scripts for benchmark data visualisation + - name: Checkout UR + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: oneapi-src/unified-runtime + path: ur-repo + fetch-depth: 1 + fetch-tags: false + + - name: Install pip packages for benchmarking scripts from UR + run: | + pip install --force-reinstall -r ${{github.workspace}}/ur-repo/third_party/benchmark_requirements.txt + + - name: Run dedicated for UMF benchmarking scripts from UR + id: benchmarks + working-directory: ${{env.BUILD_DIR}} + run: > + ${{ github.workspace }}/ur-repo/scripts/benchmarks/main.py + ~/bench_workdir + --umf ${{env.BUILD_DIR}} + ${{ inputs.upload_report && '--output-html' || '' }} + ${{ inputs.bench_script_params }} + + - name: Add comment to PR + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + if: ${{ always() && inputs.pr_no != 0 }} + with: + script: | + let markdown = "" + try { + const fs = require('fs'); + markdown = fs.readFileSync('${{env.BUILD_DIR}}/benchmark_results.md', 'utf8'); + } catch(err) { + } + + const pr_no = '${{ inputs.pr_no }}'; + const url = '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}'; + const test_status = '${{ steps.benchmarks.outcome }}'; + const job_status = '${{ job.status }}'; + const params = '${{ inputs.bench_script_params }}'; + const body = `Compute Benchmarks run (${params}):\n${url}\nJob status: ${job_status}. Test status: ${test_status}.\n ${markdown}`; + + github.rest.issues.createComment({ + issue_number: pr_no, + owner: context.repo.owner, + repo: context.repo.repo, + body: body + }) + + - name: Upload HTML report + if: ${{ always() && inputs.upload_report }} + uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 + with: + path: ${{env.BUILD_DIR}}/benchmark_results.html + key: benchmark-results-${{ github.run_id }} diff --git a/.github/workflows/reusable_docs_build.yml b/.github/workflows/reusable_docs_build.yml index e90ca87aed..c27045c5b7 100644 --- a/.github/workflows/reusable_docs_build.yml +++ b/.github/workflows/reusable_docs_build.yml @@ -45,6 +45,21 @@ jobs: -DUMF_DISABLE_HWLOC=ON cmake --build build --target docs + - name: Download benchmark HTML before uploading with documentation on GitHub pages + # If the benchmark results are meant to be uploaded on GH pages + if: ${{ inputs.upload == true }} + id: download-bench-html + uses: actions/cache/restore@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 + with: + path: ${{github.workspace}}/build/benchmark_results.html + key: benchmark-results- + + - name: Move benchmark HTML + if: ${{ inputs.upload == true && steps.download-bench-html.outputs.cache-hit != '' }} + # exact or partial cache hit + run: | + mv ${{ github.workspace }}/build/benchmark_results.html ${{ github.workspace }}/build/docs_build/generated/html + - name: Upload artifact if: ${{ inputs.upload == true }} uses: actions/upload-pages-artifact@0252fc4ba7626f0298f0cf00902a25c6afc77fa8 # v3.0.0