From 3d9741b0aca24ea7979a2c54ab3fefc990685241 Mon Sep 17 00:00:00 2001 From: Piotr Balcer Date: Thu, 20 Feb 2025 10:08:05 +0000 Subject: [PATCH 1/2] benchmarks workflow --- .github/workflows/benchmarks-reusable.yml | 197 ++++++++++++++++++ .github/workflows/benchmarks.yml | 53 +++++ unified-runtime/scripts/benchmarks/main.py | 2 +- .../scripts/benchmarks/requirements.txt | 4 + 4 files changed, 255 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/benchmarks-reusable.yml create mode 100644 .github/workflows/benchmarks.yml create mode 100644 unified-runtime/scripts/benchmarks/requirements.txt diff --git a/.github/workflows/benchmarks-reusable.yml b/.github/workflows/benchmarks-reusable.yml new file mode 100644 index 0000000000000..4b39a96608ae1 --- /dev/null +++ b/.github/workflows/benchmarks-reusable.yml @@ -0,0 +1,197 @@ +name: Benchmarks Reusable + +on: + workflow_call: + inputs: + str_name: + required: true + type: string + pr_no: + required: true + # even though this is a number, this is a workaround for issues with + # reusable workflow calls that result in "Unexpected value '0'" error. + type: string + bench_script_params: + required: false + type: string + default: '' + sycl_config_params: + required: false + type: string + default: '' + upload_report: + required: false + type: boolean + default: false + compute_runtime_commit: + required: false + type: string + default: '' + +permissions: + contents: read + pull-requests: write + +jobs: + e2e-build-hw: + name: Build SYCL, Run Benchmarks + strategy: + matrix: + adapter: [ + {str_name: "${{ inputs.str_name }}", + sycl_config: "${{ inputs.sycl_config_params }}" + } + ] + build_type: [Release] + compiler: [{c: clang, cxx: clang++}] + + runs-on: "PERF" + + steps: + - name: Add comment to PR + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + if: ${{ always() && inputs.pr_no != 0 }} + with: + script: | + const pr_no = '${{ inputs.pr_no }}'; + const adapter = '${{ matrix.adapter.str_name }}'; + const url = '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}'; + const params = '${{ inputs.bench_script_params }}'; + const body = `Compute Benchmarks ${adapter} run (with params: ${params}):\n${url}`; + + github.rest.issues.createComment({ + issue_number: pr_no, + owner: context.repo.owner, + repo: context.repo.repo, + body: body + }) + + - name: Checkout SYCL + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + path: sycl-repo + + # We need to fetch special ref for proper PR's merge commit. Note, this ref may be absent if the PR is already merged. + - name: Fetch PR's merge commit + if: ${{ inputs.pr_no != 0 }} + working-directory: ${{github.workspace}}/sycl-repo + env: + PR_NO: ${{ inputs.pr_no }} + run: | + git fetch -- https://github.com/${{github.repository}} +refs/pull/${PR_NO}/*:refs/remotes/origin/pr/${PR_NO}/* + git checkout origin/pr/${PR_NO}/merge + git rev-parse origin/pr/${PR_NO}/merge + + - name: Install pip packages + run: | + pip install --force-reinstall -r ${{github.workspace}}/sycl-repo/unified-runtime/scripts/benchmarks/requirements.txt + + - name: Configure SYCL + run: > + python3 sycl-repo/buildbot/configure.py + -t ${{matrix.build_type}} + -o ${{github.workspace}}/sycl_build + --cmake-gen "Ninja" + --cmake-opt="-DLLVM_INSTALL_UTILS=ON" + --cmake-opt="-DSYCL_PI_TESTS=OFF" + --cmake-opt=-DCMAKE_C_COMPILER_LAUNCHER=ccache + --cmake-opt=-DCMAKE_CXX_COMPILER_LAUNCHER=ccache + ${{matrix.adapter.sycl_config}} + + - name: Build SYCL + run: cmake --build ${{github.workspace}}/sycl_build -j + + # We need a complete installed UR for compute-benchmarks. + - name: Configure UR + run: > + cmake -DCMAKE_BUILD_TYPE=Release + -S${{github.workspace}}/sycl-repo/unified-runtime + -B${{github.workspace}}/ur_build + -DCMAKE_INSTALL_PREFIX=${{github.workspace}}/ur_install + -DUR_BUILD_TESTS=OFF + -DUR_BUILD_ADAPTER_L0=ON + -DUR_BUILD_ADAPTER_L0_V2=ON + -DUMF_DISABLE_HWLOC=ON + + - name: Build UR + run: cmake --build ${{github.workspace}}/ur_build -j $(nproc) + + - name: Install UR + run: cmake --install ${{github.workspace}}/ur_build + + - name: Compute core range + run: | + # Compute the core range for the first NUMA node; second node is for UMF jobs. + # Skip the first 4 cores - the kernel is likely to schedule more work on these. + CORES=$(lscpu | awk ' + /NUMA node0 CPU|On-line CPU/ {line=$0} + END { + split(line, a, " ") + split(a[4], b, ",") + sub(/^0/, "4", b[1]) + print b[1] + }') + echo "Selected core: $CORES" + echo "CORES=$CORES" >> $GITHUB_ENV + + ZE_AFFINITY_MASK=0 + echo "ZE_AFFINITY_MASK=$ZE_AFFINITY_MASK" >> $GITHUB_ENV + + # this is missing these options due to a runner issue: + # --compute-runtime ${{ inputs.compute_runtime_commit }} + # --build-igc + - name: Run benchmarks + working-directory: ${{ github.workspace }} + id: benchmarks + run: > + taskset -c ${{ env.CORES }} ${{ github.workspace }}/sycl-repo/unified-runtime/scripts/benchmarks/main.py + ~/bench_workdir + --sycl ${{ github.workspace }}/sycl_build + --ur ${{ github.workspace }}/ur_install + --adapter ${{ matrix.adapter.str_name }} + --compare baseline + ${{ inputs.upload_report && '--output-html' || '' }} + ${{ inputs.pr_no != 0 && '--output-markdown' || '' }} + ${{ inputs.bench_script_params }} + + - name: Print benchmark results + run: | + cat ${{ github.workspace }}/benchmark_results.md || true + + - name: Add comment to PR + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + if: ${{ always() && inputs.pr_no != 0 }} + with: + script: | + let markdown = "" + try { + const fs = require('fs'); + markdown = fs.readFileSync('benchmark_results.md', 'utf8'); + } catch(err) { + } + + const pr_no = '${{ inputs.pr_no }}'; + const adapter = '${{ matrix.adapter.str_name }}'; + const url = '${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}'; + const test_status = '${{ steps.benchmarks.outcome }}'; + const job_status = '${{ job.status }}'; + const params = '${{ inputs.bench_script_params }}'; + const body = `Benchmarks ${adapter} run (${params}):\n${url}\nJob status: ${job_status}. Test status: ${test_status}.\n ${markdown}`; + + github.rest.issues.createComment({ + issue_number: pr_no, + owner: context.repo.owner, + repo: context.repo.repo, + body: body + }) + + - name: Upload HTML report + if: ${{ always() && inputs.upload_report }} + uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 + with: + path: benchmark_results_${{ inputs.pr_no }}.html + key: benchmark-results-${{ inputs.pr_no }}-${{ matrix.adapter.str_name }}-${{ github.run_id }} + + - name: Get information about platform + if: ${{ always() }} + run: ${{github.workspace}}/sycl-repo/unified-runtime/.github/scripts/get_system_info.sh diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 0000000000000..c94681ef36c49 --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,53 @@ +name: Benchmarks + +on: + workflow_dispatch: + inputs: + str_name: + description: Adapter + type: choice + required: true + default: 'level_zero' + options: + - level_zero + - level_zero_v2 + pr_no: + description: PR number (if 0, it'll run on the main) + type: number + required: true + bench_script_params: + description: Parameters passed to the script executing benchmark + type: string + required: false + default: '' + sycl_config_params: + description: Extra params for SYCL configuration + type: string + required: false + default: '' + compute_runtime_commit: + description: 'Compute Runtime commit' + type: string + required: false + default: '' + upload_report: + description: 'Upload HTML report' + type: boolean + required: false + default: false + +permissions: + contents: read + pull-requests: write + +jobs: + manual: + name: Compute Benchmarks + uses: ./.github/workflows/benchmarks-reusable.yml + with: + str_name: ${{ inputs.str_name }} + pr_no: ${{ inputs.pr_no }} + bench_script_params: ${{ inputs.bench_script_params }} + sycl_config_params: ${{ inputs.sycl_config_params }} + compute_runtime_commit: ${{ inputs.compute_runtime_commit }} + upload_report: ${{ inputs.upload_report }} diff --git a/unified-runtime/scripts/benchmarks/main.py b/unified-runtime/scripts/benchmarks/main.py index 0b932a888dca2..dd44d500cff1a 100755 --- a/unified-runtime/scripts/benchmarks/main.py +++ b/unified-runtime/scripts/benchmarks/main.py @@ -263,7 +263,7 @@ def main(directory, additional_env_vars, save_name, compare_names, filter): if options.output_html: html_content = generate_html( - history.runs, "oneapi-src/unified-runtime", compare_names + history.runs, "intel/llvm", compare_names ) with open("benchmark_results.html", "w") as file: diff --git a/unified-runtime/scripts/benchmarks/requirements.txt b/unified-runtime/scripts/benchmarks/requirements.txt new file mode 100644 index 0000000000000..99ba0caab55c2 --- /dev/null +++ b/unified-runtime/scripts/benchmarks/requirements.txt @@ -0,0 +1,4 @@ +matplotlib==3.9.2 +mpld3==0.5.10 +dataclasses-json==0.6.7 +PyYAML==6.0.1 From 792552be869c9e4223798a44e564ef84f70f5756 Mon Sep 17 00:00:00 2001 From: Piotr Balcer Date: Thu, 20 Feb 2025 12:02:01 +0100 Subject: [PATCH 2/2] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6368c8ecda960..371070ff694d8 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -This is the Intel staging area for llvm.org contributions and the home for + This is the Intel staging area for llvm.org contributions and the home for Intel LLVM-based projects: - [oneAPI DPC++ compiler](#oneapi-dpc-compiler)