|
10 | 10 | name: Benchmark libc++ |
11 | 11 |
|
12 | 12 | permissions: |
13 | | - contents: read # Default everything to read-only |
| 13 | + contents: read |
14 | 14 |
|
15 | 15 | on: |
16 | 16 | issue_comment: |
|
21 | 21 | env: |
22 | 22 | CC: clang-22 |
23 | 23 | CXX: clang++-22 |
24 | | - COMMENT_BODY: ${{ github.event.comment.body }} |
25 | | - PULL_REQUEST_HEAD: ${{ github.event.issue.pull_request.head.sha }} |
26 | | - PULL_REQUEST_BASE: ${{ github.event.issue.pull_request.base.sha }} |
27 | 24 |
|
28 | 25 | jobs: |
29 | 26 | run-benchmarks: |
| 27 | + permissions: |
| 28 | + pull-requests: write |
| 29 | + |
30 | 30 | if: >- |
31 | 31 | github.event.issue.pull_request && |
32 | 32 | contains(github.event.comment.body, '/libcxx-bot benchmark') |
33 | 33 |
|
34 | 34 | runs-on: llvm-premerge-libcxx-next-runners # TODO: This should run on a dedicated set of machines |
35 | 35 | steps: |
36 | | - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 |
37 | | - with: |
38 | | - ref: ${PULL_REQUEST_HEAD} |
39 | | - fetch-depth: 0 |
40 | | - fetch-tags: true # This job requires access to all the Git branches so it can diff against (usually) main |
41 | | - |
42 | | - - uses: actions/setup-python@v6 |
| 36 | + - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 |
43 | 37 | with: |
44 | 38 | python-version: '3.10' |
45 | 39 |
|
46 | | - - name: Install dependencies |
| 40 | + - name: Extract information from the PR |
| 41 | + id: vars |
| 42 | + env: |
| 43 | + COMMENT_BODY: ${{ github.event.comment.body }} |
47 | 44 | run: | |
48 | 45 | python3 -m venv .venv |
49 | 46 | source .venv/bin/activate |
50 | | - python -m pip install -r libcxx/utils/requirements.txt |
| 47 | + python -m pip install pygithub |
| 48 | +
|
| 49 | + cat <<EOF | python >> ${GITHUB_OUTPUT} |
| 50 | + import github |
| 51 | + repo = github.Github("${{ github.token }}").get_repo("${{ github.repository }}") |
| 52 | + pr = repo.get_pull(${{ github.event.issue.number }}) |
| 53 | + print(f"pr_base={pr.base.sha}") |
| 54 | + print(f"pr_head={pr.head.sha}") |
| 55 | + EOF |
| 56 | + BENCHMARKS=$(echo "$COMMENT_BODY" | sed -nE 's/\/libcxx-bot benchmark (.+)/\1/p') |
| 57 | + echo "benchmarks=${BENCHMARKS}" >> ${GITHUB_OUTPUT} |
| 58 | +
|
| 59 | + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 |
| 60 | + with: |
| 61 | + ref: ${{ steps.vars.outputs.pr_head }} |
| 62 | + fetch-depth: 0 |
| 63 | + fetch-tags: true # This job requires access to all the Git branches so it can diff against (usually) main |
| 64 | + path: repo # Avoid nuking the workspace, where we have the Python virtualenv |
51 | 65 |
|
52 | 66 | - name: Run baseline |
53 | 67 | run: | |
54 | | - BENCHMARKS=$(echo "${COMMENT_BODY}" | sed -nE 's/\/libcxx-bot benchmark (.+)/\1/p') |
55 | | - baseline_commit=$(git merge-base ${PULL_REQUEST_BASE} ${PULL_REQUEST_SHA}) |
56 | | - ./libcxx/utils/test-at-commit --commit ${baseline_commit} -B build/baseline -- -sv -j1 --param optimization=speed ${BENCHMARKS} |
| 68 | + source .venv/bin/activate && cd repo |
| 69 | + python -m pip install -r libcxx/utils/requirements.txt |
| 70 | + baseline_commit=$(git merge-base ${{ steps.vars.outputs.pr_base }} ${{ steps.vars.outputs.pr_head }}) |
| 71 | + ./libcxx/utils/test-at-commit --commit ${baseline_commit} -B build/baseline -- -sv -j1 --param optimization=speed ${{ steps.vars.outputs.benchmarks }} |
| 72 | + ./libcxx/utils/consolidate-benchmarks build/baseline | tee baseline.lnt |
57 | 73 |
|
58 | 74 | - name: Run candidate |
59 | 75 | run: | |
60 | | - BENCHMARKS=$(echo "${COMMENT_BODY}" | sed -nE 's/\/libcxx-bot benchmark (.+)/\1/p') |
61 | | - ./libcxx/utils/test-at-commit --commit ${PULL_REQUEST_SHA} -B build/candidate -- -sv -j1 --param optimization=speed ${BENCHMARKS} |
| 76 | + source .venv/bin/activate && cd repo |
| 77 | + ./libcxx/utils/test-at-commit --commit ${{ steps.vars.outputs.pr_head }} -B build/candidate -- -sv -j1 --param optimization=speed ${{ steps.vars.outputs.benchmarks }} |
| 78 | + ./libcxx/utils/consolidate-benchmarks build/candidate | tee candidate.lnt |
62 | 79 |
|
63 | 80 | - name: Compare baseline and candidate runs |
64 | | - run: ./libcxx/utils/compare-benchmarks <(./libcxx/utils/consolidate-benchmarks build/baseline) \ |
65 | | - <(./libcxx/utils/consolidate-benchmarks build/candidate) |
| 81 | + run: | |
| 82 | + source .venv/bin/activate && cd repo |
| 83 | + ./libcxx/utils/compare-benchmarks baseline.lnt candidate.lnt | tee results.txt |
| 84 | +
|
| 85 | + - name: Update comment with results |
| 86 | + run: | |
| 87 | + source .venv/bin/activate && cd repo |
| 88 | + cat <<EOF | python |
| 89 | + import github |
| 90 | + repo = github.Github("${{ github.token }}").get_repo("${{ github.repository }}") |
| 91 | + pr = repo.get_pull(${{ github.event.issue.number }}) |
| 92 | + comment = pr.get_issue_comment(${{ github.event.comment.id }}) |
| 93 | + with open('results.txt', 'r') as f: |
| 94 | + benchmark_results = f.read() |
| 95 | +
|
| 96 | + new_comment_text = f""" |
| 97 | + {comment.body} |
| 98 | +
|
| 99 | + <details> |
| 100 | + <summary> |
| 101 | + Benchmark results: |
| 102 | + </summary> |
| 103 | +
|
| 104 | + \`\`\` |
| 105 | + {benchmark_results} |
| 106 | + \`\`\` |
| 107 | +
|
| 108 | + </details> |
| 109 | + """ |
| 110 | +
|
| 111 | + comment.edit(new_comment_text) |
| 112 | + EOF |
0 commit comments