Skip to content

Commit 482f9b0

Browse files
huydhnhuydhn
authored
Create upload_benchmark_results reusable workflow (#7269)
This is needed to unblock pytorch/helion#732. By unravelled the current upload_benchmark_results GHA, the new `.github/workflows/upload_benchmark_results.yml` workflow needs 4 inputs: 1. The benchmark artifact itself, uploaded to GitHub via `actions/upload-artifact` 2. The benchmark metadata including workflow ID, job ID, repo, and branch name. This is collected by the new `gather-benchmark-metadata` GHA 3. The runners info including GPU or CPU type. This is collected by the new `gather-runners-info` GHA 4. The list of dependencies. This is collected by the new `gather-dependencies` GHA Two examples on how to use the new reusable workflow are at: * `.github/workflows/test_upload_benchmark_results.yml` * Helion pytorch/helion#758 --------- Signed-off-by: Huy Do <[email protected]>
1 parent cf868af commit 482f9b0

File tree

6 files changed

+315
-3
lines changed

6 files changed

+315
-3
lines changed

‎.github/actions/gather-benchmark-metadata/action.yml‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
name: Gather benchmark metadata
2+
3+
description: Gather the metadata about the benchmark run
4+
5+
inputs:
6+
github-token:
7+
default: ''
8+
schema-version:
9+
default: 'v3'
10+
venv:
11+
description: 'Path to virtual environment to activate'
12+
required: false
13+
default: ''
14+
15+
outputs:
16+
benchmark-metadata:
17+
description: The benchmark metadata in JSON format
18+
value: ${{ steps.gather-metadata.outputs.metadata }}
19+
20+
runs:
21+
using: composite
22+
steps:
23+
- name: Check that GITHUB_TOKEN is defined
24+
env:
25+
GITHUB_TOKEN: ${{ inputs.github-token }}
26+
shell: bash
27+
run: |
28+
set -eux
29+
30+
if [[ -z "${GITHUB_TOKEN}" ]]; then
31+
echo "Missing github-token input"
32+
exit 1
33+
fi
34+
35+
- name: Get workflow job id
36+
if: ${{ inputs.github-token != '' }}
37+
id: get-job-id
38+
uses: pytorch/test-infra/.github/actions/get-workflow-job-id@main
39+
with:
40+
github-token: ${{ inputs.github-token }}
41+
42+
- name: Gather the metadata
43+
id: gather-metadata
44+
shell: bash
45+
env:
46+
SCHEMA_VERSION: ${{ inputs.schema-version }}
47+
REPO: ${{ github.repository }}
48+
HEAD_BRANCH: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.ref }}
49+
HEAD_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
50+
WORKFLOW_RUN_ID: ${{ github.run_id }}
51+
RUN_ATTEMPT: ${{ github.run_attempt }}
52+
JOB_ID: ${{ inputs.github-token != '' && steps.get-job-id.outputs.job-id || '0' }}
53+
JOB_NAME: ${{ inputs.github-token != '' && steps.get-job-id.outputs.job-name || '' }}
54+
run: |
55+
set -eux
56+
57+
if [[ -n "${{ inputs.venv }}" ]]; then
58+
source "${{ inputs.venv }}"
59+
fi
60+
61+
python3 "${GITHUB_ACTION_PATH}/../../scripts/benchmarks/gather_metadata.py" \
62+
--schema-version "${SCHEMA_VERSION}" \
63+
--repo "${REPO}" \
64+
--head-branch "${HEAD_BRANCH}" \
65+
--head-sha "${HEAD_SHA}" \
66+
--workflow-id "${WORKFLOW_RUN_ID}" \
67+
--run-attempt "${RUN_ATTEMPT}" \
68+
--job-id "${JOB_ID}" \
69+
--job-name "${JOB_NAME}"

‎.github/actions/gather-dependencies/action.yml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
name: Gather dependencies
2+
3+
description: Gather the list of all dependencies
4+
5+
inputs:
6+
venv:
7+
description: 'Path to virtual environment to activate'
8+
required: false
9+
default: ''
10+
11+
outputs:
12+
dependencies:
13+
description: The list of all dependencies in JSON format
14+
value: ${{ steps.gather-dependencies.outputs.dependencies }}
15+
16+
runs:
17+
using: composite
18+
steps:
19+
- name: Gather the dependencies information
20+
id: gather-dependencies
21+
shell: bash
22+
run: |
23+
set -eux
24+
25+
# TODO (huydhn): Implement this part
26+
echo "dependencies={}" >> "${GITHUB_OUTPUT}"

‎.github/actions/gather-runners-info/action.yml‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
name: Gather runners info
2+
3+
description: Gather the information about the runners
4+
5+
inputs:
6+
venv:
7+
description: 'Path to virtual environment to activate'
8+
required: false
9+
default: ''
10+
11+
outputs:
12+
runners-info:
13+
description: The runners info in JSON format
14+
value: ${{ steps.gather-runners-info.outputs.runners }}
15+
16+
runs:
17+
using: composite
18+
steps:
19+
- name: Get device name
20+
shell: bash
21+
run: |
22+
set -eux
23+
24+
if command -v nvidia-smi; then
25+
DEVICE_NAME=cuda
26+
nvidia-smi
27+
elif command -v rocm-smi; then
28+
DEVICE_NAME=rocm
29+
rocm-smi
30+
else
31+
DEVICE_NAME=cpu
32+
lscpu
33+
fi
34+
echo "DEVICE_NAME=$DEVICE_NAME" >> $GITHUB_ENV
35+
36+
- name: Get device type
37+
shell: bash
38+
run: |
39+
set -eux
40+
41+
if [[ "${DEVICE_NAME}" == "cuda" ]]; then
42+
DEVICE_TYPE=$(nvidia-smi -i 0 --query-gpu=name --format=csv,noheader | awk '{print $2}')
43+
elif [[ "${DEVICE_NAME}" == "rocm" ]]; then
44+
DEVICE_TYPE=$(rocminfo | grep "Marketing Name" | tail -n1 | awk -F':' '{print $2}' | xargs)
45+
elif [[ "${DEVICE_NAME}" == "cpu" ]]; then
46+
DEVICE_TYPE=$(lscpu | grep 'Model name' | cut -f 2 -d ":" | awk '{$1=$1}1' | cut -f 2 -d " ")
47+
fi
48+
echo "DEVICE_TYPE=$DEVICE_TYPE" >> $GITHUB_ENV
49+
50+
- name: Gather the runners information
51+
id: gather-runners-info
52+
shell: bash
53+
run: |
54+
set -eux
55+
56+
if [[ -n "${{ inputs.venv }}" ]]; then
57+
source "${{ inputs.venv }}"
58+
fi
59+
60+
python3 -mpip install psutil==7.0.0 nvidia-ml-py==13.580.82
61+
python3 "${GITHUB_ACTION_PATH}/../../scripts/benchmarks/gather_runners_info.py"

‎.github/workflows/test_gather_benchmark_info.yml‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
name: Test all GHA to gather different benchmark info
2+
3+
on:
4+
workflow_call:
5+
outputs:
6+
benchmark-metadata:
7+
value: ${{ jobs.test-gather-benchmark-info.outputs.benchmark-metadata }}
8+
runners-info:
9+
value: ${{ jobs.test-gather-benchmark-info.outputs.runners-info }}
10+
dependencies:
11+
value: ${{ jobs.test-gather-benchmark-info.outputs.dependencies }}
12+
13+
jobs:
14+
test-gather-benchmark-info:
15+
runs-on: linux.2xlarge
16+
outputs:
17+
benchmark-metadata: ${{ steps.gather-benchmark-metadata.outputs.benchmark-metadata }}
18+
runners-info: ${{ steps.gather-runners-info.outputs.runners-info }}
19+
dependencies: ${{ steps.gather-dependencies.outputs.dependencies }}
20+
steps:
21+
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
22+
23+
- name: Gather benchmark metadata
24+
id: gather-benchmark-metadata
25+
uses: ./.github/actions/gather-benchmark-metadata
26+
with:
27+
github-token: ${{ secrets.GITHUB_TOKEN }}
28+
29+
- name: Gather runners info
30+
id: gather-runners-info
31+
uses: ./.github/actions/gather-runners-info
32+
33+
- name: Gather dependencies
34+
id: gather-dependencies
35+
uses: ./.github/actions/gather-dependencies
36+
37+
- name: Upload the mock benchmark results
38+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
39+
with:
40+
name: benchmark-results
41+
path: .github/scripts/benchmark-results-dir-for-testing/v3

‎.github/workflows/test_upload_benchmark_results.yml‎

Lines changed: 32 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,20 +3,49 @@ name: Test upload-benchmark-results
33
on:
44
pull_request:
55
paths:
6+
# GHA route
7+
- .github/scripts/benchmarks/*
68
- .github/scripts/upload_benchmark_results.py
7-
- .github/workflows/test_upload_benchmark_results.ym
89
- .github/actions/upload-benchmark-results/*
10+
# Reusable workflow route
11+
- .github/actions/gather-benchmark-metadata/*
12+
- .github/actions/gather-runners-info/*
13+
- .github/actions/gather-dependencies/*
14+
- .github/workflows/upload_benchmark_results.yml
15+
# The test workflow itself
16+
- .github/workflows/test_upload_benchmark_results.yml
17+
18+
permissions:
19+
id-token: write
20+
contents: read
921

1022
jobs:
11-
test:
23+
test-upload-benchmark-results-gha:
1224
runs-on: linux.2xlarge
1325
steps:
1426
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
1527

16-
- name: Test upload the benchmark results (v3)
28+
- name: Test upload the benchmark results
1729
uses: ./.github/actions/upload-benchmark-results
1830
with:
1931
benchmark-results-dir: .github/scripts/benchmark-results-dir-for-testing/v3
2032
schema-version: v3
2133
dry-run: true
2234
github-token: ${{ secrets.GITHUB_TOKEN }}
35+
36+
test-gather-benchmark-info:
37+
uses: ./.github/workflows/test_gather_benchmark_info.yml
38+
39+
test-upload-benchmark-results-reusable-workflow:
40+
needs: test-gather-benchmark-info
41+
uses: ./.github/workflows/upload_benchmark_results.yml
42+
permissions:
43+
id-token: write
44+
contents: read
45+
with:
46+
benchmark-artifact: benchmark-results
47+
benchmark-metadata: ${{ needs.test-gather-benchmark-info.outputs.benchmark-metadata }}
48+
runners-info: ${{ needs.test-gather-benchmark-info.outputs.runners-info }}
49+
dependencies: ${{ needs.test-gather-benchmark-info.outputs.dependencies }}
50+
schema-version: v3
51+
dry-run: true

‎.github/workflows/upload_benchmark_results.yml‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
name: Upload benchmark results
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
benchmark-artifact:
7+
description: The name of the benchmark artifact to upload
8+
required: true
9+
type: string
10+
benchmark-metadata:
11+
description: The benchmark metadata provided by gather-benchmark-metadata GHA
12+
required: true
13+
type: string
14+
runners-info:
15+
description: The runners info gathered by gather-runners-info GHA
16+
required: true
17+
type: string
18+
dependencies:
19+
description: The list of dependencies (not yet implemented)
20+
required: false
21+
type: string
22+
default: '{}'
23+
schema-version:
24+
default: 'v3'
25+
type: string
26+
dry-run:
27+
default: true
28+
type: boolean
29+
30+
jobs:
31+
job:
32+
name: Uploading ${{ inputs.benchmark-artifact }}
33+
runs-on: linux.2xlarge
34+
# Keep this in case we need to use OIDC later on, so the permission is there
35+
permissions:
36+
id-token: write
37+
contents: read
38+
steps:
39+
- name: Checkout test-infra
40+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
41+
with:
42+
repository: pytorch/test-infra
43+
ref: main
44+
path: test-infra
45+
46+
- name: Download benchmark artifacts
47+
uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
48+
with:
49+
name: ${{ inputs.benchmark-artifact }}
50+
path: ${{ runner.temp }}/benchmark-artifact
51+
52+
- name: Upload benchmark results
53+
shell: bash
54+
working-directory: test-infra
55+
env:
56+
BENCHMARK_RESULTS_DIR: ${{ runner.temp }}/benchmark-artifact
57+
DRY_RUN: ${{ inputs.dry-run }}
58+
# Additional information about the benchmarks
59+
BENCHMARK_METADATA: ${{ inputs.benchmark-metadata }}
60+
RUNNERS_INFO: ${{ inputs.runners-info }}
61+
DEPENDENCIES: ${{ inputs.dependencies }}
62+
run: |
63+
set -eux
64+
65+
if [[ ! -d "${BENCHMARK_RESULTS_DIR}" ]]; then
66+
echo "${BENCHMARK_RESULTS_DIR} does not exist, skipping"
67+
# We don't want the job to fail if the directory doesn't exist
68+
exit 0
69+
fi
70+
71+
python3 -mpip install boto3==1.35.33
72+
73+
if [[ "${DRY_RUN}" == "true" ]]; then
74+
python3 ".github/scripts/upload_benchmark_results.py" \
75+
--benchmark-results-dir "${BENCHMARK_RESULTS_DIR}" \
76+
--metadata "${BENCHMARK_METADATA}" \
77+
--runners "${RUNNERS_INFO}" \
78+
--dependencies "${DEPENDENCIES}" \
79+
--dry-run
80+
else
81+
python3 ".github/scripts/upload_benchmark_results.py" \
82+
--benchmark-results-dir "${BENCHMARK_RESULTS_DIR}" \
83+
--metadata "${BENCHMARK_METADATA}" \
84+
--runners "${RUNNERS_INFO}" \
85+
--dependencies "${DEPENDENCIES}"
86+
fi

0 commit comments

Comments
 (0)