Skip to content

Commit f269c9a

Browse files
zeshengzongzeshengzong
authored
Merge pull request #1 from shink/init
Add Ascend NPU
2 parents db12eea + bff16d7 commit f269c9a

21 files changed

+1500
-2
lines changed

‎.ci/benchmark.py‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
import argparse
2+
import os
3+
import sys
4+
from src.benchmark.utils import read_metrics, to_markdown_table
5+
6+
7+
def parse_args():
8+
parser = argparse.ArgumentParser()
9+
parser.add_argument("--path", type=str, required=True, help="Report path.")
10+
parser.add_argument("--write-gh-job-summary", action="store_true", help="Write to GitHub job summary.")
11+
parser.add_argument("--update-readme", action="store_true", help="Update statistics report in README.md.")
12+
return parser.parse_args()
13+
14+
15+
def generate_report(path: str):
16+
metrics = read_metrics(path, metric="accuracy")
17+
html_table = to_markdown_table(metrics)
18+
return html_table
19+
20+
21+
def write_job_summary(report):
22+
summary_path = os.environ["GITHUB_STEP_SUMMARY"]
23+
with open(summary_path, "a") as f:
24+
f.write("## Torchbenchmark statistics report\n")
25+
f.write(report)
26+
27+
28+
def update_readme(report):
29+
project_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
30+
readme_path = os.path.join(project_path, "README.md")
31+
print(readme_path)
32+
with open(readme_path, "r") as f:
33+
readme_content = f.read()
34+
35+
start_marker = "<!-- Torchbenchmark start -->"
36+
end_marker = "<!-- Torchbenchmark end -->"
37+
start_index = readme_content.find(start_marker)
38+
end_index = readme_content.find(end_marker)
39+
assert start_index != -1
40+
assert end_index != -1
41+
42+
start_index += len(start_marker)
43+
new_readme_content = (
44+
readme_content[:start_index] + "\n\n" +
45+
report + "\n\n" +
46+
readme_content[end_index:]
47+
)
48+
with open(readme_path, "w") as f:
49+
f.write(new_readme_content)
50+
51+
52+
if __name__ == "__main__":
53+
args = parse_args()
54+
55+
# Generate statistics report
56+
report = generate_report(args.path)
57+
58+
# Write to workflow job summary
59+
if args.write_gh_job_summary:
60+
write_job_summary(report)
61+
62+
# Update README.md
63+
if args.update_readme:
64+
update_readme(report)

‎.github/actions/install-artifact/action.yml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
name: "Install artifact with pip"
2+
description: "Install artifact with pip"
3+
inputs:
4+
artifact:
5+
description: "The distribution artifact name"
6+
type: string
7+
required: true
8+
9+
# TODO: https://github.com/actions/runner/issues/3620
10+
runs:
11+
using: "composite"
12+
steps:
13+
- name: Download artifact
14+
uses: actions/download-artifact@v4
15+
with:
16+
name: ${{ inputs.artifact }}
17+
18+
- name: Install artifact
19+
shell: bash
20+
run: |
21+
pip install ${{ inputs.artifact }}

‎.github/actions/list-pr/action.yml‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
name: 'List pull requests'
2+
description: 'list and put output pull requests'
3+
inputs:
4+
owner:
5+
description: 'The repository owner'
6+
required: true
7+
repository:
8+
description: 'The repository name'
9+
required: true
10+
token:
11+
description: 'The GitHub token used to create an authenticated client'
12+
default: ${{ github.token }}
13+
required: false
14+
labels:
15+
description: 'The labels on pull requests'
16+
required: false
17+
default: ''
18+
hours:
19+
description: 'Pull requests created within this many hours will be listed'
20+
required: false
21+
default: '24'
22+
outputs:
23+
prs:
24+
description: 'Pull requests to be listed'
25+
value: ${{ steps.list-pr.outputs.result }}
26+
runs:
27+
using: "composite"
28+
steps:
29+
# See https://docs.github.com/en/rest/pulls/pulls?apiVersion=2022-11-28#list-pull-requests
30+
- name: List PRs
31+
uses: actions/github-script@v7
32+
id: list-pr
33+
with:
34+
github-token: ${{ inputs.token }}
35+
result-encoding: string
36+
script: |
37+
const timeRange = parseInt("${{ inputs.hours }}") * 60 * 60 * 1000;
38+
const sinceTime = new Date(Date.now() - timeRange).toISOString();
39+
const input_labels = "${{ inputs.labels }}";
40+
const labels = input_labels
41+
? input_labels.split(/[,\n]/).map(label => label.trim())
42+
: [];
43+
44+
const iterator = await github.paginate.iterator(github.rest.pulls.list, {
45+
owner: "${{ inputs.owner }}",
46+
repo: "${{ inputs.repository }}",
47+
state: "open",
48+
sort: "created",
49+
direction: "desc",
50+
per_page: 100,
51+
});
52+
53+
const prs = [];
54+
for await (const resp of iterator) {
55+
const filtered_prs = resp.data.filter(pr => {
56+
const hasLabel = labels.length === 0
57+
|| labels.every(label => pr.labels.some(prLabel => prLabel.name === label));
58+
59+
const createdAt = new Date(pr.created_at);
60+
const isWithinTimeRange = createdAt >= new Date(sinceTime);
61+
62+
return hasLabel && isWithinTimeRange;
63+
});
64+
65+
prs.push(...filtered_prs);
66+
}
67+
68+
if (prs.length > 0) {
69+
const pr_urls = prs.map(pr => pr.html_url);
70+
core.info(`prs: ${pr_urls.join("\n")}`);
71+
} else {
72+
core.warning("No pull requests found!");
73+
}
74+
75+
const result = prs.map(pr => ({
76+
pull_request: {
77+
number: pr.number,
78+
title: pr.title,
79+
state: pr.state,
80+
draft: pr.draft,
81+
created_at: pr.created_at,
82+
updated_at: pr.updated_at,
83+
closed_at: pr.closed_at,
84+
merged_at: pr.merged_at,
85+
html_url: pr.html_url,
86+
base: pr.base.ref,
87+
head: pr.head.ref,
88+
labels: pr.labels.map(label => label.name),
89+
}
90+
}));
91+
92+
return JSON.stringify(result);

‎.github/dependabot.yml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
version: 2
2+
updates:
3+
- package-ecosystem: "github-actions"
4+
directory: "/"
5+
schedule:
6+
# Check for updates to GitHub Actions every week
7+
interval: "weekly"
8+
open-pull-requests-limit: 2
9+
reviewers:
10+
- "shink"

‎.github/workflows/_ascend_npu_benchmark.yml‎

Lines changed: 204 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,204 @@
1+
name: '_ascend_npu_benchmark'
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
runner:
7+
required: true
8+
type: string
9+
description: 'The runner selected to run on'
10+
image:
11+
required: true
12+
type: string
13+
description: 'The docker image which will be loaded'
14+
device:
15+
required: true
16+
type: string
17+
description: 'The device selected to run on'
18+
torch-artifact:
19+
required: false
20+
type: string
21+
description: 'The distribution artifact name of torch'
22+
torch-npu-artifact:
23+
required: true
24+
type: string
25+
description: 'The distribution artifact name of torch_npu'
26+
secrets:
27+
pr-token:
28+
description: 'A token used to create a pull request'
29+
required: true
30+
31+
# Bash shells do not use ~/.profile or ~/.bashrc so these shells need to be explicitly
32+
# declared as "shell: bash -el {0}" on steps that need to be properly activated.
33+
# It's used to activate ascend-toolkit environment variables.
34+
defaults:
35+
run:
36+
shell: bash -el {0}
37+
38+
jobs:
39+
test:
40+
name: run benchmarks for torch_npu
41+
runs-on: ${{ inputs.runner }}
42+
container:
43+
image: ${{ inputs.image }}
44+
env:
45+
HF_ENDPOINT: https://hf-mirror.com
46+
steps:
47+
- name: Show NPU info
48+
run: |
49+
npu-smi info
50+
51+
- name: Config mirrors
52+
run: |
53+
sed -i 's|ports.ubuntu.com|mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list
54+
pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
55+
56+
- name: Install system dependencies
57+
run: |
58+
apt-get update
59+
apt-get install -y \
60+
git gcc g++ make cmake ninja-build curl \
61+
libgl1 libglib2.0-0 libsndfile1
62+
63+
# See: https://github.com/actions/checkout/issues/363#issuecomment-1915075699
64+
# See: https://github.com/hunshcn/gh-proxy/issues/28#issuecomment-773769630
65+
- name: Config git
66+
run: |
67+
git config --global --add safe.directory "$GITHUB_WORKSPACE"
68+
git config --global url."https://gh-proxy.test.osinfra.cn/https://github.com/".insteadOf https://github.com/
69+
70+
- name: Checkout
71+
uses: actions/checkout@v4
72+
73+
- name: Checkout benchmark
74+
uses: actions/checkout@v4
75+
with:
76+
repository: pytorch/benchmark
77+
path: benchmark
78+
79+
# TODO
80+
# - name: Install torch
81+
# id: install-torch
82+
# uses: ./.github/actions/install-artifact
83+
# with:
84+
# artifact: ${{ inputs.torch-artifact }}
85+
86+
- name: Download torch artifact
87+
if: ${{ inputs.torch-artifact }}
88+
uses: actions/download-artifact@v4
89+
with:
90+
name: ${{ inputs.torch-artifact }}
91+
92+
- name: Install torch
93+
if: ${{ inputs.torch-artifact }}
94+
run: |
95+
pip install ${{ inputs.torch-artifact }}
96+
97+
- name: Install torch_npu dependencies
98+
if: ${{ !inputs.torch-artifact }}
99+
run: |
100+
pip install -r https://raw.githubusercontent.com/Ascend/pytorch/refs/heads/master/requirements.txt
101+
102+
- name: List torch version
103+
id: list-torch-version
104+
shell: bash
105+
run: |
106+
torch_version=$(python -c "import torch; print(torch.__version__)")
107+
echo "torch-version=${torch_version}" >> $GITHUB_OUTPUT
108+
109+
- name: Download torch_npu artifact
110+
uses: actions/download-artifact@v4
111+
with:
112+
name: ${{ inputs.torch-npu-artifact }}
113+
path: ascend_npu
114+
115+
- name: Install torch_npu
116+
working-directory: ascend_npu
117+
run: |
118+
pip install ${{ inputs.torch-npu-artifact }}
119+
120+
- name: Install benchmark dependencies
121+
run: |
122+
pip install -r benchmark/requirements.txt \
123+
torch==${{ steps.list-torch-version.outputs.torch-version }} \
124+
numpy==1.*
125+
126+
- name: Install dependencies for all the models
127+
run: |
128+
python benchmark/install.py --userbenchmark test_bench --continue_on_fail
129+
130+
- name: Install nightly torchvision and torchaudio
131+
run: |
132+
pip install --pre torchvision torchaudio --no-deps --index-url https://download.pytorch.org/whl/nightly/cpu
133+
134+
- name: Install project dependencies
135+
run: |
136+
pip install -r requirements.txt
137+
138+
- name: Show environment info
139+
run: |
140+
npu_is_available=$(python -c "import torch; print(torch.npu.is_available())")
141+
npu_count=$(python -c "import torch; print(torch.npu.device_count())")
142+
echo "NPU is available: ${npu_is_available}"
143+
echo "NPU count: ${npu_count}"
144+
pip list | grep -E 'torch|numpy'
145+
146+
- name: Run benchmarks
147+
working-directory: benchmark
148+
run: |
149+
python run_benchmark.py test_bench --accuracy --device npu --test eval \
150+
--output ascend_npu_benchmark.json
151+
152+
- name: Upload the benchmark report file
153+
id: upload-report
154+
uses: actions/upload-artifact@v4
155+
with:
156+
name: ascend_npu_benchmark.json
157+
path: benchmark/ascend_npu_benchmark.json
158+
if-no-files-found: error
159+
retention-days: 1
160+
overwrite: true
161+
162+
- name: Write to workflow job summary
163+
run: |
164+
python .ci/benchmark.py --write-gh-job-summary --path benchmark/ascend_npu_benchmark.json
165+
166+
- name: Update README.md
167+
if: ${{ contains(fromJSON('["push", "schedule", "workflow_dispatch"]'), github.event_name) }}
168+
id: update-readme
169+
run: |
170+
python .ci/benchmark.py --update-readme --path benchmark/ascend_npu_benchmark.json
171+
if git diff --quiet README.md; then
172+
echo "README.md not changed"
173+
echo "changed=false" >> $GITHUB_OUTPUT
174+
else
175+
echo "README.md changed"
176+
echo "changed=true" >> $GITHUB_OUTPUT
177+
fi
178+
179+
# See: https://github.com/peter-evans/create-pull-request
180+
- name: Create a pull request for changes to README.md
181+
if: ${{ steps.update-readme.outputs.changed == 'true' }}
182+
uses: peter-evans/create-pull-request@v7
183+
with:
184+
token: ${{ secrets.pr-token }}
185+
base: ${{ github.head_ref }}
186+
committer: 'cosdt-bot <[email protected]>'
187+
author: 'cosdt-bot <[email protected]>'
188+
commit-message: 'Update torchbenchmark report in README.md'
189+
add-paths: README.md
190+
branch: ascend-npu/benchmark
191+
title: '[Ascend NPU] Update torchbenchmark report in README.md'
192+
body: |
193+
The torchbenchmark results running on Ascend NPU have changed, I'm updating the report in README.md.
194+
195+
Please check out the following resources for more information:
196+
197+
- [Workflow run][1]
198+
- [Torchbenchmark report][2] (click to download)
199+
200+
[1]: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
201+
[2]: ${{ steps.upload-report.outputs.artifact-url }}
202+
203+
cc: @Yikun @hipudding @FFFrog @Zhenbin-8 @zeshengzong @wjunLu @MengqingCao @shink
204+
reviewers: shink

0 commit comments

Comments
 (0)