Skip to content

Commit 1f1a036

Browse files
shinkshink
authored
Add torchbenchmark (#11)
1 parent facaec6 commit 1f1a036

16 files changed

+458
-156
lines changed

‎.ci/benchmark.py

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
import os
2+
import sys
3+
from src.benchmark.utils import read_metrics, to_markdown_table
4+
5+
if __name__ == "__main__":
6+
# Generate statistics report
7+
statistics_path = sys.argv[1]
8+
metrics = read_metrics(statistics_path, metric="accuracy")
9+
html_table = to_markdown_table(metrics)
10+
11+
# Write to workflow job summary
12+
summary_path = os.environ["GITHUB_STEP_SUMMARY"]
13+
with open(summary_path, "a") as f:
14+
f.write("## Torchbenchmark statistics report\n")
15+
f.write(html_table)

‎.github/workflows/_ascend_npu_benchmark.yml

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
name: '_ascend_npu_benchmark'
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
runner:
7+
required: true
8+
type: string
9+
description: 'The runner selected to run on'
10+
image:
11+
required: true
12+
type: string
13+
description: 'The docker image which will be loaded'
14+
device:
15+
required: true
16+
type: string
17+
description: 'The device selected to run on'
18+
artifact_name:
19+
required: true
20+
type: string
21+
description: 'The torch_npu distribution artifact name'
22+
23+
# Bash shells do not use ~/.profile or ~/.bashrc so these shells need to be explicitly
24+
# declared as "shell: bash -el {0}" on steps that need to be properly activated.
25+
# It's used to activate ascend-toolkit environment variables.
26+
defaults:
27+
run:
28+
shell: bash -el {0}
29+
30+
jobs:
31+
test:
32+
name: run benchmarks for torch_npu
33+
runs-on: ${{ inputs.runner }}
34+
container:
35+
image: ${{ inputs.image }}
36+
volumes:
37+
- /usr/local/dcmi:/usr/local/dcmi
38+
- /usr/local/bin/npu-smi:/usr/local/bin/npu-smi
39+
- /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/
40+
- /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info
41+
- /etc/ascend_install.info:/etc/ascend_install.info
42+
options: >-
43+
--network host
44+
--device ${{ inputs.device }}
45+
--device /dev/davinci_manager
46+
--device /dev/devmm_svm
47+
--device /dev/hisi_hdc
48+
env:
49+
HTTP_PROXY: http://127.0.0.1:10809
50+
HTTPS_PROXY: http://127.0.0.1:10809
51+
ALL_PROXY: socks5://127.0.0.1:10808
52+
SOCKS_PROXY: socks5://127.0.0.1:10808
53+
steps:
54+
- name: Show NPU info
55+
run: |
56+
npu-smi info
57+
58+
- name: Config mirrors
59+
run: |
60+
sed -i 's|ports.ubuntu.com|mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list
61+
pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
62+
63+
- name: Install system dependencies
64+
run: |
65+
apt update
66+
apt install --no-install-recommends -y \
67+
git gcc g++ make cmake ninja-build curl \
68+
libgl1 libglib2.0-0 libsndfile1
69+
70+
- name: Checkout
71+
uses: actions/checkout@v4
72+
73+
# TODO(shink): Update once PR merged
74+
# https://github.com/pytorch/benchmark/pull/2550
75+
- name: Checkout benchmark
76+
uses: actions/checkout@v4
77+
with:
78+
repository: shink/benchmark
79+
ref: feat/test_bench/continue_on_error
80+
path: benchmark
81+
82+
- name: Download ${{ inputs.artifact_name }}
83+
uses: actions/download-artifact@v4
84+
with:
85+
name: ${{ inputs.artifact_name }}
86+
path: ascend_npu
87+
88+
- name: Install torch_npu
89+
working-directory: ascend_npu
90+
run: |
91+
pip install ${{ inputs.artifact_name }}
92+
93+
- name: Install torch_npu dependencies
94+
working-directory: ascend_npu
95+
run: |
96+
curl -fsSL -O https://raw.githubusercontent.com/Ascend/pytorch/refs/heads/master/requirements.txt
97+
pip install -r requirements.txt
98+
99+
- name: Install benchmark dependencies
100+
run: |
101+
pip install -r benchmark/requirements.txt --constraint ascend_npu/requirements.txt "numpy==1.*"
102+
python benchmark/install.py --userbenchmark test_bench --continue_on_fail
103+
104+
- name: Install project dependencies
105+
run: |
106+
pip install -r requirements.txt
107+
108+
- name: Show environment info
109+
run: |
110+
npu_is_available=$(python -c "import torch; print(torch.npu.is_available())")
111+
npu_count=$(python -c "import torch; print(torch.npu.device_count())")
112+
echo "NPU is available: ${npu_is_available}"
113+
echo "NPU count: ${npu_count}"
114+
pip list | grep -E 'torch|numpy'
115+
116+
- name: Run benchmarks
117+
working-directory: benchmark
118+
run: |
119+
python run_benchmark.py test_bench --accuracy --device npu --test eval \
120+
--output ascend_npu_benchmark.json
121+
122+
- name: Upload the output file
123+
id: upload-output
124+
uses: actions/upload-artifact@v4
125+
with:
126+
name: ascend_npu_benchmark.json
127+
path: benchmark/ascend_npu_benchmark.json
128+
if-no-files-found: error
129+
retention-days: 1
130+
overwrite: true
131+
132+
- name: Write to workflow job summary
133+
id: report
134+
run: |
135+
set -x
136+
realpath benchmark/ascend_npu_benchmark.json
137+
ls benchmark
138+
cat benchmark/ascend_npu_benchmark.json
139+
140+
output_path=$(realpath benchmark/ascend_npu_benchmark.json)
141+
python .ci/benchmark.py ${output_path}
142+
143+
# TODO(shink)
144+
- name: Update README.md
145+
if: ${{ github.event_name == 'push' }}
146+
run: |
147+
echo "${{ github.event_name }}"
148+
echo "${{ github.event_name == 'push' }}"

‎.github/workflows/_ascend_npu_build.yml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,13 @@ jobs:
2929
runs-on: ${{ inputs.runner }}
3030
container:
3131
image: ${{ inputs.image }}
32+
options: >-
33+
--network host
34+
env:
35+
HTTP_PROXY: http://127.0.0.1:10809
36+
HTTPS_PROXY: http://127.0.0.1:10809
37+
ALL_PROXY: socks5://127.0.0.1:10808
38+
SOCKS_PROXY: socks5://127.0.0.1:10808
3239
outputs:
3340
dist_name: ${{ steps.list-dist.outputs.dist_name }}
3441
steps:

‎.github/workflows/_ascend_npu_test.yml

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ defaults:
2929

3030
jobs:
3131
test:
32-
name: test torch_npu in ${{ inputs.image }} with ${{ inputs.device }}
32+
name: test torch_npu
3333
runs-on: ${{ inputs.runner }}
3434
container:
3535
image: ${{ inputs.image }}
@@ -45,6 +45,11 @@ jobs:
4545
--device /dev/davinci_manager
4646
--device /dev/devmm_svm
4747
--device /dev/hisi_hdc
48+
env:
49+
HTTP_PROXY: http://127.0.0.1:10809
50+
HTTPS_PROXY: http://127.0.0.1:10809
51+
ALL_PROXY: socks5://127.0.0.1:10808
52+
SOCKS_PROXY: socks5://127.0.0.1:10808
4853
steps:
4954
- name: Show NPU info
5055
run: |
@@ -67,7 +72,6 @@ jobs:
6772
# repository: Ascend/pytorch
6873
repository: shink/torchnpu
6974
ref: feat/autoload
70-
submodules: recursive
7175
path: torch_npu
7276

7377
- name: Download distribution artifact

‎.github/workflows/ascend_npu_test.yml

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ on:
4848
- /dev/davinci6
4949
- /dev/davinci7
5050
- /dev/davinci8
51-
default: '/dev/davinci6'
51+
default: '/dev/davinci5'
5252
description: 'The device selected to run on'
5353

5454
# Only cancel the previous runs when triggered by a pull request
@@ -70,7 +70,7 @@ jobs:
7070
run: |
7171
set -e
7272
echo "runner=${{ github.event.inputs.runner || 'self-hosted' }}" >> $GITHUB_OUTPUT
73-
echo "device=${{ github.event.inputs.device || '/dev/davinci6' }}" >> $GITHUB_OUTPUT
73+
echo "device=${{ github.event.inputs.device || '/dev/davinci5' }}" >> $GITHUB_OUTPUT
7474
echo "image=${{ github.event.inputs.image || 'ascendai/cann:latest' }}" >> $GITHUB_OUTPUT
7575
7676
build:
@@ -93,3 +93,16 @@ jobs:
9393
image: ${{ needs.prepare.outputs.image }}
9494
device: ${{ needs.prepare.outputs.device }}
9595
artifact_name: ${{ needs.build.outputs.artifact_name }}
96+
97+
benchmark:
98+
name: Run benchmarks
99+
needs:
100+
- prepare
101+
- build
102+
- test
103+
uses: ./.github/workflows/_ascend_npu_benchmark.yml
104+
with:
105+
runner: ${{ needs.prepare.outputs.runner }}
106+
image: ${{ needs.prepare.outputs.image }}
107+
device: ${{ needs.prepare.outputs.device }}
108+
artifact_name: ${{ needs.build.outputs.artifact_name }}

‎.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,3 @@
11
.idea
2+
.benchmarks
3+
__pycache__

‎README.md

Lines changed: 109 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,122 @@ across various devices by running comprehensive GitHub workflows.
77

88
## Accelerator Integration Test Results
99

10-
| [torch_npu][1] |
11-
|----------------------------------|
12-
| [![Ascend NPU Test Suite][2]][3] |
10+
<!-- Start -->
11+
12+
| | [torch_npu][1] |
13+
|---------------------------------|----------------|
14+
| simple_gpt | ❌ |
15+
| detectron2_fasterrcnn_r_50_dc5 | ❌ |
16+
| LearningToPaint | ✅ |
17+
| hf_GPT2_large | ✅ |
18+
| dcgan | ✅ |
19+
| nanogpt | ✅ |
20+
| fastNLP_Bert | ✅ |
21+
| moondream | ❌ |
22+
| mobilenet_v2_quantized_qat | ❌ |
23+
| functorch_dp_cifar10 | ✅ |
24+
| simple_gpt_tp_manual | ❌ |
25+
| speech_transformer | ✅ |
26+
| yolov3 | ✅ |
27+
| resnet50_quantized_qat | ❌ |
28+
| sam_fast | ❌ |
29+
| alexnet | ✅ |
30+
| timm_efficientnet | ✅ |
31+
| pyhpc_isoneutral_mixing | ✅ |
32+
| basic_gnn_edgecnn | ✅ |
33+
| nvidia_deeprecommender | ❌ |
34+
| opacus_cifar10 | ✅ |
35+
| dlrm | ✅ |
36+
| hf_Bert | ✅ |
37+
| hf_T5_generate | ✅ |
38+
| resnet50 | ✅ |
39+
| hf_BigBird | ✅ |
40+
| resnext50_32x4d | ✅ |
41+
| pyhpc_turbulent_kinetic_energy | ✅ |
42+
| llama | ✅ |
43+
| detectron2_maskrcnn_r_50_c4 | ❌ |
44+
| Super_SloMo | ✅ |
45+
| moco | ❌ |
46+
| stable_diffusion_unet | ❌ |
47+
| microbench_unbacked_tolist_sum | ✅ |
48+
| detectron2_maskrcnn_r_101_c4 | ❌ |
49+
| hf_distil_whisper | ✅ |
50+
| mnasnet1_0 | ✅ |
51+
| detectron2_fasterrcnn_r_50_fpn | ❌ |
52+
| timm_resnest | ✅ |
53+
| hf_GPT2 | ✅ |
54+
| squeezenet1_1 | ✅ |
55+
| basic_gnn_gin | ✅ |
56+
| hf_clip | ✅ |
57+
| mobilenet_v2 | ✅ |
58+
| drq | ✅ |
59+
| hf_Roberta_base | ✅ |
60+
| detectron2_maskrcnn_r_50_fpn | ❌ |
61+
| timm_nfnet | ✅ |
62+
| timm_vovnet | ✅ |
63+
| doctr_det_predictor | ✅ |
64+
| sam | ✅ |
65+
| hf_T5_large | ✅ |
66+
| mobilenet_v3_large | ✅ |
67+
| detectron2_fcos_r_50_fpn | ❌ |
68+
| soft_actor_critic | ✅ |
69+
| llava | ❌ |
70+
| timm_regnet | ✅ |
71+
| functorch_maml_omniglot | ✅ |
72+
| detectron2_fasterrcnn_r_101_c4 | ❌ |
73+
| hf_DistilBert | ✅ |
74+
| tts_angular | ✅ |
75+
| detectron2_maskrcnn | ❌ |
76+
| basic_gnn_sage | ✅ |
77+
| tacotron2 | ❌ |
78+
| detectron2_maskrcnn_r_101_fpn | ❌ |
79+
| lennard_jones | ✅ |
80+
| pytorch_unet | ✅ |
81+
| vgg16 | ✅ |
82+
| BERT_pytorch | ✅ |
83+
| timm_efficientdet | ❌ |
84+
| pyhpc_equation_of_state | ✅ |
85+
| maml | ✅ |
86+
| detectron2_fasterrcnn_r_50_c4 | ❌ |
87+
| resnet152 | ✅ |
88+
| phlippe_densenet | ✅ |
89+
| maml_omniglot | ✅ |
90+
| phlippe_resnet | ✅ |
91+
| pytorch_CycleGAN_and_pix2pix | ✅ |
92+
| hf_Whisper | ✅ |
93+
| hf_T5 | ✅ |
94+
| densenet121 | ✅ |
95+
| cm3leon_generate | ✅ |
96+
| detectron2_fasterrcnn_r_101_fpn | ❌ |
97+
| hf_Bert_large | ✅ |
98+
| stable_diffusion_text_encoder | ❌ |
99+
| hf_Reformer | ❌ |
100+
| detectron2_fasterrcnn_r_101_dc5 | ❌ |
101+
| demucs | ✅ |
102+
| pytorch_stargan | ✅ |
103+
| hf_T5_base | ✅ |
104+
| torch_multimodal_clip | ✅ |
105+
| vision_maskrcnn | ❌ |
106+
| timm_vision_transformer_large | ✅ |
107+
| hf_Bart | ✅ |
108+
| shufflenet_v2_x1_0 | ✅ |
109+
| llama_v2_7b_16h | ❌ |
110+
| basic_gnn_gcn | ✅ |
111+
| resnet18 | ✅ |
112+
| Background_Matting | ✅ |
113+
| doctr_reco_predictor | ✅ |
114+
| timm_vision_transformer | ✅ |
115+
| hf_Albert | ✅ |
116+
| hf_Longformer | ✅ |
13117

14118
[1]: https://github.com/ascend/pytorch
15119

16120
[2]: https://github.com/cosdt/pytorch-integration-tests/actions/workflows/ascend_npu_test.yml/badge.svg
17121

18122
[3]: https://github.com/cosdt/pytorch-integration-tests/actions/workflows/ascend_npu_test.yml
19123

124+
<!-- End -->
125+
20126
## Overview
21127

22128
This repository contains workflows and scripts that automate the testing

‎ascend_npu/README.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# Huawei Ascend NPU
2+

‎ascend_npu/metadata.json

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{
2+
"device": "npu",
3+
"test": [
4+
"train",
5+
"eval"
6+
],
7+
"models": [
8+
"BERT_pytorch",
9+
"hf_GPT2"
10+
]
11+
}

‎requirements.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
tabulate

0 commit comments

Comments
 (0)