Skip to content

Commit 477cb24

Browse files
shinkshink
authored
Add support for building torch from source (#31)
1 parent 13fab09 commit 477cb24

File tree

6 files changed

+296
-49
lines changed

6 files changed

+296
-49
lines changed

‎.github/actions/install-artifact/action.yml

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
name: "Install artifact with pip"
2+
description: "Install artifact with pip"
3+
inputs:
4+
artifact:
5+
description: "The distribution artifact name"
6+
type: string
7+
required: true
8+
9+
# TODO: https://github.com/actions/runner/issues/3620
10+
runs:
11+
using: "composite"
12+
steps:
13+
- name: Download artifact
14+
uses: actions/download-artifact@v4
15+
with:
16+
name: ${{ inputs.artifact }}
17+
18+
- name: Install artifact
19+
shell: bash
20+
run: |
21+
pip install ${{ inputs.artifact }}

‎.github/workflows/_ascend_npu_benchmark.yml

Lines changed: 50 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,14 @@ on:
1515
required: true
1616
type: string
1717
description: 'The device selected to run on'
18-
artifact_name:
18+
torch-artifact:
19+
required: false
20+
type: string
21+
description: 'The distribution artifact name of torch'
22+
torch-npu-artifact:
1923
required: true
2024
type: string
21-
description: 'The torch_npu distribution artifact name'
25+
description: 'The distribution artifact name of torch_npu'
2226
secrets:
2327
pr-token:
2428
description: 'A token used to create a pull request'
@@ -87,29 +91,61 @@ jobs:
8791
repository: pytorch/benchmark
8892
path: benchmark
8993

90-
- name: Download ${{ inputs.artifact_name }}
94+
# TODO
95+
# - name: Install torch
96+
# id: install-torch
97+
# uses: ./.github/actions/install-artifact
98+
# with:
99+
# artifact: ${{ inputs.torch-artifact }}
100+
101+
- name: Download torch artifact
102+
if: ${{ inputs.torch-artifact }}
91103
uses: actions/download-artifact@v4
92104
with:
93-
name: ${{ inputs.artifact_name }}
94-
path: ascend_npu
105+
name: ${{ inputs.torch-artifact }}
95106

96-
- name: Install torch_npu
97-
working-directory: ascend_npu
107+
- name: Install torch
108+
if: ${{ inputs.torch-artifact }}
98109
run: |
99-
pip install ${{ inputs.artifact_name }}
110+
pip install ${{ inputs.torch-artifact }}
100111
101112
- name: Install torch_npu dependencies
113+
if: ${{ !inputs.torch-artifact }}
114+
run: |
115+
pip install -r https://raw.githubusercontent.com/Ascend/pytorch/refs/heads/master/requirements.txt
116+
117+
- name: List torch version
118+
id: list-torch-version
119+
shell: bash
120+
run: |
121+
torch_version=$(python -c "import torch; print(torch.__version__)")
122+
echo "torch-version=${torch_version}" >> $GITHUB_OUTPUT
123+
124+
- name: Download torch_npu artifact
125+
uses: actions/download-artifact@v4
126+
with:
127+
name: ${{ inputs.torch-npu-artifact }}
128+
path: ascend_npu
129+
130+
- name: Install torch_npu
102131
working-directory: ascend_npu
103132
run: |
104-
curl -fsSL -O https://raw.githubusercontent.com/Ascend/pytorch/refs/heads/master/requirements.txt
105-
pip install -r requirements.txt
133+
pip install ${{ inputs.torch-npu-artifact }}
106134
107-
# TODO: We must use numpy 1.x
108135
- name: Install benchmark dependencies
109136
run: |
110-
pip install -r benchmark/requirements.txt --constraint ascend_npu/requirements.txt "numpy==1.*"
137+
pip install -r benchmark/requirements.txt \
138+
torch==${{ steps.list-torch-version.outputs.torch-version }} \
139+
numpy==1.*
140+
141+
- name: Install dependencies for all the models
142+
run: |
111143
python benchmark/install.py --userbenchmark test_bench --continue_on_fail
112144
145+
- name: Install nightly torchvision and torchaudio
146+
run: |
147+
pip install --pre torchvision torchaudio --no-deps --index-url https://download.pytorch.org/whl/nightly/cpu
148+
113149
- name: Install project dependencies
114150
run: |
115151
pip install -r requirements.txt
@@ -128,14 +164,15 @@ jobs:
128164
python run_benchmark.py test_bench --accuracy --device npu --test eval \
129165
--output ascend_npu_benchmark.json
130166
131-
- name: Upload the output file
167+
- name: Upload the benchmark report file
132168
id: upload-output
133169
uses: actions/upload-artifact@v4
134170
with:
135171
name: ascend_npu_benchmark.json
136172
path: benchmark/ascend_npu_benchmark.json
137173
if-no-files-found: error
138174
retention-days: 1
175+
overwrite: true
139176

140177
- name: Write to workflow job summary
141178
run: |

‎.github/workflows/_ascend_npu_build_torch.yml

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
name: '_ascend_npu_build_torch'
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
runner:
7+
required: true
8+
type: string
9+
description: 'The runner selected to run on'
10+
image:
11+
required: true
12+
type: string
13+
description: 'The docker image which will be used to build'
14+
ref:
15+
required: false
16+
type: string
17+
default: 'refs/heads/main'
18+
description: 'The branch, tag or SHA to checkout'
19+
outputs:
20+
torch-artifact:
21+
description: 'The distribution artifact name of torch'
22+
value: ${{ jobs.build.outputs.dist-name }}
23+
24+
# Bash shells do not use ~/.profile or ~/.bashrc so these shells need to be explicitly
25+
# declared as "shell: bash -el {0}" on steps that need to be properly activated.
26+
# It's used to activate ascend-toolkit environment variables.
27+
defaults:
28+
run:
29+
shell: bash -el {0}
30+
31+
jobs:
32+
build:
33+
name: build torch
34+
runs-on: ${{ inputs.runner }}
35+
container:
36+
image: ${{ inputs.image }}
37+
options: >-
38+
--network host
39+
env:
40+
HTTP_PROXY: http://127.0.0.1:10809
41+
HTTPS_PROXY: http://127.0.0.1:10809
42+
ALL_PROXY: socks5://127.0.0.1:10808
43+
SOCKS_PROXY: socks5://127.0.0.1:10808
44+
outputs:
45+
dist-name: ${{ steps.list-dist.outputs.dist-name }}
46+
steps:
47+
- name: Config mirrors
48+
run: |
49+
sed -i 's|ports.ubuntu.com|mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list
50+
pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
51+
52+
- name: Install system dependencies
53+
run: |
54+
apt-get update
55+
apt-get install -y git gcc g++ make cmake ninja-build
56+
57+
# See: https://github.com/actions/checkout/issues/363#issuecomment-1915075699
58+
- name: Config git
59+
run: |
60+
git config --global --add safe.directory "$GITHUB_WORKSPACE"
61+
62+
- name: Checkout PyTorch
63+
uses: actions/checkout@v4
64+
with:
65+
repository: pytorch/pytorch
66+
ref: ${{ inputs.ref }}
67+
submodules: recursive
68+
path: pytorch
69+
70+
- name: View commit history
71+
working-directory: pytorch
72+
run: |
73+
git log -n 10 --graph | cat
74+
75+
- name: Install torch dependencies
76+
working-directory: pytorch
77+
run: |
78+
pip install -r requirements.txt
79+
80+
- name: Build torch
81+
working-directory: pytorch
82+
run: |
83+
python setup.py build bdist_wheel
84+
85+
- name: List distribution package
86+
id: list-dist
87+
working-directory: pytorch/dist
88+
run: |
89+
dist_name=$(ls torch*.whl)
90+
dist_path=$(pwd)/${dist_name}
91+
echo "dist-name=${dist_name}" >> $GITHUB_OUTPUT
92+
echo "dist-path=${dist_path}" >> $GITHUB_OUTPUT
93+
94+
- name: Upload distribution artifact
95+
id: upload-dist
96+
continue-on-error: true
97+
uses: actions/upload-artifact@v4
98+
with:
99+
name: ${{ steps.list-dist.outputs.dist-name }}
100+
path: ${{ steps.list-dist.outputs.dist-path }}
101+
if-no-files-found: error
102+
retention-days: 1
103+
overwrite: true
104+
105+
- name: Write to workflow job summary
106+
if: ${{ steps.upload-dist.outputs.artifact-url }}
107+
run: |
108+
echo "## torch built successfully! :rocket:" >> $GITHUB_STEP_SUMMARY
109+
echo "You can download the distribution package [here](${{ steps.upload-dist.outputs.artifact-url }})." >> $GITHUB_STEP_SUMMARY

‎.github/workflows/_ascend_npu_build.yml renamed to ‎.github/workflows/_ascend_npu_build_torch_npu.yml

Lines changed: 33 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,14 @@ on:
1111
required: true
1212
type: string
1313
description: 'The docker image which will be used to build'
14+
torch-artifact:
15+
required: false
16+
type: string
17+
description: 'The distribution artifact name of torch'
1418
outputs:
15-
artifact_name:
16-
description: 'The torch_npu distribution artifact name'
17-
value: ${{ jobs.build.outputs.dist_name }}
19+
torch-npu-artifact:
20+
description: 'The distribution artifact name of torch_npu'
21+
value: ${{ jobs.build.outputs.dist-name }}
1822

1923
# Bash shells do not use ~/.profile or ~/.bashrc so these shells need to be explicitly
2024
# declared as "shell: bash -el {0}" on steps that need to be properly activated.
@@ -37,14 +41,13 @@ jobs:
3741
ALL_PROXY: socks5://127.0.0.1:10808
3842
SOCKS_PROXY: socks5://127.0.0.1:10808
3943
outputs:
40-
dist_name: ${{ steps.list-dist.outputs.dist_name }}
44+
dist-name: ${{ steps.list-dist.outputs.dist-name }}
4145
steps:
4246
- name: Config mirrors
4347
run: |
4448
sed -i 's|ports.ubuntu.com|mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list
4549
pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
4650
47-
# TODO(shink): Should we add these dependencies to the image?
4851
- name: Install system dependencies
4952
run: |
5053
apt-get update
@@ -69,40 +72,58 @@ jobs:
6972
submodules: recursive
7073
path: torch_npu
7174

72-
- name: Install pip dependencies
75+
- name: Install torch_npu dependencies
7376
working-directory: torch_npu
7477
run: |
7578
pip install wheel
7679
pip install -r requirements.txt
7780
81+
# TODO
82+
# - name: Install torch
83+
# id: install-torch
84+
# uses: ./.github/actions/install-artifact
85+
# with:
86+
# artifact: ${{ inputs.torch-artifact }}
87+
88+
- name: Download torch artifact
89+
if: ${{ inputs.torch-artifact }}
90+
uses: actions/download-artifact@v4
91+
with:
92+
name: ${{ inputs.torch-artifact }}
93+
94+
- name: Install torch
95+
if: ${{ inputs.torch-artifact }}
96+
run: |
97+
pip install ${{ inputs.torch-artifact }}
98+
7899
- name: List Python version
79100
id: list-py-version
80101
working-directory: torch_npu
81102
run: |
82103
py_version=$(python --version | awk '{print $2}' | cut -d '.' -f 1,2)
83-
echo "py_version=${py_version}" >> $GITHUB_OUTPUT
104+
echo "py-version=${py_version}" >> $GITHUB_OUTPUT
84105
85106
- name: Build torch_npu
86107
working-directory: torch_npu
87108
run: |
88-
bash ci/build.sh --python=${{ steps.list-py-version.outputs.py_version }}
109+
bash ci/build.sh --python=${{ steps.list-py-version.outputs.py-version }}
89110
90111
- name: List distribution package
91112
id: list-dist
92113
working-directory: torch_npu/dist
93114
run: |
94115
dist_name=$(ls torch_npu*.whl)
95116
dist_path=$(pwd)/${dist_name}
96-
echo "dist_name=${dist_name}" >> $GITHUB_OUTPUT
97-
echo "dist_path=${dist_path}" >> $GITHUB_OUTPUT
117+
echo "dist-name=${dist_name}" >> $GITHUB_OUTPUT
118+
echo "dist-path=${dist_path}" >> $GITHUB_OUTPUT
98119
99120
- name: Upload distribution artifact
100121
id: upload-dist
101122
continue-on-error: true
102123
uses: actions/upload-artifact@v4
103124
with:
104-
name: ${{ steps.list-dist.outputs.dist_name }}
105-
path: ${{ steps.list-dist.outputs.dist_path }}
125+
name: ${{ steps.list-dist.outputs.dist-name }}
126+
path: ${{ steps.list-dist.outputs.dist-path }}
106127
if-no-files-found: error
107128
retention-days: 1
108129
overwrite: true

0 commit comments

Comments
 (0)