Skip to content

Commit 643430b

Browse files
shinkshink
authored
[CI] Refactor workflows (#6)
1 parent 4acf7f5 commit 643430b

File tree

3 files changed

+189
-68
lines changed

3 files changed

+189
-68
lines changed

‎.github/actions/dependencies-action/action.yml

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
# An action for installing dependencies. You can set
2+
# custom pip dependencies as following:
3+
#
4+
# Usage:
5+
#
6+
# - name: Install dependencies
7+
# uses: ./.github/actions/dependencies-action
8+
# with:
9+
# pip_packages: |
10+
# - torch
11+
# - triton
12+
# pip_requirements: |
13+
# - /path/a/requirements.txt
14+
# - /path/b/requirements.txt --no-deps
15+
16+
name: Install dependencies
17+
description: Install OS and pip dependencies
18+
19+
inputs:
20+
pip_packages:
21+
description: List of pip packages to install
22+
required: false
23+
default: ''
24+
pip_requirements:
25+
description: List of requirements files to install
26+
required: false
27+
default: ''
28+
29+
runs:
30+
using: composite
31+
steps:
32+
- name: Install system dependencies
33+
shell: bash
34+
env:
35+
DEBIAN_FRONTEND: noninteractive
36+
run: |
37+
echo "::group::Install system dependencies"
38+
sed -i 's|ports.ubuntu.com|mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list
39+
apt update
40+
apt install --no-install-recommends -y \
41+
git \
42+
gcc-10 \
43+
g++-10 \
44+
make \
45+
cmake \
46+
ninja-build
47+
echo "::endgroup::"
48+
49+
- name: Create symlinks for gcc and g++
50+
shell: bash
51+
run: |
52+
ln -s /usr/bin/gcc-10 /usr/bin/gcc
53+
ln -s /usr/bin/g++-10 /usr/bin/g++
54+
55+
- name: Show versions
56+
shell: bash
57+
run: |
58+
set -x
59+
python --version
60+
pip --version
61+
gcc --version
62+
g++ --version
63+
make --version
64+
cmake --version
65+
ninja --version
66+
67+
- name: Install extra pip packages
68+
if: ${{ inputs.pip_packages != '' }}
69+
shell: bash
70+
run: |
71+
echo "${{ inputs.pip_packages }}" | while read -r package; do
72+
package="${package#- }"
73+
if [ -n "$(echo "$package" | xargs)" ]; then
74+
echo "::group::Installing pip package: $package"
75+
pip install $package -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
76+
echo "::endgroup::"
77+
fi
78+
done
79+
80+
- name: Install extra pip requirements
81+
if: ${{ inputs.pip_requirements != '' }}
82+
shell: bash
83+
run: |
84+
echo "${{ inputs.pip_requirements }}" | while read -r requirement; do
85+
requirement="${requirement#- }"
86+
if [ -n "$(echo "$requirement" | xargs)" ]; then
87+
echo "::group::Installing from requirements: $requirement"
88+
pip install -r $requirement -i https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
89+
echo "::endgroup::"
90+
fi
91+
done

‎.github/workflows/_build-and-test.yml

Lines changed: 92 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -16,22 +16,81 @@ on:
1616
type: string
1717
description: The docker image which will be loaded
1818

19+
# Bash shells do not use ~/.profile or ~/.bashrc so these shells need to be explicitly
20+
# declared as "shell: bash -el {0}" on steps that need to be properly activated.
21+
defaults:
22+
run:
23+
shell: bash -el {0}
24+
1925
jobs:
20-
build-and-test:
21-
name: build and test in ${{ inputs.image }} with ${{ inputs.device }}
26+
build:
27+
name: build torch_npu
2228
runs-on: ${{ inputs.runner }}
23-
defaults:
24-
run:
25-
shell: bash -el {0}
2629
container:
2730
image: ${{ inputs.image }}
28-
env:
29-
HOME: /root
31+
volumes:
32+
- /home/runner/actions-runner/codes:/root/codes
33+
outputs:
34+
dist_name: ${{ steps.list-dist.outputs.dist_name }}
35+
steps:
36+
- name: Prepare the codes
37+
run: |
38+
cp -rf /root/codes /root/build
39+
40+
- name: Checkout
41+
uses: actions/checkout@v4
42+
43+
- name: Install dependencies
44+
uses: ./.github/actions/dependencies-action
45+
with:
46+
pip_packages: |
47+
- wheel
48+
pip_requirements: |
49+
- /root/build/npu/pytorch/requirements.txt
50+
51+
- name: Build torch_npu
52+
working-directory: /root/build/npu/pytorch
53+
run: |
54+
py_version=$(python --version | awk '{print $2}' | cut -d '.' -f 1,2)
55+
bash ci/build.sh --python=${py_version}
56+
57+
- name: List distribution package
58+
id: list-dist
59+
working-directory: /root/build/npu/pytorch/dist
60+
run: |
61+
dist_name=$(ls torch_npu*.whl)
62+
dist_path=$(pwd)/${dist_name}
63+
echo "dist_name=${dist_name}" >> $GITHUB_OUTPUT
64+
echo "dist_path=${dist_path}" >> $GITHUB_OUTPUT
65+
66+
- name: Upload distribution artifact
67+
id: upload-dist
68+
continue-on-error: true
69+
uses: actions/upload-artifact@v4
70+
with:
71+
name: ${{ steps.list-dist.outputs.dist_name }}
72+
path: ${{ steps.list-dist.outputs.dist_path }}
73+
retention-days: 1
74+
75+
- name: Write to workflow job summary
76+
if: ${{ steps.upload-dist.outputs.artifact-url }}
77+
run: |
78+
echo "## torch_npu built successfully! :rocket:" >> $GITHUB_STEP_SUMMARY
79+
echo "You can download the distribution package [here](${{ steps.upload-dist.outputs.artifact-url }})." >> $GITHUB_STEP_SUMMARY
80+
81+
test:
82+
name: test torch_npu in ${{ inputs.image }} with ${{ inputs.device }}
83+
runs-on: ${{ inputs.runner }}
84+
needs:
85+
- build
86+
container:
87+
image: ${{ inputs.image }}
3088
volumes:
3189
- /usr/local/dcmi:/usr/local/dcmi
3290
- /usr/local/bin/npu-smi:/usr/local/bin/npu-smi
3391
- /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/
3492
- /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info
93+
- /etc/ascend_install.info:/etc/ascend_install.info
3594
- /home/runner/actions-runner/codes:/root/codes
3695
options: >-
3796
--network host
@@ -40,73 +99,43 @@ jobs:
4099
--device /dev/devmm_svm
41100
--device /dev/hisi_hdc
42101
steps:
43-
#- name: Set environment
44-
# run: |
45-
# source /root/.bashrc && conda activate torch_npu
102+
- name: Show NPU info
103+
run: |
104+
npu-smi info
46105
47106
- name: Prepare the codes
48107
run: |
49108
cp -rf /root/codes /root/build
50109
51-
#- name: Compile torch
52-
# working-directory: /root/build/pytorch/pytorch
53-
# run: |
54-
# pip3 install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
55-
# export _GLIBCXX_USE_CXX11_ABI=0
56-
# export USE_CUDA=0
57-
# export USE_XNNPACK=0
58-
# python setup.py develop
59-
60-
- name: Install system dependencies
61-
env:
62-
DEBIAN_FRONTEND: noninteractive
63-
run: |
64-
sed -i 's|ports.ubuntu.com|mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list
65-
apt update
66-
apt install --no-install-recommends -y \
67-
git \
68-
gcc-10 \
69-
g++-10 \
70-
make \
71-
cmake \
72-
ninja-build
73-
74-
- name: Create symlinks for gcc and g++
75-
run: |
76-
ln -s /usr/bin/gcc-10 /usr/bin/gcc
77-
ln -s /usr/bin/g++-10 /usr/bin/g++
78-
79-
- name: Install pip dependencies
80-
run: |
81-
pip install -i https://pypi.tuna.tsinghua.edu.cn/simple \
82-
wheel \
83-
setuptools \
84-
unittest-xml-reporting
110+
- name: Download distribution artifact
111+
uses: actions/download-artifact@v4
112+
with:
113+
name: ${{ needs.build.outputs.dist_name }}
114+
path: /root/build
85115

86-
- name: Show versions
87-
run: |
88-
set -x
89-
python --version
90-
pip --version
91-
gcc --version
92-
g++ --version
93-
make --version
94-
cmake --version
95-
ninja --version
116+
- name: Checkout
117+
uses: actions/checkout@v4
96118

97-
- name: Show NPU info
98-
run: |
99-
npu-smi info
119+
- name: Install dependencies
120+
uses: ./.github/actions/dependencies-action
121+
with:
122+
pip_packages: |
123+
- wheel
124+
- unittest-xml-reporting
125+
pip_requirements: |
126+
- /root/build/npu/pytorch/requirements.txt
127+
- /root/build/npu/pytorch/requirements.txt
128+
- /root/build/npu/pytorch/test/requirements.txt --no-deps
100129
101-
- name: Compile and install torch_npu
102-
working-directory: /root/build/npu/pytorch
130+
- name: Install torch_npu
131+
working-directory: /root/build
103132
run: |
104-
pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple
105-
bash ci/build.sh --python=`python --version | awk '{print $2}' | cut -d '.' -f 1,2`
106-
pip install dist/torch_npu*.whl
133+
pip install ${{ needs.build.outputs.dist_name }}
107134
108135
- name: Do the test
136+
continue-on-error: true # Skip if failed
109137
working-directory: /root/build
110138
run: |
111-
pip install -r npu/pytorch/test/requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple --no-deps
112139
python npu/pytorch/ci/access_control_test.py
140+
env:
141+
DISABLED_TESTS_FILE: /root/build/npu/pytorch/test/unsupported_test_cases/.pytorch-disabled-tests.json

‎.github/workflows/ascend_npu_test.yml

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ on:
2828
- self-hosted
2929
- npu-arm64
3030
default: 'self-hosted'
31-
description: The runner selected to run on
31+
description: 'The runner selected to run on'
3232
device:
3333
required: true
3434
type: choice
@@ -47,13 +47,14 @@ on:
4747
required: true
4848
type: choice
4949
options:
50-
- 'ascendai/cann:7.1-openeuler2203sp2'
51-
- 'ascendai/cann:8.0.rc2.alpha003-910b-ubuntu22.04-py3.9'
50+
- ascendai/cann:7.1-openeuler2203sp2
51+
- ascendai/cann:8.0.rc2.alpha003-910b-ubuntu22.04-py3.9
5252
default: 'ascendai/cann:8.0.rc2.alpha003-910b-ubuntu22.04-py3.9'
53-
description: The docker image which will be loaded
53+
description: 'The docker image which will be loaded'
5454

55+
# Only cancel the previous runs when triggered by a pull request
5556
concurrency:
56-
group: ${{ github.workflow }}
57+
group: '${{ github.workflow }}-${{ github.event_name }}'
5758
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
5859

5960
jobs:

0 commit comments

Comments
 (0)