Skip to content

Commit 1347885

Browse files
karlhigleyoliverholworthy
karlhigley
authored and
oliverholworthy
committed
Use cibuildwheel to build binary distributions for PyPI (#1754)
* Use `cibuildwheel` to build binary distributions for PyPI * Fix env variable formatting for `cibuildwheel` action * Split running CPU tests and building packages into two separate actions * Rename the CPU and GPU test actions (for clarity) * Stop building packages on every PR
1 parent 7a87ffd commit 1347885

File tree

3 files changed

+201
-135
lines changed

3 files changed

+201
-135
lines changed

‎.github/workflows/cpu-ci.yml‎

Lines changed: 0 additions & 135 deletions
This file was deleted.

‎.github/workflows/cpu-packages.yml‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
name: Build NVTabular Packages (CPU)
2+
3+
on:
4+
workflow_dispatch:
5+
push:
6+
branches: [main]
7+
tags:
8+
- v*
9+
10+
jobs:
11+
build:
12+
runs-on: ${{ matrix.os }}
13+
strategy:
14+
matrix:
15+
python-version: [3.8]
16+
os: [ubuntu-latest]
17+
18+
steps:
19+
- uses: actions/checkout@v3
20+
with:
21+
fetch-depth: 0
22+
- name: Set up Python ${{ matrix.python-version }}
23+
uses: actions/setup-python@v4
24+
with:
25+
python-version: ${{ matrix.python-version }}
26+
- name: Install Ubuntu packages
27+
run: |
28+
sudo apt-get update -y
29+
sudo apt-get install -y protobuf-compiler
30+
- name: Install and upgrade python packages
31+
run: |
32+
python -m pip install --upgrade pip setuptools==59.4.0 wheel tox pybind11
33+
python -m pip uninstall protobuf -y
34+
python -m pip install --no-binary=protobuf protobuf
35+
- name: Build source package for PyPI
36+
run: |
37+
python setup.py sdist
38+
- name: Check distribution is valid
39+
run: |
40+
./ci/check_dist.sh
41+
- name: Build wheels for PyPI
42+
uses: pypa/[email protected]
43+
env:
44+
CIBW_SKIP: cp36-* cp37-* pp* # Don't build wheels for 3.6, 3.7, or PyPy
45+
with:
46+
output-dir: dist
47+
- name: Upload PyPI artifacts to Github
48+
uses: actions/upload-artifact@v3
49+
with:
50+
name: dist
51+
path: dist
52+
- name: Generate package for conda
53+
id: conda_build
54+
run: |
55+
conda update conda
56+
conda install conda-build pybind11
57+
conda build --python ${{ matrix.python-version }} . -c defaults -c conda-forge -c numba -c rapidsai -c nvidia --output-folder ./conda_packages
58+
export CONDA_PACKAGE=$(conda build --python ${{ matrix.python-version }} . -c defaults -c conda-forge -c numba -c rapidsai -c nvidia --output-folder ./conda_packages --output)
59+
echo "conda_package : $CONDA_PACKAGE"
60+
echo "conda_package=$CONDA_PACKAGE" >> $GITHUB_OUTPUT
61+
62+
- name: Upload conda artifacts to github
63+
uses: actions/upload-artifact@v3
64+
with:
65+
name: conda
66+
path: ${{ steps.conda_build.outputs.conda_package }}
67+
68+
# Build docs, treat warnings as errors
69+
- name: Building docs
70+
run: |
71+
tox -e docs
72+
- name: Upload HTML
73+
uses: actions/upload-artifact@v3
74+
with:
75+
name: html-build-artifact
76+
path: docs/build/html
77+
if-no-files-found: error
78+
retention-days: 1
79+
- name: Store PR information
80+
run: |
81+
mkdir ./pr
82+
echo ${{ github.event.number }} > ./pr/pr.txt
83+
echo ${{ github.event.pull_request.merged }} > ./pr/merged.txt
84+
echo ${{ github.event.action }} > ./pr/action.txt
85+
- name: Upload PR information
86+
uses: actions/upload-artifact@v3
87+
with:
88+
name: pr
89+
path: pr/
90+
91+
release:
92+
name: Release
93+
runs-on: ubuntu-latest
94+
if: "startsWith(github.ref, 'refs/tags/')"
95+
needs: [build]
96+
steps:
97+
- uses: actions/download-artifact@v3
98+
with:
99+
name: dist
100+
path: dist
101+
- name: Create GitHub Release
102+
uses: fnkr/[email protected]
103+
env:
104+
GHR_PATH: ./dist
105+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
106+
- uses: actions/setup-python@v4
107+
with:
108+
python-version: 3.9
109+
- name: Push to PyPi
110+
env:
111+
TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
112+
TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
113+
run: |
114+
pip install --upgrade wheel pip setuptools twine
115+
twine upload dist/*
116+
- uses: actions/download-artifact@v2
117+
with:
118+
name: conda
119+
path: conda
120+
- uses: conda-incubator/setup-miniconda@v2
121+
with:
122+
auto-update-conda: true
123+
- name: Install conda dependencies
124+
shell: bash -l {0}
125+
run: |
126+
conda install -y anaconda-client conda-build
127+
- name: Push to anaconda
128+
shell: bash -l {0}
129+
env:
130+
ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }}
131+
run: |
132+
anaconda -t $ANACONDA_TOKEN upload -u nvidia conda/*.tar.bz2

‎.github/workflows/cpu-tests.yml‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
name: CPU Tests
2+
3+
on:
4+
workflow_dispatch:
5+
push:
6+
branches: [main]
7+
tags:
8+
- v*
9+
pull_request:
10+
branches: [main]
11+
12+
jobs:
13+
cpu-tests:
14+
runs-on: ${{ matrix.os }}
15+
strategy:
16+
matrix:
17+
python-version: [3.8, 3.9]
18+
os: [ubuntu-latest]
19+
20+
steps:
21+
- uses: actions/checkout@v3
22+
with:
23+
fetch-depth: 0
24+
- name: Set up Python ${{ matrix.python-version }}
25+
uses: actions/setup-python@v4
26+
with:
27+
python-version: ${{ matrix.python-version }}
28+
- name: Install Ubuntu packages
29+
run: |
30+
sudo apt-get update -y
31+
sudo apt-get install -y protobuf-compiler
32+
- name: Install and upgrade python packages
33+
run: |
34+
python -m pip install --upgrade pip setuptools==59.4.0 wheel tox pybind11
35+
python -m pip uninstall protobuf -y
36+
python -m pip install --no-binary=protobuf protobuf
37+
- name: Run tests
38+
run: |
39+
ref_type=${{ github.ref_type }}
40+
branch=main
41+
if [[ $ref_type == "tag"* ]]
42+
then
43+
raw=$(git branch -r --contains ${{ github.ref_name }})
44+
branch=${raw/origin\/}
45+
fi
46+
tox -e test-cpu -- $branch
47+
48+
# Build docs, treat warnings as errors
49+
- name: Building docs
50+
run: |
51+
tox -e docs
52+
- name: Upload HTML
53+
uses: actions/upload-artifact@v3
54+
with:
55+
name: html-build-artifact
56+
path: docs/build/html
57+
if-no-files-found: error
58+
retention-days: 1
59+
- name: Store PR information
60+
run: |
61+
mkdir ./pr
62+
echo ${{ github.event.number }} > ./pr/pr.txt
63+
echo ${{ github.event.pull_request.merged }} > ./pr/merged.txt
64+
echo ${{ github.event.action }} > ./pr/action.txt
65+
- name: Upload PR information
66+
uses: actions/upload-artifact@v3
67+
with:
68+
name: pr
69+
path: pr/

0 commit comments

Comments
 (0)