Skip to content

Commit 6217cbe

Browse files
robtandyrobtandy
authored
Add CI to build python wheels (#76)
1 parent 0d76298 commit 6217cbe

File tree

5 files changed

+565
-0
lines changed

5 files changed

+565
-0
lines changed

‎.github/workflows/build.yml‎

Lines changed: 293 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,293 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
name: Python Release Build
19+
on:
20+
pull_request:
21+
branches: ["main"]
22+
push:
23+
tags: ["*-rc*"]
24+
branches: ["branch-*"]
25+
workflow_dispatch:
26+
27+
jobs:
28+
build:
29+
runs-on: ubuntu-latest
30+
steps:
31+
- uses: actions/checkout@v4
32+
33+
- uses: astral-sh/setup-uv@v5
34+
with:
35+
enable-cache: true
36+
python_version: "3.12"
37+
38+
# Use the --no-install-package to only install the dependencies
39+
# but do not yet build the rust library
40+
- name: Install dependencies
41+
run: uv sync --dev --no-install-package datafusion-ray
42+
43+
generate-license:
44+
runs-on: ubuntu-latest
45+
steps:
46+
- uses: actions/checkout@v4
47+
- uses: astral-sh/setup-uv@v5
48+
with:
49+
enable-cache: true
50+
51+
- name: Generate license file
52+
run: uv run --no-project python ./dev/create_license.py
53+
- uses: actions/upload-artifact@v4
54+
with:
55+
name: python-wheel-license
56+
path: LICENSE.txt
57+
58+
build-python-mac-win:
59+
needs: [generate-license]
60+
name: Mac/Win
61+
runs-on: ${{ matrix.os }}
62+
strategy:
63+
fail-fast: false
64+
matrix:
65+
python-version: ["3.13"]
66+
#os: [macos-latest, windows-latest]
67+
os: [macos-latest]
68+
steps:
69+
- uses: actions/checkout@v4
70+
71+
- uses: actions/setup-python@v5
72+
with:
73+
python-version: ${{ matrix.python-version }}
74+
75+
- uses: dtolnay/rust-toolchain@stable
76+
77+
- run: rm LICENSE.txt
78+
- name: Download LICENSE.txt
79+
uses: actions/download-artifact@v4
80+
with:
81+
name: python-wheel-license
82+
path: .
83+
84+
- name: Install Protoc
85+
uses: arduino/setup-protoc@v3
86+
with:
87+
version: "27.4"
88+
repo-token: ${{ secrets.GITHUB_TOKEN }}
89+
90+
- uses: astral-sh/setup-uv@v5
91+
with:
92+
enable-cache: true
93+
94+
- name: Build Python package
95+
run: |
96+
uv sync --dev --no-install-package datafusion-ray
97+
uv run --no-project maturin build --release --strip
98+
99+
- name: List Windows wheels
100+
if: matrix.os == 'windows-latest'
101+
run: dir target\wheels\
102+
# since the runner is dynamic shellcheck (from actionlint) can't infer this is powershell
103+
# so we specify it explicitly
104+
shell: powershell
105+
106+
- name: List Mac wheels
107+
if: matrix.os != 'windows-latest'
108+
run: find target/wheels/
109+
110+
- name: Archive wheels
111+
uses: actions/upload-artifact@v4
112+
with:
113+
name: dist-${{ matrix.os }}
114+
path: target/wheels/*
115+
116+
build-macos-x86_64:
117+
needs: [generate-license]
118+
name: Mac x86_64
119+
runs-on: macos-13
120+
strategy:
121+
fail-fast: false
122+
matrix:
123+
python-version: ["3.12"]
124+
steps:
125+
- uses: actions/checkout@v4
126+
127+
- uses: actions/setup-python@v5
128+
with:
129+
python-version: ${{ matrix.python-version }}
130+
131+
- uses: dtolnay/rust-toolchain@stable
132+
133+
- run: rm LICENSE.txt
134+
- name: Download LICENSE.txt
135+
uses: actions/download-artifact@v4
136+
with:
137+
name: python-wheel-license
138+
path: .
139+
140+
- name: Install Protoc
141+
uses: arduino/setup-protoc@v3
142+
with:
143+
version: "27.4"
144+
repo-token: ${{ secrets.GITHUB_TOKEN }}
145+
146+
- uses: astral-sh/setup-uv@v5
147+
with:
148+
enable-cache: true
149+
150+
- name: Build Python package
151+
run: |
152+
uv sync --dev --no-install-package datafusion-ray
153+
uv run --no-project maturin build --release --strip
154+
155+
- name: List Mac wheels
156+
run: find target/wheels/
157+
158+
- name: Archive wheels
159+
uses: actions/upload-artifact@v4
160+
with:
161+
name: dist-macos-aarch64
162+
path: target/wheels/*
163+
164+
build-manylinux-x86_64:
165+
needs: [generate-license]
166+
name: Manylinux x86_64
167+
runs-on: ubuntu-latest
168+
steps:
169+
- uses: actions/checkout@v4
170+
- run: rm LICENSE.txt
171+
- name: Download LICENSE.txt
172+
uses: actions/download-artifact@v4
173+
with:
174+
name: python-wheel-license
175+
path: .
176+
177+
- name: Build wheels
178+
uses: PyO3/maturin-action@v1
179+
env:
180+
RUST_BACKTRACE: 1
181+
with:
182+
rust-toolchain: nightly
183+
target: x86_64
184+
manylinux: auto
185+
rustup-components: rust-std rustfmt # Keep them in one line due to https://github.com/PyO3/maturin-action/issues/153
186+
args: --release --manylinux 2014
187+
before-script-linux: |
188+
yum install -y wget
189+
cd /
190+
wget https://github.com/protocolbuffers/protobuf/releases/download/v29.3/protoc-29.3-linux-x86_64.zip
191+
unzip -o proto*zip
192+
cd -
193+
which protoc
194+
protoc --version
195+
196+
- name: Archive wheels
197+
uses: actions/upload-artifact@v4
198+
with:
199+
name: dist-manylinux-x86_64
200+
path: target/wheels/*
201+
202+
build-manylinux-aarch64:
203+
needs: [generate-license]
204+
name: Manylinux arm64
205+
runs-on: ubuntu-latest
206+
steps:
207+
- uses: actions/checkout@v4
208+
- run: rm LICENSE.txt
209+
- name: Download LICENSE.txt
210+
uses: actions/download-artifact@v4
211+
with:
212+
name: python-wheel-license
213+
path: .
214+
215+
- name: Build wheels
216+
uses: PyO3/maturin-action@v1
217+
env:
218+
RUST_BACKTRACE: 1
219+
with:
220+
rust-toolchain: nightly
221+
target: aarch64
222+
# Use manylinux_2_28-cross because the manylinux2014-cross has GCC 4.8.5, which causes the build to fail
223+
manylinux: 2_28
224+
rustup-components: rust-std rustfmt # Keep them in one line due to https://github.com/PyO3/maturin-action/issues/153
225+
args: --release
226+
before-script-linux: |
227+
apt-get install -y unzip
228+
cd /
229+
wget https://github.com/protocolbuffers/protobuf/releases/download/v29.3/protoc-29.3-linux-x86_64.zip
230+
unzip -o proto*zip
231+
cd -
232+
which protoc
233+
protoc --version
234+
235+
- name: Archive wheels
236+
uses: actions/upload-artifact@v4
237+
with:
238+
name: dist-manylinux-aarch64
239+
path: target/wheels/*
240+
241+
build-sdist:
242+
needs: [generate-license]
243+
name: Source distribution
244+
runs-on: ubuntu-latest
245+
steps:
246+
- uses: actions/checkout@v4
247+
- run: rm LICENSE.txt
248+
- name: Download LICENSE.txt
249+
uses: actions/download-artifact@v4
250+
with:
251+
name: python-wheel-license
252+
path: .
253+
254+
- name: Build sdist
255+
uses: PyO3/maturin-action@v1
256+
with:
257+
rust-toolchain: stable
258+
manylinux: auto
259+
rustup-components: rust-std rustfmt
260+
args: --release --sdist --out dist
261+
before-script-linux: |
262+
yum install -y wget
263+
cd /
264+
wget https://github.com/protocolbuffers/protobuf/releases/download/v29.3/protoc-29.3-linux-x86_64.zip
265+
unzip -o proto*zip
266+
cd -
267+
which protoc
268+
protoc --version
269+
270+
- name: Assert sdist build does not generate wheels
271+
run: |
272+
if [ "$(ls -A target/wheels)" ]; then
273+
echo "Error: Sdist build generated wheels"
274+
exit 1
275+
else
276+
echo "Directory is clean"
277+
fi
278+
shell: bash
279+
280+
merge-build-artifacts:
281+
runs-on: ubuntu-latest
282+
needs:
283+
- build-python-mac-win
284+
- build-macos-x86_64
285+
- build-manylinux-x86_64
286+
- build-manylinux-aarch64
287+
- build-sdist
288+
steps:
289+
- name: Merge Build Artifacts
290+
uses: actions/upload-artifact/merge@v4
291+
with:
292+
name: dist
293+
pattern: dist-*

‎Cargo.lock‎

Lines changed: 19 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ tonic-build = { version = "0.8", default-features = false, features = [
8585
"prost",
8686
] }
8787
url = "2"
88+
protobuf-src = "2.1"
8889

8990
[dev-dependencies]
9091
tempfile = "3.17"

LICENSE renamed to LICENSE.txt

File renamed without changes.

0 commit comments

Comments
 (0)