Skip to content

Commit 8b1a6ce

Browse files
davidhewittdavidhewitt
authored
publish python package (#93)
1 parent 1df24cc commit 8b1a6ce

File tree

12 files changed

+657
-332
lines changed

12 files changed

+657
-332
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 356 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ jobs:
6565
strategy:
6666
fail-fast: false
6767
matrix:
68-
runs-on: [macos-latest, macos-latest-xlarge]
68+
runs-on: [macos-latest, macos-latest]
6969

7070
runs-on: ${{ matrix.runs-on }}
7171

@@ -129,7 +129,7 @@ jobs:
129129
strategy:
130130
fail-fast: false
131131
matrix:
132-
runs-on: [ubuntu-latest, macos-latest-xlarge]
132+
runs-on: [ubuntu-latest, macos-latest]
133133

134134
runs-on: ${{ matrix.runs-on }}
135135

@@ -180,6 +180,327 @@ jobs:
180180

181181
- run: cargo doc -F python
182182

183+
build-sdist:
184+
name: build sdist
185+
runs-on: ubuntu-latest
186+
steps:
187+
- uses: actions/checkout@v4
188+
- uses: PyO3/maturin-action@v1
189+
with:
190+
command: sdist
191+
args: --out dist
192+
rust-toolchain: stable
193+
working-directory: crates/jiter-python
194+
- uses: actions/upload-artifact@v4
195+
with:
196+
name: pypi_files_sdist
197+
path: crates/jiter-python/dist
198+
199+
build:
200+
name: build on ${{ matrix.os }} (${{ matrix.target }} - ${{ matrix.interpreter || 'all' }}${{ matrix.os == 'linux' && format(' - {0}', matrix.manylinux == 'auto' && 'manylinux' || matrix.manylinux) || '' }})
201+
# only run on push to main and on release
202+
if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build')
203+
strategy:
204+
fail-fast: false
205+
matrix:
206+
os: [linux, macos, windows]
207+
target: [x86_64, aarch64]
208+
manylinux: [auto]
209+
include:
210+
# manylinux for various platforms
211+
- os: linux
212+
manylinux: auto
213+
target: i686
214+
- os: linux
215+
manylinux: auto
216+
target: aarch64
217+
- os: linux
218+
manylinux: auto
219+
target: armv7
220+
interpreter: 3.8 3.9 3.10 3.11 3.12
221+
- os: linux
222+
manylinux: auto
223+
target: ppc64le
224+
interpreter: 3.8 3.9 3.10 3.11 3.12
225+
- os: linux
226+
manylinux: auto
227+
target: s390x
228+
interpreter: 3.8 3.9 3.10 3.11 3.12
229+
230+
# musllinux
231+
- os: linux
232+
manylinux: musllinux_1_1
233+
target: x86_64
234+
- os: linux
235+
manylinux: musllinux_1_1
236+
target: aarch64
237+
238+
# macos;
239+
# all versions x86_64
240+
# older pythons which can't be run on the arm hardware for PGO
241+
- os: macos
242+
target: x86_64
243+
- os: macos
244+
target: aarch64
245+
interpreter: 3.8 3.9
246+
247+
# windows;
248+
# aarch64 only 3.11 and up, also not PGO optimized
249+
- os: windows
250+
target: i686
251+
python-architecture: x86
252+
interpreter: 3.8 3.9 3.10 3.11 3.12
253+
# FIXME had link failures, needs investigation
254+
# - os: windows
255+
# target: aarch64
256+
# interpreter: 3.11 3.12
257+
exclude:
258+
- os: windows
259+
target: aarch64
260+
261+
runs-on: ${{ (matrix.os == 'linux' && 'ubuntu') || matrix.os }}-latest
262+
steps:
263+
- uses: actions/checkout@v4
264+
265+
- name: set up python
266+
uses: actions/setup-python@v5
267+
with:
268+
python-version: '3.11'
269+
architecture: ${{ matrix.python-architecture || 'x64' }}
270+
271+
- name: build wheels
272+
uses: PyO3/maturin-action@v1
273+
with:
274+
target: ${{ matrix.target }}
275+
manylinux: ${{ matrix.manylinux }}
276+
args: --release --out dist --interpreter ${{ matrix.interpreter || '3.8 3.9 3.10 3.11 3.12' }}
277+
rust-toolchain: stable
278+
docker-options: -e CI
279+
working-directory: crates/jiter-python
280+
281+
- run: ${{ (matrix.os == 'windows' && 'dir') || 'ls -lh' }} crates/jiter-python/dist/
282+
283+
- run: |
284+
pip install -U twine
285+
twine check --strict crates/jiter-python/dist/*
286+
287+
- uses: actions/upload-artifact@v4
288+
with:
289+
name: pypi_files_${{ matrix.os }}_${{ matrix.target }}_${{ matrix.interpreter || 'all' }}_${{ matrix.manylinux }}
290+
path: crates/jiter-python/dist
291+
292+
build-pgo:
293+
name: build pgo-optimized on ${{ matrix.os }} / ${{ matrix.interpreter }}
294+
# only run on push to main and on release
295+
if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build')
296+
strategy:
297+
fail-fast: false
298+
matrix:
299+
os: [linux, windows, macos]
300+
interpreter: ['3.8', '3.9', '3.10', '3.11', '3.12']
301+
include:
302+
# standard runners with override for macos arm
303+
- os: linux
304+
runs-on: ubuntu-latest
305+
- os: windows
306+
ls: dir
307+
runs-on: windows-latest
308+
- os: macos
309+
runs-on: macos-latest
310+
exclude:
311+
# macos arm only supported from 3.10 and up
312+
- os: macos
313+
interpreter: '3.8'
314+
- os: macos
315+
interpreter: '3.9'
316+
317+
runs-on: ${{ matrix.runs-on }}
318+
steps:
319+
- uses: actions/checkout@v4
320+
321+
- name: set up python
322+
uses: actions/setup-python@v5
323+
with:
324+
python-version: ${{ matrix.interpreter }}
325+
326+
- name: install rust stable
327+
id: rust-toolchain
328+
uses: dtolnay/rust-toolchain@stable
329+
with:
330+
components: llvm-tools
331+
332+
- run: rustc --version --verbose
333+
334+
- name: build initial wheel
335+
uses: PyO3/maturin-action@v1
336+
with:
337+
manylinux: auto
338+
args: >
339+
--release
340+
--out pgo-wheel
341+
--interpreter ${{ matrix.interpreter }}
342+
rust-toolchain: stable
343+
docker-options: -e CI
344+
working-directory: crates/jiter-python
345+
env:
346+
RUSTFLAGS: "-Cprofile-generate=${{ github.workspace }}/profdata"
347+
348+
- name: detect rust host
349+
run: echo RUST_HOST=$(rustc -Vv | grep host | cut -d ' ' -f 2) >> "$GITHUB_ENV"
350+
shell: bash
351+
352+
- name: generate pgo data
353+
run: |
354+
cd crates/jiter-python
355+
pip install -U pip
356+
pip install -r tests/requirements.txt
357+
pip install jiter --no-index --no-deps --find-links pgo-wheel --force-reinstall
358+
python bench.py jiter jiter-cache
359+
rustup run stable bash -c 'echo LLVM_PROFDATA=$RUSTUP_HOME/toolchains/$RUSTUP_TOOLCHAIN/lib/rustlib/${{ env.RUST_HOST }}/bin/llvm-profdata >> "$GITHUB_ENV"'
360+
361+
- name: merge pgo data
362+
run: |
363+
cd crates/jiter-python
364+
${{ env.LLVM_PROFDATA }} merge -o ${{ github.workspace }}/merged.profdata ${{ github.workspace }}/profdata
365+
366+
- name: build pgo-optimized wheel
367+
uses: PyO3/maturin-action@v1
368+
with:
369+
manylinux: auto
370+
args: >
371+
--release
372+
--out dist
373+
--interpreter ${{ matrix.interpreter }}
374+
rust-toolchain: stable
375+
docker-options: -e CI
376+
working-directory: crates/jiter-python
377+
env:
378+
RUSTFLAGS: "-Cprofile-use=${{ github.workspace }}/merged.profdata"
379+
380+
- run: ${{ matrix.ls || 'ls -lh' }} crates/jiter-python/dist/
381+
382+
- uses: actions/upload-artifact@v4
383+
with:
384+
name: pypi_files_${{ matrix.os }}_${{ matrix.interpreter }}
385+
path: crates/jiter-python/dist
386+
387+
inspect-pypi-assets:
388+
needs: [build, build-sdist, build-pgo]
389+
runs-on: ubuntu-latest
390+
391+
steps:
392+
- uses: actions/checkout@v4
393+
394+
- name: get dist artifacts
395+
uses: actions/download-artifact@v4
396+
with:
397+
pattern: pypi_files_*
398+
merge-multiple: true
399+
path: dist
400+
401+
- name: list dist files
402+
run: |
403+
ls -lh dist/
404+
ls -l dist/
405+
echo "`ls dist | wc -l` files"
406+
407+
- name: extract and list sdist file
408+
run: |
409+
mkdir sdist-files
410+
tar -xvf dist/*.tar.gz -C sdist-files
411+
tree -a sdist-files
412+
413+
- name: extract and list wheel file
414+
run: |
415+
ls dist/*cp310-manylinux*x86_64.whl | head -n 1
416+
python -m zipfile --list `ls dist/*cp310-manylinux*x86_64.whl | head -n 1`
417+
418+
test-builds-arch:
419+
name: test build on ${{ matrix.target }}-${{ matrix.distro }}
420+
needs: [build]
421+
runs-on: ubuntu-latest
422+
423+
strategy:
424+
fail-fast: false
425+
matrix:
426+
target: [aarch64, armv7, s390x, ppc64le]
427+
distro: ['ubuntu22.04']
428+
include:
429+
- target: aarch64
430+
distro: alpine_latest
431+
432+
steps:
433+
- uses: actions/checkout@v4
434+
435+
- name: get dist artifacts
436+
uses: actions/download-artifact@v4
437+
with:
438+
pattern: pypi_files_linux_*
439+
merge-multiple: true
440+
path: crates/jiter-python/dist
441+
442+
- uses: uraimo/[email protected]
443+
name: install & test
444+
with:
445+
arch: ${{ matrix.target }}
446+
distro: ${{ matrix.distro }}
447+
githubToken: ${{ github.token }}
448+
install: |
449+
set -x
450+
if command -v apt-get &> /dev/null; then
451+
echo "installing python & pip with apt-get..."
452+
apt-get update
453+
apt-get install -y --no-install-recommends python3 python3-pip python3-venv git
454+
else
455+
echo "installing python & pip with apk..."
456+
apk update
457+
apk add python3 py3-pip git
458+
fi
459+
run: |
460+
cd crates/jiter-python
461+
set -x
462+
# typing-extensions isn't automatically installed because of `--no-index --no-deps`
463+
python3 -m venv venv
464+
source venv/bin/activate
465+
python3 -m pip install -U pip typing-extensions
466+
python3 -m pip install -r tests/requirements.txt
467+
python3 -m pip install jiter --no-index --no-deps --find-links dist --force-reinstall
468+
python3 -c 'import jiter; print(jiter.__version__)'
469+
470+
test-builds-os:
471+
name: test build on ${{ matrix.os }}
472+
needs: [build, build-pgo]
473+
474+
strategy:
475+
fail-fast: false
476+
matrix:
477+
os: [ubuntu, macos, windows]
478+
479+
runs-on: ${{ matrix.os }}-latest
480+
steps:
481+
- uses: actions/checkout@v4
482+
483+
- name: set up python
484+
uses: actions/setup-python@v5
485+
with:
486+
python-version: '3.11'
487+
488+
- name: get dist artifacts
489+
uses: actions/download-artifact@v4
490+
with:
491+
pattern: pypi_files_*
492+
merge-multiple: true
493+
path: crates/jiter-python/dist
494+
495+
- name: run tests
496+
run: |
497+
cd crates/jiter-python
498+
pip install typing-extensions
499+
pip install -r tests/requirements.txt
500+
pip install jiter --no-index --no-deps --find-links dist --force-reinstall
501+
pytest
502+
503+
183504
# https://github.com/marketplace/actions/alls-green#why used for branch protection checks
184505
check:
185506
if: always()
@@ -193,7 +514,7 @@ jobs:
193514

194515
release:
195516
needs: [check]
196-
if: "success() && startsWith(github.ref, 'refs/tags/')"
517+
if: "success() && startsWith(github.ref, 'refs/tags/v')"
197518
runs-on: ubuntu-latest
198519
environment: release
199520

@@ -208,3 +529,35 @@ jobs:
208529
- run: cargo publish -p jiter
209530
env:
210531
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
532+
533+
release-python:
534+
needs: [test-builds-arch, test-builds-os, build-sdist, check]
535+
if: "success() && startsWith(github.ref, 'refs/tags/jiter-python-v')"
536+
runs-on: ubuntu-latest
537+
environment: release-python
538+
539+
steps:
540+
- uses: actions/checkout@v4
541+
542+
- name: set up python
543+
uses: actions/setup-python@v5
544+
with:
545+
python-version: '3.10'
546+
547+
- name: get dist artifacts
548+
uses: actions/download-artifact@v4
549+
with:
550+
pattern: pypi_files_*
551+
merge-multiple: true
552+
path: crates/jiter-python/dist
553+
554+
- name: check assets
555+
run: |
556+
pip install -U twine
557+
cd crates/jiter-python
558+
twine check --strict dist/*
559+
560+
- name: upload to pypi
561+
uses: pypa/gh-action-pypi-publish@release/v1
562+
with:
563+
packages-dir: crates/jiter-python/dist/

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,4 @@ env*/
99

1010
**/Cargo.lock
1111
profile.json
12+
__pycache__

0 commit comments

Comments
 (0)